Error: java.lang.ClassCastException: wordCountTest.WordCountTest cannot be cast to org.apache.hadoop.mapreduce.Mapper

Question

我是一名尝试使用 java 学习 map-reduce 的初学者。我正在尝试运行使用 oozie 协调器的字数统计示例。我收到一个铸造错误。如果有人帮我解决这个错误就太好了。

"Error: java.lang.ClassCastException: wordCountTest.WordCountTest cannot be cast to org.apache.hadoop.mapreduce.Mapper"

这是我的 WordCountTest.java 片段：

package wordCountTest;

import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCountTest {
 public static class TokenizerMapper
            extends Mapper<Object, Text, Text, IntWritable>{
  private final static IntWritable one = new IntWritable(1);
  private Text word = new Text();
  public void map(Object key, Text value, Context context
        ) throws IOException, InterruptedException {
   StringTokenizer itr = new StringTokenizer(value.toString());
   while (itr.hasMoreTokens()) {
    word.set(itr.nextToken());
    context.write(word, one);
   }
  }
 }
 public static class IntSumReducer
            extends Reducer<Text,IntWritable,Text,IntWritable> {
  private IntWritable result = new IntWritable();    
  public void reduce(Text key, Iterable<IntWritable> values,
                           Context context
        ) throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
    sum += val.get();
   }
   result.set(sum);
   context.write(key, result);
  }
 }
 public static void main(String[] args) throws Exception {
  Configuration conf = new Configuration();
  Job job = new Job(conf,"WordCount");
  job.setJarByClass(WordCountTest.class);
  job.setMapperClass(TokenizerMapper.class);
  job.setCombinerClass(IntSumReducer.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);
  FileInputFormat.addInputPath(job, new Path(args[0]));
  FileOutputFormat.setOutputPath(job, new Path(args[1]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
 }
}

此外，我的 workflow.xml 文件：

<?xml version="1.0" encoding="UTF-8"?>
<workflow-app xmlns="uri:oozie:workflow:0.1" name="map-reduce-wf">
 <start to="mr-node" />
 <action name="mr-node">
  <map-reduce>
   <job-tracker>${jobTracker}</job-tracker>
   <name-node>${nameNode}</name-node>
   <configuration>
    <property>
     <name>mapred.mapper.new-api</name>
     <value>true</value>
    </property>
    <property>
     <name>mapred.reducer.new-api</name>
     <value>true</value>
    </property>
    <property>
     <name>mapred.job.queue.name</name>
     <value>${queueName}</value>
    </property>
    <property>
     <name>mapreduce.map.class</name>
     <value>wordCountTest.WordCountTest.TokenizerMapper</value>
    </property>
    <property>
     <name>mapreduce.reduce.class</name>
     <value>wordCountTest.WordCountTest.IntSumReducer</value>
    </property>
    <property>
     <name>mapreduce.combine.class</name>
     <value>wordCountTestEmr.WordCountTest.IntSumReducer</value>
    </property>
    <property>
     <name>mapred.output.key.class</name>
     <value>org.apache.hadoop.io.Text</value>
    </property>
    <property>
     <name>mapred.output.value.class</name>
     <value>org.apache.hadoop.io.IntWritable</value>
    </property>
    <property>
     <name>mapred.input.dir</name>
     <value>/user/ahegde/testemr/InputData/</value>
    </property>
    <property>
     <name>mapred.output.dir</name>
     <value>/user/ahegde/testemr/OutputData</value>
    </property>
   </configuration>
  </map-reduce>
 <ok to="end" />
 <error to="fail" />
 </action>
  <kill name="fail">
   <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
  </kill>
  <end name="end" />
 </workflow-app>

Answer 1

您可能需要更改您的 Oozie 配置，因为它们是内部静态的类到：

<property>
    <name>mapreduce.map.class</name>
    <value>wordCountTest.WordCountTest$TokenizerMapper</value>
</property>

变化是 . 到 $。

Error: java.lang.ClassCastException: wordCountTest.WordCountTest cannot be cast to org.apache.hadoop.mapreduce.Mapper

Error: java.lang.ClassCastException: wordCountTest.WordCountTest cannot be cast to org.apache.hadoop.mapreduce.Mapper

java

hadoop

mapreduce

oozie

oozie-coordinator