Question: hadoop file system I am trying torun the code below in hdfs but it couldn't work well it is LetterCount similar to wordcount example import
hadoop file system
I am trying torun the code below in hdfs but it couldn't work well it is LetterCount similar to wordcount example
import java.io.IOException;
import java.util.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
public class LetterCount {
//create class for count letter frequency in separate elements (for tasks)
public static class LettersMapper
extends Mapper
//base for separate is number of line (we read text line by line), line
//output: characters and frequence
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
//go throw character of line which was converted to lower case
for (char s : line.toLowerCase().toCharArray()) {
//add 1 for frequence
context.write(s, new IntWritable(1));
}
}
}
//class for join all the tasks
public static class LetterReducer
extends Reducer
//we join all values for same key
public void reduce(Character key, Iterable
throws IOException, InterruptedException {
int count = 0;
for (IntWritable value : values) {
count += value.get();
}
context.write(key, new IntWritable(count));
}
}
//main program
public static void main(String[] args)
throws Exception {
//check for amount aruments
if (args.length != 2) {
System.err.println("Usage: LetterCount
System.exit(-1);
}
//set parameters of the job
Configuration conf;
conf=new Configuration();
Job job = new Job(conf, "letterCount");
job.setJarByClass(LetterCount.class);
job.setJobName("Letter Count without case sensetive");
job.setNumReduceTasks(2);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setMapperClass(LettersMapper.class);
job.setReducerClass(LetterReducer.class);
job.setOutputKeyClass(Character.class);
job.setOutputValueClass(IntWritable.class);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
Step by Step Solution
There are 3 Steps involved in it
Get step-by-step solutions from verified subject matter experts
