Slightly Improved Hadoop Word Count Example and Documentation

Author: thomasWeise

hadoop/README.md

@@ -20,9 +20,11 @@ If a word occurs multiple times in a line, one token is emitted for each occurre
 
     <Text (the word), Iterable<WriteableInteger> (number of occurrences)>
 
-The reducer also acts as combinator, meaning that a reduction step is also performed locally before the results of each mapper is sent to the central reduction step. It will add up all the occurrences to a single number and thus emit tuples of the form
+Hadoop as put all the `WriteableInteger` generated by the mapping step which belong to the same `Text (the word)` key into an `Iterable` list for us. Thus, for each word that the mapper has discovered, we get a list with numbers. All we have to do is to add them up and emit tuples of the form:
 
     <Text (the word), WriteableInteger (total number of occurrences)>
+    
+The reducer here also acts as combinator, meaning that a reduction step is also performed locally before the results of each mapper is sent to the central reduction step. This way we can already add up some word counts locally and the amount of data that needs to be sent to the central reducer decreases, as two tuples for the same word are already merged. This is possible in this simple form because the output of the reducer is the same as the output of the mapper, just that the `WriteableInteger` part will not necessarily have value `1` afterwards. 
 
 After the reduction step, we therefore know how often each word occurred in the text. Furthermore, since the tuples are sorted automatically before reduction, the word/occurrences list is also nicely sorted alphabetically.
 

hadoop/wordCount/pom.xml

@@ -36,7 +36,7 @@
 		<project.build.sourceEncoding>${encoding}</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>${encoding}</project.reporting.outputEncoding>
 		<jdk.version>1.7</jdk.version>
-<project.mainClass>wordCount.WordCountDriver</project.mainClass>
+		<project.mainClass>wordCount.WordCountDriver</project.mainClass>
 	</properties>
 
 	<licenses>

hadoop/wordCount/src/main/java/wordCount/WordCountDriver.java

@@ -16,21 +16,17 @@
 public class WordCountDriver extends Configured implements Tool {
 
   public static void main(final String[] args) throws Exception {
-    try {
-      final int res = ToolRunner.run(new Configuration(),
-          new WordCountDriver(), args);
-      System.exit(res);
-    } catch (final Exception e) {
-      e.printStackTrace();
-      System.exit(255);
-    }
+    System.exit(ToolRunner.run(new Configuration(), //
+        new WordCountDriver(), args));
   }
 
   @Override
   public int run(final String[] args) throws Exception {
+    final Configuration conf;
+    final Job job;
 
-    final Configuration conf = new Configuration();
-    final Job job = Job.getInstance(conf, "Your job name");
+    conf = new Configuration();
+    job = Job.getInstance(conf, "Word Count Map-Reduce");
 
     job.setJarByClass(WordCountDriver.class);
 
@@ -39,25 +35,17 @@ public int run(final String[] args) throws Exception {
     }
 
     job.setMapperClass(WordCountMapper.class);
-
-    // job.setMapOutputKeyClass(Text.class);
-    // job.setMapOutputValueClass(IntWritable.class);
-
     job.setReducerClass(WordCountReducer.class);
     job.setCombinerClass(WordCountReducer.class);
 
     job.setOutputKeyClass(Text.class);
     job.setOutputValueClass(IntWritable.class);
 
     job.setInputFormatClass(TextInputFormat.class);
-
     job.setOutputFormatClass(TextOutputFormat.class);
 
-    final Path filePath = new Path(args[0]);
-    FileInputFormat.setInputPaths(job, filePath);
-
-    final Path outputPath = new Path(args[1]);
-    FileOutputFormat.setOutputPath(job, outputPath);
+    FileInputFormat.setInputPaths(job, new Path(args[0]));
+    FileOutputFormat.setOutputPath(job, new Path(args[1]));
 
     job.waitForCompletion(true);
     return 0;