一、WordCount代码编写

1、maven 依赖

<dependencies>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.2.4</version>
            <!--hadoop内部依赖jdk.tools，需要排除，否则打包时会报错-->
            <exclusions>
                <exclusion>
                    <artifactId>jdk.tools</artifactId>
                    <groupId>jdk.tools</groupId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.2.4</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>3.2.4</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-common</artifactId>
            <version>3.2.4</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>RELEASE</version>
        </dependency>
    </dependencies>

2、Mapper代码

package cn.nuwa.hap.mp;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    Text k = new Text();
    IntWritable v = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] words = line.split(" ");
        for (String word : words) {
            k.set(word);
            context.write(k, v);
        }


    }
}

3、reduce代码

package cn.nuwa.hap.mp;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;


public class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable> {


    int sum;
    IntWritable v = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
        sum = 0;
        for (IntWritable count : values) {
            sum += count.get();
        }

        v.set(sum);
        context.write(key, v);
    }
}

4、驱动代码

package cn.nuwa.hap.mp;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class WordCountDriver {
    public static void main(String[] args) throws Exception {
        // 1 获取配置信息以及获取job对象
        Configuration entries = new Configuration();
        Job job = Job.getInstance(entries);

        // 2 关联本Driver程序的jar
        job.setJarByClass(WordCountDriver.class);

        // 3 关联Mapper和Reducer的jar
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReduce.class);

        // 4 设置Mapper输出的kv类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        // 5 设置最终输出kv类型
        job.setMapOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        // 6 设置输入和输出路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        // 7 提交job
        boolean b = job.waitForCompletion(true);
        System.exit(b ? 0 : 1);

    }
}

二、本地测试

输入文件内容

atguigu atguigu
ss ss
cls cls
jiao
banzhang
xue
hadoop

编辑启动参数 。参数1：输入文件路径，参数2：最终输出文件夹（文件夹存在会报异常，每次运行都需要将文件夹删除，或者指定一个不存在的文件夹）

 预期的输出内容

atguigu    2
banzhang    1
cls    2
hadoop    1
jiao    1
ss    2
xue    1

三、集群测试

代码打包

会在项目路径下多一个target文件夹，jar包就在target文件夹下 

将jar包拷贝到hadoop服务器上，每个hadoop节点都需要拷贝一份

将输入文件上传到集群环境的HDFS上

运行代码

# hadoop jar 语法
# lib/native/examples/hadoop-test.jar jar包的路径
# cn.nuwa.hap.mp.WordCountDriver 主程序的全类名
# /input/1.txt 输入文件路径（HDFS路径）
# /output/wc 输出文件路径（HDFS路径）
hadoop jar  lib/native/examples/hadoop-test.jar cn.nuwa.hap.mp.WordCountDriver /input/1.txt /output/wc

最终结果