1.1 打包插件

<properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <hadoop.version>3.3.3</hadoop.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-cos</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
    </dependencies>

    <build>
        <finalName>MyMapReduceExtractor</finalName>
        <resources>
            <resource>
                <directory>src/main/resources</directory>
                <excludes>
                    <exclude>*</exclude>
                </excludes>
            </resource>
        </resources>

        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <configuration>
                    <archive>
                        <manifest>
                            <mainClass>mapred.MyMapReduceExtractor</mainClass>
                        </manifest>
                    </archive>
                </configuration>
            </plugin>
        </plugins>
    </build>

2.1 示例代码

/**
 * @Date:
 * @功能描述: 功能视情况添加删减，
 * @Version:1.0
 */
public class MyMapReduceExtractor {
    public static class MyMapper extends Mapper<Object, Text, Text, NullWritable> {
        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
            context.write(value, NullWritable.get());
        }
    }

    public static class MyReducer extends Reducer<Text, NullWritable, Text, NullWritable> {
        public void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
            context.write(key, NullWritable.get());
        }
    }

    public static void main(String[] args) throws Exception {
        String dateStr;
        if (args.length == 1) {
            dateStr = args[0];
        } else {
            dateStr = DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd");
        }

        Configuration conf = new Configuration();

        conf.setBoolean("mapreduce.map.output.compress", true); //打开 map 端输出压缩
        conf.setClass("mapreduce.map.output.compress.codec", GzipCodec.class, CompressionCodec.class); // 设置 map 端输出压缩方式
        // conf.setInt("mapreduce.reduce.memory.mb", 3 * 1024);//reduce DmpMediumSideExtractor运行内存
        // conf.setInt("fs.cosn.block.size", 256 * 1024 * 1024);//map 给256m
        // conf.setInt("fs.cosn.buffer.size", 256 * 1024 * 1024);//map 给256m

        Job job = Job.getInstance(conf);
        job.setJarByClass(MyMapReduceExtractor.class);
        job.setMapperClass(MyMapper.class);
        job.setReducerClass(MyReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        job.setNumReduceTasks(48);

        FileInputFormat.setInputDirRecursive(job, true);
        FileInputFormat.setInputPaths(job, "inPutPath");
        FileOutputFormat.setOutputPath(job, "outPutPath"));
        // FileOutputFormat.setCompressOutput(job, true); // 设置 reduce 端输出压缩开启
        // FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class); // 设置压缩的方式
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}