hadoop由于NodeManager无法启动而导致执行Jar包出现running job卡住的解决方案之一...

菜鸟上手大数据一头雾水...所以先自己用三个虚拟机搭建一个hadoop集群环境了

hadoop版本2.7.3

JDK版本8u101

系统CentOS7

其中有两个值得注意的地方(或者说勘误一下)：

其一第8步的mapred-site.xml应为yarn-site.xml

其二还是第8步中关于yarn.nodemanager.resource.memory-mb的设置，请设置至少为1024，否则host的nodemanager会启动失败

hadoop集群正常启动的情况下，可以用jps查看服务的端口，其中的master应该至少启动了

SecondaryNameNode
NameNode
ResourceManager
Jps

host在没有执行任务时应该至少启动了

Jps
DataNode
NodeManager

设置完毕后执行wordcount的程序，发现还是卡住了，调hadoop的日志看(日志的位置在start-all.sh)的时候会显示，默认在hadoop文件夹的log中看yarn的日志)发现分配的空间小了，我的wordcount需要1500mb空间，而设置成1024不够，于是再次将 yarn.nodemanager.resource.memory-mb都设置为2048后程序跑通，出来了想要的wordcount结果。

其中多次重新设置hadoop,我都选择了清空之前的文件以及格式化namenode,否则容易报错。可能有些暴力...等熟悉后有更好的重新设置并启动的方案吧。

最后贴个我的wordcount留作纪念....向大数据说helloword!

/**
 * Created by jacobzhou on 2016/8/29.
 */
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;

public class HadoopTest extends Configured implements Tool {
    private static String DELIMA = "\t";
    public static class MRMapper extends Mapper<Object, Text, Text, LongWritable> {
        protected void map(Object key, Text value, Mapper.Context context) throws IOException, InterruptedException {
                context.write(value,new LongWritable(1));
        }
        protected void cleanup(Context context) throws IOException, InterruptedException {}
    }
    public static class MRReducer extends Reducer<Text,LongWritable,Text,LongWritable> {
        private LongWritable result = new LongWritable();
        public void reduce(Text key, Iterable<LongWritable> values, Context context)
                throws IOException, InterruptedException {
                long sum = 0;
                for (LongWritable val : values) {
                    sum += val.get();
                }
                result.set(sum);
                context.write(key, result);
        }
    }
    public int run(String[] args) throws Exception {
        Configuration conf = getConf();
        conf.set("mapreduce.job.queuename", "datacenter");
        conf.set("mapred.max.map.failures.percent", "5");
        int reduceTasksMax = 10;
        Job job = new Job(conf);
        job.setJobName("wordCount job");
        job.setNumReduceTasks(reduceTasksMax);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        job.setMapperClass(MRMapper.class);
        job.setReducerClass(MRReducer.class);
        job.setJarByClass(HadoopTest.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        FileInputFormat.addInputPath(job,new Path(args[0]));
        TextOutputFormat.setOutputPath(job, new Path(args[1]));
        return job.waitForCompletion(true) ? 0 : 1;
    }
    public static void main(String[] args) throws Exception {
        try{
            System.out.println("start run job!");
            int ret = ToolRunner.run(new HadoopTest(), args);
            System.exit(ret);
        }catch (Exception e){
            e.printStackTrace();
        }
    }
}