`

如何在本地eclipse运行mapreduce程序

阅读更多
因为不是太复杂,所以只贴代码了,有注释,相信朋友没都能看懂。

public static void main(String[] args) {
//在本地windows平台eclipse运行mapreduce程序

//创建远程用户,以指定的用户来运行程序
//把要运行的程序代码放到run方法体里
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hadoop");
ugi.doAs(new PrivilegedAction<Void>() {
public Void run() {

try{

//设置引用jar的分割符,linux一般用,和:,但windows一般是用;,
//所以要设置一下本地的分割符
System.setProperty("path.separator", ":");

Configuration conf = new Configuration();

//可以设置用到的第三方jar
//conf.set("tmpjars", "/tmp/jars/hbase-0.94.5.jar,/tmp/jars/protobuf-java-2.4.0a.jar,/tmp/jars/zookeeper-3.4.3.jar");

conf.set("mapred.job.tracker", "172.168.8.54:9001");
conf.set("fs.default.name", "hdfs://172.168.8.54:9000");
conf.set("hadoop.job.ugi", "hadoop");
conf.set("hbase.zookeeper.quorum","172.168.8.55,172.168.8.56,172.168.8.57");
conf.set("hbase.zookeeper.property.clientPort", "2181");
Job job = new Job(conf);
job.setJobName("ReadHbaseToHdfsAction");
job.setJarByClass(ReadHbaseToHdfsAction.class);

job.setNumReduceTasks(1);
job.setReducerClass(ReadHbaseToHdfsReduce.class);

FileSystem fs=FileSystem.get(conf);
Path outPath=new Path("/tmp/2/1");
if(fs.exists(outPath)){
fs.delete(outPath, true);
}
FileOutputFormat.setOutputPath(job, outPath);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);

Scan scan = new Scan();

TableMapReduceUtil.initTableMapperJob("misdn_catetory22", scan, ReadHbaseToHdfsMapper.class,
Text.class, Text.class, job);


job.waitForCompletion(true);
}catch(Exception e){
e.printStackTrace();
}
return null;
}});
}
分享到:
评论

相关推荐

Global site tag (gtag.js) - Google Analytics