.
import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; import java.util.List; import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; public class EJob { // To declare global field private static List<URL> classPath = new ArrayList<URL>(); // To declare method public static File createTempJar(String root) throws IOException { if (!new File(root).exists()) { return null; } Manifest manifest = new Manifest(); manifest.getMainAttributes().putValue("Manifest-Version", "1.0"); final File jarFile = File.createTempFile("EJob-", ".jar", new File(System.getProperty("java.io.tmpdir"))); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { jarFile.delete(); } }); JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile), manifest); createTempJarInner(out, new File(root), ""); out.flush(); out.close(); return jarFile; } private static void createTempJarInner(JarOutputStream out, File f, String base) throws IOException { if (f.isDirectory()) { File[] fl = f.listFiles(); if (base.length() > 0) { base = base + "/"; } for (int i = 0; i < fl.length; i++) { createTempJarInner(out, fl[i], base + fl[i].getName()); } } else { out.putNextEntry(new JarEntry(base)); FileInputStream in = new FileInputStream(f); byte[] buffer = new byte[1024]; int n = in.read(buffer); while (n != -1) { out.write(buffer, 0, n); n = in.read(buffer); } in.close(); } } public static ClassLoader getClassLoader() { ClassLoader parent = Thread.currentThread().getContextClassLoader(); if (parent == null) { parent = EJob.class.getClassLoader(); } if (parent == null) { parent = ClassLoader.getSystemClassLoader(); } return new URLClassLoader(classPath.toArray(new URL[0]), parent); } public static void addClasspath(String component) { if ((component != null) && (component.length() > 0)) { try { File f = new File(component); if (f.exists()) { URL key = f.getCanonicalFile().toURI().toURL(); if (!classPath.contains(key)) { classPath.add(key); } } } catch (IOException e) { } } } }
// mian方法中添加:
File jarFile = EJob.createTempJar("bin"); EJob.addClasspath("/usr/hadoop/conf"); ClassLoader classLoader = EJob.getClassLoader(); Thread.currentThread().setContextClassLoader(classLoader);
job.waitForCompletion(true) 提交之前加上下面一条:
((JobConf) job.getConfiguration()).setJar(jarFile.toString());
demo如下:
package mr2demo; import java.io.File; import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class WordCount { public static void main(String[] args) throws Exception { File jarFile = EJob.createTempJar("bin"); EJob.addClasspath("E:\\workspace\\hadoop\\mapreduce\\mr2demo\\conf"); ClassLoader classLoader = EJob.getClassLoader(); Thread.currentThread().setContextClassLoader(classLoader); Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args) .getRemainingArgs(); if (otherArgs.length < 2) { System.err.println("Usage: wordcount <in> [<in>...] <out>"); System.exit(2); } Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); for (int i = 0; i < otherArgs.length - 1; i++) { FileInputFormat.addInputPath(job, new Path(otherArgs[i])); } FileOutputFormat.setOutputPath(job, new Path(otherArgs[(otherArgs.length - 1)])); ((JobConf) job.getConfiguration()).setJar(jarFile.toString()); System.exit(job.waitForCompletion(true) ? 0 : 1); } public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> { private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable<IntWritable> values, Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } this.result.set(sum); context.write(key, this.result); } } public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> { private static final IntWritable one = new IntWritable(1); private Text word = new Text(); public void map(Object key, Text value, Mapper<Object, Text, Text, IntWritable>.Context context) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(value.toString()); while (itr.hasMoreTokens()) { this.word.set(itr.nextToken()); context.write(this.word, one); } } } }
以上解决了eclipse和插件因为兼容性,导致无法生成jar,windows上无法运行job。
相关推荐
一套eclipse中的hadoop开发环境搭建教程,附带了eclipse中的hadoop的两个demo,一个是wordcount、一个是sort,大家一起学习进步!
Hadoop集群-WordCount运行详解.pdf Hadoop,Hadoop集群,WordCount,pdf
使用hadoop实现WordCount详细实验报告,配有环境变量配置截图以及实验运行及结果详细过程描述与截图
hadoop2.7.1运行Wordcount错误 at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012) Exception in thread "main" java.lang.UnsatisfiedLinkError
用eclipse本地提交Hadoop任务(如WordCount)到服务器上跑的时候,会报错: Stack trace: ExitCodeException exitCode=1: at org.apache.hadoop.util.Shell.runCommand(Shell.java:538) at org.apache.hadoop.util....
1. 资源包括了eclipse Oxygen版本的hadoop插件,包括winutils 2. 文档详细介绍了windows 下的hadoop版本的配置 3. 实例程序可以跑通,有详细截图
window 下eclipse远程访问伪分布式hadoop 运行wordcount
Hadoop环境搭建及wordcount实例运行.pdf
包含配置hadoop过程中遇到的一些问题的解决办法和成功运行wordcount实例的步骤
hadoop wordcount 实验,成功运行 maven项目(全部源代码)
Hadoop环境搭建与WordCount实例浅析.pdf
MapReduce经典例子WordCount运行详解.pdfMapReduce经典例子WordCount运行详解.pdf
这是一个wordcount的一个简单实例jar包,仅仅用来做测试。...map类:org.apache.hadoop.wordcount.WordCountMapReduce$WordCountMapper reduce类 org.apache.hadoop.wordcount.WordCountMapReduce$WordCountReducer
<groupId>org.apache.hadoop <artifactId>hadoop-common <version>2.7.2 <groupId>org.apache.hadoop <artifactId>hadoop-client <version>2.7.2 <groupId>jdk.tools <artifactId>jdk.tools ...
Using the old API – org.apache.hadoop.mapred Going real — network monitoring data Getting and understanding the data Knowing the problems Solution approaches Approach 1 – Preaggregate the ...
hadoop入门级的代码 Java编写 eclipse可运行 包含 hdfs的文件操作 rpc远程调用的简单示例 map-reduce的几个例子:wordcount 学生平均成绩 手机流量统计
Hadoop的配置及运行WordCount.docxHadoop的配置及运行WordCount.docx
WordCount.java
ubuntu上面运行hadoop自带的wordcount实例
Hadoop 用mapreduce实现Wordcount实例,绝对能用