package com.bfd.util; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; public class CopyOfGidAddCartTemp { public static final String TABLE_NAME = "_AddCart_TEMP"; public static final String COLUMN_FAMILY = "ci"; private static Configuration conf = null; static { conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", Const.ZOOKEEPER_QUORAM); conf.set("zookeeper.znode.parent", Const.ZOOKEEPER_ZNODE_PARENT); } static class Mapper extends org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, ImmutableBytesWritable, LongWritable> { private ImmutableBytesWritable outKey = new ImmutableBytesWritable(); private LongWritable outValue = new LongWritable(); @Override protected void map( LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, ImmutableBytesWritable, LongWritable>.Context context) throws IOException, InterruptedException { context.write(new ImmutableBytesWritable(), new LongWritable()); } } static class Reducer extends org.apache.hadoop.mapreduce.Reducer<ImmutableBytesWritable, LongWritable, ImmutableBytesWritable, KeyValue> { public void reduce(ImmutableBytesWritable key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { context.write(key,new KeyValue()); } } public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = new Job(conf, "_AddCart_TEMP"); job.setJarByClass(CopyOfGidAddCartTemp.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(ImmutableBytesWritable.class); job.setOutputValueClass(KeyValue.class); job.setMapperClass(com.bfd.util.CopyOfGidAddCartTemp.Mapper.class); job.setReducerClass(com.bfd.util.CopyOfGidAddCartTemp.Reducer.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(HFileOutputFormat.class); job.setNumReduceTasks(4); /* 本地执行 */ // ((JobConf) job.getConfiguration()).setJar(jarFile.toString()); TextInputFormat.setInputPaths(job, Const.HDFS_BASE_INPUT + "/l_date=" + args[0] + "/*"); HFileOutputFormat.setOutputPath(job, new Path(Const.HDFS_BASE_OUTPUT + "/addcart")); Configuration HBASE_CONFIG = new Configuration(); HBASE_CONFIG.set("hbase.zookeeper.quorum", Const.ZOOKEEPER_QUORAM); HBASE_CONFIG.set("zookeeper.znode.parent", Const.ZOOKEEPER_ZNODE_PARENT); HBASE_CONFIG.set("date2", args[0]); Configuration cfg = HBaseConfiguration.create(HBASE_CONFIG); HTable htable = new HTable(cfg, TABLE_NAME); HFileOutputFormat.configureIncrementalLoad(job, htable); System.exit(job.waitForCompletion(true) ? 0 : 1); } }
相关推荐
mapreduce方式入库hbase hive hdfs,速度很快,里面详细讲述了代码的编写过程,值得下载
利用hadoop的mapreduce把oracle/mysql中的数据导入到hbase和hdfs中的两个java程序
对Hadoop中的HDFS、MapReduce、Hbase系列知识的介绍。如果想初略了解Hadoop 可下载观看
大数据技术 Hadoop开发者第二期 Nutch MapReduce HDFS Hive Mahout HBase 共64页.r
NULL 博文链接:https://jsh0401.iteye.com/blog/2096103
通过对Hadoop分布式计算平台最核心的分布式文件系统HDFS、MapReduce处理过程,以及数据仓库工具Hive和分布式数据库Hbase的介绍,基本涵盖了Hadoop分布式平台的所有技术核心。通过这一阶段的调研总结,从内部机理的...
8、短评:HDFS、MapReduce和HBase三者相辅相成、各有长处 ..... - 34 - 9、HDFS在web开发中的应用................................. - 35 - 10、Mapreduce中value集合的二次排序 ....................... - 38 - 11...
Hadoop-0.20.0-HDFS+MapReduce+Hive+HBase十分钟快速入门
HBase – Hadoop Database,是一...Google运行MapReduce来处理Bigtable中的海量数据,HBase同样利用Hadoop MapReduce来处理HBase中的海量数据;Google Bigtable利用 Chubby作为协同服务,HBase利用Zookeeper作为对应。
HDFS+MapReduce+Hive+HBase十分钟快速入门.pdf
google三大论文 gfs bigtable mapreduce hadoop hdfs hbase原型,学hadoop 必看
HDFS+MapReduce+Hive+HBase十分钟快速入门
该案例中主要使用MapReduce作为处理组件进行数据处理,实现的案例有如通过javaapi实现hbase数据写入hdfs、hbase表数据复制到另一个表中等操作 对应(《HBase分布式存储系统应用》胡鑫喆 张志刚著)教材中案例
上图描述了Hadoop EcoSystem中的各层系统,其中HBase位于结构化存储层,Hadoop HDFS为HBase提供了高可靠性的底层存储支持,Hadoop MapReduce为HBase提供了高性能的计算能力,Zookeeper为HBase提供了稳定服务和...
基于Python+SpringBoot+Vue+HDFS+MapReduce+HBase+Hive+Kafka+Sp
系统和 MapReduce 编程模型来应对这一挑战.提出了大数据环境中一种基于 Hadoop 分布式文件系统(HDFS)和 MapReduce 编程模型的分布式 MOLAP 技术,称为 DOLAP(distributed OLAP).DOLAP 采用一种特殊的多维模型完 成...
hadoopHA with QJM环境搭建(hdfs+hbase)
HDFS+MapReduce+Hive+HBase十分钟快速入门,包括这几个部分的简单使用
hadoop基础,hdfs,hive,mapreduce,hbase