Hadoop的HDFS和MapReduce子框架主要是针对大数据文件来设计的,在小文件的处理上不但效率低下,
而且十分消耗磁盘空间(每一个小文件占用一个Block,HDFS默认block大小为64M)。解决办法通常是选择一个容器,
将这些小文件组织起来统一存储。HDFS提供了两种类型的容器,分别是SequenceFile和MapFile。
1 SequenceFile使用
package org.tony.file; import java.io.IOException; import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.ReflectionUtils; public class SequenceFileWriter { public static String uri = "hdfs://192.168.142.128:9000"; public static String[] data = { "one,two", "three,four", "five,six", "seven,eight", "night,ten" }; public static void main(String[] args) throws IOException { write(); read(); } /** * @Title: write * @Description: (1)创建Configuration;(2)获取FileSystem;(3)创建文件输出路径Path; (4)调用SequenceFile.createWriter得到SequenceFile.Writer对象; (5)调用SequenceFile.Writer.append追加写入文件;(6)关闭流; * @return void 返回类型 * @throws */ public static void write() throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); Path path = new Path("/tmp.seq"); // Path path = new Path("/tmp1.seq"); 采用压缩 IntWritable key = new IntWritable(); Text value = new Text(); SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass()); // SequenceFile.Writer writer = // SequenceFile.createWriter(fs,conf,path,key.getClass(),value.getClass(),CompressionType.RECORD,new // BZip2Codec()); 采用压缩,用BZip2压缩算法 for (int i = 0; i < 100; i++) { // 写入100次 key.set(100 - i); value.set(data[i % data.length]); writer.append(key, value); } IOUtils.closeStream(writer); } /** * @Title: read * @Description: (1)创建Configuration;(2)获取FileSystem;(3)创建文件输出路径Path; (4)new一个SequenceFile Reader进行读取;(5)得到keyClass和valueClass;(6)关闭流; * @return void 返回类型 * @throws */ public static void read() throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); Path path = new Path("/tmp.seq"); // Path path = new Path("/tmp1.seq"); 读取压缩文件 SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); Writable key = (Writable) ReflectionUtils.newInstance( reader.getKeyClass(), conf); Writable value = (Writable) ReflectionUtils.newInstance( reader.getValueClass(), conf); while (reader.next(key, value)) { System.out.println("key = " + key); System.out.println("value = " + value); System.out.println("position = " + reader.getPosition()); } IOUtils.closeStream(reader); } }
2 MapFile使用
package org.tony.file; import java.io.IOException; import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.MapFile; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.util.ReflectionUtils; public class MapFileTest { public static String uri = "hdfs://192.168.142.128:9000"; //访问hdfs 的uri public static String[] data = { "one,two", "three,four", "five,six", "seven,eight", "night,ten" }; public static void main(String[] args) throws Exception { // write(); read(); // seqToMapFile(); } /**用Mapfile写文件 * @Title: write * @Description: (1)创建Configuration;(2)获取FileSystem;(3)创建文件输出路径Path; (4)new一个MapFile.Writer对象;(5)调用MapFile.Writer.append追加写入文件;(6)关闭流; * @return void 返回类型 * @throws */ public static void write() throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); Path path = new Path("/tmpdata.map"); IntWritable key = new IntWritable(); Text value = new Text(); MapFile.Writer writer = new MapFile.Writer(conf, fs, path.toString(), key.getClass(), value.getClass()); for (int i = 0; i < 100; i++) { key.set(i + 1); value.set(data[i % data.length]); writer.append(key, value); } IOUtils.closeStream(writer); } /**用Mapfile读文件 * @Title: read * @Description: (1)创建Configuration;(2)获取FileSystem;(3)创建文件输出路径Path; (4)new一个MapFile.Reader对象;(5)得到keyClass和valueClass;(6)关闭流; * @return void 返回类型 * @throws */ public static void read() throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); Path path = new Path("/tmpdata.map"); MapFile.Reader reader = new MapFile.Reader(fs, path.toString(), conf); WritableComparable key = (WritableComparable) ReflectionUtils .newInstance(reader.getKeyClass(), conf); Writable value = (Writable) ReflectionUtils.newInstance( reader.getValueClass(), conf); while (reader.next(key, value)) { System.out.println("key = " + key); System.out.println("value = " + value); } IOUtils.closeStream(reader); } /**将sequence文件转换为MapFile文件 * @Title: seqToMapFile * @Description: TODO * @return void 返回类型 * @throws * 1 创建tmp1.map文件夹 * 2 复制tmp1.seq SequenceFile文件到tmp1.map文件夹下面,并重命名data $./hadoop fs -mv /tmp1.seq /tmp1.map/data * 3 运行程序 */ public static void seqToMapFile() throws Exception{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); Path map = new Path("/tmp1.map"); //文件夹位置 //MapFile.DATA_FILE_NAME 为seq文件移动到tmp1.map文件夹下面的文件名称 Path mapData = new Path(map,MapFile.DATA_FILE_NAME); SequenceFile.Reader reader = new SequenceFile.Reader(fs,mapData,conf); Class key = reader.getKeyClass(); Class value = reader.getValueClass(); reader.close(); long entries = MapFile.fix(fs, map, key, value, false, conf); System.out.printf("Created MapFile %s with %d entries\n",map,entries); } }
相关推荐
本文介绍使用MapReduce读写文件,包括:读写SequenceFile、MapFile、ORCFile和ParquetFile文件。 本文前提:hadoop环境可正常使用。pom.xml文件内容参考本专栏中的其他文章内容。 本文分为四部分,即MapReduce读写...
sequencefile&mapfile相关测试代码
Hadoop中将SequenceFile转换成MapFile的主要方法:给Sequencefile重建索引的程序
spark-SequenceFile及MapFile讲解
1.1 SequenceFile概述 1.2 SequenceFile压缩 1.4 SequenceFile读取文件 1.5 SequenceFile总结
22、MapReduce使用Gzip压缩、Snappy压缩和Lzo压缩算法写文件和读取相应的文件 网址:https://blog.csdn.net/chenwewi520feng/article/details/130456088 本文的前提是hadoop环境正常。 本文最好和MapReduce操作常见...
使用Pig和Wukong来探索10亿数量级边的 网络图 测量社区 每个人都在和我说话:Twitter回复关系图 (度)degree 对称链接 社区提取 附录A 安装Apache Hadoop 附录B Cloudera’s Distribution for Hadoop 附录C 准备...
使用Pig和Wukong来探索10亿数量级边的 网络图 测量社区 每个人都在和我说话:Twitter回复关系图 degree(度) 对称链接 社区提取 附录A 安装Apache Hadoop 先决条件 安装 配置 本机模式 伪分布...
search-1047基于Nutch和Hadoop简易搜索引擎,排序的依据主要是PageRank以及由倒排索引文件计算的url page与输入模式的余弦距离值。Nutch & HadoopNutch-1.9:. Nutch爬取产生的链接数据库(MapFile Format)linkdb,...
SequenceFile MapFile 5. Developing a MapReduce Application . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 145 The Configuration API Combining Resources Variable Expansion ...
SequenceFile 132 MapFile 139 5. Developing a MapReduce Application . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 145 The Configuration API 146 Combining Resources 147 ...