hadoop Mapper이용 text파일을 SequenceFileFormat으로 저장

Notice

Recent Posts

Recent Comments

Link

« 2024/09 »
일	월	화	수	목	금	토
1	2	3	4	5	6	7
8	9	10	11	12	13	14
15	16	17	18	19	20	21
22	23	24	25	26	27	28
29	30

Tags more

Archives

Today

Total

관리 메뉴

DBILITY

hadoop Mapper이용 text파일을 SequenceFileFormat으로 저장 본문

bigdata/hadoop

hadoop Mapper이용 text파일을 SequenceFileFormat으로 저장

DBILITY 2017. 3. 6. 21:28

아무 이유 없이 그냥 한번 해봤다.

윈도 환경이라 압축코덱을 설정하기 어려워 DefaultCodec은 사용이 되길래 사용하였고, eclipse에서 테스트되었습니다.

Txt2SequenceFileConverter는 csv형태의 Text파일을 Mapper를 통해 SequenceFile로 변환 후 저장하고,

SequenceFileRead는 SequenceFIle경로를 읽어 각 파일의 압축 정보를 출력하고, 데이터는 한 라인만 출력.
결과에 CompressorType을 보니 BuiltInZlibDeflater네요. zlib를 사용하나 보네요.
분명 native-zlib library 로딩이 안된다고 나왔는데...
누군가에겐 당연한 얘기겠지만, CompressionType에 따라 헤더가 달라지나 봅니다.
Enum으로 NONE, BLOCK, RECORD가 있네요.
대상 데이터는 ASA운항 기록 2007~8년 자료입니다.

Txt2SequenceFileConverter.java

다운로드

package com.dbility.hadoop.execise;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 *
 * Description
 *
 *
 * @author hyperrookie@gmail.com
 *
 * @version 1.0.0
 * @date 2017. 3. 6.
 */
public class Txt2SequenceFileConverter extends Configured implements Tool {

	public int run(String[] args) throws Exception {

		String[] remainArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();

		if ( remainArgs.length != 2 ) {
			System.err.println("Usage : hadoop jar jarFile [mainClass] <input_path> <output_path>");
			return -1;
		}

		Job job = new Job(getConf());
		job.setJobName("Txt2SequenceFileJob");
		job.setJarByClass(Txt2SequenceFileConverter.class);

		job.setInputFormatClass(TextInputFormat.class);

		job.setMapperClass(ConvertMapper.class);

		job.setNumReduceTasks(0);
		job.setOutputFormatClass(SequenceFileOutputFormat.class);
		job.setOutputKeyClass(NullWritable.class);
		job.setOutputValueClass(Text.class);

		SequenceFileOutputFormat.setCompressOutput(job, true);
		SequenceFileOutputFormat.setOutputCompressorClass(job, DefaultCodec.class);
		//SequenceFileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
		SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);

		Path input_path = new Path(remainArgs[0]);
		Path output_path = new Path(remainArgs[1]);

		TextInputFormat.addInputPath(job, input_path);
		SequenceFileOutputFormat.setOutputPath(job, output_path);

		FileSystem hdfs = FileSystem.get(getConf());

		if ( hdfs.exists(output_path) ) hdfs.delete(output_path, true);

		return job.waitForCompletion(true) ? 0 : -2;
	}

	public static void main(String[] args) throws Exception {

		Configuration conf = new Configuration();

		conf.set("fs.default.name", "file:///");
		conf.set("mapred.job.reducer", "local");
		conf.set(
				"io.serializations",
				"org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
		conf.set("fs.file.impl", "com.dbility.hadoop.execise.Txt2SequenceFileConverter$WindowsLocalFileSystem");
		conf.set("io.sort.mb","512");

		args = new String[] {"d:/hadoop_test/input_data/","d:/hadoop_test/sequence_data/"};

		int result = ToolRunner.run(conf, new Txt2SequenceFileConverter(), args);

		if ( result == 0 ) {
			FileSystem hdfs = FileSystem.get(conf);
			Path path = new Path(args[1]+"_SUCCESS");
			if ( hdfs.exists(path) ) {
				hdfs.delete(path, true);
			}
		}

		Runtime.getRuntime().exit(result);
	}

	public static class WindowsLocalFileSystem extends LocalFileSystem {

		public WindowsLocalFileSystem() {
			super();
		}

		public boolean mkdirs(final Path f, final FsPermission permission)
				throws IOException {
			final boolean result = super.mkdirs(f);
			this.setPermission(f, permission);
			return result;
		}

		public void setPermission(final Path p, final FsPermission permission)
				throws IOException {
			try {
				super.setPermission(p, permission);
			} catch ( final IOException ioe ) {
				System.err.println(ioe.getMessage());
			}
		}

	}

	public static class ConvertMapper extends Mapper<LongWritable, Text, NullWritable, Text> {

		@Override
		protected void map(LongWritable key, Text value,Context context)
				throws IOException, InterruptedException {
			context.write(NullWritable.get(), value);
		}
	}
}

SequenceFileRead.java

다운로드

package com.dbility.hadoop.execise;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.util.ReflectionUtils;

/**
 *
 * Description
 *
 *
 * @author hyperrookie@gmail.com
 *
 * @version 1.0.0
 * @date 2017. 3. 6.
 */
public class SequenceFileRead {

	public static void main(String[] args) throws Exception {

		Configuration conf = new Configuration();

		conf.set("fs.default.name", "file:///");
		conf.set("mapred.job.reducer", "local");
		conf.set(
				"io.serializations",
				"org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
		conf.set("fs.file.impl", "com.dbility.hadoop.execise.SequenceFileRead$WindowsLocalFileSystem");
		conf.set("io.sort.mb","512");

		args = new String[] {"d:/hadoop_test/sequence_data/"};

		FileSystem hdfs = FileSystem.get(conf);

		Path input_path = new Path(args[0]);
		int result = 0;

		if ( !hdfs.exists(input_path) ){
			System.out.printf("%s is not exists!!",args[0]);
			result = -1;
		} else {
			Path[] paths = FileUtil.stat2Paths(hdfs.listStatus(input_path));
			System.out.printf("File Count : %d\n",paths.length);
			int line = 0;
			for (Path path : paths) {

				System.out.printf("file path : %s\n",path);

				SequenceFile.Reader reader = new SequenceFile.Reader(hdfs, path, conf);

				Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
				Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
				CompressionCodec codec = reader.getCompressionCodec();

				System.out.printf("Compression Codec : %s\n",codec.getClass());
				System.out.printf("Compressor Type : %s\n",codec.getCompressorType());
				System.out.printf("Decompressor Type : %s\n",codec.getDecompressorType());
				line = 0;
				while ( reader.next(key, value) && line < 1  ) {
					System.out.printf("key : %s , value : %s\n", key,value);
					line++;
				}
				reader.close();
				System.out.printf("\n");

			}

		}

		Runtime.getRuntime().exit(result);
	}

	public static class WindowsLocalFileSystem extends LocalFileSystem {

		public WindowsLocalFileSystem() {
			super();
		}

		public boolean mkdirs(final Path f, final FsPermission permission)
				throws IOException {
			final boolean result = super.mkdirs(f);
			this.setPermission(f, permission);
			return result;
		}


		public void setPermission(final Path p, final FsPermission permission)
				throws IOException {
			try {
				super.setPermission(p, permission);
			} catch ( final IOException ioe ) {
				System.err.println(ioe.getMessage());
			}
		}

	}

}

결과

File Count : 42
file path : file:/d:/hadoop_test/sequence_data/part-m-00000
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,1,1,1,1232,1225,1341,1340,WN,2891,N351,69,75,54,1,7,SMF,ONT,389,4,11,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00001
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,1,1,1,1646,1447,1902,1703,FL,574,N982AT,136,136,98,119,119,ATL,EWR,745,7,31,0,,0,0,0,0,0,119

file path : file:/d:/hadoop_test/sequence_data/part-m-00002
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,2,18,7,1724,1730,2037,2035,XE,2287,N11150,133,125,102,2,-6,IAH,ATL,689,10,21,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00003
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,2,15,4,846,850,1158,1205,AA,476,N416AA,132,135,115,-7,-4,DFW,CMH,927,6,11,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00004
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,3,26,1,559,600,614,625,UA,349,N330UA,75,85,60,-11,-1,CLE,ORD,316,4,11,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00005
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,3,3,6,658,700,1002,1010,CO,1554,N16647,184,190,165,-8,-2,EWR,PBI,1024,4,15,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00006
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,4,24,2,1303,1307,1553,1605,DL,856,N613DL,110,118,88,-12,-4,SEA,SLC,689,7,15,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00007
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,5,20,7,1454,1455,1653,1700,WN,114,N645SW,59,65,49,-7,-1,RNO,BOI,335,2,8,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00008
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,5,27,7,944,945,1239,1245,MQ,4021,N682AE,115,120,93,-6,-1,ORD,SWF,714,8,14,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00009
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,6,13,3,938,940,1110,1105,YV,7454,N858MJ,92,85,50,5,-2,CLT,IAD,321,10,32,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00010
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,6,14,4,1930,1855,2105,2000,AA,351,N549AA,95,65,55,65,35,SAT,DFW,247,13,27,0,,0,0,0,33,0,32

file path : file:/d:/hadoop_test/sequence_data/part-m-00011
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,7,25,3,715,700,928,915,OO,4053,N804SK,193,195,163,13,15,MEM,SLC,1261,5,25,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00012
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,7,2,1,1118,1120,1358,1410,B6,576,N639JB,160,170,143,-12,-2,MCO,SWF,989,6,11,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00013
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,8,18,6,914,848,1627,1610,US,567,N915AW,253,262,226,17,26,PHX,MCO,1848,4,23,0,,0,17,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00014
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,8,12,7,1805,1805,2055,2105,DL,1027,N922DL,170,180,152,-10,0,MCO,BOS,1121,8,10,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00015
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,9,3,1,1436,1401,1729,1658,FL,436,N910AT,113,117,93,31,35,MCI,ATL,692,6,14,0,,0,0,0,0,0,31

file path : file:/d:/hadoop_test/sequence_data/part-m-00016
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,10,16,2,856,855,955,932,XE,2724,N15985,119,97,80,23,1,CLE,BNA,448,13,26,0,,0,0,0,23,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00017
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,10,7,7,1848,1850,2143,2110,NW,953,N319US,115,80,68,33,-2,MEM,IND,381,19,28,0,,0,0,0,33,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00018
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,11,7,3,849,855,950,947,OO,5765,N292SW,61,52,33,3,-6,SEA,PDX,129,8,20,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00019
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,11,10,6,1055,1045,1321,1326,AS,550,N975AS,146,161,132,-5,10,SEA,SAN,1050,2,12,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00020
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,1,3,4,2003,1955,2211,2225,WN,335,N712SW,128,150,116,-14,8,IAD,TPA,810,4,8,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00021
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,1,29,2,1733,1735,1910,1925,EV,4823,N631AS,97,110,79,-15,-2,MYR,ATL,317,16,2,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00022
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,2,27,3,1910,1905,2009,2005,WN,1677,N767SW,59,60,46,4,5,LAS,BUR,223,1,12,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00023
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,2,11,1,1235,1240,1424,1435,NW,1571,N777NC,109,115,83,-11,-5,CLT,DTW,500,15,11,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00024
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,3,16,7,1037,1029,1159,1202,OO,6518,N413SW,82,93,61,-3,8,LAX,PHX,370,4,17,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00025
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,3,6,4,850,850,1735,1712,AS,18,N581AS,345,322,310,23,0,SEA,MCO,2553,12,23,0,,0,0,0,23,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00026
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,4,11,5,1528,1530,1642,1700,US,2038,N766US,74,90,57,-18,-2,DCA,BOS,399,6,11,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00027
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,5,6,2,1228,1220,1404,1400,WN,296,N508SW,96,100,84,4,8,BNA,MCI,491,5,7,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00028
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,5,1,4,1302,1240,1553,1529,FL,659,N971AT,171,169,153,24,22,PBI,HPN,1057,11,7,0,,0,22,0,2,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00029
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,6,28,6,1135,1135,1232,1240,WN,3196,N493WN,57,65,47,-8,0,PHX,ONT,325,4,6,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00030
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,6,8,7,2053,1910,47,2152,NW,312,N557NW,174,102,126,175,103,MSP,IND,503,34,14,0,,0,94,0,72,0,9

file path : file:/d:/hadoop_test/sequence_data/part-m-00031
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,7,4,5,825,831,1047,1032,OH,5168,N685BR,82,61,40,15,-6,HSV,ATL,151,32,10,0,,0,0,0,15,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00032
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,7,9,3,841,845,1100,1110,AA,661,N447AA,139,145,119,-10,-4,MSP,DFW,852,12,8,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00033
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,8,29,5,1432,1437,1536,1539,OO,6655,N927SW,124,122,105,-3,-5,CID,DEN,692,10,9,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00034
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,8,24,7,543,550,710,713,B6,37,N523JB,87,83,59,-3,-7,BUF,JFK,301,17,11,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00035
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,9,4,4,1838,1844,1933,1927,EV,4201,N820AS,115,103,82,6,-6,ATL,FSM,579,6,27,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00036
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,10,30,4,625,625,851,905,WN,1629,N395SW,86,100,76,-14,0,MSY,MCO,550,4,6,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00037
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,10,3,5,2125,2130,2302,2315,9E,4743,87979E,97,105,71,-13,-5,MSP,SGF,528,6,20,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00038
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,11,5,3,2114,2117,2230,2258,UA,815,N852UA,136,161,117,-28,-3,DEN,LAX,862,7,12,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00039
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,12,9,2,1953,1930,2028,2020,WN,3186,N510SW,95,110,88,8,23,MCO,MSY,550,2,5,0,,0,NA,NA,NA,NA,NA

file path : file:/d:/hadoop_test/sequence_data/part-m-00040
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2007,12,16,7,1726,1730,1903,1855,US,44,N619AW,97,85,67,8,-4,DCA,CLT,331,11,19,0,,0,0,0,0,0,0

file path : file:/d:/hadoop_test/sequence_data/part-m-00041
Compression Codec : class org.apache.hadoop.io.compress.DefaultCodec
Compressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater
Decompressor Type : class org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater
key : (null) , value : 2008,12,25,4,1642,1635,1822,1830,MQ,3579,N942MQ,100,115,87,-8,7,DFW,CID,685,4,9,0,,0,NA,NA,NA,NA,NA

'bigdata > hadoop' 카테고리의 다른 글

hadoop partial sort exercise 2 ( 부분 정렬 실습 2 ) (0)	2017.03.08
hadoop Text -> SequenceFile -> MapFile로 변환 (0)	2017.03.07
hadoop total sort ( 전체 정렬 ) (0)	2017.03.01
hadoop secondary sort exercise 3 ( 보조 정렬 실습 3 ) (0)	2017.02.28
hadoop secondary sort exercise 2 ( 보조 정렬 실습 2 ) (0)	2017.02.21

공유하기 링크

페이스북
카카오스토리
트위터

'bigdata/hadoop' Related Articles

Comments

DBILITY

hadoop Mapper이용 text파일을 SequenceFileFormat으로 저장 본문

hadoop Mapper이용 text파일을 SequenceFileFormat으로 저장

'bigdata > hadoop' 카테고리의 다른 글

티스토리툴바