Use sequncefile input file in Mr

Convert original data to block-compressed sequencefile

 

Import Org. apache. hadoop. conf. configuration; import Org. apache. hadoop. conf. configured; import Org. apache. hadoop. FS. filesystem; import Org. apache. hadoop. FS. path; import Org. apache. hadoop. io. longwritable; import Org. apache. hadoop. io. sequencefile. compressiontype; import Org. apache. hadoop. io. text; import Org. apache. hadoop. io. compress. gzipcodec; import Org. apache. hadoop. mapreduce. job; import Org. apache. hadoop. ma Preduce. lib. input. fileinputformat; import Org. apache. hadoop. mapreduce. lib. output. fileoutputformat; import Org. apache. hadoop. mapreduce. lib. output. sequencefileoutputformat; import Org. apache. hadoop. util. tool; import Org. apache. hadoop. util. toolrunner; import COM. hadoop. compression. lzo. lzocodec; public class toseqfile extends configured implements tool {@ override public int run (string [] arg0) throws limit t Ion {job = new job (); job. setjarbyclass (getclass (); configuration conf = getconf (); filesystem FS = filesystem. get (CONF); fileinputformat. setinputpaths (job, "/home/hadoop/tmp/tmplzo.txt"); Path outdir = New Path ("/home/hadoop/tmp/tmplzo. out "); FS. delete (outdir, true); fileoutputformat. setoutputpath (job, outdir); // job. setmapperclass (indentitymapper); job. setnumreducetasks (0); job. setoutputkeyclas S (longwritable. class); job. setoutputvalueclass (text. class); // set outputformat to sequencefileoutputformat job. setoutputformatclass (sequencefileoutputformat. class); // allows the compression of sequencefileoutputformat. setcompressoutput (job, true); // The compression algorithm is gzip sequencefileoutputformat. setoutputcompressorclass (job, lzocodec. class); // The compression mode is block sequencefileoutputformat. setoutputcompressiontype (job, compressiontype. block); Re Turn job. waitforcompletion (true )? 0: 1;} public static void main (string [] ARGs) throws exception {int res = toolrunner. run (new configuration (), new toseqfile (), argS); system. exit (RES );}}

 

Mr processes the compressed sequencefile

Import Org. apache. hadoop. io. text; import Java. io. file; import Java. io. ioexception; import java.net. uri; import Java. util. iterator; import Java. util. stringtokenizer; import Org. apache. hadoop. conf. configuration; import Org. apache. hadoop. conf. configured; import Org. apache. hadoop. FS. filesystem; import Org. apache. hadoop. FS. path; import Org. apache. hadoop. io. intwritable; import Org. apache. hadoop. io. nullwritable; impo RT Org. apache. hadoop. io. text; import Org. apache. hadoop. io. compress. *; import Org. apache. hadoop. mapreduce. contextfactory; import Org. apache. hadoop. mapreduce. inputsplit; import Org. apache. hadoop. mapreduce. job; import Org. apache. hadoop. mapreduce. mapper; import Org. apache. hadoop. mapreduce. reducer; import Org. apache. hadoop. mapreduce. lib. input. fileinputformat; import Org. apache. hadoop. mapreduce. lib. input. filesp Logging; import Org. apache. hadoop. mapreduce. lib. input. sequencefileinputformat; import Org. apache. hadoop. mapreduce. lib. input. textinputformat; import Org. apache. hadoop. mapreduce. lib. output. fileoutputformat; import Org. apache. hadoop. mapreduce. lib. output. mapfileoutputformat; import Org. apache. hadoop. mapreduce. lib. output. multipleoutputs; import Org. apache. hadoop. mapreduce. lib. output. sequencefileoutputformat; impo RT Org. apache. hadoop. mapreduce. lib. output. textoutputformat; import Org. apache. hadoop. util. genericoptionsparser; import Org. apache. hadoop. util. progressable; import Org. apache. hadoop. util. tool; import Org. apache. hadoop. util. toolrunner; import Org. apache. commons. logging. log; import Org. apache. commons. logging. logfactory; // import Org. apache. hadoop. mapred. deprecatedlzotextinputformat; import COM. hadoop. compres Sion. lzo. lzocodec; import COM. hadoop. mapreduce. lzotextinputformat; public class compress extends configured implements tool {Private Static final log = logfactory. getlog (compress. class); Private Static class provincemapper extendsmapper <object, text >{@ overrideprotected void map (Object key, text value, context) throws ioexception, interruptedexception {// system. out. println (valu E); // inputsplit = context. getinputsplit (); // string filename = (filesplit) inputsplit ). getpath (). tostring (); // system. out. println (filename); context. write (value, value) ;}} Private Static class provincereducer extendscer CER <text, text >{@ overrideprotected void reduce (Text key, iterable <text> values, context context) throws ioexception, interruptedexception {for (Text VA: Value S) {// system. out. println ("reduce" + key); context. write (key, key) ;}} public static void main (string [] ARGs) throws exception {toolrunner. run (new configuration (), new compress (), argS);} public static final string reduces_per_host = "mapreduce. sort. reducesperhost "; @ overridepublic int run (string [] ARGs) throws exception {log.info ("My Service Query starts ............................. ........ "); long beg = system. cu Rrenttimemillis (); int result = 0; configuration conf = new configuration (); Conf. set ("Io. compression. codecs "," org. apache. hadoop. io. compress. defaultcodec, org. apache. hadoop. io. compress. gzipcodec, Com. hadoop. compression. lzo. lzopcodec "); Conf. set ("Io. compression. codec. lzo. class "," com. hadoop. compression. lzo. lzocodec "); Conf. setboolean ("mapreduce. map. output. compress ", true); Conf. setclass ("mapreduce. map. outp Ut. compression. codec ", snappycodec. class, compressioncodec. class); // Conf. setboolean ("mapreduce. output. fileoutputformat. compress ", true); // whether to compress the output Conf. setclass ("mapreduce. output. fileoutputformat. compress. codec ", snappycodec. class, compressioncodec. class); string [] argarray = new genericoptionsparser (Conf, argS ). getremainingargs (); If (argarray. length! = 2) {system. err. println ("Usage: compress <in> <out>"); system. exit (1);} // hadoop has a total of five jobs. java // hadoop-2.0.0-cdh4.5.0/src/hadoop-mapreduce-Project/hadoop-mapreduce-Client/hadoop-mapreduce-client-core/src/main/Java/org/Apache/hadoop/ mapreduce/job. javajob job = new job (Conf, "compress"); job. setjarbyclass (compress. class); job. setmapperclass (provincemapper. class); job. setreducerclass (provincereducer. CIA SS); job. setmapoutputkeyclass (text. class); job. setmapoutputvalueclass (text. class); job. setoutputkeyclass (text. class); job. setoutputvalueclass (text. class); // job. setinputformatclass (lzotextinputformat. class); // textinputformat // myfileinput // use the lzo index file as the input file // job. setinputformatclass (lzotextinputformat. class); job. setinputformatclass (sequencefileinputformat. class); // sequencefileoutputformat. set (job, l Zocodec. class); // test the block size // fileinputformat. setmininputsplitsize (job, 150*1024*1024); // fileinputformat. setmininputsplitsize (job, 301349250); // fileinputformat. setmaxinputsplitsize (job, 10000); // the speculative execution switch also has the corresponding switch for map and reduce. // job. setspeculativeexecution (false); fileinputformat. addinputpath (job, new path (argarray [0]); fileoutputformat. setoutputpath (job, new path (argarray [1]); string uri = argarray [1 ]; Path = New Path (URI); filesystem FS = filesystem. get (URI. create (URI), conf); If (FS. exists (PATH) {FS. delete (PATH);} result = job. waitforcompletion (true )? 0: 1; // try {// result = job. waitforcompletion (true )? 0: 1; //} catch (classnotfoundexception | interruptedexception e) {// E. printstacktrace (); //} long end = (system. currenttimemillis ()-Beg); system. out. println ("Time consumed:" + end); return result ;}}

 

Test Results

File Size: 544 MB (no compression is used)
Duration: 73805

Use seqencefile (the block uses lzo for compression, and the intermediate result uses snappy for compression)

44207 s

 

Use sequncefile input file in Mr