Java real-time listening log write Kafka

Original link: http://www.sjsjw.com/kf_cloud/article/020376ABA013802.asp purpose

Real-time monitoring of a directory of log files, such as the new file to switch to a new file, and synchronously write to Kafka, while recording the log file line location, in order to deal with the process of abnormal exit, can be read from the last file location (considering the efficiency, here is every 100 records, can be adjusted)

Source:

ImportJava.io.BufferedReader;ImportJava.io.BufferedWriter;ImportJava.io.File;ImportJava.io.FileInputStream;Importjava.io.FileNotFoundException;ImportJava.io.FileReader;ImportJava.io.FileWriter;Importjava.io.IOException;ImportJava.io.LineNumberReader;ImportJava.io.PrintWriter;ImportJava.io.RandomAccessFile;Importjava.net.NoRouteToHostException;Importjava.util.ArrayList; Importjava.util.Collection; Importjava.util.List; Importjava.util.Properties; ImportJava.util.Random;Importjava.util.concurrent.Executors;ImportJava.util.concurrent.ScheduledExecutorService;ImportJava.util.concurrent.TimeUnit; ImportKafka.javaapi.producer.Producer; ImportKafka.producer.KeyedMessage; ImportKafka.producer.ProducerConfig; /** Own in the source server writes the producer to Kafka inserts the data, note the file "Producer.properties puts under the Linux the jar file same directory * listens to a directory the file data then writes Kafka * Nohup Java-jar Portallog_producer.jar portallog/var/apache/logs portallog.position >/home/sre/portalhandler/handler.log 2> &1 & * **/ Public classPortallogtail_line {PrivateProducer<string,string>inner; Java.util.Random ran=NewRandom ();  PublicPortallogtail_line ()throwsFileNotFoundException, IOException {Properties Properties=NewProperties (); //Properties.load (Classloader.getsystemresourceasstream ("producer.properties")); Properties.load (NewFileInputStream ("Producer.properties")); Producerconfig Config=NewProducerconfig (properties); Inner=NewProducer<string, string>(config); }               Public voidSend (String topicname,string message) {if(Topicname = =NULL|| Message = =NULL){              return; }       //keyedmessage<string, string> km = new keyedmessage<string, string> (topicname,message); //randomly dispersed as key,hash to each partitionKeyedmessage<string, string> km =NewKeyedmessage<string, String> (topicname,string.valueof (Ran.nextint (9) ), message); //keyedmessage<string, string> km = new keyedmessage<string, string> (topicname,message,message);inner.send (km); }             Public voidSend (String topicname,collection<string>messages) {          if(Topicname = =NULL|| Messages = =NULL){              return; }          if(Messages.isempty ()) {return; } List<keyedmessage<string, string>> kms =NewArraylist<keyedmessage<string, string>>();  for(String entry:messages) {keyedmessage<string, string> km =NewKeyedmessage<string, string>(Topicname,entry);          Kms.add (km);      } inner.send (KMS); }             Public voidClose () {inner.close (); }               PublicString getnewfile (file file) {file[] fs=File.listfiles (); LongMaxtime=0; String NewFileName="";  for(inti=0;i<fs.length;i++)        {            if(Fs[i].lastmodified () >maxtime && fs[i].getname (). Contains ("Access") ) {MaxTime=fs[i].lastmodified (); NewFileName=Fs[i].getabsolutepath (); }        }        returnNewFileName; }      //write file name and line number     Public voidWriteposition (String Path,intrn,string Positionpath) {        Try{BufferedWriter out=NewBufferedWriter (NewFileWriter (Positionpath)); Out.write (Path+","+RN);        Out.close (); } Catch(IOException e) {}} LineNumberReader Randomfile=NULL; String NewFile=NULL; String Thisfile=NULL; String Prefile=NULL; intLn=0; intBeginln=0;  Public voidRealtimeshowlog (FinalFile file,FinalString Topicname,FinalString Positionpath)throwsioexception{//start a thread to read the new log information every 1 seconds       NewThread (NewRunnable () { Public voidrun () {thisfile=getnewfile (file); Prefile=Thisfile; //access the position file, if the file path is logged, and the line number is located, otherwise use the latest file                 Try{bufferedreader br=NewBufferedReader (NewFileReader (Positionpath)); String Line=Br.readline (); if(line!=NULL&&line.contains (",") ) {Thisfile=line.split (",") [0]; Prefile=Thisfile; Beginln=integer.parseint (Line.split (",") [1]); }                                                           } Catch(filenotfoundexception E2) {//TODO auto-generated Catch blockE2.printstacktrace (); }                  Catch(IOException E2) {//TODO auto-generated Catch blockE2.printstacktrace (); }                                  //Specifies that the file is readable and writable                     Try{randomfile=NewLineNumberReader (NewFileReader (thisfile)); } Catch(FileNotFoundException e) {//TODO auto-generated Catch blockE.printstacktrace (); }                    while(true)              {                  Try{Thread.Sleep (100); } Catch(interruptedexception E1) {//TODO auto-generated Catch blockE1.printstacktrace (); }                  Try {                           //get the change part of the//Randomfile.seek (lasttimefilesize); String tmp = "";  while(TMP = Randomfile.readline ())! =NULL) {                            intcurrln=Randomfile.getlinenumber (); //Beginln defaults to 0                          if(currln>beginln) Send (Topicname,NewString (Tmp.getbytes ("UTF8"))); ln++; //each occurrence of a write once impact efficiency, 100 consecutive times before recording position                          if(ln>100) {writeposition (Thisfile,currln,positionpath); ln=0; }} thisfile=getnewfile (file); if(!thisfile.equals (Prefile))                         {Randomfile.close (); Randomfile=NewLineNumberReader (NewFileReader (thisfile)); Prefile=Thisfile; Beginln=0; }                                                             } Catch(IOException e) {Throw NewRuntimeException (e);         }}}). Start (); }               /**      * @paramargs *@throwsException*/       Public Static voidMain (string[] args)throwsException {portallogtail_line producer=NewPortallogtail_line (); if(args.length!=3) {System.out.println ("Usage:topicname Pathname Positionpath"); System.exit (1); } String topicname=args[0]; String Pathname=args[1]; String Positionpath=args[2]; FinalFile Tmplogfile =NewFile (pathname);                  Producer.realtimeshowlog (Tmplogfile,topicname,positionpath); }    } 

The producer.properties file is placed in a sibling directory metadata.broker.list=xxx:10909,xxx:10909# name of the Partitionerclass  forpartitioning events;defaultpartition spreads data randomly#partitioner.class=# Specifies whether the messages is sent asynchronously (async) or synchronously (sync) producer.type=Sync#producer.type=async# Specify the compression codec forAll data Generated:none, gzip, snappy.# the old config values work as well:0, 1, 2 forNone, gzip, snappy, Respectivallycompression.codec=None#compression.codec=gzip# message Encoderserializer.class=kafka.serializer.stringencoder

Test

Final execution:

Nohup Java-jar portallog_producer.jar portallog/var/apache/logs portallog.position  >/home/sre/portalhandler /handler.log 2>&1 &

Java real-time listening log write Kafka (RPM)