Multithreaded crawler java call wget download file, independent thread read output buffer

Write a crawl AppStore, to crawl a large number of apps, it was originally used httpclient, but the effect is not ideal, so the direct call wget download, but because the standard output, error output causes will lead to stuck, in addition wget will be inexplicable stuck.

So I use:

One, the independent thread reads the output information;

Second, implement the Dowaitfor method instead of the WAITFOR () method provided by the API to avoid the child process to die.

Three, set timeout, kill wget child process, do not return correctly, retry once, and the timeout time doubled;

With the above operation, wget will not be stuck, even if the card will be killed because of the time-out and then try again, so most of the app can be crawled down.

ImportCom.google.common.io.Files;ImportCom.xxx.appstore.service.crawler.CalcMD5Service;Importorg.apache.commons.lang.StringUtils;Importorg.apache.commons.lang.math.RandomUtils;ImportOrg.slf4j.Logger;Importorg.slf4j.LoggerFactory;ImportJava.io.File;Importjava.io.IOException;Importjava.util.ArrayList;Importjava.util.List;ImportJava.util.concurrent.TimeUnit; Public classCrawlerutils { Public Static FinalString Apk_download_path = "/data/appstore/category/"; Private StaticLogger Logger = Loggerfactory.getlogger (crawlerutils.class); /*** Download files using wget * *@paramdisplayName AppName *@paramCategory Categories *@paramDownload_url *@returnsuccessfully returned file path, failed to return null*/     Public Staticstring Downloadfilebywget (String displayName, String category, String download_url) {if(Stringutils.isblank (displayName) | | Stringutils.isblank (category) | |Stringutils.isblank (Download_url)) {Logger.info ("Downloadfilebywget ERROR, displayname:{}, category:{}, download_url:{}",Newobject[]{displayname, category, Download_url}); return NULL; } String fileName= Calcmd5service.encoder (DisplayName + randomutils.nextint (1000)); String seed=Calcmd5service.encoder (category); String Midpath= Stringutils.left (seed, 10); String FilePath= Apk_download_path + Midpath + "/" + FileName + ". APK"; File File=NewFile (FilePath); Try{files.createparentdirs (file); } Catch(IOException e) {Logger.warn ("IOException", E); return NULL; }        intRetry = 2; intres =-1; intTime = 1;  while(retry--> 0) {Processbuilder PB=NewProcessbuilder ("wget", Download_url, "-T", "2", "-T", "ten", "-O", FilePath); Logger.info ("Wget Shell: {}", Pb.command ()); Process PS=NULL; Try{PS=Pb.start (); } Catch(IOException e) {logger.error ("IOException", E); } Res=dowaitfor(PS, * time++); if(Res! = 0) {Logger.warn ("Wget Download Failed ..."); } Else {                 Break; }        }        if(Res! = 0) {            return NULL; }        returnFilePath; }    /**     * @paramPS Sub Process *@paramTimeout time-out, SECONDS *@returnnormal end return 0*/    Private Static intDowaitfor (Process PS,intTimeout) {        intres =-1; if(PS = =NULL) {            returnRes; } List<String> stdoutlist =NewArraylist<>(); List<String> erroroutlist =NewArraylist<>(); BooleanFinished =false; intTime = 0; Threadutil Stdoututil=NewThreadutil (Ps.getinputstream (), stdoutlist); Threadutil Erroroututil=NewThreadutil (Ps.geterrorstream (), erroroutlist); //start thread read buffer dataStdoututil.start ();        Erroroututil.start ();  while(!finished) { time++; if(Time >=Timeout) {Logger.info ("Process wget timeout 30s, destroyed!");                Ps.destroy ();  Break; }            Try{res=Ps.exitvalue (); Finished=true; } Catch(illegalthreadstateexception e) {Try{TimeUnit.SECONDS.sleep (1); } Catch(Interruptedexception E1) {}}} returnRes; }}

Importorg.apache.commons.io.Charsets;ImportJava.io.BufferedReader;Importjava.io.IOException;ImportJava.io.InputStream;ImportJava.io.InputStreamReader;Importjava.util.List; Public classThreadutilImplementsRunnable {//set the character encoding to read    PrivateString character =Charsets.UTF_8.displayName (); PrivateList<string>list; PrivateInputStream InputStream;  PublicThreadutil (InputStream InputStream, list<string>list) {         This. InputStream =InputStream;  This. List =list; }     Public voidstart () {Thread thread=NewThread ( This); Thread.setdaemon (true);//set it as the daemon threadThread.Start (); }     Public voidrun () {BufferedReader br=NULL; Try{br=NewBufferedReader (NewInputStreamReader (InputStream, character)); String Line=NULL;  while(line = Br.readline ())! =NULL) {List.add (line); }        } Catch(IOException e) {e.printstacktrace (); } finally {            Try {                //Freeing ResourcesInputstream.close (); if(BR! =NULL) {br.close (); }            } Catch(IOException e) {e.printstacktrace (); }        }    }}

Multithreaded crawler java call wget download file, independent thread read output buffer