A simple Java crawler that knows a picture under a topic

Package Util;
Import Java.io.BufferedInputStream;
Import Java.io.BufferedOutputStream;
Import Java.io.BufferedReader;
Import Java.io.File;
Import Java.io.FileOutputStream;
Import Java.io.FileReader;
Import Java.io.FileWriter;
Import java.io.IOException;
Import Java.io.InputStreamReader;
Import Java.io.PrintWriter;
Import java.net.HttpURLConnection;
Import Java.net.URL;
Import Java.util.regex.Matcher;

Import Java.util.regex.Pattern; public class Urlget {public static void main (string[] args) throws Exception {//TODO auto-generated Metho
        D stub//want crawler URL String url= "https://www.zhihu.com/question/38376393";
        Returns a folder name that stores a valid URL String urlfilename = GetUrl (URL);
        Download a valid URL corresponding to the photo downpic (Urlfilename);

    System.out.println ("Picture download Completed"); ///From a valid URL that has already been downloaded public static void Downpic (String urlfilename) throws Exception {//TODO auto-generate D method stub//url url = new URL (nulL);
        BufferedReader bfr= New BufferedReader (New FileReader (Urlfilename));
        String URL1 = null;
        int num=0;
            while ((Url1=bfr.readline ())!=null) {num++;
        Downpic (BfR, url1,num);
    } bfr.close (); /** * Download pictures via URL/public static void Downpic (BufferedReader bfr, String url1, int num) throws Ioexceptio
        n {URL url = new URL (URL1);
        HttpURLConnection URLConnection = (httpurlconnection) url.openconnection ();
        Gets the download input stream bufferedinputstream bis= new Bufferedinputstream (Urlconnection.getinputstream ()); Write out the stream in the Downpic folder bufferedoutputstream bos = new Bufferedoutputstream (New FileOutputStream (NE
        W File ("f:/downpic1/" +num+ ". jpg"));
        int ch=0;
        byte[] buf = new byte[1024];

        Write to Downpic while (Ch=bis.read (BUF)!=-1) {bos.write (buf,0,ch);


    } bos.close ();//Get Web page valid URL public static string GetUrl (String url) throws Exception {//TODO auto-generated method Stu
        b url u=new url (url);
        HttpURLConnection URLConnection = (httpurlconnection) u.openconnection ();

        BufferedReader bus= New BufferedReader (New InputStreamReader (Urlconnection.getinputstream ()));
        PrintWriter pw = new PrintWriter (New FileWriter ("F:/zhihu_pic_url.txt"), true);

        PrintWriter PW1 = new PrintWriter (New FileWriter ("F:/zhihu_content.txt"), true);
        String regex= "[https|http]+://+[pic]+[a-za-z0-9.//_%=-]*";

        Pattern p = pattern.compile (regex);
        String line = null;
            while ((Line=bus.readline ())!=null) {Matcher m = p.matcher (line);
            Pw1.println (line);
            while (M.find ()) {pw.println (M.group ());
        } bus.close ();
        Pw.close ();
        Pw1.close ();
        System.out.println ("Urlget completed");String urlfilename= "F:/zhihu_pic_url.txt";
    return urlfilename;
 }
}

This method applies only if it is known, because the regular expression is based only on the source code of the Web page, but the URL is still duplicated, but the size and resolution are different, and the results are as follows

This is a useful URL to obtain and the resulting photos are as follows

From the above you can see that the downloaded photos are duplicated, carefully compared to the URL can be seen, but the size of the photos, the resolution is different, so you can also improve the regular expression.