Java crawler (1) uses GET and POST to send requests to obtain the server's returned information

Java crawler (1) uses GET and POST to send requests to obtain the server's returned information
I use the software eclipse fiddle UC browser to analyze the request information. For example, to simulate a login request and obtain the homepage after login, I first analyze the request information. Use the UC browser F12, click Network, and press F5 to refresh. After logging on to zhihu with your account and clicking www.zhihu.com, the following page appears:

In General, we can see that the request method is GET. In the request construction in fiddle, The GET method is selected.
After the drop-down, you can see the Request Header. Copy all the content and paste it into the Request structure of fiddle.

Click Execute, click the accessed URL in fiddle, click sniffing, and click TextView below. NO content is displayed.

Delete some useless Request headers and click Execute. The returned data is successful!

Send the Request information and obtain the data. From the analysis above, you can see that the Request Header required for access is sufficient only for COOKIE. Therefore, we will copy the content formatted in the request structure to the txt file.

The next step is to use Java to send request information. Sending request information is simple, so you can directly post the code for discussion.

Package Main;

Import java. io. IOException;
Import java. io. InputStream;
Import java.net. HttpURLConnection;
Import java. util. HashMap;
Import java. util. Map;

Import Utils. HttpUtils;
Import Utils. HttpUtils. OnVisitingListener;
Import Utils. StreamUtils;
Import Utils. StreamUtils. OnGetStringListener;

Public class Main {
Public static void main (String [] args ){
// Obtain webpage data
GetWebData ();
// Set parameters
// Get the returned data
}

private static void getWebData() {    HttpUtils httpUtils = HttpUtils.newInstance();    httpUtils.setOnVisitingListener(new OnVisitingListener() {        @Override        public void onSuccess(HttpURLConnection conn) {            try {                InputStream inputStream = conn.getInputStream();                String string = StreamUtils.getString(inputStream);                System.out.println(string);            } catch (IOException e) {                e.printStackTrace();            }        }        @Override        public void onSetDetails(HttpURLConnection conn, HttpUtils httpUtils) {            Map
  
    map = new HashMap
   
    ();            StreamUtils.getString("requestheader.txt", new OnGetStringListener() {                @Override                public void onGeted() {                }                @Override                public void onGetString(String line) {                    System.out.println(line);                    String[] strings = line.split(":");                    map.put(strings[0], strings[1]);                }            });            httpUtils.setRequestProperties(map);        }        @Override        public void onFail(IOException e) {        }    }).startConnenction("https://www.zhihu.com/", "GET");}}
   
  

Utils encapsulation Tool
Package Utils;

Import java. io. IOException;
Import java. io. PrintWriter;
Import java.net. HttpURLConnection;
Import java.net. URL;
Import java. util. Iterator;
Import java. util. Map;
Import java. util. Set;

/**
* Created by admin on 2016/3/2.
*/
Public class HttpUtils {
Private HttpURLConnection conn;

public void setConnection(String fileUrl, String method) throws IOException {    URL url = new URL(fileUrl);    conn = (HttpURLConnection) url.openConnection();    conn.setRequestMethod(method);    conn.setConnectTimeout(5000);    conn.setReadTimeout(5000);    listener.onSetDetails(conn, this);    conn.connect();}OnVisitingListener listener;public interface OnVisitingListener {    void onSuccess(HttpURLConnection conn);    void onSetDetails(HttpURLConnection conn, HttpUtils httpUtils);    void onFail(IOException e);}public HttpUtils setOnVisitingListener(OnVisitingListener listener) {    this.listener = listener;    return this;}public void startConnenction(String url, String method) {    try {        setConnection(url, method);        if (conn.getResponseCode() == 200) {            listener.onSuccess(conn);        } else {            throw new IOException();        }    } catch (IOException e) {        listener.onFail(e);    }    // if (conn != null) {    // conn.disconnect();    // }}public void setRequestProperties(Map
  
    map) {    String key;    String value;    Set
   
     set = map.keySet();    Iterator
    
      it = set.iterator();    while (it.hasNext()) {        key = it.next();        value = map.get(key);        conn.setRequestProperty(key, value);    }}public void setRequestBody(String body) {    try {        PrintWriter writer = new PrintWriter(conn.getOutputStream());        writer.write(body);        writer.flush();        writer.close();    } catch (IOException e) {        e.printStackTrace();    }}public void setRequestProperty(String type, String value) {    conn.setRequestProperty(type, value);}public static HttpUtils newInstance() {    return new HttpUtils();}
    
   
  

}

Package Utils;

Import java. util. ArrayList;
Import java. util. List;
Import java. util. regex. Matcher;
Import java. util. regex. Pattern;

Public class RegexUtils {
Public static String RegexGroup (String targetStr, String patternStr, int which ){
Pattern pattern = Pattern. compile (patternStr );
Matcher matcher = pattern. matcher (targetStr );
If (matcher. find ()){
Return matcher. group (which );
}
Return "Nothing !";
}

public static List
  
    RegexGroups(String targetStr, String patternStr, int which) {    Pattern pattern = Pattern.compile(patternStr);    Matcher matcher = pattern.matcher(targetStr);    List
   
     list = new ArrayList
    
     ();    while (matcher.find()) {        list.add(matcher.group(which));    }    return list;}public static String RegexString(String targetStr, String patternStr) {    Pattern pattern = Pattern.compile(patternStr);    Matcher matcher = pattern.matcher(targetStr);    if (matcher.find()) {        return matcher.group();    }    return "Nothing!";}}
    
   
  

Package Utils;

Import java. io. BufferedReader;
Import java. io. BufferedWriter;
Import java. io. ByteArrayOutputStream;
Import java. io. File;
Import java. io. FileInputStream;
Import java. io. FileNotFoundException;
Import java. io. FileOutputStream;
Import java. io. FileReader;
Import java. io. FileWriter;
Import java. io. IOException;
Import java. io. InputStream;
Import java. io. InputStreamReader;
Import java. io. OutputStream;
Import java. io. OutputStreamWriter;
Import java. io. Reader;
Import java. io. UnsupportedEncodingException;
Import java. io. Writer;

/**
* Created by admin on 2016/2/18.
*/
Public class StreamUtils {
Public static String readFromStream (InputStream inputStream) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream ();
Int len = 0;
Byte [] buffer = new byte [1024];

    while ((len = inputStream.read(buffer)) != -1) {        outputStream.write(buffer, 0, len);    }    String result = outputStream.toString();    inputStream.close();    outputStream.close();    return result;}private static String line;public static FileReader createFileReader(File file) throws FileNotFoundException {    return new FileReader(file);}public static FileWriter createFileWriter(File file) throws IOException {    return new FileWriter(file);}public static InputStreamReader createInputStreamReader(Object obj) {    if (obj instanceof File)        if (!((File) obj).exists())            ((File) obj).getParentFile().mkdirs();    try {        return new InputStreamReader(new FileInputStream((File) obj), "utf-8");    } catch (UnsupportedEncodingException e) {        e.printStackTrace();    } catch (FileNotFoundException e) {        e.printStackTrace();    }    return null;}public static OutputStreamWriter createOutputStreamWriter(Object obj) {    if (obj instanceof File)        if (!((File) obj).exists())            ((File) obj).getParentFile().mkdirs();    try {        return new OutputStreamWriter(new FileOutputStream((File) obj, true), "utf-8");    } catch (UnsupportedEncodingException e) {        e.printStackTrace();    } catch (FileNotFoundException e) {        e.printStackTrace();    }    return null;}public static BufferedReader createBufferedReader(Object obj, String cd) throws IOException {    if (obj instanceof String)        return new BufferedReader(createInputStreamReader(new File((String) obj)));    if (obj instanceof InputStream) {        if (cd == null)            return new BufferedReader(new InputStreamReader((InputStream) obj));        else            return new BufferedReader(new InputStreamReader((InputStream) obj, cd));    }    if (obj instanceof File) {        if (!((File) obj).exists())            ((File) obj).createNewFile();        return new BufferedReader(createFileReader((File) obj));    }    if (obj instanceof Reader)        return new BufferedReader((Reader) obj);    if (obj instanceof BufferedReader)        return (BufferedReader) obj;    return null;}public static BufferedWriter createBufferedWriter(Object obj) throws IOException {    if (obj instanceof String)        return new BufferedWriter(createOutputStreamWriter(new File((String) obj)));    if (obj instanceof OutputStream)        return new BufferedWriter(new OutputStreamWriter((OutputStream) obj, "utf-8"));    if (obj instanceof File)        return new BufferedWriter(createOutputStreamWriter(obj));    if (obj instanceof Writer)        return new BufferedWriter((Writer) obj);    if (obj instanceof BufferedWriter)        return (BufferedWriter) obj;    return null;}public interface OnGetStringListener {    void onGetString(String line);    void onGeted();}public static void getString(Object obj, OnGetStringListener listener) {    BufferedReader br;    try {        br = createBufferedReader(obj, null);        if (br != null) {            while ((line = br.readLine()) != null) {                listener.onGetString(line);            }            listener.onGeted();            br.close();        }    } catch (IOException e) {        e.printStackTrace();    }}public static String getString(Object obj) {    BufferedReader br;    String str = "";    try {        br = createBufferedReader(obj, "utf-8");        if (br != null) {            while ((line = br.readLine()) != null) {                str += line + "\n";            }        }    } catch (IOException e) {        e.printStackTrace();    }    return str;}public static void writeString(Object obj, String str) {    BufferedWriter bw;    try {        bw = createBufferedWriter(obj);        if (bw != null) {            bw.write(str);            bw.close();        }    } catch (IOException e) {        e.printStackTrace();    }}

There is no big difference between sending a POST Request and a GET Request by sending a POST Request, except that the Request Body must be set for a POST Request. Before the connection, get the output stream and write the Request Body data in fiddle.