Java Crawl Web content implementation code _java

Copy Code code as follows:

Package test;

Import Java.io.BufferedReader;

Import java.io.IOException;

Import Java.io.InputStream;

Import Java.io.InputStreamReader;

Import Java.net.Authenticator;

Import java.net.HttpURLConnection;

Import java.net.PasswordAuthentication;

Import Java.net.URL;

Import java.net.URLConnection;

Import java.util.Properties;

public class Urltest {

A public method that returns a string, and an error returns an "Error open URL"

public static string GetContent (string strurl) {

try {

URL url = new URL (strurl);

BufferedReader br = new BufferedReader (new InputStreamReader (URL

. OpenStream ());

String s = "";

StringBuffer sb = new StringBuffer ("");

while ((s = br.readline ())!= null) {

Sb.append (S + "/r/n");

}

Br.close ();

return sb.tostring ();

catch (Exception e) {

Return "Error open URL:" + strurl;

}

}

public static void Initproxy (string host, int port, final String username,

Final String password) {

Authenticator.setdefault (New Authenticator () {

Protected Passwordauthentication getpasswordauthentication () {

Return to New Passwordauthentication (username,

New String (password). ToCharArray ());

}

});

System.setproperty ("Http.proxytype", "4");

System.setproperty ("Http.proxyport", integer.tostring (port));

System.setproperty ("Http.proxyhost", host);

System.setproperty ("Http.proxyset", "true");

}

public static void Main (string[] args) throws IOException {

String url = "Http://www.jb51.net";

String proxy = "http://192.168.22.81";

int port = 80;

String username = "username";

String password = "password";

String curline = "";

String content = "";

URL server = new URL (URL);

Initproxy (proxy, port, username, password);

HttpURLConnection connection = (httpurlconnection) server

. OpenConnection ();

Connection.connect ();

InputStream is = Connection.getinputstream ();

BufferedReader reader = new BufferedReader (new

InputStreamReader (IS));

while ((CurLine = Reader.readline ())!= null) {

Content = content + curline+ "/r/n";

}

System.out.println ("content=" + content);

Is.close ();

System.out.println (getcontent (URL));

}

}