16. Obtain webpage data from the network

When obtaining webpage data from the network, the webpage may use GZIP compression technology to compress the webpage. This will reduce the amount of data transmitted over the network and increase the browsing speed. Therefore, when obtaining network data, You must judge it and use GZIPInputStream for special processing of GZIP data. Otherwise, garbled characters may occur when obtaining data.

The following is the case code for retrieving webpage data on the network.

 

Package com. ljq. test;

Import java. io. ByteArrayOutputStream;
Import java. io. InputStream;
Import java.net. HttpURLConnection;
Import java.net. URL;
Import java.util.zip. GZIPInputStream;
Import java.util.zip. GZIPOutputStream;

/**
* Retrieving webpage data from the network
*
* @ Author jiqinlin
*
*/
Public class InternetTest2 {

@ SuppressWarnings ("static-access ")
Public static void main (String [] args) throws Exception {
String result = "";
// URL url = new URL ("http://www.sohu.com ");
URL url = new URL ("http://www.ku6.com /");
HttpURLConnection conn = (HttpURLConnection) url. openConnection ();
Conn. setConnectTimeout (6*1000); // sets the connection timeout.
If (conn. getResponseCode ()! = 200) throw new RuntimeException ("request url failed ");
InputStream is = conn. getInputStream (); // gets the input stream returned by the network.
If ("gzip". equals (conn. getContentEncoding ())){
Result = new InternetTest2 (). readDataForZgip (is, "GBK ");
} Else {
Result = new InternetTest2 (). readData (is, "GBK ");
}
Conn. disconnect ();
System. out. println (result );
System. err. println ("ContentEncoding:" + conn. getContentEncoding ());
}

// The first parameter is the input stream, and the second parameter is character set encoding.
Public static String readData (InputStream inSream, String charsetName) throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream ();
Byte [] buffer = new byte [1024];
Int len =-1;
While (len = inSream. read (buffer ))! =-1 ){
OutStream. write (buffer, 0, len );
}
Byte [] data = outStream. toByteArray ();
OutStream. close ();
InSream. close ();
Return new String (data, charsetName );
}

// The first parameter is the input stream, and the second parameter is character set encoding.
Public static String readDataForZgip (InputStream inStream, String charsetName) throws Exception {
GZIPInputStream gzipStream = new GZIPInputStream (inStream );
ByteArrayOutputStream outStream = new ByteArrayOutputStream ();
Byte [] buffer = new byte [1024];
Int len =-1;
While (len = gzipStream. read (buffer ))! =-1 ){
OutStream. write (buffer, 0, len );
}
Byte [] data = outStream. toByteArray ();
OutStream. close ();
GzipStream. close ();
InStream. close ();
Return new String (data, charsetName );
}

}