Using Jsoup as a reptile, crawl through all the headlines and links in my blog, with the following code examples:
Package Com.test.jsoup;import Java.io.ioexception;import Org.jsoup.jsoup;import org.jsoup.nodes.document;import Org.jsoup.nodes.element;import Org.jsoup.select.elements;public class TestDemo4 {public String BASEURL = "/http www.cnblogs.com/zhangfei/p/";p ublic String pager ="? page=%s ";p ublic int getallpagecount () {int count = 0;try {Document do c = Jsoup.connect (BASEURL). get (); String Counttext = Doc.select ("#myposts >div.pager:nth-of-type (1);. Pager "). Text (); counttext = Counttext.replacefirst (" \\d+ (\\d+). * "," $ "); count = Integer.valueof (Counttext);} catch (IOException e) {e.printstacktrace ();} return count;} public void crawler () {int count = This.getallpagecount (), for (int i = 1; I <= count; i++) {String URL = baseUrl + strin G.format (Pager, i); This.testjsop (URL);}} public void Testjsop (String url) {try {Document doc = jsoup.connect (URL). get (); Elements element = Doc.select ("Div. Postlist a "); for (Element e:element) {String text = E.text (); String href = e.attr ("href"); System.out.priNtln (text+ ":" +href);}} catch (IOException e) {e.printstacktrace ();}} public static void Main (string[] args) {TestDemo4 t = new TestDemo4 (); T.crawler ();}}
Jsoup Crawler Example