(C #) Use ScrapySharp to concurrently download Tianya images,

(C #) Download the End of the World images in parallel with ScrapySharp,

   
       Recently, because of an assignment that required the completion of the CNKI crawler, I discovered this famous open source crawler framework Scrapy ScrapySharp when I researched the crawler architecture. However, I only found this F # Demo after searching online, and I used the example website in the original text. Wrote this C # version of the code.

    
       PS: After research, I found that the gap between ScrapySharp and Scrapy is still quite large. There are no eight components as complete as Scrapy. It only contains web page acquisition functions and web page analysis functions based on the extension of HtmlAgilityPack.

using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using HtmlAgilityPack;
using ScrapySharp.Extensions;
using ScrapySharp.Network;

namespace ScrapySharpDemo
{
    class Program
    {
        static void Main (string [] args)
        {
            // Example website address
            var url = "http://bbs.tianya.cn/post-12-563201-1.shtml";
            var web = new ScrapingBrowser ();
            var html = web.DownloadString (new Uri (url));
            var doc = new HtmlDocument ();
            doc.LoadHtml (html);
            // Get the picture address in the website
            var urls = doc.DocumentNode.CssSelect ("div.bbs-content> img"). Select (node => node.GetAttributeValue ("original")). ToList ();
            // Download pictures in parallel
            Parallel.ForEach (urls, SavePic);
        }

        public static void SavePic (string url)
        {
            var web = new ScrapingBrowser ();
            // Because of the limit of the Tianya website, all off-site sources cannot access the picture, so first set the Refer header of the request header to the current page address
            web.Headers.Add ("Referer", "http://bbs.tianya.cn/post-12-563201-1.shtml");
            var pic = web.NavigateToPage (new Uri (url)). RawResponse.Body;
            var file = url.Substring (url.LastIndexOf ("/", StringComparison.Ordinal));
            if (! Directory.Exists ("imgs"))
                Directory.CreateDirectory ("imgs");
            File.WriteAllBytes ("imgs" + file, pic);
        }
    }
}