How to use CasperJS in Python to obtain HTML content generated by JS rendering-Python tutorial

This article mainly introduces how to use CasperJS in Python to obtain HTML content generated by JS rendering. you need to create an interface file with JavaScript first. if you need it, refer to the following article abstract: in fact, casperjs is not directly related to python. it mainly relies on casperjs to call phantomjs webkit to obtain html file content. For a long time, it has been extremely difficult for crawlers to capture html pages generated by javascript rendering on the client. HtmlUnit exists in Java, while in Python, we can use an independent cross-platform CasperJS.

Create site. js (interface file, input: url, output: html file)

// USAGE: E: \ toolkit \ n1k0-casperjs-e3a77d0 \ bin> python casperjs site. js -- url = http://spys.ru/free-proxy-list/IE/ --outputfile='temp.html 'var fs = require ('Fs'); var casper = require ('Casper '). create ({pageSettings: {loadImages: false, loadPlugins: false, userAgent: 'mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) chrome/34.0.1847.20.safari/537.36 LBBROWSER '}, logLevel: "debug", // log level verbose: true // log recorded on the console}); var url = casper. cli. raw. get ('URL'); var outputfile = casper. cli. raw. get ('outputfile'); // request page casper. start (url, function () {fs. write (outputfile, this. getHTML (), 'w') ;}); casper. run ();

Python code, checkout_proxy.py

Import json import sys # import requests. utils, pickle from bs4 import BeautifulSoup import OS. path, OS import threading # from multiprocessing import Process, Manager from datetime import traceback import logging import re, random import subprocess import shutil import platform output_file = thread ') global_log = 'http _ proxy' + datetime. now (). strftime ('% Y-% m-% d') + '. log 'if not OS. path. exists (OS. path. join (OS. path. dirname (OS. path. realpath (_ file _), 'logs'): OS. mkdir (OS. path. join (OS. path. dirname (OS. path. realpath (_ file _), 'logs') global_log = OS. path. join (OS. path. dirname (OS. path. realpath (_ file _), 'logs', global_log) logging. basicConfig (level = logging. DEBUG, format = '[% (asctime) s] [% (levelname) s] [% (module) s] [% (funcName) s] [% (lineno) d] % (message) s', filename = global_log, filemode = 'a') log = logging. getLogger (_ name _) # manager = Manager () # PROXY_LIST = manager. list () mutex = threading. lock () PROXY_LIST = [] def isWindows (): if "Windows" in str (platform. uname (): return True else: return False def getTagsByAttrs (tagName, pageContent, attrName, attrRegValue): soup = BeautifulSoup (pageContent) return soup. find_all (tagName, {attrName: re. compile (attrRegValue)}) def getTagsByAttrsExt (tagName, filename, attrName, attrRegValue): if OS. path. isfile (filename): f = open (filename, 'r') soup = BeautifulSoup (f) f. close () return soup. find_all (tagName, {attrName: re. compile (attrRegValue)}) else: return None class Site1Thread (threading. thread): def _ init _ (self, outputFilePath): threading. thread. _ init _ (self) self. outputFilePath = outputFilePath self. fileName = str (random. randint (100,1000) + ". html "self. setName ('site1thread') def run (self): site‑file = OS. path. join (OS. path. dirname (OS. path. realpath (_ file _), 'site. js') site2_file = OS. path. join (self. outputFilePath, 'site. js') if not OS. path. isfile (site2_file) and OS. path. isfile (site‑file): shutil. copy (site‑file, site2_file) # proc = subprocess. popen (["bash", "-c", "cd % s &&. /casperjs site. js -- url = http://spys.ru/free-proxy-list/IE/ -- Outputfile = % s "% (self. outputFilePath, self. fileName)], stdout = subprocess. PIPE) if isWindows (): proc = subprocess. popen (["cmd", "/c", "% s/casperjs site. js -- url = http://spys.ru/free-proxy-list/IE/ -- Outputfile = % s "% (self. outputFilePath, self. fileName)], stdout = subprocess. PIPE) else: proc = subprocess. popen (["bash", "-c", "cd % s &&. /casperjs site. js -- url = http://spys.ru/free-proxy-list/IE/ -- Outputfile = % s "% (self. outputFilePath, self. fileName)], stdout = subprocess. PIPE) out = proc. communicate () [0] htmlFileName = ''# because the output path is unknown in windows, all possible paths are added to determine if OS. path. isfile (self. fileName): htmlFileName = self. fileName elif OS. path. isfile (OS. path. join (self. outputFilePath, self. fileName): htmlFileName = OS. path. join (self. outputFilePath, self. fileName) elif OS. path. isfile (OS. path. join (OS. path. dirname (OS. path. realpath (_ file _), self. fileName): htmlFileName = OS. path. join (OS. path. dirname (OS. path. realpath (_ file _), self. fileName) if (not OS. path. isfile (htmlFileName): print 'failed' to get html content from http://spys.ru/free-proxy-list/IE/ 'Print out sys. exit (3) mutex. acquire () PROXYList = getTagsByAttrsExt ('font', htmlFileName, 'class', 'spy14 $ ') for proxy in PROXYList: tdContent = proxy. renderContents () lineElems = re. split ('[<>]', tdContent) if re. compile (r' \ d + '). search (lineElems [-1]) and re. compile ('(\ d + \. \ d + \. \ d + )'). search (lineElems [0]): print lineElems [0], lineElems [-1] PROXY_LIST.append ("% s: % s" % (lineElems [0], lineElems [-1]) mutex. Release () try: if OS. path. isfile (htmlFileName): OS. remove (htmlFileName) partition T: pass if _ name _ = '_ main _': try: if (len (sys. argv) <2: print "Usage: % s [casperjs path]" % (sys. argv [0]) sys. exit (1) if not OS. path. exists (sys. argv [1]): print "casperjs path: % s does not exist! "% (Sys. argv [1]) sys. exit (2) if OS. path. isfile (output_file): f = open (output_file) lines = f. readlines () f. close for line in lines: PROXY_LIST.append (line. strip () thread1 = Site1Thread (sys. argv [1]) thread1.start () thread1.join () f = open (output_file, 'w') for proxy in set (PROXY_LIST): f. write (proxy + "\ n") f. close () print "Done! "Failed t SystemExit: pass failed t: errMsg = traceback. format_exc () print errMsg log. error (errMsg)