在Python中使用CasperJS获取JS渲染生成的HTML内容的教程

文章摘要：其实这里casperjs与python/' target='_blank'>python没有直接关系,主要依赖casperjs调用phantomjs webkit获取html文件内容。长期以来，爬虫抓取 客户端javascript渲染生成的html页面 都极为 困难, Java里面有 HtmlUnit, 而Python里，我们可以使用独立的跨平台的CasperJS。

创建site.js(接口文件，输入:url，输出:html file)

   //USAGE: E:\toolkit\n1k0-casperjs-e3a77d0\bin>python casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile='temp.html'          var fs = require('fs');     var casper = require('casper').create({      pageSettings: {      loadImages: false,          loadPlugins: false,         userAgent: 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36 LBBROWSER'     },     logLevel: "debug",//日志等级     verbose: true  // 记录日志到控制台      });     var url = casper.cli.raw.get('url');     var outputfile = casper.cli.raw.get('outputfile');     //请求页面     casper.start(url, function () {     fs.write(outputfile, this.getHTML(), 'w');     });          casper.run(); 

python 代码, checkout_proxy.py

 import json     import sys     #import requests     #import requests.utils, pickle     from bs4 import BeautifulSoup     import os.path,os     import threading     #from multiprocessing import Process, Manager     from datetime import datetime     import traceback     import logging     import re,random     import subprocess     import shutil     import platform                          output_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'proxy.txt')     global_log = 'http_proxy' + datetime.now().strftime('%Y-%m-%d') + '.log'     if not os.path.exists(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')):       os.mkdir(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs'))     global_log = os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs',global_log)          logging.basicConfig(level=logging.DEBUG,format='[%(asctime)s] [%(levelname)s] [%(module)s] [%(funcName)s] [%(lineno)d] %(message)s',filename=global_log,filemode='a')     log = logging.getLogger(__name__)      #manager = Manager()     #PROXY_LIST = manager.list()     mutex = threading.Lock()     PROXY_LIST = []               def isWindows():       if "Windows" in str(platform.uname()):       return True       else:       return False               def getTagsByAttrs(tagName,pageContent,attrName,attrRegValue):       soup = BeautifulSoup(pageContent)                                                       return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })               def getTagsByAttrsExt(tagName,filename,attrName,attrRegValue):       if os.path.isfile(filename):       f = open(filename,'r')          soup = BeautifulSoup(f)       f.close()       return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })       else:       return None               class Site1Thread(threading.Thread):       def __init__(self,outputFilePath):         threading.Thread.__init__(self)       self.outputFilePath = outputFilePath       self.fileName = str(random.randint(100,1000)) + ".html"       self.setName('Site1Thread')             def run(self):       site1_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'site.js')       site2_file = os.path.join(self.outputFilePath,'site.js')       if not os.path.isfile(site2_file) and os.path.isfile(site1_file):         shutil.copy(site1_file,site2_file)       #proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)       if isWindows():         proc = subprocess.Popen(["cmd","/c", "%s/casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)       else:         proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)       out=proc.communicate()[0]       htmlFileName = ''       #因为输出路径在windows不确定，所以这里加了所有可能的路径判断       if os.path.isfile(self.fileName):         htmlFileName = self.fileName       elif os.path.isfile(os.path.join(self.outputFilePath,self.fileName)):         htmlFileName = os.path.join(self.outputFilePath,self.fileName)       elif os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)):         htmlFileName = os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)        if (not os.path.isfile(htmlFileName)):         print 'Failed to get html content from http://spys.ru/free-proxy-list/IE/'         print out         sys.exit(3)        mutex.acquire()       PROXYList= getTagsByAttrsExt('font',htmlFileName,'class','spy14$')       for proxy in PROXYList:         tdContent = proxy.renderContents()         lineElems = re.split('[<>]',tdContent)         if re.compile(r'\d+').search(lineElems[-1]) and re.compile('(\d+\.\d+\.\d+)').search(lineElems[0]):         print lineElems[0],lineElems[-1]         PROXY_LIST.append("%s:%s" % (lineElems[0],lineElems[-1]))       mutex.release()       try:         if os.path.isfile(htmlFileName):         os.remove(htmlFileName)       except:         pass          if __name__ == '__main__':       try:       if(len(sys.argv)) < 2:         print "Usage:%s [casperjs path]" % (sys.argv[0])         sys.exit(1)        if not os.path.exists(sys.argv[1]):         print "casperjs path: %s does not exist!" % (sys.argv[1])         sys.exit(2)        if os.path.isfile(output_file):         f = open(output_file)         lines = f.readlines()         f.close         for line in lines:         PROXY_LIST.append(line.strip())       thread1 = Site1Thread(sys.argv[1])       thread1.start()       thread1.join()              f = open(output_file,'w')       for proxy in set(PROXY_LIST):         f.write(proxy+"\n")       f.close()       print "Done!"       except SystemExit:       pass       except:         errMsg = traceback.format_exc()         print errMsg         log.error(errMsg)