安裝包下載地址： http://phantomjs.org/ ，包括 Windows ，Mac OS，linux版本，自行選擇對應(yīng) 版本下載解壓即可（ 為方便使用，可自已為phantomjs設(shè)置環(huán)境變量 ），其中帶有一個(gè)example文件夾，里面有很多已經(jīng)寫好的代碼供使用。本文假設(shè)phantomjs已經(jīng)安裝好并已設(shè)置了環(huán)境變量。
神馬是中間件？
# -*- coding: utf-8 -*-from selenium import webdriverfrom scrapy.conf import settings# from scrapy.http.response import Responsefrom scrapy.http import HtmlResponseimport timefrom scrapy import signalsfrom scrapy.xlib.pydispatch import dispatcher from telnetlib import DOclass JavaScriptMiddleware(object): def __init__(self): if settings['LOGIN_TYPE'] == 'MyCrawl': ''' self.simulation = weibo_login(settings['USERNAME'], settings['PWD'], settings['COOKIE_FILE']) cookie_file = settings['COOKIE_FILE'] cookie_jar = cookielib.LWPCookieJar(cookie_file) cookie_jar.load(ignore_discard=True, ignore_expires=True) self.driver = webdriver.PhantomJS(executable_path=settings['JS_BIN']) for c in cookie_jar: self.driver.add_cookie({'name': c.name, 'value': c.value, 'path': '/', 'domain': c.domain}) ''' # simulate user login PRocess self.driver = webdriver.PhantomJS(executable_path=settings['JS_BIN'])# 登錄# self.driver.get('http://login.sina.com.cn/')# uid = self.driver.find_element_by_id('username')# upw = self.driver.find_element_by_id('passWord')# loginBtn = self.driver.find_element_by_class_name('smb_btn')# time.sleep(1)# uid.send_keys(settings['USERNAME'])# upw.send_keys(settings['PWD'])# loginBtn.click()# time.sleep(1) elif settings['LOGIN_TYPE'] == 'other': print('add login code') pass else: self.driver = webdriver.PhantomJS(executable_path=settings['JS_BIN']) dispatcher.connect(self.spider_closed, signals.spider_closed) def process_request(self, request, spider): self.driver.get(request.url) print("頁面渲染中····開始自動下拉頁面") indexPage = 1000 while indexPage<self.driver.execute_script("return document.body.offsetHeight"): self.driver.execute_script("scroll(0,"+str(indexPage)+")") indexPage = indexPage +1000 print(indexPage) time.sleep(1) rendered_body = self.driver.page_source #編碼處理 if r'charset="GBK"' in rendered_body or r'charset=gbk' in rendered_body: coding = 'gbk' else: coding = 'utf-8' return HtmlResponse(request.url, body=rendered_body, encoding='utf-8') #關(guān)閉瀏覽器 def spider_closed(self, spider, reason): print ('close driver......') self.driver.close()新聞熱點(diǎn)
疑難解答
圖片精選
