京東圖書評論有非常豐富的信息,這里面就包含了購買日期、書名、作者、好評、中評、差評等等。以購買日期為例,使用Python + Mysql的搭配進行實現,程序不大,才100行。相關的解釋我都在程序里加注了:
from selenium import webdriver
from bs4 import BeautifulSoup
import re
import win32com.client
import threading,time
import MySQLdb
def mydebug():
driver.quit()
exit(0)
def catchDate(s):
"""頁面數據提取"""
soup = BeautifulSoup(s)
z = []
global nowtimes
m = soup.findAll("div",class_="date-buy")
for obj in m:
try:
tmp = obj.find('br').contents
except Exception, e:
continue
if(tmp != ""):
z.append(tmp)
nowtimes += 1
return z
def getTimes(n,t):
"""獲取當前進度"""
return "當前進度為:" + str(int(100*n/t)) + "%"
#―――――――――――――――――――――――――――――――――――| 程序開始 |―――――――――――――――――――――――――――――――――
#確定圖書大類
cate = {"3273":"歷史","3279":"心理學","3276":"政治軍事","3275":"國學古籍","3274":"哲學宗教","3277":"法律","3280":"文化","3281":"社會科學"}
#斷點續抓
num1 = input("bookid:")
num2 = input("pagenumber:")
#生成圖書大類鏈接,共需17355*20 = 347100次
totaltimes = 347100.0
nowtimes = 0
#開啟webdirver的PhantomJS對象
#driver = webdriver.PhantomJS()
driver = webdriver.Ie('C:/Python27/Scripts/IEDriverServer')
#driver = webdriver.Chrome('C:/Python27/Scripts/chromedriver')
#讀出Mysql中的評論頁面,進行抓取
# 連接數據庫
try:
conn = MySQLdb.connect(host='localhost',user='root',passwd='',db='jd')
except Exception, e:
print e
sys.exit()
# 獲取cursor對象
cursor = conn.cursor()
sql = "SELECT * FROM booknew ORDER BY pagenumber DESC"
cursor.execute(sql)
alldata = cursor.fetchall()
flag = 0
flag2 = 0
# 如果有數據返回就循環輸出,http://club.jd.com/review/10178500-1-154.html
if alldata:
for rec in alldata:
#rec[0]--bookid,rec[1]--cateid,rec[2]--pagenumber
if(rec[0] != str(num1) and flag == 0):
continue
else:
flag = 1
for p in range(num2,rec[2]):
if(flag2 == 0):
num2 = 0
flag2 = 1
p += 1
link = " #抓網頁
driver.get(link)
html = driver.page_source
#抓評論
buydate = catchDate(html)
#寫入數據庫
for z in buydate:
sql = "INSERT INTO ljj (id, cateid, bookid, date) VALUES (NULL, '" + rec[0] + "','" + rec[1] + "','" + z[0] + "');"
try:
cursor.execute(sql)
except Exception, e:
print e
conn.commit()
print getTimes(nowtimes,totaltimes)
driver.quit()
cursor.close()
conn.close()
新聞熱點
疑難解答
圖片精選