# -*- coding: utf-8 -*- """ Crawling pictures by selenium and urlliburl: 虎撲 馬刺 http://photo.hupu.com/nba/tag/%E9%A9%AC%E5%88%BAurl: 虎撲 陳露 http://photo.hupu.com/nba/tag/%E9%99%88%E9%9C%B2Created on 2015-10-24@author: Eastmount CSDN """  import time   import re   import os import sys import urllib import shutil import datetime from selenium import webdriver  from selenium.webdriver.common.keys import Keys  import selenium.webdriver.support.ui as ui  from selenium.webdriver.common.action_chains import ActionChains  #Open PhantomJS driver = webdriver.PhantomJS(executable_path="G:/phantomjs-1.9.1-windows/phantomjs.exe")#driver = webdriver.Firefox() wait = ui.WebDriverWait(driver,10)  #Download one Picture By urllib def loadPicture(pic_url, pic_path):  pic_name = os.path.basename(pic_url) #刪除路徑獲取圖片名字 pic_name = pic_name.replace('*','') #去除'*' 防止錯誤 invalid mode ('wb') or filename urllib.urlretrieve(pic_url, pic_path + pic_name)  #爬取具體的圖片及下一張def getScript(elem_url, path, nums): try:  #由于鏈接 http://photo.hupu.com/nba/p29556-1.html  #只需拼接 http://..../p29556-數字.html 省略了自動點擊"下一張"操作  count = 1  t = elem_url.find(r'.html')  while (count <= nums):   html_url = elem_url[:t] + '-' + str(count) + '.html'   #print html_url   '''   driver_pic.get(html_url)   elem = driver_pic.find_element_by_xpath("//div[@class='pic_bg']/div/img")   url = elem.get_attribute("src")   '''   #采用正則表達式獲取第3個<div></div> 再獲取圖片URL進行下載   content = urllib.urlopen(html_url).read()   start = content.find(r'<div class="flTab">')   end = content.find(r'<div class="comMark" style>')   content = content[start:end]   div_pat = r'<div.*?>(.*?)<//div>'   div_m = re.findall(div_pat, content, re.S|re.M)   #print div_m[2]   link_list = re.findall(r"(?<=href=/").+?(?=/")|(?<=href=/').+?(?=/')", div_m[2])   #print link_list   url = link_list[0] #僅僅一條url鏈接   loadPicture(url, path)   count = count + 1 except Exception,e:   print 'Error:',e  finally:   print 'Download ' + str(count) + ' pictures/n'    #爬取主頁圖片集的URL和主題 def getTitle(url):  try:   #爬取URL和標題   count = 0   print 'Function getTitle(key,url)'   driver.get(url)   wait.until(lambda driver: driver.find_element_by_xpath("//div[@class='piclist3']"))  print 'Title: ' + driver.title + '/n'    #縮略圖片url(此處無用) 圖片數量 標題(文件名) 注意順序  elem_url = driver.find_elements_by_xpath("//a[@class='ku']/img")  elem_num = driver.find_elements_by_xpath("//div[@class='piclist3']/table/tbody/tr/td/dl/dd[1]")  elem_title = driver.find_elements_by_xpath("//div[@class='piclist3']/table/tbody/tr/td/dl/dt/a")  for url in elem_url:    pic_url = url.get_attribute("src")   html_url = elem_title[count].get_attribute("href")   print elem_title[count].text   print html_url    print pic_url   print elem_num[count].text      #創建圖片文件夾   path = "E://Picture_HP//" + elem_title[count].text + "//"   m = re.findall(r'(/w*[0-9]+)/w*', elem_num[count].text) #python/95398.html">爬蟲圖片張數   nums = int(m[0])   count = count + 1    if os.path.isfile(path):   #Delete file     os.remove(path)    elif os.path.isdir(path):  #Delete dir     shutil.rmtree(path, True)    os.makedirs(path)    #create the file directory    getScript(html_url, path, nums) #visit pages      except Exception,e:   print 'Error:',e  finally:   print 'Find ' + str(count) + ' pages with key/n'   #Enter Function def main():  #Create Folder  basePathDirectory = "E://Picture_HP"  if not os.path.exists(basePathDirectory):   os.makedirs(basePathDirectory)   #Input the Key for search str=>unicode=>utf-8  key = raw_input("Please input a key: ").decode(sys.stdin.encoding)  print 'The key is : ' + key   #Set URL List Sum:1-2 Pages  print 'Ready to start the Download!!!/n/n'  starttime = datetime.datetime.now()  num=1  while num<=1:  #url = 'http://photo.hupu.com/nba/tag/%E9%99%88%E9%9C%B2?p=2&o=1'  url = 'http://photo.hupu.com/nba/tag/%E9%A9%AC%E5%88%BA'    print '第'+str(num)+'頁','url:'+url   #Determine whether the title contains key   getTitle(url)   time.sleep(2)   num = num + 1  else:   print 'Download Over!!!'   #get the runtime  endtime = datetime.datetime.now()  print 'The Running time : ',(endtime - starttime).seconds    main()