# python 2.7import requestsfrom lxml import htmlimport os

# 獲取首頁中子頁的url鏈接def get_page_urls(url):  response = requests.get(url).content  # 通過lxml的html來構(gòu)建選擇器  selector = html.fromstring(response)  urls = []  for i in selector.xpath("/html/body//div[@class='feed-item _j_feed_item']/a/@href"):    urls.append(i)  return urls

# get title from a child's html(div[@class='title'])def get_page_a_title(url):  '''url is ziyouxing's a@href'''  response = requests.get(url).content  selector = html.fromstring(response)  # get xpath by chrome's tool --> /html/body//div[@class='title']/text()  a_title = selector.xpath("/html/body//div[@class='title']/text()")  return a_title

# 獲取頁面選擇器(通過lxml的html構(gòu)建)def get_selector(url):  response = requests.get(url).content  selector = html.fromstring(response)  return selector

# 通過chrome的開發(fā)者工具分析html頁面結(jié)構(gòu)后發(fā)現(xiàn)，我們需要獲取的文本內(nèi)容主要顯示在div[@class='l-topic']和div[@class='p-section']中

# 獲取所需的文本內(nèi)容 def get_page_content(selector):   # /html/body/div[2]/div[2]/div[1]/div[@class='l-topic']/p/text()   page_title = selector.xpath("//div[@class='l-topic']/p/text()")   # /html/body/div[2]/div[2]/div[1]/div[2]/div[15]/div[@class='p-section']/text()   page_content = selector.xpath("//div[@class='p-section']/text()")   return page_title,page_content

# 獲取頁面中的圖片url地址def get_image_urls(selector):  imagesrcs = selector.xpath("//img[@class='_j_lazyload']/@src")  return imagesrcs

# 獲取圖片的標(biāo)題def get_image_title(selector, num)  # num 是從2開始的  url = "/html/body/div[2]/div[2]/div[1]/div[2]/div["+num+"]/span[@class='img-an']/text()"  if selector.xpath(url) is not None:    image_title = selector.xpath(url)  else:    image_title = "map"+str(num) # 沒有就起一個(gè)  return image_title

# 下載圖片def downloadimages(selector,number):  '''number是用來計(jì)數(shù)的'''  urls = get_image_urls()  num = 2  amount = len(urls)  for url in urls:    image_title = get_image_title(selector, num)    filename = "/home/WorkSpace/tour/words/result"+number+"/+"image_title+".jpg"    if not os.path.exists(filename):      os.makedirs(filename)    print('downloading %s image %s' %(number, image_title))    with open(filename, 'wb') as f:      f.write(requests.get(url).content)    num += 1  print "已經(jīng)下載了%s張圖" %num

# 入口，啟動(dòng)并把獲取的數(shù)據(jù)存入文件中if __name__ =='__main__':  url = 'https://www.mafengwo.cn/gonglve/ziyouxing/2033.html'  urls = get_page_urls(url)  # turn to get response from html  number = 1  for i in urls:    selector = get_selector(i)    # download images    downloadimages(selector,number)    # get text and write into a file    page_title, page_content = get_page_content(selector)    result = page_title+'/n'+page_content+'/n/n'    path = "/home/WorkSpace/tour/words/result"+num+"/"    if not os.path.exists(filename):      os.makedirs(filename)    filename = path + "num"+".txt"    with open(filename,'wb') as f:      f.write(result)    print result