# -*- coding: utf-8 -*-import scrapyclass TencentItem(scrapy.Item):  # 職位名  positionname = scrapy.Field()  # 詳情連接  positionlink = scrapy.Field()  # 職位類別  positionType = scrapy.Field()  # 招聘人數  peopleNum = scrapy.Field()  # 工作地點  workLocation = scrapy.Field()  # 發布時間  publishTime = scrapy.Field()

# tencentPostion為爬蟲名，tencent.com為爬蟲作用范圍scrapy genspider tencentPostion "tencent.com"

# -*- coding: utf-8 -*-import scrapyfrom tencent.items import TencentItemclass TencentpositionSpider(scrapy.Spider):  """  功能：爬取騰訊社招信息  """  # 爬蟲名  name = "tencentPosition"  # 爬蟲作用范圍  allowed_domains = ["tencent.com"]  url = "http://hr.tencent.com/position.php?&start="  offset = 0  # 起始url  start_urls = [url + str(offset)]  def parse(self, response):    for each in response.xpath("//tr[@class='even'] | //tr[@class='odd']"):      # 初始化模型對象      item = TencentItem()      # 職位名稱      item['positionname'] = each.xpath("./td[1]/a/text()").extract()[0]      # 詳情連接      item['positionlink'] = each.xpath("./td[1]/a/@href").extract()[0]      # 職位類別      item['positionType'] = each.xpath("./td[2]/text()").extract()[0]      # 招聘人數      item['peopleNum'] = each.xpath("./td[3]/text()").extract()[0]      # 工作地點      item['workLocation'] = each.xpath("./td[4]/text()").extract()[0]      # 發布時間      item['publishTime'] = each.xpath("./td[5]/text()").extract()[0]      yield item    if self.offset < 1680:      self.offset += 10    # 每次處理完一頁的數據之后，重新發送下一頁頁面請求    # self.offset自增10，同時拼接為新的url，并調用回調函數self.parse處理Response    yield scrapy.Request(self.url + str(self.offset), callback = self.parse)

# -*- coding: utf-8 -*-import jsonclass TencentPipeline(object):　　"""     功能：保存item數據   """  def __init__(self):    self.filename = open("tencent.json", "w")  def process_item(self, item, spider):    text = json.dumps(dict(item), ensure_ascii = False) + ",/n"    self.filename.write(text.encode("utf-8"))    return item  def close_spider(self, spider):    self.filename.close()

# 設置請求頭部，添加urlDEFAULT_REQUEST_HEADERS = {  "User-Agent" : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;",  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}# 設置item——pipelinesITEM_PIPELINES = {  'tencent.pipelines.TencentPipeline': 300,}

# tencentPosition為爬蟲名scrapy crwal tencentPosition

# 創建項目scrapy startproject TencentSpider# 進入項目目錄下，創建爬蟲文件scrapy genspider -t crawl tencent tencent.comitem等文件寫法不變，主要是爬蟲文件的編寫# -*- coding:utf-8 -*-import scrapy# 導入CrawlSpider類和Rulefrom scrapy.spiders import CrawlSpider, Rule# 導入鏈接規則匹配類，用來提取符合規則的連接from scrapy.linkextractors import LinkExtractorfrom TencentSpider.items import TencentItemclass TencentSpider(CrawlSpider):  name = "tencent"  allow_domains = ["hr.tencent.com"]  start_urls = ["http://hr.tencent.com/position.php?&start=0#a"]  # Response里鏈接的提取規則，返回的符合匹配規則的鏈接匹配對象的列表  pagelink = LinkExtractor(allow=("start=/d+"))  rules = [    # 獲取這個列表里的鏈接，依次發送請求，并且繼續跟進，調用指定回調函數處理    Rule(pagelink, callback = "parseTencent", follow = True)  ]  # 指定的回調函數  def parseTencent(self, response):    for each in response.xpath("//tr[@class='even'] | //tr[@class='odd']"):      item = TencentItem()      # 職位名稱      item['positionname'] = each.xpath("./td[1]/a/text()").extract()[0]      # 詳情連接      item['positionlink'] = each.xpath("./td[1]/a/@href").extract()[0]      # 職位類別      item['positionType'] = each.xpath("./td[2]/text()").extract()[0]      # 招聘人數      item['peopleNum'] = each.xpath("./td[3]/text()").extract()[0]      # 工作地點      item['workLocation'] = each.xpath("./td[4]/text()").extract()[0]      # 發布時間      item['publishTime'] = each.xpath("./td[5]/text()").extract()[0]      yield item