# coding=utf-8__author__ = 'scaleworld'import requestsimport reimport sysreload(sys)sys.setdefaultencoding("utf-8")class Spider:    def __init__(self):        PRint '開始爬取極客學(xué)院課程信息。。。'    # 獲取源代碼    def getSource(self, url):        html = requests.get(url)        return html.text    # 獲取每個(gè)課程塊信息    def getLessons(self, source):        lessons = re.findall('deg="0" >(.*?)</li>', source, re.S)        return lessons    # 獲取課程信息，如課程名稱、課程介紹、課程時(shí)間、課程等級(jí)、學(xué)習(xí)人數(shù)    def getLessonInfo(self, lesson):        info = {}        info['title'] = re.search('<h2 class="lesson-info-h2"><a(.*?)>(.*?)</a></h2>', lesson, re.S).group(2).strip()        info['desc'] = re.search('<p style="height: 0px; opacity: 0; display: none;">(.*?)</p>', lesson, re.S).group(            1).strip()        timeandlevel = re.findall('<em>(.*?)</em>', lesson, re.S)        info['time'] = timeandlevel[0].strip().replace("/n", "").replace("    ", "")        info['level'] = timeandlevel[1].strip()        info['learnNumber'] = re.search('"learn-number">(.*?)</em>', lesson, re.S).group(1).strip()        return info    # 保存課程信息到文件LessionInfos.txt    def saveLessionInfos(self, lessonInfos):        # 'w'：只寫，會(huì)覆蓋之前寫入的內(nèi)容        # 也可以用'a'：追加到文件末尾        # 如果文件不存在，則自動(dòng)創(chuàng)建文件        f = open('LessionInfos.txt', 'w')        i = 0        for each in lessonInfos:            i += 1            f.writelines('第' + str(i) + '個(gè)課程：/n')            f.writelines('title:' + each['title'] + '/n')            f.writelines('desc:' + each['desc'] + '/n')            f.writelines('time:' + each['time'] + '/n')            f.writelines('level:' + each['level'] + '/n')            f.writelines('learnNumber:' + each['learnNumber'] + '/n/n')        f.close()if __name__ == '__main__':    # 定義課程信息數(shù)組    lessonInfos = []    # 課程信息頁(yè)面url    url = 'http://www.jikexueyuan.com/course/'    # 實(shí)例化爬蟲    spider = Spider()    # 取[1,21)及1到20頁(yè)的課程信息    for i in range(1, 21):        # 構(gòu)建分頁(yè)URL        pageUrl = url + '?pageNum=' + str(i)        print '正在處理頁(yè)面：' + pageUrl        source = spider.getSource(pageUrl)        lessons = spider.getLessons(source)        for lesson in lessons:            lessonInfo = spider.getLessonInfo(lesson)            lessonInfos.append(lessonInfo)            # print 'title:'+lessonInfo.get('title')            # print 'desc:'+lessonInfo.get('desc')            # print 'time:'+lessonInfo.get('time')            # print 'level:'+lessonInfo.get('level')            # print 'learnNumber:'+lessonInfo.get('learnNumber')        print '已處理' + str(lessons.__len__()) + '個(gè)課程信息。'    print '極客學(xué)院課程信息爬取完畢，正在保存課程信息。。。'    spider.saveLessionInfos(lessonInfos)    print '極客學(xué)院課程信息保存完畢。'