import urllib2content = urllib2.urlopen('http://XXXX').read()

import urllib2proxy_support = urllib2.ProxyHandler({'http':'http://XX.XX.XX.XX:XXXX'})opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)urllib2.install_opener(opener)content = urllib2.urlopen('http://XXXX').read()

import urllib2, cookielibcookie_support= urllib2.HTTPCookieProcessor(cookielib.CookieJar())opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)urllib2.install_opener(opener)content = urllib2.urlopen('http://XXXX').read()

opener = urllib2.build_opener(proxy_support, cookie_support, urllib2.HTTPHandler)

import urllibpostdata=urllib.urlencode({ 'username':'XXXXX', 'password':'XXXXX', 'continueURI':'http://www.verycd.com/', 'fk':fk, 'login_submit':'登錄'})

req = urllib2.Request( url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/', data = postdata)result = urllib2.urlopen(req).read()

headers = { 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}req = urllib2.Request( url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/', data = postdata, headers = headers)

headers = { 'Referer':'http://www.cnbeta.com/articles'}

from threading import Threadfrom Queue import Queuefrom time import sleep#q是任務(wù)隊(duì)列#NUM是并發(fā)線程總數(shù)#JOBS是有多少任務(wù)q = Queue()NUM = 2JOBS = 10#具體的處理函數(shù)，負(fù)責(zé)處理單個(gè)任務(wù)def do_somthing_using(arguments): print arguments#這個(gè)是工作進(jìn)程，負(fù)責(zé)不斷從隊(duì)列取數(shù)據(jù)并處理def working(): while True:  arguments = q.get()  do_somthing_using(arguments)  sleep(1)  q.task_done()#fork NUM個(gè)線程等待隊(duì)列for i in range(NUM): t = Thread(target=working) t.setDaemon(True) t.start()#把JOBS排入隊(duì)列for i in range(JOBS): q.put(i)#等待所有JOBS完成q.join()

import urllib2from gzip import GzipFilefrom StringIO import StringIOclass ContentEncodingProcessor(urllib2.BaseHandler): """A handler to add gzip capabilities to urllib2 requests """  # add headers to requests def http_request(self, req): req.add_header("Accept-Encoding", "gzip, deflate") return req  # decode def http_response(self, req, resp): old_resp = resp # gzip if resp.headers.get("content-encoding") == "gzip":  gz = GzipFile(     fileobj=StringIO(resp.read()),     mode="r"     )  resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)  resp.msg = old_resp.msg # deflate if resp.headers.get("content-encoding") == "deflate":  gz = StringIO( deflate(resp.read()) )  resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and  resp.msg = old_resp.msg return resp # deflate supportimport zlibdef deflate(data): # zlib only provides the zlib compress format, not the deflate format; try:    # so on top of all there's this workaround: return zlib.decompress(data, -zlib.MAX_WBITS) except zlib.error: return zlib.decompress(data)

encoding_support = ContentEncodingProcessoropener = urllib2.build_opener( encoding_support, urllib2.HTTPHandler ) #直接用opener打開網(wǎng)頁，如果服務(wù)器支持gzip/defalte則自動(dòng)解壓縮content = opener.open(url).read()

from twisted.web.client import getPagefrom twisted.internet import reactor links = [ 'http://www.verycd.com/topics/%d/'%i for i in range(5420,5430) ] def parse_page(data,url): print len(data),url def fetch_error(error,url): print error.getErrorMessage(),url # 批量抓取鏈接for url in links: getPage(url,timeout=5) /  .addCallback(parse_page,url) / #成功則調(diào)用parse_page方法  .addErrback(fetch_error,url)  #失敗則調(diào)用fetch_error方法 reactor.callLater(5, reactor.stop) #5秒鐘后通知reactor結(jié)束程序reactor.run()

f = Fetcher(threads=10) #設(shè)定下載線程數(shù)為10for url in urls: f.push(url) #把所有url推入下載隊(duì)列while f.taskleft(): #若還有未完成下載的線程 content = f.pop() #從下載完成隊(duì)列中取出結(jié)果 do_with(content) # 處理content內(nèi)容

import urllib2from threading import Thread,Lockfrom Queue import Queueimport time class Fetcher: def __init__(self,threads):  self.opener = urllib2.build_opener(urllib2.HTTPHandler)  self.lock = Lock() #線程鎖  self.q_req = Queue() #任務(wù)隊(duì)列  self.q_ans = Queue() #完成隊(duì)列  self.threads = threads  for i in range(threads):   t = Thread(target=self.threadget)   t.setDaemon(True)   t.start()  self.running = 0  def __del__(self): #解構(gòu)時(shí)需等待兩個(gè)隊(duì)列完成  time.sleep(0.5)  self.q_req.join()  self.q_ans.join()  def taskleft(self):  return self.q_req.qsize()+self.q_ans.qsize()+self.running  def push(self,req):  self.q_req.put(req)  def pop(self):  return self.q_ans.get()  def threadget(self):  while True:   req = self.q_req.get()   with self.lock: #要保證該操作的原子性，進(jìn)入critical area    self.running += 1   try:    ans = self.opener.open(req).read()   except Exception, what:    ans = ''    print what   self.q_ans.put((req,ans))   with self.lock:    self.running -= 1   self.q_req.task_done()   time.sleep(0.1) # don't spam if __name__ == "__main__": links = [ 'http://www.verycd.com/topics/%d/'%i for i in range(5420,5430) ] f = Fetcher(threads=10) for url in links:  f.push(url) while f.taskleft():  url,content = f.pop()  print url,len(content)

from threading import stack_sizestack_size(32768*16)

 def get(self,req,retries=3):  try:   response = self.opener.open(req)   data = response.read()  except Exception , what:   print what,req   if retries>0:    return self.get(req,retries-1)   else:    print 'GET Failed',req    return ''  return data

 import socket socket.setdefaulttimeout(10) #設(shè)置10秒后連接超時(shí)

def login(self,username,password): import urllib data=urllib.urlencode({'username':username,       'password':password,       'continue':'http://www.verycd.com/',       'login_submit':u'登錄'.encode('utf-8'),       'save_cookie':1,}) url = 'http://www.verycd.com/signin' self.opener.open(url,data).read()