'''Created on 2012-3-13Get Page using GET methodDefault using HTTP Protocol , http port 80@author: xiaojay'''import socketimport statisticsimport datetimeimport threadingsocket.setdefaulttimeout(statistics.timeout)class Error404(Exception):  '''Can not find the page.'''  passclass ErrorOther(Exception):  '''Some other exception'''  def __init__(self,code):    #print 'Code :',code    passclass ErrorTryTooManyTimes(Exception):  '''try too many times'''  passdef downPage(hostname ,filename , trytimes=0):  try :    #To avoid too many tries .Try times can not be more than max_try_times    if trytimes >= statistics.max_try_times :      raise ErrorTryTooManyTimes  except ErrorTryTooManyTimes :    return statistics.RESULTTRYTOOMANY,hostname+filename  try:    s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)    #DNS cache    if statistics.DNSCache.has_key(hostname):      addr = statistics.DNSCache[hostname]    else:      addr = socket.gethostbyname(hostname)      statistics.DNSCache[hostname] = addr    #connect to http server ,default port 80    s.connect((addr,80))    msg = 'GET '+filename+' HTTP/1.0/r/n'    msg += 'Host: '+hostname+'/r/n'    msg += 'User-Agent:xiaojay/r/n/r/n'    code = ''    f = None    s.sendall(msg)    first = True    while True:      msg = s.recv(40960)      if not len(msg):        if f!=None:          f.flush()          f.close()        break      # Head information must be in the first recv buffer      if first:        first = False        headpos = msg.index("/r/n/r/n")        code,other = dealwithHead(msg[:headpos])        if code=='200':          #statistics.fetched_url += 1          f = open('pages/'+str(abs(hash(hostname+filename))),'w')          f.writelines(msg[headpos+4:])        elif code=='301' or code=='302':          #if code is 301 or 302 , try down again using redirect location          if other.startswith("http") :            hname, fname = parse(other)            downPage(hname,fname,trytimes+1)#try again          else :            downPage(hostname,other,trytimes+1)        elif code=='404':          raise Error404        else :          raise ErrorOther(code)      else:        if f!=None :f.writelines(msg)    s.shutdown(socket.SHUT_RDWR)    s.close()    return statistics.RESULTFETCHED,hostname+filename  except Error404 :    return statistics.RESULTCANNOTFIND,hostname+filename  except ErrorOther:    return statistics.RESULTOTHER,hostname+filename  except socket.timeout:    return statistics.RESULTTIMEOUT,hostname+filename  except Exception, e:    return statistics.RESULTOTHER,hostname+filenamedef dealwithHead(head):  '''deal with HTTP HEAD'''  lines = head.splitlines()  fstline = lines[0]  code =fstline.split()[1]  if code == '404' : return (code,None)  if code == '200' : return (code,None)  if code == '301' or code == '302' :    for line in lines[1:]:      p = line.index(':')      key = line[:p]      if key=='Location' :        return (code,line[p+2:])  return (code,None)def parse(url):  '''Parse a url to hostname+filename'''  try:    u = url.strip().strip('/n').strip('/r').strip('/t')    if u.startswith('http://') :      u = u[7:]    elif u.startswith('https://'):      u = u[8:]    if u.find(':80')>0 :      p = u.index(':80')      p2 = p + 3    else:      if u.find('/')>0:        p = u.index('/')        p2 = p      else:        p = len(u)        p2 = -1    hostname = u[:p]    if p2>0 :      filename = u[p2:]    else : filename = '/'    return hostname, filename  except Exception ,e:    print "Parse wrong : " , url    print edef PrintDNSCache():  '''print DNS dict'''  n = 1  for hostname in statistics.DNSCache.keys():    print n,'/t',hostname, '/t',statistics.DNSCache[hostname]    n+=1def dealwithResult(res,url):  '''Deal with the result of downPage'''  statistics.total_url+=1  if res==statistics.RESULTFETCHED :    statistics.fetched_url+=1    print statistics.total_url , '/t fetched :', url  if res==statistics.RESULTCANNOTFIND :    statistics.failed_url+=1    print "Error 404 at : ", url  if res==statistics.RESULTOTHER :    statistics.other_url +=1    print "Error Undefined at : ", url  if res==statistics.RESULTTIMEOUT :    statistics.timeout_url +=1    print "Timeout ",url  if res==statistics.RESULTTRYTOOMANY:    statistics.trytoomany_url+=1    print e ,"Try too many times at", urlif __name__=='__main__':  print 'Get Page using GET method'

'''Created on 2012-3-16@author: xiaojay'''import fetchPageimport threadpoolimport datetimeimport statisticsimport urllib2'''one thread'''def usingOneThread(limit):  urlset = open("input.txt","r")  start = datetime.datetime.now()  for u in urlset:    if limit <= 0 : break    limit-=1    hostname , filename = parse(u)    res= fetchPage.downPage(hostname,filename,0)    fetchPage.dealwithResult(res)  end = datetime.datetime.now()  print "Start at :/t" , start  print "End at :/t" , end  print "Total Cost :/t" , end - start  print 'Total fetched :', statistics.fetched_url'''threadpoll and GET method'''def callbackfunc(request,result):  fetchPage.dealwithResult(result[0],result[1])def usingThreadpool(limit,num_thread):  urlset = open("input.txt","r")  start = datetime.datetime.now()  main = threadpool.ThreadPool(num_thread)  for url in urlset :    try :      hostname , filename = fetchPage.parse(url)      req = threadpool.WorkRequest(fetchPage.downPage,args=[hostname,filename],kwds={},callback=callbackfunc)      main.putRequest(req)    except Exception:      print Exception.message  while True:    try:      main.poll()      if statistics.total_url >= limit : break    except threadpool.NoResultsPending:      print "no pending results"      break    except Exception ,e:      print e  end = datetime.datetime.now()  print "Start at :/t" , start  print "End at :/t" , end  print "Total Cost :/t" , end - start  print 'Total url :',statistics.total_url  print 'Total fetched :', statistics.fetched_url  print 'Lost url :', statistics.total_url - statistics.fetched_url  print 'Error 404 :' ,statistics.failed_url  print 'Error timeout :',statistics.timeout_url  print 'Error Try too many times ' ,statistics.trytoomany_url  print 'Error Other faults ',statistics.other_url  main.stop()'''threadpool and urllib2 '''def downPageUsingUrlib2(url):  try:    req = urllib2.Request(url)    fd = urllib2.urlopen(req)    f = open("pages3/"+str(abs(hash(url))),'w')    f.write(fd.read())    f.flush()    f.close()    return url ,'success'  except Exception:    return url , Nonedef writeFile(request,result):  statistics.total_url += 1  if result[1]!=None :    statistics.fetched_url += 1    print statistics.total_url,'/tfetched :', result[0],  else:    statistics.failed_url += 1    print statistics.total_url,'/tLost :',result[0],def usingThreadpoolUrllib2(limit,num_thread):  urlset = open("input.txt","r")  start = datetime.datetime.now()  main = threadpool.ThreadPool(num_thread)  for url in urlset :    try :      req = threadpool.WorkRequest(downPageUsingUrlib2,args=[url],kwds={},callback=writeFile)      main.putRequest(req)    except Exception ,e:      print e  while True:    try:      main.poll()      if statistics.total_url >= limit : break    except threadpool.NoResultsPending:      print "no pending results"      break    except Exception ,e:      print e  end = datetime.datetime.now()  print "Start at :/t" , start  print "End at :/t" , end  print "Total Cost :/t" , end - start  print 'Total url :',statistics.total_url  print 'Total fetched :', statistics.fetched_url  print 'Lost url :', statistics.total_url - statistics.fetched_url  main.stop()if __name__ =='__main__':  '''too slow'''  #usingOneThread(100)  '''use Get method'''  #usingThreadpool(3000,50)  '''use urllib2'''  usingThreadpoolUrllib2(3000,50)