python代理下载

#!/usr/bin/env python
# encoding:utf-8
import sys
import time
import urllib2
import socket
import struct

def get_dest_ip(domain):
    ip_addr = socket.gethostbyname(domain) #just for ipv4
    #ip_addr = socket.getaddrinfo(domain, 'http')[0][4][0]
    #[(2, 1, 6, '', ('14.215.177.38', 80)), (2, 2, 17, '', ('14.215.177.38', 80)), (2, 1, 6, '', ('14.215.177.37', 80)), (2, 2, 17, '', ('14.215.177.37', 80))]
    uint32_binary_str = socket.inet_aton(str(ip_addr))
    unpack_result = struct.unpack("!I", uint32_binary_str)
    ip_int = socket.htonl(unpack_result[0]) 
    return ip_int

def crawl_page(url, dest_ip, cur_proxy):
    '''
    '''
    content = ''
    myheaders = {"User-Agent": "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)", 
        "Proxy-Connection": "Keep-Alive",
        "dest_ip": dest_ip}
    p = "http://%s" % cur_proxy
    h = urllib2.ProxyHandler({"http": p})
    o = urllib2.build_opener(h, urllib2.HTTPHandler)
    o.addheaders  = myheaders.items()
    try:
        r = o.open(url, timeout=5)
        content = r.read()
    except urllib2.HTTPError, e:
        print "Error Code:", e.code
        if e.code == 404:
            print "No page: %s" % url
    except urllib2.URLError, e:
        print "Error Reason:",e.reason
    except Exception as e:
        print "Error",str(e)
    if len(content) > 10:
        print "Good\t%s" % p
    else:
        print "Bad\t%s" % p
    return content

def crawl_page2(url, cur_proxy=''):
    '''
    get
    '''
#    print "-->crawl comment: %s" % url
#    print "-->cur_proxy: %s" % cur_proxy
    myheaders = {"User-Agent": "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"}
    content = ''
    try:
        if cur_proxy:
            proxy_handler = urllib2.ProxyHandler({'http': cur_proxy})
            opener = urllib2.build_opener(proxy_handler)
            f = opener.open(url, timeout = 5)
            content = f.read()
        else:
            req = urllib2.Request(url, headers = myheaders)
            f = urllib2.urlopen(req, timeout = 5)
            content = f.read()
    except urllib2.HTTPError, e:
        print "Error Code:", e.code
        if e.code == 404:
            print "No page: %s" % url
    except urllib2.URLError, e:
        print "Error Reason:",e.reason
    except Exception as e:
        print "Error",str(e)
    time.sleep(1)
    return content

if __name__ == '__main__': # open proxys
    proxy_list = open(sys.argv[1]).readlines()
    url = "http://www.baidu.com"
    host = "www.baidu.com"
    dest_ip = str(get_dest_ip(host))
    for p in proxy_list:
        p = p.strip()
        n1 = len(crawl_page(url, dest_ip, p))
        print "crawl_page len:", n1
        n2 = len(crawl_page2(url, p))
        print "crawl_page2 len:", n2