count_ip_key='spider_ip'
ip_limit_num = 500
def count_ip(ip): #统计ip次数
    count = r.hget(count_ip_key,ip)
    if isinstance(count,bytes):r.hset(count_ip_key,ip, eval(count)+1)
    else:r.hset(count_ip_key,ip, 1)
def num_ip(ip): 获取ip的次数与限制值比较
    count = r1.hget(count_ip_key, ip)
    if isinstance(count,bytes):return eval(count)
    else:return 0
while True:
    if r.scard(proxy_key) > 0:
        ip = r.spop(proxy_key)
        if not isinstance(ip,bytes):continue
        ip=ip.decode()
        print('ip:',ip)
        if num_ip(ip) > ip_limit_num:
            continue
        proxies = {'http': 'http://{}'.format(ip), 'https': 'http://{}'.format(ip)}
    爬虫函数
    count_ip(ip)
    if num_ip(ip) < ip_limit_num:
        r.sadd(proxy_key,ip)

更多推荐