python利用proxybroker構(gòu)建爬蟲免費IP代理池的實現(xiàn)

proxybroker find --types HTTP -l 10

proxybroker grab -c CN -o ./proxies.txt -l 5

proxybroker serve --host localhost --port 5000 --types HTTP --lvl High

curl http://httpbin.org/get

curl --proxy http://localhost:5000 http://httpbin.org/get

proxybroker serve --host localhost --port 5000 --types HTTP --lvl High

import requests

proxies = {
  "http": "http://localhost:5000",
  "https": "http://localhost:5000"
}

res = requests.get('http://httpbin.org/get', proxies=proxies)

print(res.json())

scrapy startproject proxy_demo
cd proxy_demo

scrapy genspider proxy_spider_demo httpbin.org

import json
import scrapy
# from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware


class ProxySpiderDemoSpider(scrapy.Spider):
  name = 'proxy_spider_demo'
  allowed_domains = ['httpbin.org']
  # start_urls = ['http://httpbin.org/']

  def start_requests(self):
    meta = dict(proxy='http://localhost:5000')
    # 請求三次該網(wǎng)址，查看IP是否不同
    for _ in range(3):
      # 得加上dont_filter=True，否則第2、3個請求將被dupefilter過濾
      # 請求可以通過meta攜帶代理服務(wù)器地址，與HttpProxyMiddleware下載器中間件配合達(dá)到代理請求的目的
      yield scrapy.Request('http://httpbin.org/get', dont_filter=True, meta=meta, callback=self.parse)

  def parse(self, response):
    json_body = json.loads(response.text)
    print('當(dāng)前請求的IP為：', json_body['origin'])

scrapy crawl --nolog proxy_spider_demo

import asyncio
from proxybroker import Broker

async def view_proxy(proxies_queue):
  while True:
    proxy = await proxies_queue.get()
    
    if proxy is None:
      print('done...')
      break
      
    print(proxy)

# 異步隊列
proxies_queue = asyncio.Queue()

broker = Broker(proxies_queue)

tasks = asyncio.gather(
  # 使用grab子命令獲取3條IP
  broker.grab(limit=3),
  view_proxy(proxies_queue))

loop = asyncio.get_event_loop()
_ = loop.run_until_complete(tasks)