設定IP代理池

requests設定代理

import requests

proxy = \'127.0.0.1:9743\'
proxies = {
    \'http\': \'http://\' + proxy,
    \'https\': \'https://\' + proxy,
}
try:
    response = requests.get(\'http://httpbin.org/get\', proxies=proxies)
    print(response.text)
except requests.exceptions.ConnectionError as e:
    print(\'Error\', e.args)
           

selenium設定代理

from selenium import webdriver

proxy = \'127.0.0.1:9743\'
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument(\'--proxy-server=http://\' + proxy)
browser = webdriver.Chrome(chrome_options=chrome_options)
browser.get(\'http://httpbin.org/get\')
           

scrapy設定代理

class ProxyMiddleware(object):

    def get_random_proxy(self):
        # 自己維護在本地的IP代理池
        try:
            proxy = requests.get(\'http://127.0.0.1:5555/random\')
            if proxy.status_code == 200:
                return proxy.text
        except:
            return None
        
    def process_request(self, request, spider):
        proxy = self.get_random_proxy()
        if proxy:
            request.meta[\'proxy\'] = proxy