from urllib.error import URLError from urllib.request import ProxyHandler, build_openerproxy = '127.0.0.1:8888'#需要认证的代理#proxy = 'username:password@127.0.0.1:8888'#使用ProxyHandler设置代理proxy_handler = ProxyHandler({ 'http': 'http://' + proxy, 'https': 'https://' + proxy})#传入参数创建Opener对象opener = build_opener(proxy_handler)try: response = opener.open('http://httpbin.org/get') print(response.read().decode('utf-8'))except URLError as e: print(e.reason)
import requests proxy = '127.0.0.1:8888'#需要认证的代理#proxy = 'username:password@127.0.0.1:8888'proxies = { 'http': 'http://' + proxy, 'https': 'https://' + proxy,}try: response = requests.get('http://httpbin.org/get', proxies=proxies) print(response.text)except requests.exceptions.ConnectionError as e: print('Error', e.args)
from selenium import webdriver service_args = [ '--proxy=127.0.0.1:9743', '--proxy-type=http', #'--proxy-auth=username:password' #带认证代理]browser = webdriver.PhantomJS(service_args=service_args)browser.get('http://httpbin.org/get')print(browser.page_source) 使用的是Chrome from selenium import webdriver proxy = '127.0.0.1:9743'chrome_options = webdriver.ChromeOptions()chrome_options.add_argument('--proxy-server=http://' + proxy)chrome = webdriver.Chrome(chrome_options=chrome_options)chrome.get('http://httpbin.org/get')
#在Scrapy的Downloader Middleware中间件里 ... def process_request(self, request, spider): request.meta['proxy'] = 'http://127.0.0.1:9743' ...
免费代理IP的使用
import requests,random #定义代理池proxy_list = [ '182.39.6.245:38634', '115.210.181.31:34301', '123.161.152.38:23201', '222.85.5.187:26675', '123.161.152.31:23127',]# 随机选择一个代理proxy = random.choice(proxy_list)proxies = { 'http': 'http://' + proxy, 'https': 'https://' + proxy,}try: response = requests.get('http://httpbin.org/get', proxies=proxies) print(response.text)except requests.exceptions.ConnectionError as e: print('Error', e.args)
import requests # 从代理服务中获取一个代理IPproxy = requests.get("http://tvp.daxiangdaili.com/ip/?tid=559775358931681&num=1").textproxies = { 'http': 'http://' + proxy, 'https': 'https://' + proxy,}try: response = requests.get('http://httpbin.org/get', proxies=proxies) print(response.text)except requests.exceptions.ConnectionError as e: print('Error', e.args)
本站文章如无特殊说明,均为本站原创,如若转载,请注明出处:python爬虫数据采集ip被封一篇解决 - Python技术站