Scrapy中间件user

一、定义实现随机User-Agent的下载中间件

1.在middlewares.py中完善代码

1 1 import random
2 2 from Tencent.settings import USER_AGENTS_LIST # 注意导入路径,请忽视pycharm的错误提示
3 3 
4 4 class UserAgentMiddleware(object):
5 5     def process_request(self, request, spider):
6 6         user_agent = random.choice(USER_AGENTS_LIST)
7 7         request.headers['User-Agent'] = user_agent
8 8         # 不写return
9 9 
1010 class CheckUA:
1111     def process_response(self,request,response,spider):
1212         print(request.headers['User-Agent'])
1313         return response # 不能少！

2.在settings中设置开启自定义的下载中间件，设置方法同管道

11 DOWNLOADER_MIDDLEWARES = {
22    'Tencent.middlewares.UserAgentMiddleware': 543, # 543是权重值
33    'Tencent.middlewares.CheckUA': 600, # 先执行543权重的中间件，再执行600的中间件
44 }

3.在settings中添加UA的列表

11 USER_AGENTS_LIST = [
22     "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
33     "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
44     "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
55     "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
66     "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5"
77 ]

二、代理ip的使用

1.在middlewares.py中完善代码

1 1 class RandomProxy(object):
2 2 
3 3     def process_request(self, request, spider):
4 4 
5 5         proxy = random.choice(PROXY_LIST)
6 6         print(proxy)
7 7 
8 8         if 'user_passwd' in proxy:
9 9             # 对账号密码进行编码，基础认证,python3中需要是bytes类型的数据才能编码
1010             b64_up = base64.b64encode(proxy['user_passwd'].encode())
1111 
1212             # 进行代理认证
1313             request.headers['Proxy-Authorization'] = 'Basic ' + b64_up.decode()
1414 
1515             # 设置代理
1616             request.meta['proxy'] = proxy['ip_port']
1717         else:
1818             #设置代理
1919             request.meta['proxy'] = proxy['ip_port']

2.检测代理ip是否可用

在使用了代理ip的情况下可以在下载中间件的process_response()方法中处理代理ip的使用情况，如果该代理ip不能使用可以替换其他代理ip

11 class ProxyMiddleware(object):
22     ......
33     def process_response(self, request, response, spider):
44         if response.status != '200':
55             request.dont_filter = True # 重新发送的请求对象能够再次进入队列
66             return requst

3.在settings中添加代理ip的列表

11 PROXY_LIST = [
22     {"ip_port": "139.199.121.163:16818", "user_passwd": "user:password"},#收费代理
33     # {"ip_port": "114.234.81.72:9000"} # 免费代理
44 ]

三. 在中间件中使用selenium

以github登陆为例

1. 完成爬虫代码

1 1 import scrapy
2 2 
3 3 class Login4Spider(scrapy.Spider):
4 4     name = 'login4'
5 5     allowed_domains = ['github.com']
6 6     start_urls = ['https://github.com/returnes'] # 直接对验证的url发送请求
7 7 
8 8     def parse(self, response):
9 9         with open('check.html', 'w') as f:
1010             f.write(response.body.decode())

2.在middlewares.py中使用selenium获取cookie信息

1 1 import time
2 2 from selenium import webdriver
3 3 
4 4 
5 5 def getCookies():
6 6     # 使用selenium模拟登陆，获取并返回cookie
7 7     username = input('输入github账号:')
8 8     password = input('输入github密码:')
9 9     options = webdriver.ChromeOptions()
1010     options.add_argument('--headless')
1111     options.add_argument('--disable-gpu')
1212     driver = webdriver.Chrome('/home/worker/Desktop/driver/chromedriver',
1313                               chrome_options=options)
1414     driver.get('https://github.com/login')
1515     time.sleep(1)
1616     driver.find_element_by_xpath('//*[@id="login_field"]').send_keys(username)
1717     time.sleep(1)
1818     driver.find_element_by_xpath('//*[@id="password"]').send_keys(password)
1919     time.sleep(1)
2020     driver.find_element_by_xpath('//*[@id="login"]/form/div[3]/input[3]').click()
2121     time.sleep(2)
2222     cookies_dict = {cookie['name']: cookie['value'] for cookie in driver.get_cookies()}
2323     driver.quit()
2424     return cookies_dict
2525 
2626 class LoginDownloaderMiddleware(object):
2727 
2828     def process_request(self, request, spider):
2929         cookies_dict = getCookies()
3030         print(cookies_dict)
3131         request.cookies = cookies_dict # 对请求对象的cookies属性进行替换

3.在middlewares.py中使用selenium获取指定页面渲染后的html源码

1 1 class SelMiddleWare(object):
2 2 
3 3     def process_request(self, request, spider):
4 4 
5 5         url = request.url
6 6         # 过滤需要渲染的请求对象
7 7         if 'daydata' in url:
8 8 
9 9             driver = webdriver.Chrome()
1010 
1111             driver.get(url)
1212             time.sleep(3)
1313 
1414             data = driver.page_source
1515             driver.close()
1616 
1717             res = HtmlResponse(
1818                 url=url,
1919                 body=data,
2020                 encoding='utf-8',
2121                 request=request
2222             )
2323 
2424             return res