scrapy模拟用户登录

scrapy框架编写模拟用户登录的三种方式：

方式一：携带cookie登录，携带cookie一般请求的url为登录后的页面，获取cookie信息应在登录后的页面获取，cookie参数应转成字典形式

# -*- coding: utf-8 -*-
import re
import scrapyclass RenrenSpider(scrapy.Spider):name = 'renren'allowed_domains = ['renren.com']start_urls = ['http://www.renren.com/966403607/profile']#要对star_urls进行请求，则要重写start_requestsdef start_requests(self):cookies = 'anonymid=joz9buh7-q7cfyi; depovince=GUZ; _r01_=1; _de=A10BB6D966D15FBA1F90E79AB0D2FDF8; ln_uact=18520877258; ln_hurl=http://head.xiaonei.com/photos/0/0/men_main.gif; jebe_key=b605eb90-21b9-4072-9d48-b75b233c1cea%7Cb156ee0bfd56183e1b8eb9e5994eb5ef%7C1543293909743%7C1%7C1543293910671; jebecookies=9700aefc-77a1-49a7-8d74-882aa173e271|||||; JSESSIONID=abcxtZqTI1hOu4WzN0tDw; ick_login=21361cc0-986f-49bc-95f9-de3a9ed54a69; p=29e3cce85947859ee0e1d6264160539f7; first_login_flag=1; t=b6e6ac604c66019acf93cb471550349e7; societyguester=b6e6ac604c66019acf93cb471550349e7; id=966403607; xnsid=ac2d6a1a; loginfrom=syshome; wp_fold=0'# 将cookies分割成字典形式cookies = {i.split("=")[0]:i.split("=")[1] for i in cookies.split("; ")}# 回调中携带cookies的值yield scrapy.Request(self.start_urls[0],cookies=cookies,callback=self.parse,)def parse(self, response):item = {}name = re.findall("尚学堂",response.body.decode())item["name"] = nameyield item

为了可以知道cookie确实是在不同的解析函数中传递在setting中设置一个参数如下
# 该设置可以知道cookie确实是在不同的解析函数中传递
COOKIES_DEBUG=True

方式二：表单dataform的post请求，

# -*- coding: utf-8 -*-
import re
import scrapyclass GithubSpider(scrapy.Spider):name = 'github'allowed_domains = ['github.com']start_urls = ['https://github.com/login']def parse(self, response):authenticity_token = response.xpath('//input[@name="authenticity_token"]/@value').extract_first()# form表单提交postformdata = {"commit": "Sign in","utf8": "✓","authenticity_token": authenticity_token,"login": "sxtpython","password": "sxt123456"}yield scrapy.FormRequest(# 回调提交表单跳转后的url地址'https://github.com/session',formdata=formdata,callback=self.parse_item,)def parse_item(self,response):item = {}item["name"] = re.findall('sxtpython',response.body.decode())yield item

方式三：自动从f响应中找到form表单进行登录

# -*- coding: utf-8 -*-
import re
import scrapy# 方式三
class Github2Spider(scrapy.Spider):name = 'github2'allowed_domains = ['github.com']start_urls = ['https://github.com/login']def parse(self, response):yield scrapy.FormRequest.from_response(response,#自动从该响应中获取form表单formdata={'login':'sxtpython','password':'sxt123456'},callback=self.parse_item,)def parse_item(self,response):item = {}item["name"] = re.findall('sxtpython',response.body.decode())yield item