python中利用cookie免登录获取网页
·
完整代码:
from bs4 import BeautifulSoup
import urllib.request as urllib
url = 'https://book.douban.com/'
cookie = ''
headers = {
# 'Host': 'www.douban.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
# 'Connection': 'keep-alive',
'Cookie': cookie
}
req = urllib.Request(url, headers=headers)
page = urllib.urlopen(req)
soup = BeautifulSoup(page, 'lxml')
# html = page.read()
print(soup)
with open('rr.html', 'w', encoding='utf-8') as f:
f.write(res.content.decode("GBK","ignore"))
用request方法:
import requests
from bs4 import BeautifulSoup
from datetime import datetime
import json
import xlwings as xw
# 新闻链接
url = 'https://book.douban.com/'
session=requests.session()
cookie=''
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
'Cookie': cookie
}
res=session.get(url=url,headers=headers)
soup = BeautifulSoup(res.text, 'lxml')
# html = page.read()
print(soup)
# 'Host': 'www.douban.com',
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
# 'Connection': 'keep-alive',
with open('rr.html', 'w', encoding='utf-8') as f:
f.write(res.content.decode("GBK","ignore"))
更多推荐


所有评论(0)