python爬虫之爬取携程景点评价（5）

一、景点部分评价爬取

【携程攻略】携程旅游攻略,自助游,自驾游,出游,自由行攻略指南 (ctrip.com)

import requests
from bs4 import BeautifulSoupif __name__ == '__main__':url = 'https://m.ctrip.com/webapp/you/commentWeb/commentList?seo=0&businessId=22176&businessType=sight&hideStatusBar=1&DistrictName=%E9%BB%84%E9%BE%99%E6%BA%AA&isBack=yes&from=https%3A%2F%2Fm.ctrip.com%2Fwebapp%2Fyou%2Fgspoi%2Fsight%2F104%2F0.html%3FpoiId%3D81011%26seo%3D0%26isHideNavBar%3DYES%26scene%3DDISTRICT%26ctm_ref%3Dch5_hp_bs_lst&'#目标访问网站urlheader = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"}req = requests.get(url=url,headers = header)#获取该网页内容req.encoding = 'utf-8'#防止中文乱码，还可以改成gbkhtml = req.textbes = BeautifulSoup(html,"lxml")div_contents = bes.find_all('div', class_='content_wrap')#找到里面的所有div标签if div_contents:count = 1for div_content in div_contents:all_info = div_content.textprint(f'{count}: {all_info}')count += 1else:print('未找到指定的<div class="content_wrap">标签')

二、景点所有评价爬取

通过搜索，发现请求方法不在是get,而是post,且预览并得不到我们需要的评论，反而是响应界面才可以，则针对这种情况，我们需要重新更改代码。

更改代码如下

import requests
import json
import pandas as pd
from tqdm import tqdmuserNames = []
commentDetails = []
commentTimes = []total_pages = 1for pagen in tqdm(range(0, total_pages), desc='爬取进度', unit='页'):#payload参数实质上就是网络下的负载payload = {"arg": {"channelType": 7,"collapseTpte": 1,"commentTagId": 0,"pageIndex": pagen,"pageSize": 10,"resourceId":22176,"resourceType":11,"sourseType": 1,"sortType": 3,"starType": 0},"head": {"cid": "09031081213865125571","ctok": "","cver": "1.0","lang": "01","sid": "8888","syscode": "09","auth": "","xsid": "","extension": []}}#网络的标头中的url路径，采用POST请求方法，其？后面的内容就是payloadpostUrl = "https://m.ctrip.com/restapi/soa2/13444/json/getCommentCollapseList"html = requests.post(postUrl, data=json.dumps(payload)).texthtml_1 = json.loads(html)#html_1实质就是网络下面的响应界面# 检查响应中是否存在'items'if 'items' in html_1["result"]:commentItems = html_1["result"]["items"]for i in range(0, len(commentItems)):# 在访问元素之前检查当前项是否不为Noneif commentItems[i] is not None and 'userInfo' in commentItems[i] and 'userNick' in commentItems[i]['userInfo']:userName = commentItems[i]['userInfo']['userNick']commentDetail = commentItems[i]['content']commentTime = commentItems[i]['publishTypeTag']userNames.append(userName)commentDetails.append(commentDetail)commentTimes.append(commentTime)# 创建 DataFrame
df = pd.DataFrame({'用户评论内容': commentDetails,'用户名': userNames,'用户评论时间': commentTimes
})# 保存到 Excel 文件
df.to_excel('只爬黄龙溪评论1223url.xlsx', index=False, encoding='utf-8')