python-scrapy框架示例

参考：https://blog.csdn.net/qq_44907926/article/details/119531324

创建项目步骤：
1.目标网站:www.itcast.cn
2.安装虚拟环境

pip install virtualenv

3.创建虚拟环境

virtualenv --always-copy --system-site-packages venv

4.激活虚拟环境

venv\scripts\activate

5.安装twisted（Twisted是基于事件驱动的网络引擎框架）

pip install twisted

6.安装scrapy 爬虫框架

pip install scrapy

7.创建项目

scrapy startproject itcastScrapy
cd itcastScrapy
#itcastSpider  爬虫名字 itcast.cn 爬虫网站
scrapy genspider itcastSpider itcast.cn

在这里插入图片描述
8.创建 main.py 执行scrapy

from scrapy import cmdline
cmdline.execute("scrapy crawl itcastSpider".split())

在这里插入图片描述
9.各文件代码
9.1 items.py 需要爬取的内容

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.htmlimport scrapy# 需要爬取的内容 建模
class ItcastscrapyItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()passclass ItcastItem(scrapy.Item):# 爬取老师信息name = scrapy.Field()# 爬取老师职位title = scrapy.Field()# 爬取 老师信息info = scrapy.Field()

9.2 pipelines.py 管道，保存数据

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import json# 管道，保存数据
class ItcastscrapyPipeline:def process_item(self, item, spider):return itemclass UbuntuPipeline(object):def __init__(self):self.file = open('itcast.json', 'w', encoding='utf-8')def process_item(self, item, spider):# 将item对象强制转为字典，该操作只能在scrapy中使用item = dict(item)# 爬虫文件中提取数据的方法每yield一次，就会运行一次# 该方法为固定名称函数# 默认使用完管道，需要将数据返回给引擎# 1.将字典数据序列化'''ensure_ascii=False 将unicode类型转化为str类型，默认为True'''json_data = json.dumps(item, ensure_ascii=False, indent=2) + ',\n'# 2.将数据写入文件self.file.write(json_data)return itemdef __del__(self):self.file.close()

9.3 middlewares.py 自定义中间件文件

9.4 settings.py 设置文件

# Obey robots.txt rules
ROBOTSTXT_OBEY = False# 设置浏览器UA
DEFAULT_REQUEST_HEADERS = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'en','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36',}
# 300 优先级，越小执行越早
ITEM_PIPELINES = {# "itcastScrapy.pipelines.ItcastscrapyPipeline": 300,"itcastScrapy.pipelines.UbuntuPipeline": 300,
}

9.5 ItcastspiderSpider.py

import scrapy
from ..items import ItcastItem
from bs4 import BeautifulSoupclass ItcastspiderSpider(scrapy.Spider):name = "itcastSpider"allowed_domains = ["itcast.cn"]start_urls = ["https://itcast.cn"]def parse(self, response):# 获取网页源代码# print(response.body.decode())soup = BeautifulSoup(response.body.decode(), 'lxml')li_list = soup.select('div.head_nav>ul>li')for li in li_list:if li.text.strip() == '教研团队':for a in li.select('a'):href = a['href']yield scrapy.Request(url=href, callback=self.parse_teacher)breakdef parse_teacher(self, response):# 获取网页源代码# print(response.body.decode())items = []document = BeautifulSoup(response.body.decode(), 'lxml')li_list = document.select('div.tea_con div.tea_txt_cur ul li')# 遍历教师节点列表for li in li_list:item = ItcastItem()name = li.select(' div.li_txt h3')[0].texttitle = li.select(' div.li_txt h4')[0].textinfo = li.select(' div.li_txt p')[0].textitem['name'] = nameitem['title'] = titleitem['info'] = infoitems.append(item)# 提交所有收集的itemsfor item in items:yield item