每天15分钟玩转Scrapy - Spider 与选择器

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

import scrapy


class QuotesSpider(scrapy.Spider):
    """Spider 的四要素与回调链。"""

    name = "quotes"                              # 要素一：唯一的名字
    allowed_domains = ["quotes.toscrape.com"]    # 要素二：域白名单
    start_urls = ["https://quotes.toscrape.com/"]  # 要素三：起始请求

    def parse(self, response):                   # 要素四：默认回调
        self.logger.info("正在处理 %s", response.url)

        # 产出数据
        for quote in response.css("div.quote"):
            yield {
                "text": quote.css("span.text::text").get(),
                "author": quote.css("small.author::text").get(),
            }

        # 产出新任务：作者主页另找函数处理，别挤在 parse 里
        for link in response.css("div.quote span a::attr(href)").getall():
            yield response.follow(link, callback=self.parse_author)

    def parse_author(self, response):
        yield {
            "name": response.css("h3.author-title::text").get(default="").strip(),
            "birthdate": response.css(".author-born-date::text").get(default="").strip(),
        }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

>>> response.css("title").get()
'<title>Quotes to Scrape</title>'

>>> response.css("title::text").get()
'Quotes to Scrape'

>>> response.css("div.quote span.text::text").get()[:40]
'“The world as we have created it is a pro'

>>> response.css("div.tags a.tag::text").getall()[:4]
['change', 'deep-thoughts', 'thinking', 'world']

>>> response.css("li.next a").attrib
{'href': '/page/2/'}

1
2
3
4
5
6

>>> first = response.css("div.quote")[0]
>>> first.css("span.text::text").get()[:30]
'“The world as we have created '

>>> first.css("small.author::text").get()
'Albert Einstein'

1
2
3
4
5
6
7
8

>>> response.xpath('//small[@class="author"]/text()').getall()[:3]
['Albert Einstein', 'J.K. Rowling', 'Albert Einstein']

>>> response.xpath('//a[contains(., "Next")]/@href').get()
'/page/2/'

>>> response.xpath('//div[@class="tags"]/a/@href').getall()[:3]
['/tag/change/page/1/', '/tag/deep-thoughts/page/1/', '/tag/thinking/page/1/']

1
2
3

>>> quotes = response.xpath('//div[@class="quote"]')   # 10 条名言
>>> len(quotes.xpath('.//small[@class="author"]'))     # 在每条名言内部找作者
10

1
2
3
4

>>> len(response.xpath('//div[@class="quote"]'))
10
>>> len(response.xpath('(//div[@class="quote"])[1]'))
1

1
2
3
4
5

>>> response.css("span.text::text").re(r"([A-Z][a-z]+) is")
['It', 'One', 'Imperfection', 'It']

>>> response.css("span.text::text").re_first(r"world")
'world'

1
2
3
4

# 三种等价写法，任选一种
yield response.follow("/page/2/", callback=self.parse)
yield response.follow(response.css("li.next a"), callback=self.parse)
yield response.follow(response.css("li.next a::attr(href)").get(), callback=self.parse)

1
2

anchors = response.css("div.quote span a")
yield from response.follow_all(anchors, callback=self.parse_author)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class AuthorCrawlSpider(CrawlSpider):
    """规则化爬取：翻页和作者页都交给 Rule。"""

    name = "author_crawl"
    allowed_domains = ["quotes.toscrape.com"]
    start_urls = ["https://quotes.toscrape.com/"]

    rules = (
        # 作者主页：交给 parse_author，不再往下跟
        Rule(LinkExtractor(allow=r"/author/"), callback="parse_author"),
        # 分页：没有 callback，follow 默认为 True，会一直翻到没有下一页
        Rule(LinkExtractor(allow=r"/page/")),
    )

    def parse_author(self, response):
        yield {
            "name": response.css("h3.author-title::text").get(default="").strip(),
            "birthdate": response.css(".author-born-date::text").get(default="").strip(),
        }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

import scrapy


class AuthorsSpider(scrapy.Spider):
    """两段式抓取：列表页取作者链接，作者页补生平档案。"""

    name = "authors"
    allowed_domains = ["quotes.toscrape.com"]
    start_urls = ["https://quotes.toscrape.com/"]

    def parse(self, response):
        # 第一段：名言条目里的作者主页链接。
        # dict.fromkeys 保序去重，同一页重复出现的作者只发一次请求
        links = response.css("div.quote span a::attr(href)").getall()
        for link in dict.fromkeys(links):
            yield response.follow(link, callback=self.parse_author)

        # 第二段：翻页，把 10 页都走一遍
        yield from response.follow_all(css="li.next a", callback=self.parse)

    def parse_author(self, response):
        """作者详情页：字段与页面结构一一对应，缺字段用 default 兜底。"""
        yield {
            "name": response.css("h3.author-title::text").get(default="").strip(),
            "birthdate": response.css(".author-born-date::text").get(default="").strip(),
            "birthplace": response.css(".author-born-location::text").get(default="").strip(),
            "bio": response.css(".author-description::text").get(default="").strip(),
        }

1

scrapy crawl authors -O authors.json

1
2
3
4
5

'item_scraped_count': 50,
'downloader/request_count': 111,
'dupefilter/filtered': 37,
'elapsed_time_seconds': 132.32,
'finish_reason': 'finished',

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

"""读 authors.json，统计出生年代分布与出生地 Top 榜。"""
import json
from collections import Counter

with open("authors.json", encoding="utf-8") as f:
    data = json.load(f)

years = [int(item["birthdate"][-4:]) for item in data if item["birthdate"]]
decade = Counter(f"{y // 10 * 10}s" for y in years)
places = Counter(item["birthplace"].removeprefix("in ").rsplit(",", 1)[-1].strip()
                 for item in data if item["birthplace"])

print(f"作者档案: {len(data)} 位")
print(f"出生年份跨度: {min(years)} - {max(years)}")
print("\n出生年代分布:")
for d, n in sorted(decade.items()):
    print(f"  {d:<8} {'█' * n} {n}")
print("\n出生地 Top 5:")
for p, n in places.most_common(5):
    print(f"  {p:<16} {n} 位")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

作者档案: 50 位
出生年份跨度: 1775 - 1973

出生年代分布:
  1770s    █ 1
  1800s    ██ 2
  1810s    █ 1
  ...
  1920s    ███████ 7
  1940s    █████████ 9
  1960s    ███ 3
  1970s    █ 1

出生地 Top 5:
  The United States 25 位
  The United Kingdom 9 位
  Germany          3 位
  France           2 位
  Ireland          2 位