一篇文章搞定 Scrapy 爬虫框架

copying src\twisted\words\xish\xpathparser.g -> build\lib.win-amd64-3.5\twisted\words\xish running build_ext building 'twisted.test.raiser' extension error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools解决方案是，下载编译好的twisted，https://www.lfd.uci.edu/~gohlke/pythonlibs/#twistedpython3.5 下载 Twisted-18.4.0-cp35-cp35m-win_amd64.whlpython3.6 下载 Twisted-18.4.0-cp36-cp36m-win_amd64.whl
安装twisted$ pip install Twisted-18.4.0-cp35-cp35m-win_amd64.whl之后在安装scrapy就没有什么问题了

> scrapyScrapy 1.5.0 - no active project
Usage:  scrapy  [options] [args]  Available commands:  bench Run    quick benchmark test  check       Check spider contracts  crawl       Run a spider  edit       Edit spider  fetch       Fetch a URL using the Scrapy downloader  genspider     Generate new spider using pre-defined templates  list       List available spiders  parse       Parse URL (using its spider) and print the results  runspider     Run a self-contained spider (without creating a project)  settings     Get settings values  shell       Interactive scraping console  startproject   Create new project  version     Print Scrapy version  view       Open URL in browser, as seen by Scrapy

first ├─ scrapy.cfg └─ first   ├─ items.py   ├─ middlewares.py   ├─ pipelines.py   ├─ settings.py   ├─ __init__.py   └─ spiders     └─ __init__.py

# first/settings.py参考BOT_NAME = 'first'SPIDER_MODULES = ['first.spiders']NEWSPIDER_MODULE = 'first.spiders'
USER_AGENT = "Mozilla/5.0 (Windows NT 6.1)AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36"ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 3
# Disable cookies (enabled by default)COOKIES_ENABLED = False

import scrapy
class BookItem(scrapy.Item):  title = scrapy.Field() # 书名  rate = scrapy.Field() # 评分

# scrapy源码中class Spider():  def parse(self, response): # 解析返回的内容    raise NotImplementedError

import scrapy
class BookSpider(scrapy.Spider): # BookSpider   name = 'doubanbook' # 爬虫名，可修改，重要   allowed_domains = ['douban.com'] # 爬虫爬取范围   url = 'https://book.douban.com/tag/%E7%BC%96%E7%A8%8B?start=0&type=T'   start_urls = [url] # 起始URL     # 下载器获取了WEB Server的response就行了，parse就是解析响应的内容   def parse(self, response):     print(type(response), '~~~~~~~~~') #scrapy.http.response.html.HtmlResponse     print(response)     print('-' * 30)

$ scrapy list$ scrapy crawl -hscrapy crawl [options] 
指定爬虫名称开始爬取$ scrapy crawl doubanbook
可以不打印日志$ scrapy crawl doubanbook --nolog

import scrapyfrom scrapy.http.response.html import HtmlResponse
class BookSpider(scrapy.Spider): # BookSpider  name = 'doubanbook' # 爬虫名   allowed_domains = ['douban.com'] # 爬虫爬取范围   url = 'https://book.douban.com/tag/%E7%BC%96%E7%A8%8B?start=0&type=T'   start_urls = [url] # 起始URL     # 下载器获取了WEB Server的response就行了，parse就是解析响应的内容   def parse(self, response:HtmlResponse):     print(type(response)) #scrapy.http.response.html.HtmlResponse     print('-'*30)     print(type(response.text), type(response.body))    print('-'*30)    print(response.encoding)    with open('o:/testbook.html', 'w', encoding='utf-8') as f:      try:         f.write(response.text)         f.flush()       except Exception as e:         print(e)

import scrapyfrom scrapy.http.response.html import HtmlResponse
response = HtmlResponse('file:///O:/testbook.html', encoding='utf-8') # 构造对象
with open('o:/testbook.html', encoding='utf8') as f:   response._set_body(f.read()) # 填充数据   #print(response.text)   # 获取所有标题及评分  # xpath解析   subjects = response.xpath('//li[@class="subject-item"]')   for subject in subjects:     title = subject.xpath('.//h2/a/text()').extract() # list     print(title[0].strip())        rate = subject.xpath('.//span[@class="rating_nums"]/text()').extract()    print(rate[0].strip()) 
  print('-'*30)   # css解析   subjects = response.css('li.subject-item')   for subject in subjects:     title = subject.css('h2 a::text').extract()     print(title[0].strip())       rate = subject.css('span.rating_nums::text').extract()     print(rate[0].strip())   print('-'*30)     # xpath和css混合使用、正则表达式匹配   subjects = response.css('li.subject-item')   for subject in subjects:  # 提取链接    href =subject.xpath('.//h2').css('a::attr(href)').extract()    print(href[0])        # 使用正则表达式    id = subject.xpath('.//h2/a/@href').re(r'\d*99\d*')    if id:       print(id[0])         # 要求显示9分以上数据     rate = subject.xpath('.//span[@class="rating_nums"]/text()').re(r'^9.*')     # rate = subject.css('span.rating_nums::text').re(r'^9\..*')     if rate:       print(rate)

# spiders/bookspider.pyimport scrapyfrom scrapy.http.response.html import HtmlResponsefrom ..items import BookItem
class BookSpider(scrapy.Spider): # BookSpider  name = 'doubanbook' # 爬虫名   allowed_domains = ['douban.com'] # 爬虫爬取范围   url = 'https://book.douban.com/tag/%E7%BC%96%E7%A8%8B?start=0&type=T'  start_urls = [url] # 起始URL   # 下载器获取了WEB Server的response就行了，parse就是解析响应的内容   def parse(self, response:HtmlResponse):     items = []     # xpath解析     subjects = response.xpath('//li[@class="subject-item"]')     for subject in subjects:       title = subject.xpath('.//h2/a/text()').extract()           rate = subject.xpath('.//span[@class="rating_nums"]/text()').extract_first()           item = BookItem()           item['title'] = title[0].strip()           item['rate'] = rate.strip()           items.append(item)           print(items)     return items # 一定要return，否则保存不下来    # 使用命令保存return的数据# scrapy crawl -h# --output=FILE, -o FILE dump scraped items into FILE (use - for stdout)# 文件扩展名支持'json', 'jsonlines', 'jl', 'csv', 'xml', 'marshal', 'pickle'# scrapy crawl doubanbook -o dbbooks.json

# Configure item pipelines# See https://doc.scrapy.org/en/latest/topics/item-pipeline.htmlITEM_PIPELINES = {  'first.pipelines.FirstPipeline': 300,}

class FirstPipeline(object):   def __init__(self): # 全局设置     print('~~~~~~~~~~ init ~~~~~~~~~~~~')       def open_spider(self, spider): # 当某spider开启时调用     print(spider,'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')     def process_item(self, item, spider):     # item 获取的item；spider 获取该item的spider     return item 
  def close_spider(self, spider): # 当某spider关闭时调用     print(spider,'========================================')

# spider/bookspider.py
                            

                            






                            

                                请到「今天看啥」查看全文