目标数据： zol桌面壁纸，[风景] [1920*1080] 分类下19页每个图册的图片
items.py
1 import scrapy 2 3 4 class zol2item(scrapy.item): 5 # define the fields for your item here like: 6 # name = scrapy.field() 7 image_urls = scrapy.field() 8 images = scrapy.field() 9 10 image_title = scrapy.field()
pipelines.py
1 from scrapy import request
2 from scrapy.pipelines.images import imagespipeline
3
4 class zolpipeline(imagespipeline):
5 # num = 1
6 def get_media_requests(self, item, info):
7 image_url = item["image_urls"]
8 if image_url:
9 # self.num + 1
10 yield request(url=image_url, meta={"item": item})
11
12 def file_path(self, request, response=none, info=none):
13 ## start of deprecation warning block (can be removed in the future)
14 def _warn():
15 from scrapy.exceptions import scrapydeprecationwarning
16 import warnings
17 warnings.warn('imagespipeline.image_key(url) and file_key(url) methods are deprecated, '
18 'please use file_path(request, response=none, info=none) instead',
19 category=scrapydeprecationwarning, stacklevel=1)
20
21 # check if called from image_key or file_key with url as first argument
22 if not isinstance(request, request):
23 _warn()
24 url = request
25 else:
26 url = request.url
27
28 # detect if file_key() or image_key() methods have been overridden
29 if not hasattr(self.file_key, '_base'):
30 _warn()
31 return self.file_key(url)
32 elif not hasattr(self.image_key, '_base'):
33 _warn()
34 return self.image_key(url)
35 ## end of deprecation warning block
36
37 return 'desk/{}.jpg'.format(request.meta["item"]["image_title"])
middlewares.py
1 from scrapy import signals
2 from zol2.useragents import agents
3
4
5 class zol2spidermiddleware(object):
6 # not all methods need to be defined. if a method is not defined,
7 # scrapy acts as if the spider middleware does not modify the
8 # passed objects.
9
10 @classmethod
11 def from_crawler(cls, crawler):
12 # this method is used by scrapy to create your spiders.
13 s = cls()
14 crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
15 return s
16
17 def process_spider_input(self, response, spider):
18 # called for each response that goes through the spider
19 # middleware and into the spider.
20
21 # should return none or raise an exception.
22 return none
23
24 def process_spider_output(self, response, result, spider):
25 # called with the results returned from the spider, after
26 # it has processed the response.
27
28 # must return an iterable of request, dict or item objects.
29 for i in result:
30 yield i
31
32 def process_spider_exception(self, response, exception, spider):
33 # called when a spider or process_spider_input() method
34 # (from other spider middleware) raises an exception.
35
36 # should return either none or an iterable of response, dict
37 # or item objects.
38 pass
39
40 def process_start_requests(self, start_requests, spider):
41 # called with the start requests of the spider, and works
42 # similarly to the process_spider_output() method, except
43 # that it doesn’t have a response associated.
44
45 # must return only requests (not items).
46 for r in start_requests:
47 yield r
48
49 def spider_opened(self, spider):
50 spider.logger.info('spider opened: %s' % spider.name)
51
52
53 class zol2downloadermiddleware(object):
54 # not all methods need to be defined. if a method is not defined,
55 # scrapy acts as if the downloader middleware does not modify the
56 # passed objects.
57
58 @classmethod
59 def from_crawler(cls, crawler):
60 # this method is used by scrapy to create your spiders.
61 s = cls()
62 crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
63 return s
64
65 def process_request(self, request, spider):
66 # called for each request that goes through the downloader
67 # middleware.
68
69 # must either:
70 # - return none: continue processing this request
71 # - or return a response object
72 # - or return a request object
73 # - or raise ignorerequest: process_exception() methods of
74 # installed downloader middleware will be called
75 return none
76
77 def process_response(self, request, response, spider):
78 # called with the response returned from the downloader.
79
80 # must either;
81 # - return a response object
82 # - return a request object
83 # - or raise ignorerequest
84 return response
85
86 def process_exception(self, request, exception, spider):
87 # called when a download handler or a process_request()
88 # (from other downloader middleware) raises an exception.
89
90 # must either:
91 # - return none: continue processing this exception
92 # - return a response object: stops process_exception() chain
93 # - return a request object: stops process_exception() chain
94 pass
95
96 def spider_opened(self, spider):
97 spider.logger.info('spider opened: %s' % spider.name)
settings.py
1 # -*- coding: utf-8 -*-
2
3 # scrapy settings for zol2 project
4 #
5 # for simplicity, this file contains only settings considered important or
6 # commonly used. you can find more settings consulting the documentation:
7 #
8 # https://doc.scrapy.org/en/latest/topics/settings.html
9 # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
10 # https://doc.scrapy.org/en/latest/topics/spider-middleware.html
11
12 bot_name = 'zol2'
13
14 spider_modules = ['zol2.spiders']
15 newspider_module = 'zol2.spiders'
16
17
18 # crawl responsibly by identifying yourself (and your website) on the user-agent
19 user_agent = 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/73.0.3683.75 safari/537.36'
20
21 # obey robots.txt rules
22 # robotstxt_obey = true
23
24 # configure maximum concurrent requests performed by scrapy (default: 16)
25 #concurrent_requests = 32
26
27 # configure a delay for requests for the same website (default: 0)
28 # see https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
29 # see also autothrottle settings and docs
30 download_delay = 0.5
31 # the download delay setting will honor only one of:
32 #concurrent_requests_per_domain = 16
33 #concurrent_requests_per_ip = 16
34
35 # disable cookies (enabled by default)
36 #cookies_enabled = false
37
38 # disable telnet console (enabled by default)
39 #telnetconsole_enabled = false
40
41 # override the default request headers:
42 #default_request_headers = {
43 # 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
44 # 'accept-language': 'en',
45 #}
46
47 # enable or disable spider middlewares
48 # see https://doc.scrapy.org/en/latest/topics/spider-middleware.html
49 #spider_middlewares = {
50 # 'zol2.middlewares.zol2spidermiddleware': 543,
51 #}
52
53 # enable or disable downloader middlewares
54 # see https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
55 #downloader_middlewares = {
56 # 'zol2.middlewares.zol2downloadermiddleware': 543,
57 #}
58
59 # enable or disable extensions
60 # see https://doc.scrapy.org/en/latest/topics/extensions.html
61 #extensions = {
62 # 'scrapy.extensions.telnet.telnetconsole': none,
63 #}
64
65 # configure item pipelines
66 # see https://doc.scrapy.org/en/latest/topics/item-pipeline.html
67 item_pipelines = {
68 'zol2.pipelines.zol2pipeline': 300,
69 }
70 images_store = "/home/pyvip/env_spider/zol2/zol2/images"
71
72 # enable and configure the autothrottle extension (disabled by default)
73 # see https://doc.scrapy.org/en/latest/topics/autothrottle.html
74 #autothrottle_enabled = true
75 # the initial download delay
76 #autothrottle_start_delay = 5
77 # the maximum download delay to be set in case of high latencies
78 #autothrottle_max_delay = 60
79 # the average number of requests scrapy should be sending in parallel to
80 # each remote server
81 #autothrottle_target_concurrency = 1.0
82 # enable showing throttling stats for every response received:
83 #autothrottle_debug = false
84
85 # enable and configure http caching (disabled by default)
86 # see https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
87 #httpcache_enabled = true
88 #httpcache_expiration_secs = 0
89 #httpcache_dir = 'httpcache'
90 #httpcache_ignore_http_codes = []
91 #httpcache_storage = 'scrapy.extensions.httpcache.filesystemcachestorage'
pazol2.py
1 # -*- coding: utf-8 -*-
2 import scrapy
3 from scrapy.linkextractors import linkextractor
4 from scrapy.spiders import crawlspider, rule
5 from zol2.items import zol2item
6
7 class pazol2spider(crawlspider):
8 name = 'pazol2'
9 # allowed_domains = ['desk.zol.com.cn']
10 start_urls = ['http://desk.zol.com.cn/fengjing/1920x1080/']
11 front_url = "http://desk.zol.com.cn"
12 num = 1
13
14 rules = (
15 # 1.解决翻页
16 rule(linkextractor(allow=r'/fengjing/1920x1080/[0-1]?[0-9]?.html'), callback='parse_album', follow=true),
17 # 2.进入各个图库的每一张图片页
18 rule(linkextractor(allow=r'/bizhi/\d+_\d+_\d+.html', restrict_xpaths=("//div[@class='main']/ul[@class='pic-list2 clearfix']/li", "//div[@class='photo-list-box']")), follow=true),
19 # 3.点击各个图片1920*1080按钮，获得html
20 rule(linkextractor(allow=r'/showpic/1920x1080_\d+_\d+.html'), callback='get_img', follow=true),
21 )
22
23 def get_img(self, response):
24 item = zol2item()
25 item['image_urls'] = response.xpath("//body/img[1]/@src").extract_first()
26 item['image_title'] = str(self.num)
27 self.num += 1
28 yield item
爬取结果
共爬取了4517张图片，用时108分钟
放在桌面图库，半小时换一张，美滋滋。
