https://doc-ja-scrapy.readthedocs.io/ja/latest/index.html
#Project creation
$ scrapy startproject <project name>
#Setting
$ cat setting.py
DOWNLOAD_DELAY = 1
FEED_EXPORT_ENCODING = "utf-8"
#spider creation
$ scrapy genspider <mydomain> <mydomain.com>
#Write parse process and execute crawling
$ scrappy crawl <spider name>
def parse(self, response):
for sel in response.css('#gmap_list > li > a'):
next_page = response.urljoin(sel.css('a::attr("href")').get())
yield scrapy.Request(next_page, callback=self.parse_detail)
def parse_detail(self, response):
'''
Detail page parse processing
'''
ORM If you use it, Orator seems to be simple and easy to use. https://orator-orm.com/docs/0.9/basic_usage.html
Recommended Posts