import scrapy
from scrapy.http import Request
from scrapy.utils.project import get_project_settings
class MouserSpider(scrapy.Spider):
name = 'mouser'
allowed_domains = ['mouser.cn']
start_urls = ['https://www.mouser.cn/electronic-components/']
def parse(self, response):
# 解析器件信息
for product in response.css('.search-results .result-item'):
yield {
'name': product.css('.result-title::text').get(),
'price': product.css('.result-price::text').get(),
'link': product.css('a::attr(href)').get(),
}
# 翻页逻辑
next_page = response.css('a.pagination-next::attr(href)').get()
if next_page:
yield Request(url=response.urljoin(next_page), callback=self.parse)
def start_requests(self):
settings = get_project_settings()
proxy = settings.get('HTTP_PROXY')
for url in self.start_urls:
yield Request(url=url, callback=self.parse, meta={'proxy': proxy})