Scrapy Python 脚本给出 raise TypeError("Cannot mix str and non-str arguments")
Scrapy Python script gives raise TypeError("Cannot mix str and non-str arguments")
嗨，我是编程新手，运行 遇到了这个看似极其常见的问题，但老实说，none 我看到的答案对我的情况有所帮助。
我的代码是：
import json
import scrapy
class MoreKeysSpider(scrapy.Spider):
name = 'getoffers'
def __init__(self):
with open(r'C:\Users\magnu\brickset-scraper\postscrape\postscrape\prod.json', encoding='utf-8') as data_file:
self.data = json.load(data_file)
def start_requests(self):
for item in self.data:
request = scrapy.Request(item['url'], callback=self.parse)
request.meta['item'] = item
yield request
def parse(self, response):
item = response.meta['item']
item['details'] = []
item['details'].append({
"Name" : response.css('span[itemprop=name]::text').extract_first(),
"Release" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[1]/div[2]/text()').extract_first(),
"Website" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[2]/div[2]/a/@href').extract_first(),
"Entwickler" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[3]/div[2]/text()').extract_first(),
"Publisher" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[4]/div[2]/text()').extract_first(),
"Tags" : response.xpath('//*[@id="info"]/div[2]/div[2]/div[3]/div[2]/descendant').getall(),
"Systemanforderungenmin" : response.xpath('//*[@id="config"]/ul[1]/descendant').getall(),
"Systemanforderungenmax" : response.xpath('//*[@id="config"]/ul[2]/descendant').getall(),
})
yield item
item['offer'] = []
for div in response.css('#offers_table'):
for offer_row in div.css('div.offers-table-row'):
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
url_str = ''.join(map(str, url)) #coverts list to str
item['offer'].append({
"offer:"
"Shop": offer_row.css('div[itemprop ~= seller] div.offers-merchant::attr(title)').extract_first(),
"Typ": offer_row.css('div.offers-edition-region::text').extract_first(),
"Edition": offer_row.css("div[data-toggle=tooltip]::attr(data-content)"),
"Link": response.follow(url_str, self.parse_topics),
})
yield item
我收到了回复
DEBUG: Scraped from <200 https://www.keyforsteam.de/kaufen-crusader-kings-2-cd-key-preisvergleich/>
{'url': 'https://www.keyforsteam.de/kaufen-crusader-kings-2-cd-key-preisvergleich/', 'details': [{'Name': '\n\t\t\t\t\tCrusader Kings 2\n\t\t\t\t', 'Release': '\n 14. Februar 2012\n ', 'Website': 'https://www.paradoxplaza.com/crusader-kings-ii/CKCK02GSK-MASTER.html', 'Entwickler': '\n Paradox Development Studio\n
', 'Publisher': '\n Paradox Interactive\n
', 'Tags': [], 'Systemanforderungenmin': [], 'Systemanforderungenmax': []}]}
2021-03-22 21:47:22 [scrapy.core.scraper] ERROR: Spider error processing <GET https://www.keyforsteam.de/kaufen-crusader-kings-2-cd-key-preisvergleich/> (referer: None)
Traceback (most recent call last):
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\utils\defer.py", line 120, in iter_errback
yield next(it)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\utils\python.py", line 353, in __next__
return next(self.data)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\utils\python.py", line 353, in __next__
return next(self.data)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 29, in process_spider_output
for x in result:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 340, in <genexpr>
return (_set_referer(r) for r in result or ())
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\depth.py",
line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "C:\Users\magnu\brickset-scraper\postscrape\postscrape\spiders\keysint.py", line 40, in parse
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\http\response\text.py", line
102, in urljoin
return urljoin(get_base_url(self), url)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\urllib\parse.py", line 524, in urljoin
base, url, _coerce_result = _coerce_args(base, url)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\urllib\parse.py", line 122, in _coerce_args
raise TypeError("Cannot mix str and non-str arguments")
TypeError: Cannot mix str and non-str arguments
所以第一部分似乎有效，我很确定错误在第二项中的某处，但我似乎找不到它
item['offer'] = []
for div in response.css('#offers_table'):
for offer_row in div.css('div.offers-table-row'):
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
url_str = ''.join(map(str, url)) #coverts list to str
item['offer'].append({
"offer:"
"Shop": offer_row.css('div[itemprop ~= seller] div.offers-merchant::attr(title)').extract_first(),
"Typ": offer_row.css('div.offers-edition-region::text').extract_first(),
"Edition": offer_row.css("div[data-toggle=tooltip]::attr(data-content)"),
"Link": response.follow(url_str, self.parse_topics),
})
yield item
有一种循环路线可以得到这个，但我认为调试过程会很有启发性。
如果没有程序正在调用的 json 文件，诊断起来会比较困难，但看起来您的问题出在这一行：url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
来自How Can I Fix "TypeError: Cannot mix str and non-str arguments"?
According to the Scrapy documentation, the .css(selector) method that you're using, returns a SelectorList instance. If you want the actual (unicode) string version of the url, call the extract() method:
所以我尝试了：
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').extract()).get(),
但我仍然得到同样的错误。奇怪！
为了诊断，我在此处将 breakpoint() 放入蜘蛛中：
for div in response.css('#offers_table'):
for offer_row in div.css('div.offers-table-row'):
breakpoint()
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').extract()).get(),
运行 蜘蛛又来了，我可以试下一行：
(Pdb) offer_row.css('div.buy-btn-cell a::attr(href)').extract()
['https://www.keyforsteam.de/outgoinglink/keyforsteam/37370?merchant=1', 'https://www.keyforsteam.de/outgoinglink/keyforsteam/37370?merchant=1']
啊，所以 extract() 返回的是字符串列表而不是单个字符串。必须有两个元素匹配。然而，它们是相同的，所以我们不关心我们得到的是哪一个。查看 https://docs.scrapy.org/en/latest/topics/selectors.html 的 scrapy 文档，我们看到还有一个 extract-first() 函数。
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').extract-first()).get(),
虽然，查看 scrapy 文档，您可能想使用 get() 而不是 extract-first()
那时我终于注意到你唯一的错误是把 get() 放在错误的括号外。
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').get())
嗨，我是编程新手，运行 遇到了这个看似极其常见的问题，但老实说，none 我看到的答案对我的情况有所帮助。
我的代码是：
import json
import scrapy
class MoreKeysSpider(scrapy.Spider):
name = 'getoffers'
def __init__(self):
with open(r'C:\Users\magnu\brickset-scraper\postscrape\postscrape\prod.json', encoding='utf-8') as data_file:
self.data = json.load(data_file)
def start_requests(self):
for item in self.data:
request = scrapy.Request(item['url'], callback=self.parse)
request.meta['item'] = item
yield request
def parse(self, response):
item = response.meta['item']
item['details'] = []
item['details'].append({
"Name" : response.css('span[itemprop=name]::text').extract_first(),
"Release" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[1]/div[2]/text()').extract_first(),
"Website" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[2]/div[2]/a/@href').extract_first(),
"Entwickler" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[3]/div[2]/text()').extract_first(),
"Publisher" : response.xpath('//*[@id="info"]/div[2]/div[1]/div[4]/div[2]/text()').extract_first(),
"Tags" : response.xpath('//*[@id="info"]/div[2]/div[2]/div[3]/div[2]/descendant').getall(),
"Systemanforderungenmin" : response.xpath('//*[@id="config"]/ul[1]/descendant').getall(),
"Systemanforderungenmax" : response.xpath('//*[@id="config"]/ul[2]/descendant').getall(),
})
yield item
item['offer'] = []
for div in response.css('#offers_table'):
for offer_row in div.css('div.offers-table-row'):
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
url_str = ''.join(map(str, url)) #coverts list to str
item['offer'].append({
"offer:"
"Shop": offer_row.css('div[itemprop ~= seller] div.offers-merchant::attr(title)').extract_first(),
"Typ": offer_row.css('div.offers-edition-region::text').extract_first(),
"Edition": offer_row.css("div[data-toggle=tooltip]::attr(data-content)"),
"Link": response.follow(url_str, self.parse_topics),
})
yield item
我收到了回复
DEBUG: Scraped from <200 https://www.keyforsteam.de/kaufen-crusader-kings-2-cd-key-preisvergleich/>
{'url': 'https://www.keyforsteam.de/kaufen-crusader-kings-2-cd-key-preisvergleich/', 'details': [{'Name': '\n\t\t\t\t\tCrusader Kings 2\n\t\t\t\t', 'Release': '\n 14. Februar 2012\n ', 'Website': 'https://www.paradoxplaza.com/crusader-kings-ii/CKCK02GSK-MASTER.html', 'Entwickler': '\n Paradox Development Studio\n
', 'Publisher': '\n Paradox Interactive\n
', 'Tags': [], 'Systemanforderungenmin': [], 'Systemanforderungenmax': []}]}
2021-03-22 21:47:22 [scrapy.core.scraper] ERROR: Spider error processing <GET https://www.keyforsteam.de/kaufen-crusader-kings-2-cd-key-preisvergleich/> (referer: None)
Traceback (most recent call last):
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\utils\defer.py", line 120, in iter_errback
yield next(it)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\utils\python.py", line 353, in __next__
return next(self.data)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\utils\python.py", line 353, in __next__
return next(self.data)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 29, in process_spider_output
for x in result:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 340, in <genexpr>
return (_set_referer(r) for r in result or ())
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\spidermiddlewares\depth.py",
line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "C:\Users\magnu\brickset-scraper\postscrape\postscrape\spiders\keysint.py", line 40, in parse
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
File "c:\users\magnu\appdata\local\programs\python\python39\lib\site-packages\scrapy\http\response\text.py", line
102, in urljoin
return urljoin(get_base_url(self), url)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\urllib\parse.py", line 524, in urljoin
base, url, _coerce_result = _coerce_args(base, url)
File "c:\users\magnu\appdata\local\programs\python\python39\lib\urllib\parse.py", line 122, in _coerce_args
raise TypeError("Cannot mix str and non-str arguments")
TypeError: Cannot mix str and non-str arguments
所以第一部分似乎有效，我很确定错误在第二项中的某处，但我似乎找不到它
item['offer'] = []
for div in response.css('#offers_table'):
for offer_row in div.css('div.offers-table-row'):
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
url_str = ''.join(map(str, url)) #coverts list to str
item['offer'].append({
"offer:"
"Shop": offer_row.css('div[itemprop ~= seller] div.offers-merchant::attr(title)').extract_first(),
"Typ": offer_row.css('div.offers-edition-region::text').extract_first(),
"Edition": offer_row.css("div[data-toggle=tooltip]::attr(data-content)"),
"Link": response.follow(url_str, self.parse_topics),
})
yield item
有一种循环路线可以得到这个，但我认为调试过程会很有启发性。
如果没有程序正在调用的 json 文件，诊断起来会比较困难，但看起来您的问题出在这一行：url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)')).get(),
来自How Can I Fix "TypeError: Cannot mix str and non-str arguments"?
According to the Scrapy documentation, the .css(selector) method that you're using, returns a SelectorList instance. If you want the actual (unicode) string version of the url, call the extract() method:
所以我尝试了：
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').extract()).get(),
但我仍然得到同样的错误。奇怪！
为了诊断，我在此处将 breakpoint() 放入蜘蛛中：
for div in response.css('#offers_table'):
for offer_row in div.css('div.offers-table-row'):
breakpoint()
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').extract()).get(),
运行 蜘蛛又来了，我可以试下一行：
(Pdb) offer_row.css('div.buy-btn-cell a::attr(href)').extract()
['https://www.keyforsteam.de/outgoinglink/keyforsteam/37370?merchant=1', 'https://www.keyforsteam.de/outgoinglink/keyforsteam/37370?merchant=1']
啊，所以 extract() 返回的是字符串列表而不是单个字符串。必须有两个元素匹配。然而，它们是相同的，所以我们不关心我们得到的是哪一个。查看 https://docs.scrapy.org/en/latest/topics/selectors.html 的 scrapy 文档，我们看到还有一个 extract-first() 函数。
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').extract-first()).get(),
虽然，查看 scrapy 文档，您可能想使用 get() 而不是 extract-first()
那时我终于注意到你唯一的错误是把 get() 放在错误的括号外。
url = response.urljoin(offer_row.css('div.buy-btn-cell a::attr(href)').get())