DropItem if parsed url 包含关键字（管道）

Question

我正在尝试为我从 allrecipes.com 抓取食谱的学校项目构建一个蜘蛛。一切都运行良好，但是我似乎无法删除重复的食谱，其中一个 url 包含实际食谱，另一个包含相同的 url 并附加了 "video=true"。

这是我在 pipelines.py 中处理这个问题的尝试：

from scrapy.exceptions import DropItem
from scrapy import log



class DuplicatesPipeline(object):

# minCal = 50

def __init__(self):
    self.urls_seen = set()

def process_vids(self, item, spider):
    video = "video=true"
    url = str(item.get('url'))
    if video in url:
        raise DropItem("Contains video")
    else:   
        return item

def process_item(self, item, spider):
    unique_id = item.get('url')
    if unique_id in self.urls_seen:
        raise DropItem("Duplicate Item found (%s)" % unique_id)
    else:
        self.urls_seen.add('url')
        return item

settings.py:

# Scrapy settings for dirbot project
BOT_NAME = 'dirbot'

SPIDER_MODULES = ['dirbot.spiders']
NEWSPIDER_MODULE = 'dirbot.spiders'
DEFAULT_ITEM_CLASS = 'dirbot.items.Website'

ITEM_PIPELINES = {'dirbot.pipelines.DuplicatesPipeline': 300,}

items.py:

from scrapy.item import Item, Field


class Website(Item):

    name = Field()
    url = Field()
    description = Field()
    kcal = Field()
    carbs = Field()
    fat = Field()
    protein = Field()
    main = Field()
    sugar = Field()
    fibre = Field()
    author = Field()
    rating = Field()
    img = Field()

dnot.py:

from scrapy.spider import Spider
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from scrapy.http import Request
from scrapy.http.request import Request
from dirbot.items import Website
from scrapy.contrib.spiders import CrawlSpider,Rule
import urlparse
import scrapy

page = "http://allrecipes.com/recipes/main.aspx?Page=%d#recipes"

class DmozSpider(Spider):
    name = "dnot"
    allowed_domains = ["allrecipes.com"]
    start_urls = [page % 1]

    rules = [Rule(SgmlLinkExtractor(allow=('allrecipes.com'), restrict_xpaths = '//a[contains(.,"NEXT")]'),
    callback="parse", follow= True),
    ]

    def __init__(self):
        self.page_number = 1

    def parse(self, response):
        print "-------------------------------------------------"
        print self.page_number
        print "-------------------------------------------------"


        sel = Selector(response)
        sites = response.xpath('//div[@id="divGridItemWrapper"]')
        items = []

        for site in sites:
            item = Website()
            recipe = response.xpath('//a[contains(@href, "/Recipe/")]/@href').extract()
            url = "http://www.allrecipes.com" 
            for nth in recipe:
                go = urlparse.urljoin(url, str(nth))
                items.append(item)
                for link in go:
                    yield Request(go, self.recipes)
        if self.page_number <= 3:
            self.page_number += 1
            yield Request(page % self.page_number)
        else:
            pass

    def recipes(self,response):
        item = Website()
        sel = Selector(response)
        recipe = response.xpath('//div[@id="content-wrapper"]')
        items = []
        print "second page - %s" % response.url

        for i in recipe:
            item['url'] = response.url
            item['description'] = i.xpath('//span[@itemprop="description"]/text()').extract()
            item['name'] = i.xpath('//h1[@itemprop="name"]/text()').extract()
            item['kcal'] = i.xpath('//ul/li[contains(.,"kcal")]/span/text()').extract()
            item['carbs'] = i.xpath('//ul/li[contains(.,"Carbohydrates")]/following-sibling::li[1]//span[@id="lblNutrientValue"]/text()').extract()
            item['fat'] = i.xpath('//ul/li[contains(.,"Fat")]/following-sibling::li[1]//span[@id="lblNutrientValue"]/text()').extract()
            item['protein'] = i.xpath('//ul/li[contains(.,"Protein")]/following-sibling::li[1]//span[@id="lblNutrientValue"]/text()').extract()
            item['main'] = "allrecipes.com"
            item['sugar'] = i.xpath('//li/span[@itemprop="sugarContent"]/text()').extract()
            item['fibre'] = i.xpath('//li/span[@itemprop="proteinContent"]/text()').extract()
            item['author'] = i.xpath('//span[@id="lblUser0"]/text()').extract()
            item['rating'] = i.xpath('//div[@class="rating-stars-img"][1]/meta[1][@itemprop="ratingValue"]/@content').extract()
            item['img'] = i.xpath('//img[@id="imgPhoto"]/@src').extract()
            items.append(item)
            yield item

我对Python有点陌生，我不确定是否需要将项目['url']转换为字符串；但是我试过使用 "str" 和不使用。我还尝试了其他一些其他方法来做类似的事情，但到目前为止对我没有任何效果。

希望有人能指出我正确的方向。提前致谢！

示例：

项目['url'] = http://allrecipes.com/Recipe/Delicious-Ham-and-Potato-Soup/Detail.aspx?evt19=1&referringHubId=1 项目['url'] = http://allrecipes.com/Recipe/Delicious-Ham-and-Potato-Soup/Detail.aspx?evt19=1&referringHubId=1&video=true

Answer 1

您需要创建一个 class 来实现 pipelines.py 文件中的 process_item 方法，例如：

from urllib import urlencode
from urlparse import urlparse, urlunparse, parse_qs

class DuplicatesPipeline(object):
    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        url = item['url']
        u = urlparse(url)
        query = parse_qs(u.query)
        query.pop('video', None)
        u = u._replace(query=urlencode(query, True))
        unique_id = urlunparse(u)
        if unique_id and unique_id in self.ids_seen:
            raise DropItem("Duplicate Item found (%s)" % unique_id)
        else:
            self.ids_seen.add(unique_id)
        return item

然后您需要将 class 添加到 settings.py

ITEM_PIPELINES = {
    'yourproject.pipelines.DuplicatesPipeline': 300,
}

此外，您的 process_vids 方法未被使用。

如果对你有帮助，请告诉我。

DropItem if parsed url 包含关键字（管道）

DropItem if parsed url contains key words (pipeline)

python

scrapy