Scrapy：不要抓取其他域页面上的链接

Question

下面是我创建的蜘蛛 id，以获取 NecToday.com 上的所有 links。

import socket
import scrapy

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import HtmlXPathSelector

class PropertiesItem(scrapy.Item):
    # Primary fields
    title = scrapy.Field()
    url = scrapy.Field()

class NecSpider(CrawlSpider):
    name = "NecSpider"
    #allowed_domains = ["nectoday.com"]
    start_urls = ["http://nectoday.com"]

    rules = (
        Rule(SgmlLinkExtractor(allow=(), restrict_xpaths=('//a',)), callback="parse_items", follow= True),
    )

    def parse_items(self, response):
        hxs = HtmlXPathSelector(response)
        print(response.url)
        item = PropertiesItem()
        item["title"] = response.xpath("//title/text()").extract()
        item["url"] = response.url
        return(item)

此代码开始获取网站上存在的所有 link。一些页面也有 YouTube link。问题是，一旦第一个 YouTube link 被抓取，它就会开始抓取从第一个 YouTube link 引用的其他 YouTube link。

我想抓取第一个 YouTube link，但不想抓取其他。 YouTube 只是一个例子。明天也可以是另一个站点。如何实现？

Answer 1

为什么不尝试以下方法：

start_urls=["http://nectoday.com"] 

def parse(self, response):
    #parse whatever you need

    for url in response.selector.xpath('//@href').extract():
        if 'youtube.com' in url:
            yield scrapy.Request(url, callback=self.parse_no_follow)
        else:
            yield scrapy.Request(url, callback=self.parse)

def parse_no_follow(self, response):
    #parse whatever you want and not follow anymore links

Answer 2

这只会从您允许的域中抓取。

class QuotesSpider(CrawlSpider):
    name = "your app name"
    n=0

    allowed_domains = ['domain']
    start_urls=['anywebpage']
    rules = (
        Rule(LinkExtractor(), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        QuotesSpider.n=QuotesSpider.n+1
        if (len(response.body)>100):

            h = html2text.HTML2Text()
            h.ignore_links = True
            h.ignore_images = True
            h.body_width = 0
            dd = response.body.decode("utf-8")
            init=dd.find("<p>")
        while init>0:
            end = dd.find("</p>", init)
            if end>0:
                o=h.handle(dd[init:end+4]+"\n")
                supersentences=o.split('\n')

Scrapy：不要抓取其他域页面上的链接

Scrapy: Do not crawl links on other domains page

python

scrapy

python-2.7

scrapy-spider