Scrapy 获取 Start_Urls

Question

好的，长话短说，要赶路开会了

我正尝试在 scrapy 中开始 urls，但无论我如何尝试，我似乎都无法完成。这是我的代码（蜘蛛）。

import scrapy
import csv

from scrapycrawler.items import DmozItem
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor
from scrapy.selector import Selector
from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request


class DmozSpider(CrawlSpider):
    name = "dmoz"
    allowed_domains = ["snipplr.com"]


def start_requests(self):
    for i in range(1, 230):
        yield self.make_requests_from_url("http://www.snipplr.com/view/%d" % i)




def make_requests_from_url(self, url):
    item = DmozItem()

    # assign url
    item['link'] = url
    request = Request(url, dont_filter=True)

    # set the meta['item'] to use the item in the next call back
    request.meta['item'] = item
    return request


#Rules only apply before
rules = (
    Rule (LxmlLinkExtractor(deny_domains=('http:\/\/www.snipplr.com\/snippet-not-found\/',)),callback="parse", follow= True),
)


def parse(self, response):
    sel = Selector(response)
    item = response.meta['item']
    item['title'] = sel.xpath('//div[@class="post"]/h1/text()').extract()
    #start_url
    item['link'] = response.url

我已经尝试了所有方法，直到现在，我在我的数据库中得到了一个 "h"，url 列。

这是我的数据库：

import csv
from scrapy.exceptions import DropItem
from scrapy import log
import sys
 import mysql.connector

class CsvWriterPipeline(object):

    def __init__(self):
        self.connection = mysql.connector.connect(host='localhost', user='ws', passwd='ps', db='ws')
        self.cursor = self.connection.cursor()

    def process_item(self, item, spider):
        self.cursor.execute("SELECT title,url FROM items WHERE title= %s", item['title'])
        result = self.cursor.fetchone()
        if result:

            log.msg("Item already in database: %s" % item, level=log.DEBUG)
        else:
            self.cursor.execute(
               "INSERT INTO items (title, url) VALUES (%s, %s)",
                    (item['title'][0], item['link'][0]))
            self.connection.commit()

            log.msg("Item stored : " % item, level=log.DEBUG)
        return item

    def handle_error(self, e):
            log.err(e)

正如你从这里看到的，它显然在工作。

我将如何开始 url 或者我将如何赞美它。我相信 h 表示该字段为空。数据库是 mysql.

感谢您的阅读和帮助

此致，查理

Answer 1

item['link']，相对于item['title']，只是一个字符串，而不是一个列表：

self.cursor.execute("INSERT INTO items (title, url) VALUES (%s, %s)",
                    (item['title'][0], item['link']))

Scrapy 获取 Start_Urls

Scrapy Getting Start_Urls

python

mysql

scrapy

web-crawler

pipeline