Scrapy + pyqt5: signal only works in main thread 错误

Scrapy + pyqt5: signal only works in main thread Error

我用 scrapy 写了一个蜘蛛,它使用了 CrawlerProcess 并且运行良好。

下一步,我想要一个 GUI 对话框,用于使用 pyqt5 制作的一些输入数据,并在 scrapy 工作时设置一个微调器。此任务是使用虚拟进程完成的,并且工作正常。

我知道多线程与 CrawlerProcess 不兼容,所以我将其更改为 CrawlerRunner。无论如何,它失败了:

signal.signal(signal.SIGINT, self.sigInt)
  File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
    handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread

我创建了一个最小示例来重现错误:

from PyQt5.QtWidgets import QApplication, QDialog, QTabWidget, QWidget, QGroupBox, QPushButton, QVBoxLayout
from PyQt5.QtCore import Qt, QRunnable, QMetaObject, pyqtSlot, QThreadPool
from PyQt5.QtGui import QIcon
from scrapy.crawler import CrawlerRunner
from scrapy import Request, Spider, Item, Field
from twisted.internet import reactor
from waitingspinnerwidget import QtWaitingSpinner

class RequestRunnable(QRunnable):
    def __init__(self, dialog):
        super(RequestRunnable, self).__init__()
        self.w = dialog
        print('runnable __init__')

    def run(self):
        print('run')
        crawler = CrawlerRunner({
            'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
            'FEED_FORMAT': 'json',
            'FEED_URI': 'output.json',
            'DOWNLOAD_DELAY': 3,
            'LOG_STDOUT': True,
            'LOG_FILE': 'scrapy_output.txt',
            'ROBOTSTXT_OBEY': False,
            'RETRY_ENABLED': True,
            'RETRY_HTTP_CODES': [500, 503, 504, 400, 404, 408],
            'RETRY_TIMES': 5
        })

        # instantiate a spider
        spider = CustomSpider()
        d = crawler.crawl(spider)
        d.addBoth(lambda _: reactor.stop()) # I think this is for manually stopping the reactor when we are done
        reactor.run() #TODO gives error
        QMetaObject.invokeMethod(self.w, "FinishedDownload", Qt.QueuedConnection)


class CustomSpider(Spider):
    name = 'quotes_spider'
    allowed_domains = ['quotes.toscrape.com']
    start_urls = ['http://quotes.toscrape.com/']

    def parse(self, response):
        quotes = response.xpath("//div[@class='quote']//span[@class='text']/text()").extract()
        yield {'quotes': quotes}


class DownloadDataDialog(QDialog):
    def __init__(self, parent=None):
        super(DownloadDataDialog, self).__init__(parent)

        self.spinner = QtWaitingSpinner(self, True, True, Qt.ApplicationModal)

        tabWidget = QTabWidget(self)
        tabWidget.addTab(MyTab(tabWidget), "MyTab")

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(tabWidget)
        self.setLayout(mainLayout)

        self.setWindowTitle("Download option chain data from web")


class MyTab(QWidget):
    def __init__(self, parent=None):
        super(MyTab, self).__init__(parent)

        dataGroup = QGroupBox('Data')

        getButton = QPushButton('Download')
        getButton.clicked.connect(self.download_data)

        dataLayout = QVBoxLayout()
        dataLayout.addWidget(getButton)
        dataGroup.setLayout(dataLayout)

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(dataGroup)
        mainLayout.addStretch(1)
        self.setLayout(mainLayout)

    def download_data(self):
        self.parentWidget().window().spinner.start()
        runnable = RequestRunnable(self)
        QThreadPool.globalInstance().start(runnable)

    @pyqtSlot()
    def FinishedDownload(self):
        self.parentWidget().window().spinner.stop()

if __name__ == '__main__':

    import sys

    app = QApplication(sys.argv)

    tabdialog = DownloadDataDialog()
    tabdialog.show()
    sys.exit(app.exec_())

为了执行它,您需要 waitingspinnerwidget.py:

# -*- coding: utf-8 -*-
"""
The MIT License (MIT)

Copyright (c) 2012-2014 Alexander Turkin
Copyright (c) 2014 William Hallatt
Copyright (c) 2015 Jacob Dawid
Copyright (c) 2016 Luca Weiss

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import math

from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *


class QtWaitingSpinner(QWidget):
    def __init__(self, parent, centerOnParent=True, disableParentWhenSpinning=False, modality=Qt.NonModal):
        super().__init__(parent, Qt.Dialog | Qt.FramelessWindowHint)

        self._centerOnParent = centerOnParent
        self._disableParentWhenSpinning = disableParentWhenSpinning

        # WAS IN initialize()
        self._color = QColor(Qt.black)
        self._roundness = 100.0
        self._minimumTrailOpacity = 3.14159265358979323846
        self._trailFadePercentage = 80.0
        self._revolutionsPerSecond = 1.57079632679489661923
        self._numberOfLines = 20
        self._lineLength = 10
        self._lineWidth = 2
        self._innerRadius = 10
        self._currentCounter = 0
        self._isSpinning = False

        self._timer = QTimer(self)
        self._timer.timeout.connect(self.rotate)
        self.updateSize()
        self.updateTimer()
        self.hide()
        # END initialize()

        self.setWindowModality(modality)
        self.setAttribute(Qt.WA_TranslucentBackground)

    def paintEvent(self, QPaintEvent):
        self.updatePosition()
        painter = QPainter(self)
        painter.fillRect(self.rect(), Qt.transparent)
        painter.setRenderHint(QPainter.Antialiasing, True)

        if self._currentCounter >= self._numberOfLines:
            self._currentCounter = 0

        painter.setPen(Qt.NoPen)
        for i in range(0, self._numberOfLines):
            painter.save()
            painter.translate(self._innerRadius + self._lineLength, self._innerRadius + self._lineLength)
            rotateAngle = float(360 * i) / float(self._numberOfLines)
            painter.rotate(rotateAngle)
            painter.translate(self._innerRadius, 0)
            distance = self.lineCountDistanceFromPrimary(i, self._currentCounter, self._numberOfLines)
            color = self.currentLineColor(distance, self._numberOfLines, self._trailFadePercentage,
                                          self._minimumTrailOpacity, self._color)
            painter.setBrush(color)
            painter.drawRoundedRect(QRect(0, -self._lineWidth / 2, self._lineLength, self._lineWidth), self._roundness,
                                    self._roundness, Qt.RelativeSize)
            painter.restore()

    def start(self):
        self.updatePosition()
        self._isSpinning = True
        self.show()

        if self.parentWidget and self._disableParentWhenSpinning:
            self.parentWidget().setEnabled(False)

        if not self._timer.isActive():
            self._timer.start()
            self._currentCounter = 0

    def stop(self):
        self._isSpinning = False
        self.hide()

        if self.parentWidget() and self._disableParentWhenSpinning:
            self.parentWidget().setEnabled(True)

        if self._timer.isActive():
            self._timer.stop()
            self._currentCounter = 0

    def setNumberOfLines(self, lines):
        self._numberOfLines = lines
        self._currentCounter = 0
        self.updateTimer()

    def setLineLength(self, length):
        self._lineLength = length
        self.updateSize()

    def setLineWidth(self, width):
        self._lineWidth = width
        self.updateSize()

    def setInnerRadius(self, radius):
        self._innerRadius = radius
        self.updateSize()

    def color(self):
        return self._color

    def roundness(self):
        return self._roundness

    def minimumTrailOpacity(self):
        return self._minimumTrailOpacity

    def trailFadePercentage(self):
        return self._trailFadePercentage

    def revolutionsPersSecond(self):
        return self._revolutionsPerSecond

    def numberOfLines(self):
        return self._numberOfLines

    def lineLength(self):
        return self._lineLength

    def lineWidth(self):
        return self._lineWidth

    def innerRadius(self):
        return self._innerRadius

    def isSpinning(self):
        return self._isSpinning

    def setRoundness(self, roundness):
        self._roundness = max(0.0, min(100.0, roundness))

    def setColor(self, color=Qt.black):
        self._color = QColor(color)

    def setRevolutionsPerSecond(self, revolutionsPerSecond):
        self._revolutionsPerSecond = revolutionsPerSecond
        self.updateTimer()

    def setTrailFadePercentage(self, trail):
        self._trailFadePercentage = trail

    def setMinimumTrailOpacity(self, minimumTrailOpacity):
        self._minimumTrailOpacity = minimumTrailOpacity

    def rotate(self):
        self._currentCounter += 1
        if self._currentCounter >= self._numberOfLines:
            self._currentCounter = 0
        self.update()

    def updateSize(self):
        size = (self._innerRadius + self._lineLength) * 2
        self.setFixedSize(size, size)

    def updateTimer(self):
        self._timer.setInterval(1000 / (self._numberOfLines * self._revolutionsPerSecond))

    def updatePosition(self):
        if self.parentWidget() and self._centerOnParent:
            self.move(self.parentWidget().width() / 2 - self.width() / 2,
                      self.parentWidget().height() / 2 - self.height() / 2)
            dialogCenter = self.mapToGlobal(self.rect().center())
            parentWindowCenter = self.parentWidget().window().mapToGlobal(self.parentWidget().window().rect().center())
            self.move(parentWindowCenter - dialogCenter)
            parentRect = QRect(self.parentWidget().mapToGlobal(QPoint(0, 0)), self.parentWidget().size())
            self.move(QStyle.alignedRect(Qt.LeftToRight, Qt.AlignCenter, self.size(), parentRect).topLeft())

    def lineCountDistanceFromPrimary(self, current, primary, totalNrOfLines):
        distance = primary - current
        if distance < 0:
            distance += totalNrOfLines
        return distance

    def currentLineColor(self, countDistance, totalNrOfLines, trailFadePerc, minOpacity, colorinput):
        color = QColor(colorinput)
        if countDistance == 0:
            return color
        minAlphaF = minOpacity / 100.0
        distanceThreshold = int(math.ceil((totalNrOfLines - 1) * trailFadePerc / 100.0))
        if countDistance > distanceThreshold:
            color.setAlphaF(minAlphaF)
        else:
            alphaDiff = color.alphaF() - minAlphaF
            gradient = alphaDiff / float(distanceThreshold + 1)
            resultAlpha = color.alphaF() - gradient * countDistance
            # If alpha is out of bounds, clip it.
            resultAlpha = min(1.0, max(0.0, resultAlpha))
            color.setAlphaF(resultAlpha)
        return color

完整追溯:

Traceback (most recent call last):
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 428, in fireEvent
    DeferredList(beforeResults).addCallback(self._continueFiring)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 322, in addCallback
    callbackKeywords=kw)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 311, in addCallbacks
    self._runCallbacks()
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 654, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
--- <exception caught here> ---
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 441, in _continueFiring
    callable(*args, **kwargs)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1256, in _reallyStartRunning
    self._handleSignals()
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\posixbase.py", line 295, in _handleSignals
    _SignalReactorMixin._handleSignals(self)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1221, in _handleSignals
    signal.signal(signal.SIGINT, self.sigInt)
  File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
    handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread

知道发生了什么以及如何解决它吗?

不要使用线程,只需使用 qt reactor 进行扭曲,这样你的扭曲应用程序 (scrapy) 和你的 qt5 应用程序就可以 运行 在同一个事件循环中:

安装 qt5reactor:

pip install qt5reactor

然后添加代码来安装反应器和运行它:

app = QApplication(sys.argv)

import qt5reactor
qt5reactor.install()

reactor.run()