tornado curl_httpclient: TypeError: unsetopt() is not supported for this option?

Question

谁能帮我分析错误堆栈：

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/site-packages/tornado/web.py", line 1144, in _when_complete
    if result.result() is not None:
  File "/usr/local/lib/python2.7/site-packages/tornado/concurrent.py", line 129, in result
    raise_exc_info(self.__exc_info)
  File "/usr/local/lib/python2.7/site-packages/tornado/stack_context.py", line 302, in wrapped
    ret = fn(*args, **kwargs)
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 550, in inner
    self.set_result(key, result)
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 476, in set_result
    self.run()
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 505, in run
    yielded = self.gen.throw(*exc_info)
  File "get_image.py", line 53, in async_fetch
    yield _f
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 496, in run
    next = self.yield_point.get_result()
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 395, in get_result
    return self.runner.pop_result(self.key).result()
  File "/usr/local/lib/python2.7/site-packages/tornado/concurrent.py", line 129, in result
    raise_exc_info(self.__exc_info)
  File "/usr/local/lib/python2.7/site-packages/tornado/stack_context.py", line 302, in wrapped
    ret = fn(*args, **kwargs)
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 550, in inner
    self.set_result(key, result)
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 476, in set_result
    self.run()
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 505, in run
    yielded = self.gen.throw(*exc_info)
  File "get_image.py", line 60, in async_fetch
    yield _f
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 496, in run
    next = self.yield_point.get_result()
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 395, in get_result
    return self.runner.pop_result(self.key).result()
  File "/usr/local/lib/python2.7/site-packages/tornado/concurrent.py", line 129, in result
    raise_exc_info(self.__exc_info)
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 221, in wrapper
    runner.run()
  File "/usr/local/lib/python2.7/site-packages/tornado/gen.py", line 507, in run
    yielded = self.gen.send(next)
  File "get_image.py", line 43, in async_fetch
    request_timeout=request_timeout)
  File "/usr/local/lib/python2.7/site-packages/tornado/httpclient.py", line 199, in fetch
    self.fetch_impl(request, handle_response)
  File "/usr/local/lib/python2.7/site-packages/tornado/curl_httpclient.py", line 95, in fetch_impl
    self._process_queue()
  File "/usr/local/lib/python2.7/site-packages/tornado/curl_httpclient.py", line 233, in _process_queue
    curl.info["headers"])
  File "/usr/local/lib/python2.7/site-packages/tornado/curl_httpclient.py", line 295, in _curl_setup_request
    curl.setopt(pycurl.URL, native_str(request.url))
TypeError: unsetopt() is not supported for this option

这里是 async_fetch 函数：

@tornado.gen.coroutine
def async_fetch(self, ourl, callback, headers=None, connect_timeout=3, request_timeout=12, replace=False):
    assert callable(callback)
    if ourl not in FETCHING or replace:
        future = GlobalAsyncClient.fetch(ourl, headers=headers, connect_timeout=connect_timeout,
                                         request_timeout=request_timeout)
        FETCHING[ourl] = future
    else:
        future = FETCHING[ourl]
    try:
        response = yield future
    except Exception as exc:
        if isinstance(exc, HTTPError) and exc.response is not None:
            _f = callback(exc.response)
            if _f is not None:
                yield _f
        else:
            self.log_exception(*sys.exc_info())
            self.return_error(404, 10, "can't get original image")
    else:
        _f = callback(response)
        if _f is not None:
            yield _f
    finally:
        FETCHING.pop(ourl, None)

回调之一可能是这样的：

def get_image_size(self, response, dest_file, original_file, ourl=None, whole=False):
    if err_ourl(response.effective_url):
        self.return_error(404, 10, "no such image")

    callback = functools.partial(self.get_image_size, dest_file=dest_file, original_file=original_file, whole=True)

    if response.error:
        if response.error.code == 416:
            return self.async_fetch(ourl, callback, replace=True)
        self.return_error(404, 10, str(response.error))

    try:
        if getimagesize.format_fromstring(response.body) == 'WEBP':
            return self.async_fetch(ourl, callback, replace=True)
        # noinspection PyStringFormat
        result = '{"w":%d, "h":%d}' % getimagesize.fromfileobj(response.buffer)[1:]
    except IOError:
        self.return_error(500, 10, "read/write image error")
    except:
        self.log_exception(*sys.exc_info())
        self.return_error(400, 10, "can't recognise img")
    else:
        self.set_header("Content-Type", "application/json")
        self.success_return(result)

        safe_write(dest_file, result)

        if whole:
            safe_write(original_file, response.body)

情况如下：我在龙卷风中使用 curl_httpclient，获取图像文件并获取大小。首先使用 Range header 获取 som 字节，如果远程服务器不支持这个，然后尝试在没有 Range 的情况下重新下载，当图像是 webp 格式时，去下载整个文件。

持有 fetch Future 对象以便在相同的请求同时到来时重用它。

它工作正常，但我有时会收到 TypeError。

我正在使用 python 2.7, tornado 3.1.1, pycurl 7.19.0, libcurl.x86_64 7.19.7-40.el6_6.4

Answer 1

这是来自 pycurl 的错误消息；它的意思是"URL may not be None"。 URL 有时会在重试链中丢失（例如，当您构建 functools.partial(self.get_image_size)）