在 python 请求流中幸存的 icinga2 重启
Surviving icinga2 restart in a python requests stream
我一直在研究 icinga2 的聊天机器人界面，但还没有找到在 icinga2 服务器的 restart/reload 中生存的持久方法。在移动 try/except 个区块、使用请求会话等一周之后，是时候接触社区了。
这是请求函数的当前迭代：
def i2api_request(url, headers={}, data={}, stream=False, *, auth=api_auth, ca=api_ca):
''' Do not call this function directly; it's a helper for the i2* command functions '''
# Adapted from http://docs.icinga.org/icinga2/latest/doc/module/icinga2/chapter/icinga2-api
# Section 11.10.3.1
try:
r = requests.post(url,
headers=headers,
auth=auth,
data=json.dumps(data),
verify=ca,
stream=stream
)
except (requests.exceptions.ChunkedEncodingError,requests.packages.urllib3.exceptions.ProtocolError, http.client.IncompleteRead,ValueError) as drop:
return("No connection to Icinga API")
if r.status_code == 200:
for line in r.iter_lines():
try:
if stream == True:
yield(json.loads(line.decode('utf-8')))
else:
return(json.loads(line.decode('utf-8')))
except:
debug("Could not produce JSON from "+line)
continue
else:
#r.raise_for_status()
debug('Received a bad response from Icinga API: '+str(r.status_code))
print('Icinga2 API connection lost.')
（调试函数只是标记错误并将指示的错误打印到控制台。）
此代码可以很好地处理来自 API 的事件并将它们发送到聊天机器人，但是如果重新加载 icinga 服务器，则需要在 /etc/icinga2 中添加新的服务器定义。 ..，监听器崩溃了。
这是我在重新启动服务器时得到的错误响应：
Exception in thread Thread-11:
Traceback (most recent call last):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 447, in _update_chunk_length
self.chunk_left = int(line, 16)
ValueError: invalid literal for int() with base 16: b''
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 228, in _error_catcher
yield
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 498, in read_chunked
self._update_chunk_length()
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 451, in _update_chunk_length
raise httplib.IncompleteRead(line)
http.client.IncompleteRead: IncompleteRead(0 bytes read)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/models.py", line 664, in generate
for chunk in self.raw.stream(chunk_size, decode_content=True):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 349, in stream
for line in self.read_chunked(amt, decode_content=decode_content):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 526, in read_chunked
self._original_response.close()
File "/usr/lib64/python3.4/contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 246, in _error_catcher
raise ProtocolError('Connection broken: %r' % e, e)
requests.packages.urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib64/python3.4/threading.py", line 920, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.4/threading.py", line 868, in run
self._target(*self._args, **self._kwargs)
File "/home/errbot/plugins/icinga2bot.py", line 186, in report_events
for line in queue:
File "/home/errbot/plugins/icinga2bot.py", line 158, in i2events
for line in queue:
File "/home/errbot/plugins/icinga2bot.py", line 98, in i2api_request
for line in r.iter_lines():
File "/home/errbot/err3/lib/python3.4/site-packages/requests/models.py", line 706, in iter_lines
for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/models.py", line 667, in generate
raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
对于 Icinga2.4，每次重新启动服务器时都会发生此崩溃。我以为我们升级到 2.5 后问题就消失了，但现在它似乎变成了一个 heisenbug。
我最终在 IRC 上获得了重新排序 try/except 块并确保它们位于正确位置的建议。这是工作结果。
def i2api_request(url, headers={}, data={}, stream=False, *, auth=api_auth, ca=api_ca):
''' Do not call this function directly; it's a helper for the i2* command functions '''
# Adapted from http://docs.icinga.org/icinga2/latest/doc/module/icinga2/chapter/icinga2-api
# Section 11.10.3.1
debug(url)
debug(headers)
debug(data)
try:
r = requests.post(url,
headers=headers,
auth=auth,
data=json.dumps(data),
verify=ca,
stream=stream
)
debug("Connecting to Icinga server")
debug(r)
if r.status_code == 200:
try:
for line in r.iter_lines():
debug('in i2api_request: '+str(line))
try:
if stream == True:
yield(json.loads(line.decode('utf-8')))
else:
return(json.loads(line.decode('utf-8')))
except:
debug("Could not produce JSON from "+line)
return("Could not produce JSON from "+line)
except (requests.exceptions.ChunkedEncodingError,ConnectionRefusedError):
return("Connection to Icinga lost.")
else:
debug('Received a bad response from Icinga API: '+str(r.status_code))
print('Icinga2 API connection lost.')
except (requests.exceptions.ConnectionError,
requests.packages.urllib3.exceptions.NewConnectionError) as drop:
debug("No connection to Icinga API. Error received: "+str(drop))
sleep(5)
return("No connection to Icinga API.")
我一直在研究 icinga2 的聊天机器人界面，但还没有找到在 icinga2 服务器的 restart/reload 中生存的持久方法。在移动 try/except 个区块、使用请求会话等一周之后，是时候接触社区了。
这是请求函数的当前迭代：
def i2api_request(url, headers={}, data={}, stream=False, *, auth=api_auth, ca=api_ca):
''' Do not call this function directly; it's a helper for the i2* command functions '''
# Adapted from http://docs.icinga.org/icinga2/latest/doc/module/icinga2/chapter/icinga2-api
# Section 11.10.3.1
try:
r = requests.post(url,
headers=headers,
auth=auth,
data=json.dumps(data),
verify=ca,
stream=stream
)
except (requests.exceptions.ChunkedEncodingError,requests.packages.urllib3.exceptions.ProtocolError, http.client.IncompleteRead,ValueError) as drop:
return("No connection to Icinga API")
if r.status_code == 200:
for line in r.iter_lines():
try:
if stream == True:
yield(json.loads(line.decode('utf-8')))
else:
return(json.loads(line.decode('utf-8')))
except:
debug("Could not produce JSON from "+line)
continue
else:
#r.raise_for_status()
debug('Received a bad response from Icinga API: '+str(r.status_code))
print('Icinga2 API connection lost.')
（调试函数只是标记错误并将指示的错误打印到控制台。）
此代码可以很好地处理来自 API 的事件并将它们发送到聊天机器人，但是如果重新加载 icinga 服务器，则需要在 /etc/icinga2 中添加新的服务器定义。 ..，监听器崩溃了。
这是我在重新启动服务器时得到的错误响应：
Exception in thread Thread-11:
Traceback (most recent call last):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 447, in _update_chunk_length
self.chunk_left = int(line, 16)
ValueError: invalid literal for int() with base 16: b''
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 228, in _error_catcher
yield
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 498, in read_chunked
self._update_chunk_length()
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 451, in _update_chunk_length
raise httplib.IncompleteRead(line)
http.client.IncompleteRead: IncompleteRead(0 bytes read)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/models.py", line 664, in generate
for chunk in self.raw.stream(chunk_size, decode_content=True):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 349, in stream
for line in self.read_chunked(amt, decode_content=decode_content):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 526, in read_chunked
self._original_response.close()
File "/usr/lib64/python3.4/contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "/home/errbot/err3/lib/python3.4/site-packages/requests/packages/urllib3/response.py", line 246, in _error_catcher
raise ProtocolError('Connection broken: %r' % e, e)
requests.packages.urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib64/python3.4/threading.py", line 920, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.4/threading.py", line 868, in run
self._target(*self._args, **self._kwargs)
File "/home/errbot/plugins/icinga2bot.py", line 186, in report_events
for line in queue:
File "/home/errbot/plugins/icinga2bot.py", line 158, in i2events
for line in queue:
File "/home/errbot/plugins/icinga2bot.py", line 98, in i2api_request
for line in r.iter_lines():
File "/home/errbot/err3/lib/python3.4/site-packages/requests/models.py", line 706, in iter_lines
for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode):
File "/home/errbot/err3/lib/python3.4/site-packages/requests/models.py", line 667, in generate
raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
对于 Icinga2.4，每次重新启动服务器时都会发生此崩溃。我以为我们升级到 2.5 后问题就消失了，但现在它似乎变成了一个 heisenbug。
我最终在 IRC 上获得了重新排序 try/except 块并确保它们位于正确位置的建议。这是工作结果。
def i2api_request(url, headers={}, data={}, stream=False, *, auth=api_auth, ca=api_ca):
''' Do not call this function directly; it's a helper for the i2* command functions '''
# Adapted from http://docs.icinga.org/icinga2/latest/doc/module/icinga2/chapter/icinga2-api
# Section 11.10.3.1
debug(url)
debug(headers)
debug(data)
try:
r = requests.post(url,
headers=headers,
auth=auth,
data=json.dumps(data),
verify=ca,
stream=stream
)
debug("Connecting to Icinga server")
debug(r)
if r.status_code == 200:
try:
for line in r.iter_lines():
debug('in i2api_request: '+str(line))
try:
if stream == True:
yield(json.loads(line.decode('utf-8')))
else:
return(json.loads(line.decode('utf-8')))
except:
debug("Could not produce JSON from "+line)
return("Could not produce JSON from "+line)
except (requests.exceptions.ChunkedEncodingError,ConnectionRefusedError):
return("Connection to Icinga lost.")
else:
debug('Received a bad response from Icinga API: '+str(r.status_code))
print('Icinga2 API connection lost.')
except (requests.exceptions.ConnectionError,
requests.packages.urllib3.exceptions.NewConnectionError) as drop:
debug("No connection to Icinga API. Error received: "+str(drop))
sleep(5)
return("No connection to Icinga API.")