Why do I get BioPython HTTPError: HTTP Error 400: Bad Request when I use Esearch and Efetch
Why do I get BioPython HTTPError: HTTP Error 400: Bad Request when I use Esearch and Efetch
我正在尝试访问脊索动物门的生物体列表，这些生物体已经从 Entrez 的 "assembly" 数据库中对染色体进行了测序。我正在尝试使用 biopython 中的 E-utilities 来做到这一点。我能够使用 esearch 搜索生物体，然后出现正确的列表（它与我进行在线搜索时出现的生物体及其登录 ID 相匹配）。但是，我无法通过 efetch 下载有机体。
我已将下面的代码写入 ipython 终端并继续收到错误 400。我在阵亡将士纪念日尝试了此操作，当时 ncbi 服务器不忙，但我仍然不断收到此错误。我正在使用 biopython 教程手册，并按照手册中的 esearch、epost 和 efetch 指南获取生物。
In [1]: from Bio import Entrez
In [2]: from Bio import SeqIO
In [3]: Entrez.email = "chahnapatel2798@gmail.com"
In [4]: handle = Entrez.esearch(db = "assembly", term = "chordata[orgn] AND chro
...: mosome", retmax = 2700, idtype = "acc")
In [5]: genome_ids = Entrez.read(handle)['IdList']
In [6]: print(genome_ids)
['1716181', '1699511', '1678341', '1677391', '1676551', '1668981', '1615891', '1597161', '1560261', '1559891', '1556721', '1551371', '1543341', '1529631', '1528571', '1528541', '1523321', '1516161', '1512011', '1510921', '1497921', '1493941', '1493281', '1470421', '1460771', '1459101', '1448961', '1426091', '1424411', '1252331', '1225811', '1198761', '1161601', '1161551', '1161541', '1154981', '1134921', '1132361', '1117251', '1116181', '1104621', '1086041', '1086031', '1082401', '1080921', '1062661', '1034061', '1033621', '1024761', '1020831', '1004191', '1002641', '998221', '954671', '944911', '905851', '905421', '905371', '905331', '895291', '895201', '893611', '872241', '859411', '802141', '788871', '786311', '779791', '768701', '763971', '763951', '763931', '763271', '738491', '738481', '738461', '738451', '733711', '731351', '731341', '731331', '731321', '731311', '731301', '731291', '731281', '731271', '731261', '731251', '731241', '707541', '706168', '705028', '704988', '700758', '654721', '634151', '632211', '618441', '599081', '598778', '595851', '588981', '585171', '585021', '582731', '561668', '558528', '527578', '524908', '524258', '524218', '521878', '516978', '508288', '506498', '504458', '503571', '497601', '487471', '487001', '486691', '485801', '485691', '474701', '474211', '457978', '448378', '448068', '443538', '440818', '418928', '415668', '399268', '397958', '382928', '368578', '365298', '362278', '355991', '355941', '354508', '347931', '331778', '327908', '327618', '326968', '326171', '320101', '317958', '317138', '315421', '313728', '313678', '310698', '310688', '304538', '304498', '303998', '298528', '294518', '293148', '286598', '284398', '281188', '280818', '280798', '280718', '279808', '255628', '254138', '250841', '249188', '247028', '238918', '238558', '238058', '237648', '237618', '237598', '237548', '237518', '237428', '234761', '232401', '228231', '227361', '226861', '221321', '221311', '216221', '215291', '210611', '210341', '203101', '202458', '202158', '202018', '201781', '195491', '191871', '189361', '182491', '175841', '169768', '167788', '165668', '150471', '138171', '132581', '132211', '111518', '111478', '111338', '95041', '88331', '84681', '81581', '80571', '52361', '37871', '37801', '6608', '6408', '6328', '5758', '5728', '5678', '5448', '5298', '5238', '5178', '4898', '4548', '4528', '4418', '4348', '4038', '4018', '3328', '3298', '3188', '3168', '2928', '2908', '2758', '2748', '2728', '2718', '2708', '2698', '2618', '2598', '2578', '2568']
In [7]: print(Entrez.epost("assembly", id = ",".join(genome_ids)).read())
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE ePostResult PUBLIC "-//NLM//DTD epost 20090526//EN" "https://eutils.ncbi.nlm.nih.gov/eutils/dtd/20090526/epost.dtd"><ePostResult>
<QueryKey>1</QueryKey>
<WebEnv>NCID_1_210557518_130.14.22.215_9001_1527649525_320674132_0MetA0_S_MegaStore</WebEnv>
</ePostResult>
In [8]: search_results = Entrez.read(Entrez.epost("assembly", id = ",".join(geno
...: me_ids)))
In [9]: webenv = search_results["WebEnv"]
In [10]: query_key = search_results["QueryKey"]
In [11]: search_handle = Entrez.esearch(db = "assembly", term = "chordata[orgn]
...: AND chromosome", usehistory = "y", idtype = "acc")
In [12]: search_results = Entrez.read(search_handle)
In [13]: search_handle.close()
In [14]: acc_list = search_results["IdList"]
In [15]: count = int(search_results["Count"])
In [16]: count == len(acc_list)
Out[16]: False
In [17]: batch_size = 3
In [18]: from urllib2 import HTTPError
...: out_handle = open("chordata_sequence.fasta","w")
...: for start in range(0,count,batch_size):
...: end = min(count, start+batch_size)
...: print ("Going to download record %i to %i" % (start + 1,end))
...: attempt = 0
...: while attempt < 3:
...: attempt += 1
...: try:
...: fetch_handle = Entrez.efetch(db = "assembly", rettype = "fa
...: sta", retmode = "text", retstart = start, retmax = batch_size, webenv =
...: webenv, query_key = query_key, idtype = "acc")
...: except HTTPError as err:
...: if 500 <= err.code <= 599:
...: print("Received error from server %s" % err)
...: print("Attempt %i of 3" % attempt)
...: time.sleep(15)
...: else:
...: raise
...: data = fetch_handle.read()
...: fetch_handle.close()
...: out_handle.write(data)
...: out_handle.close()
...:
In [18]: from urllib2 import HTTPError
...: out_handle = open("chordata_sequence.fasta","w")
...: for start in range(0,count,batch_size):
...: end = min(count, start+batch_size)
...: print ("Going to download record %i to %i" % (start + 1,end))
...: attempt = 0
...: while attempt < 3:
...: attempt += 1
...: try:
...: fetch_handle = Entrez.efetch(db = "assembly", rettype = "fa
...: sta", retmode = "text", retstart = start, retmax = batch_size, webenv =
...: webenv, query_key = query_key, idtype = "acc")
...: except HTTPError as err:
...: if 500 <= err.code <= 599:
...: print("Received error from server %s" % err)
...: print("Attempt %i of 3" % attempt)
...: time.sleep(15)
...: else:
...: raise
...: data = fetch_handle.read()
...: fetch_handle.close()
...: out_handle.write(data)
...: out_handle.close()
...:
Going to download record 1 to 3
下面的错误是我在 运行 上面的代码时得到的：
HTTPError Traceback (most recent call last)
<ipython-input-18-4726db68aa54> in <module>()
12 attempt += 1
13 try:
---> 14 fetch_handle = Entrez.efetch(db = "assembly", rettype = "fasta", retmode = "text", retstart = start, retmax = batch_size, webenv = webenv, query_key = query_key, idtype = "acc")
15 except HTTPError as err:
16 if 500 <= err.code <= 599:
~/anaconda3/lib/python3.6/site-packages/Bio/Entrez/__init__.py in efetch(db, **keywords)
178 # more than about 200 IDs
179 post = True
--> 180 return _open(cgi, variables, post=post)
181
182
~/anaconda3/lib/python3.6/site-packages/Bio/Entrez/__init__.py in _open(cgi, params, post, ecitmatch)
528 handle = _urlopen(cgi)
529 except _HTTPError as exception:
--> 530 raise exception
531
532 return _binary_to_string_handle(handle)
~/anaconda3/lib/python3.6/site-packages/Bio/Entrez/__init__.py in _open(cgi, params, post, ecitmatch)
526 handle = _urlopen(cgi, data=_as_bytes(options))
527 else:
--> 528 handle = _urlopen(cgi)
529 except _HTTPError as exception:
530 raise exception
~/anaconda3/lib/python3.6/urllib/request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
221 else:
222 opener = _opener
--> 223 return opener.open(url, data, timeout)
224
225 def install_opener(opener):
~/anaconda3/lib/python3.6/urllib/request.py in open(self, fullurl, data, timeout)
530 for processor in self.process_response.get(protocol, []):
531 meth = getattr(processor, meth_name)
--> 532 response = meth(req, response)
533
534 return response
~/anaconda3/lib/python3.6/urllib/request.py in http_response(self, request, response)
640 if not (200 <= code < 300):
641 response = self.parent.error(
--> 642 'http', request, response, code, msg, hdrs)
643
644 return response
~/anaconda3/lib/python3.6/urllib/request.py in error(self, proto, *args)
568 if http_err:
569 args = (dict, 'default', 'http_error_default') + orig_args
--> 570 return self._call_chain(*args)
571
572 # XXX probably also want an abstract factory that knows when it makes
~/anaconda3/lib/python3.6/urllib/request.py in _call_chain(self, chain, kind, meth_name, *args)
502 for handler in handlers:
503 func = getattr(handler, meth_name)
--> 504 result = func(*args)
505 if result is not None:
506 return result
~/anaconda3/lib/python3.6/urllib/request.py in http_error_default(self, req, fp, code, msg, hdrs)
648 class HTTPDefaultErrorHandler(BaseHandler):
649 def http_error_default(self, req, fp, code, msg, hdrs):
--> 650 raise HTTPError(req.full_url, code, msg, hdrs, fp)
651
652 class HTTPRedirectHandler(BaseHandler):
HTTPError: HTTP Error 400: Bad Request
您问题的答案似乎是 "fasta" 不是 assembly 的有效 rettype。如果将第 14 行更改为：
fetch_handle = Entrez.efetch(db = "assembly", rettype='docsum',
retmode = "xml", retstart = start, retmax = batch_size,
webenv = webenv, query_key = query_key, idtype = "acc")
你会得到一个结果。不可否认，这可能不是您要寻找的结果，但我不确定解决方案是什么，因为 eutils 文档……不充分。如果我是你，我会开始寻找有用的 rettype here。您可能需要 link 到不同的数据库才能获取 fasta 文件。
我正在尝试访问脊索动物门的生物体列表，这些生物体已经从 Entrez 的 "assembly" 数据库中对染色体进行了测序。我正在尝试使用 biopython 中的 E-utilities 来做到这一点。我能够使用 esearch 搜索生物体，然后出现正确的列表（它与我进行在线搜索时出现的生物体及其登录 ID 相匹配）。但是，我无法通过 efetch 下载有机体。 我已将下面的代码写入 ipython 终端并继续收到错误 400。我在阵亡将士纪念日尝试了此操作，当时 ncbi 服务器不忙，但我仍然不断收到此错误。我正在使用 biopython 教程手册，并按照手册中的 esearch、epost 和 efetch 指南获取生物。
In [1]: from Bio import Entrez
In [2]: from Bio import SeqIO
In [3]: Entrez.email = "chahnapatel2798@gmail.com"
In [4]: handle = Entrez.esearch(db = "assembly", term = "chordata[orgn] AND chro
...: mosome", retmax = 2700, idtype = "acc")
In [5]: genome_ids = Entrez.read(handle)['IdList']
In [6]: print(genome_ids)
['1716181', '1699511', '1678341', '1677391', '1676551', '1668981', '1615891', '1597161', '1560261', '1559891', '1556721', '1551371', '1543341', '1529631', '1528571', '1528541', '1523321', '1516161', '1512011', '1510921', '1497921', '1493941', '1493281', '1470421', '1460771', '1459101', '1448961', '1426091', '1424411', '1252331', '1225811', '1198761', '1161601', '1161551', '1161541', '1154981', '1134921', '1132361', '1117251', '1116181', '1104621', '1086041', '1086031', '1082401', '1080921', '1062661', '1034061', '1033621', '1024761', '1020831', '1004191', '1002641', '998221', '954671', '944911', '905851', '905421', '905371', '905331', '895291', '895201', '893611', '872241', '859411', '802141', '788871', '786311', '779791', '768701', '763971', '763951', '763931', '763271', '738491', '738481', '738461', '738451', '733711', '731351', '731341', '731331', '731321', '731311', '731301', '731291', '731281', '731271', '731261', '731251', '731241', '707541', '706168', '705028', '704988', '700758', '654721', '634151', '632211', '618441', '599081', '598778', '595851', '588981', '585171', '585021', '582731', '561668', '558528', '527578', '524908', '524258', '524218', '521878', '516978', '508288', '506498', '504458', '503571', '497601', '487471', '487001', '486691', '485801', '485691', '474701', '474211', '457978', '448378', '448068', '443538', '440818', '418928', '415668', '399268', '397958', '382928', '368578', '365298', '362278', '355991', '355941', '354508', '347931', '331778', '327908', '327618', '326968', '326171', '320101', '317958', '317138', '315421', '313728', '313678', '310698', '310688', '304538', '304498', '303998', '298528', '294518', '293148', '286598', '284398', '281188', '280818', '280798', '280718', '279808', '255628', '254138', '250841', '249188', '247028', '238918', '238558', '238058', '237648', '237618', '237598', '237548', '237518', '237428', '234761', '232401', '228231', '227361', '226861', '221321', '221311', '216221', '215291', '210611', '210341', '203101', '202458', '202158', '202018', '201781', '195491', '191871', '189361', '182491', '175841', '169768', '167788', '165668', '150471', '138171', '132581', '132211', '111518', '111478', '111338', '95041', '88331', '84681', '81581', '80571', '52361', '37871', '37801', '6608', '6408', '6328', '5758', '5728', '5678', '5448', '5298', '5238', '5178', '4898', '4548', '4528', '4418', '4348', '4038', '4018', '3328', '3298', '3188', '3168', '2928', '2908', '2758', '2748', '2728', '2718', '2708', '2698', '2618', '2598', '2578', '2568']
In [7]: print(Entrez.epost("assembly", id = ",".join(genome_ids)).read())
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE ePostResult PUBLIC "-//NLM//DTD epost 20090526//EN" "https://eutils.ncbi.nlm.nih.gov/eutils/dtd/20090526/epost.dtd"><ePostResult>
<QueryKey>1</QueryKey>
<WebEnv>NCID_1_210557518_130.14.22.215_9001_1527649525_320674132_0MetA0_S_MegaStore</WebEnv>
</ePostResult>
In [8]: search_results = Entrez.read(Entrez.epost("assembly", id = ",".join(geno
...: me_ids)))
In [9]: webenv = search_results["WebEnv"]
In [10]: query_key = search_results["QueryKey"]
In [11]: search_handle = Entrez.esearch(db = "assembly", term = "chordata[orgn]
...: AND chromosome", usehistory = "y", idtype = "acc")
In [12]: search_results = Entrez.read(search_handle)
In [13]: search_handle.close()
In [14]: acc_list = search_results["IdList"]
In [15]: count = int(search_results["Count"])
In [16]: count == len(acc_list)
Out[16]: False
In [17]: batch_size = 3
In [18]: from urllib2 import HTTPError
...: out_handle = open("chordata_sequence.fasta","w")
...: for start in range(0,count,batch_size):
...: end = min(count, start+batch_size)
...: print ("Going to download record %i to %i" % (start + 1,end))
...: attempt = 0
...: while attempt < 3:
...: attempt += 1
...: try:
...: fetch_handle = Entrez.efetch(db = "assembly", rettype = "fa
...: sta", retmode = "text", retstart = start, retmax = batch_size, webenv =
...: webenv, query_key = query_key, idtype = "acc")
...: except HTTPError as err:
...: if 500 <= err.code <= 599:
...: print("Received error from server %s" % err)
...: print("Attempt %i of 3" % attempt)
...: time.sleep(15)
...: else:
...: raise
...: data = fetch_handle.read()
...: fetch_handle.close()
...: out_handle.write(data)
...: out_handle.close()
...:
In [18]: from urllib2 import HTTPError
...: out_handle = open("chordata_sequence.fasta","w")
...: for start in range(0,count,batch_size):
...: end = min(count, start+batch_size)
...: print ("Going to download record %i to %i" % (start + 1,end))
...: attempt = 0
...: while attempt < 3:
...: attempt += 1
...: try:
...: fetch_handle = Entrez.efetch(db = "assembly", rettype = "fa
...: sta", retmode = "text", retstart = start, retmax = batch_size, webenv =
...: webenv, query_key = query_key, idtype = "acc")
...: except HTTPError as err:
...: if 500 <= err.code <= 599:
...: print("Received error from server %s" % err)
...: print("Attempt %i of 3" % attempt)
...: time.sleep(15)
...: else:
...: raise
...: data = fetch_handle.read()
...: fetch_handle.close()
...: out_handle.write(data)
...: out_handle.close()
...:
Going to download record 1 to 3
下面的错误是我在 运行 上面的代码时得到的：
HTTPError Traceback (most recent call last)
<ipython-input-18-4726db68aa54> in <module>()
12 attempt += 1
13 try:
---> 14 fetch_handle = Entrez.efetch(db = "assembly", rettype = "fasta", retmode = "text", retstart = start, retmax = batch_size, webenv = webenv, query_key = query_key, idtype = "acc")
15 except HTTPError as err:
16 if 500 <= err.code <= 599:
~/anaconda3/lib/python3.6/site-packages/Bio/Entrez/__init__.py in efetch(db, **keywords)
178 # more than about 200 IDs
179 post = True
--> 180 return _open(cgi, variables, post=post)
181
182
~/anaconda3/lib/python3.6/site-packages/Bio/Entrez/__init__.py in _open(cgi, params, post, ecitmatch)
528 handle = _urlopen(cgi)
529 except _HTTPError as exception:
--> 530 raise exception
531
532 return _binary_to_string_handle(handle)
~/anaconda3/lib/python3.6/site-packages/Bio/Entrez/__init__.py in _open(cgi, params, post, ecitmatch)
526 handle = _urlopen(cgi, data=_as_bytes(options))
527 else:
--> 528 handle = _urlopen(cgi)
529 except _HTTPError as exception:
530 raise exception
~/anaconda3/lib/python3.6/urllib/request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
221 else:
222 opener = _opener
--> 223 return opener.open(url, data, timeout)
224
225 def install_opener(opener):
~/anaconda3/lib/python3.6/urllib/request.py in open(self, fullurl, data, timeout)
530 for processor in self.process_response.get(protocol, []):
531 meth = getattr(processor, meth_name)
--> 532 response = meth(req, response)
533
534 return response
~/anaconda3/lib/python3.6/urllib/request.py in http_response(self, request, response)
640 if not (200 <= code < 300):
641 response = self.parent.error(
--> 642 'http', request, response, code, msg, hdrs)
643
644 return response
~/anaconda3/lib/python3.6/urllib/request.py in error(self, proto, *args)
568 if http_err:
569 args = (dict, 'default', 'http_error_default') + orig_args
--> 570 return self._call_chain(*args)
571
572 # XXX probably also want an abstract factory that knows when it makes
~/anaconda3/lib/python3.6/urllib/request.py in _call_chain(self, chain, kind, meth_name, *args)
502 for handler in handlers:
503 func = getattr(handler, meth_name)
--> 504 result = func(*args)
505 if result is not None:
506 return result
~/anaconda3/lib/python3.6/urllib/request.py in http_error_default(self, req, fp, code, msg, hdrs)
648 class HTTPDefaultErrorHandler(BaseHandler):
649 def http_error_default(self, req, fp, code, msg, hdrs):
--> 650 raise HTTPError(req.full_url, code, msg, hdrs, fp)
651
652 class HTTPRedirectHandler(BaseHandler):
HTTPError: HTTP Error 400: Bad Request
您问题的答案似乎是 "fasta" 不是 assembly 的有效 rettype。如果将第 14 行更改为：
fetch_handle = Entrez.efetch(db = "assembly", rettype='docsum',
retmode = "xml", retstart = start, retmax = batch_size,
webenv = webenv, query_key = query_key, idtype = "acc")
你会得到一个结果。不可否认，这可能不是您要寻找的结果，但我不确定解决方案是什么，因为 eutils 文档……不充分。如果我是你，我会开始寻找有用的 rettype here。您可能需要 link 到不同的数据库才能获取 fasta 文件。