boto3 aws api - 列出可用的实例类型
boto3 aws api - Listing available instance types
实例类型：（t2.micro、t2.small、c4.large...）此处列出的那些：
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html
我想通过 boto3 访问这些列表。
类似于：
conn.get_all_instance_types()
甚至
conn.describe_instance_types()['InstanceTypes'][0]['Name']
在这个奇怪的 api.
中，一切看起来都像这样
我已经查看了客户端和 ServiceResource 的文档，但我找不到任何似乎接近的内容。
我什至还没有找到一个 hacky 解决方案来列出恰好代表所有实例类型的其他内容。
谁有更多boto3经验？
EC2 API 不提供获取所有 EC2 实例类型列表的方法。我希望如此。有些人通过像 this 这样的抓取网站拼凑了他们自己的有效类型列表，但目前这是唯一的方法。
试试这个
'''
Created on Mar 22, 2017
@author: ijessop
'''
import boto3
import urllib2
from bs4 import BeautifulSoup as soup
class EnumEc2():
def __init__(self, region):
self.client = boto3.client(
'ec2',
aws_access_key_id = 'YOUR_KEY' ,
aws_secret_access_key='YOUR_SECRET',
region_name = region
)
self.instance_types = None
self.instance_table_headers = None
self.max_col_width = {}
def getInstanceTypes(self):
mp = soup(urllib2.urlopen('https://aws.amazon.com/ec2/instance-types').read(),'html.parser')
imx = mp.find(id="instance-type-matrix")
trs = imx.parent.parent.parent.next_sibling.next_sibling.find_all('tr')
rt = []
first_row = True
for trow in trs:
td_strs = []
for td in trow.find_all("td"):
td_nested = []
for s in td.strings:
s.strip()
td_nested.append(s)
td_all = " ".join(td_nested).strip()
td_strs.append(td_all)
if first_row is True:
header_row = td_strs
for head in header_row:
self.max_col_width.update({head:(len(head) + 2)})
first_row = False
else:
dr = dict(zip(header_row,td_strs))
for k,v in dr.items():
cw = len(v)
if k in self.max_col_width.keys():
if cw >= self.max_col_width.get(k):
self.max_col_width.update({k:(cw +2)})
else:
self.max_col_width.update({k:cw})
rt.append(dr)
self.instance_table_headers = header_row
self.instance_types = rt
if __name__ == '__main__':
myen = EnumEc2('us-west-2')
myen.getInstanceTypes()
heads_I_want_to_see = ['Instance Type', u'vCPU', u'Memory (GiB)', u'Storage (GB)','Physical Processor', u'Clock Speed (GHz)']
out_str ="|"
for h in heads_I_want_to_see:
out_str = "%s%s|" % (out_str,h.ljust(myen.max_col_width.get(h)))
print "%s" % "-" * len(out_str)
print "%s" % out_str
print "%s" % "-" * len(out_str)
for i in myen.instance_types:
out_str ="|"
for k in myen.instance_table_headers: # to preserve the table column order
if k in heads_I_want_to_see:
out_str = "%s%s|" % (out_str, i.get(k).ljust(myen.max_col_width.get(k)))
print "%s" % out_str
print "%s" % "-" * len(out_str)
我也需要它，但是没有合适的代码用于此目的。我自己修改一个。享受！可能有人也需要。
以下代码修改自libcloud/contrib/scrape-ec2-prices.py
这个程序将生成一个关于可用实例类型的字典
#!/usr/bin/env python
import os
import re
import json
import time
from collections import defaultdict, OrderedDict
import requests
import demjson
LINUX_PRICING_URLS = [
# Deprecated instances (JSON format)
'https://aws.amazon.com/ec2/pricing/json/linux-od.json',
# Previous generation instances (JavaScript file)
'https://a0.awsstatic.com/pricing/1/ec2/previous-generation/linux-od.min.js',
# New generation instances (JavaScript file)
'https://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js'
]
EC2_REGIONS = [
'us-east-1',
'us-east-2',
'us-west-1',
'us-west-2',
'us-gov-west-1',
'eu-west-1',
'eu-west-2',
'eu-central-1',
'ca-central-1',
'ap-southeast-1',
'ap-southeast-2',
'ap-northeast-1',
'ap-northeast-2',
'ap-south-1',
'sa-east-1',
'cn-north-1',
]
INSTANCE_SIZES = [
'micro',
'small',
'medium',
'large',
'xlarge',
'x-large',
'extra-large'
]
RE_NUMERIC_OTHER = re.compile(r'(?:([0-9]+)|([-A-Z_a-z]+)|([^-0-9A-Z_a-z]+))')
PRICING_FILE_PATH = './price.json'
PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH)
def scrape_ec2_pricing():
result = {}
result['regions'] = []
result['prices'] = defaultdict(OrderedDict)
result['models'] = defaultdict(OrderedDict)
for url in LINUX_PRICING_URLS:
response = requests.get(url)
if re.match('.*?\.json$', url):
data = response.json()
elif re.match('.*?\.js$', url):
data = response.content
match = re.match('^.*callback\((.*?)\);?$', data,
re.MULTILINE | re.DOTALL)
data = match.group(1)
# demjson supports non-strict mode and can parse unquoted objects
data = demjson.decode(data)
regions = data['config']['regions']
for region_data in regions:
region_name = region_data['region']
if region_name not in result['regions']:
result['regions'].append(region_name)
libcloud_region_name = region_name
instance_types = region_data['instanceTypes']
for instance_type in instance_types:
sizes = instance_type['sizes']
for size in sizes:
price = size['valueColumns'][0]['prices']['USD']
if str(price).lower() == 'n/a':
# Price not available
continue
if not result['models'][libcloud_region_name].has_key(size['size']):
result['models'][libcloud_region_name][size['size']] = {}
result['models'][libcloud_region_name][size['size']]['CPU'] = int(size['vCPU'])
if size['ECU'] == 'variable':
ecu = 0
else:
ecu = float(size['ECU'])
result['models'][libcloud_region_name][size['size']]['ECU'] = ecu
result['models'][libcloud_region_name][size['size']]['memoryGiB'] = float(size['memoryGiB'])
result['models'][libcloud_region_name][size['size']]['storageGB'] = size['storageGB']
result['prices'][libcloud_region_name][size['size']] = float(price)
return result
def update_pricing_file(pricing_file_path, pricing_data):
## with open(pricing_file_path, 'r') as fp:
# content = fp.read()
data = {'compute': {}} # json.loads(content)
data['updated'] = int(time.time())
data['compute'].update(pricing_data)
# Always sort the pricing info
data = sort_nested_dict(data)
content = json.dumps(data, indent=4)
lines = content.splitlines()
lines = [line.rstrip() for line in lines]
content = '\n'.join(lines)
with open(pricing_file_path, 'w') as fp:
fp.write(content)
def sort_nested_dict(value):
"""
Recursively sort a nested dict.
"""
result = OrderedDict()
for key, value in sorted(value.items(), key=sort_key_by_numeric_other):
if isinstance(value, (dict, OrderedDict)):
result[key] = sort_nested_dict(value)
else:
result[key] = value
return result
def sort_key_by_numeric_other(key_value):
"""
Split key into numeric, alpha and other part and sort accordingly.
"""
return tuple((
int(numeric) if numeric else None,
INSTANCE_SIZES.index(alpha) if alpha in INSTANCE_SIZES else alpha,
other
) for (numeric, alpha, other) in RE_NUMERIC_OTHER.findall(key_value[0]))
def main():
print('Scraping EC2 pricing data')
pricing_data = scrape_ec2_pricing()
update_pricing_file(pricing_file_path=PRICING_FILE_PATH,
pricing_data=pricing_data)
print('Pricing data updated')
if __name__ == '__main__':
main()
可以在最近宣布的 AWS Price List API 提供的 JSON 中检索此信息。作为使用 Python requests 模块的简单示例：
#!/usr/bin/env python
# List EC2 Instance Types
# see: https://aws.amazon.com/blogs/aws/new-aws-price-list-api/
import requests
offers = requests.get(
'https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/index.json'
)
ec2_offer_path = offers.json()['offers']['AmazonEC2']['currentVersionUrl']
ec2offer = requests.get(
'https://pricing.us-east-1.amazonaws.com%s' % ec2_offer_path
).json()
uniq = set()
for sku, data in ec2offer['products'].items():
if data['productFamily'] != 'Compute Instance':
# skip anything that's not an EC2 Instance
continue
uniq.add(data['attributes']['instanceType'])
for itype in sorted(uniq):
print(itype)
请注意，这可能需要一段时间...截至今天，当前的 EC2 Offers JSON 文件 ( https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json ) 为 173MB，因此检索和解析都需要一段时间.当前结果是 99 个不同的实例类型。
现在有 boto3.client('ec2').describe_instance_types() 和相应的 aws-cli 命令 aws ec2 describe-instance-types:
'''EC2 describe_instance_types usage example'''
import boto3
def ec2_instance_types(region_name):
'''Yield all available EC2 instance types in region <region_name>'''
ec2 = boto3.client('ec2', region_name=region_name)
describe_args = {}
while True:
describe_result = ec2.describe_instance_types(**describe_args)
yield from [i['InstanceType'] for i in describe_result['InstanceTypes']]
if 'NextToken' not in describe_result:
break
describe_args['NextToken'] = describe_result['NextToken']
for ec2_type in ec2_instance_types('us-east-1'):
print(ec2_type)
预计 运行 时间约为 3 秒。
实例类型：（t2.micro、t2.small、c4.large...）此处列出的那些： http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html
我想通过 boto3 访问这些列表。 类似于：
conn.get_all_instance_types()
甚至
conn.describe_instance_types()['InstanceTypes'][0]['Name']
在这个奇怪的 api.
中，一切看起来都像这样我已经查看了客户端和 ServiceResource 的文档，但我找不到任何似乎接近的内容。 我什至还没有找到一个 hacky 解决方案来列出恰好代表所有实例类型的其他内容。
谁有更多boto3经验？
EC2 API 不提供获取所有 EC2 实例类型列表的方法。我希望如此。有些人通过像 this 这样的抓取网站拼凑了他们自己的有效类型列表，但目前这是唯一的方法。
试试这个
'''
Created on Mar 22, 2017
@author: ijessop
'''
import boto3
import urllib2
from bs4 import BeautifulSoup as soup
class EnumEc2():
def __init__(self, region):
self.client = boto3.client(
'ec2',
aws_access_key_id = 'YOUR_KEY' ,
aws_secret_access_key='YOUR_SECRET',
region_name = region
)
self.instance_types = None
self.instance_table_headers = None
self.max_col_width = {}
def getInstanceTypes(self):
mp = soup(urllib2.urlopen('https://aws.amazon.com/ec2/instance-types').read(),'html.parser')
imx = mp.find(id="instance-type-matrix")
trs = imx.parent.parent.parent.next_sibling.next_sibling.find_all('tr')
rt = []
first_row = True
for trow in trs:
td_strs = []
for td in trow.find_all("td"):
td_nested = []
for s in td.strings:
s.strip()
td_nested.append(s)
td_all = " ".join(td_nested).strip()
td_strs.append(td_all)
if first_row is True:
header_row = td_strs
for head in header_row:
self.max_col_width.update({head:(len(head) + 2)})
first_row = False
else:
dr = dict(zip(header_row,td_strs))
for k,v in dr.items():
cw = len(v)
if k in self.max_col_width.keys():
if cw >= self.max_col_width.get(k):
self.max_col_width.update({k:(cw +2)})
else:
self.max_col_width.update({k:cw})
rt.append(dr)
self.instance_table_headers = header_row
self.instance_types = rt
if __name__ == '__main__':
myen = EnumEc2('us-west-2')
myen.getInstanceTypes()
heads_I_want_to_see = ['Instance Type', u'vCPU', u'Memory (GiB)', u'Storage (GB)','Physical Processor', u'Clock Speed (GHz)']
out_str ="|"
for h in heads_I_want_to_see:
out_str = "%s%s|" % (out_str,h.ljust(myen.max_col_width.get(h)))
print "%s" % "-" * len(out_str)
print "%s" % out_str
print "%s" % "-" * len(out_str)
for i in myen.instance_types:
out_str ="|"
for k in myen.instance_table_headers: # to preserve the table column order
if k in heads_I_want_to_see:
out_str = "%s%s|" % (out_str, i.get(k).ljust(myen.max_col_width.get(k)))
print "%s" % out_str
print "%s" % "-" * len(out_str)
我也需要它，但是没有合适的代码用于此目的。我自己修改一个。享受！可能有人也需要。
以下代码修改自libcloud/contrib/scrape-ec2-prices.py 这个程序将生成一个关于可用实例类型的字典
#!/usr/bin/env python
import os
import re
import json
import time
from collections import defaultdict, OrderedDict
import requests
import demjson
LINUX_PRICING_URLS = [
# Deprecated instances (JSON format)
'https://aws.amazon.com/ec2/pricing/json/linux-od.json',
# Previous generation instances (JavaScript file)
'https://a0.awsstatic.com/pricing/1/ec2/previous-generation/linux-od.min.js',
# New generation instances (JavaScript file)
'https://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js'
]
EC2_REGIONS = [
'us-east-1',
'us-east-2',
'us-west-1',
'us-west-2',
'us-gov-west-1',
'eu-west-1',
'eu-west-2',
'eu-central-1',
'ca-central-1',
'ap-southeast-1',
'ap-southeast-2',
'ap-northeast-1',
'ap-northeast-2',
'ap-south-1',
'sa-east-1',
'cn-north-1',
]
INSTANCE_SIZES = [
'micro',
'small',
'medium',
'large',
'xlarge',
'x-large',
'extra-large'
]
RE_NUMERIC_OTHER = re.compile(r'(?:([0-9]+)|([-A-Z_a-z]+)|([^-0-9A-Z_a-z]+))')
PRICING_FILE_PATH = './price.json'
PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH)
def scrape_ec2_pricing():
result = {}
result['regions'] = []
result['prices'] = defaultdict(OrderedDict)
result['models'] = defaultdict(OrderedDict)
for url in LINUX_PRICING_URLS:
response = requests.get(url)
if re.match('.*?\.json$', url):
data = response.json()
elif re.match('.*?\.js$', url):
data = response.content
match = re.match('^.*callback\((.*?)\);?$', data,
re.MULTILINE | re.DOTALL)
data = match.group(1)
# demjson supports non-strict mode and can parse unquoted objects
data = demjson.decode(data)
regions = data['config']['regions']
for region_data in regions:
region_name = region_data['region']
if region_name not in result['regions']:
result['regions'].append(region_name)
libcloud_region_name = region_name
instance_types = region_data['instanceTypes']
for instance_type in instance_types:
sizes = instance_type['sizes']
for size in sizes:
price = size['valueColumns'][0]['prices']['USD']
if str(price).lower() == 'n/a':
# Price not available
continue
if not result['models'][libcloud_region_name].has_key(size['size']):
result['models'][libcloud_region_name][size['size']] = {}
result['models'][libcloud_region_name][size['size']]['CPU'] = int(size['vCPU'])
if size['ECU'] == 'variable':
ecu = 0
else:
ecu = float(size['ECU'])
result['models'][libcloud_region_name][size['size']]['ECU'] = ecu
result['models'][libcloud_region_name][size['size']]['memoryGiB'] = float(size['memoryGiB'])
result['models'][libcloud_region_name][size['size']]['storageGB'] = size['storageGB']
result['prices'][libcloud_region_name][size['size']] = float(price)
return result
def update_pricing_file(pricing_file_path, pricing_data):
## with open(pricing_file_path, 'r') as fp:
# content = fp.read()
data = {'compute': {}} # json.loads(content)
data['updated'] = int(time.time())
data['compute'].update(pricing_data)
# Always sort the pricing info
data = sort_nested_dict(data)
content = json.dumps(data, indent=4)
lines = content.splitlines()
lines = [line.rstrip() for line in lines]
content = '\n'.join(lines)
with open(pricing_file_path, 'w') as fp:
fp.write(content)
def sort_nested_dict(value):
"""
Recursively sort a nested dict.
"""
result = OrderedDict()
for key, value in sorted(value.items(), key=sort_key_by_numeric_other):
if isinstance(value, (dict, OrderedDict)):
result[key] = sort_nested_dict(value)
else:
result[key] = value
return result
def sort_key_by_numeric_other(key_value):
"""
Split key into numeric, alpha and other part and sort accordingly.
"""
return tuple((
int(numeric) if numeric else None,
INSTANCE_SIZES.index(alpha) if alpha in INSTANCE_SIZES else alpha,
other
) for (numeric, alpha, other) in RE_NUMERIC_OTHER.findall(key_value[0]))
def main():
print('Scraping EC2 pricing data')
pricing_data = scrape_ec2_pricing()
update_pricing_file(pricing_file_path=PRICING_FILE_PATH,
pricing_data=pricing_data)
print('Pricing data updated')
if __name__ == '__main__':
main()
可以在最近宣布的 AWS Price List API 提供的 JSON 中检索此信息。作为使用 Python requests 模块的简单示例：
#!/usr/bin/env python
# List EC2 Instance Types
# see: https://aws.amazon.com/blogs/aws/new-aws-price-list-api/
import requests
offers = requests.get(
'https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/index.json'
)
ec2_offer_path = offers.json()['offers']['AmazonEC2']['currentVersionUrl']
ec2offer = requests.get(
'https://pricing.us-east-1.amazonaws.com%s' % ec2_offer_path
).json()
uniq = set()
for sku, data in ec2offer['products'].items():
if data['productFamily'] != 'Compute Instance':
# skip anything that's not an EC2 Instance
continue
uniq.add(data['attributes']['instanceType'])
for itype in sorted(uniq):
print(itype)
请注意，这可能需要一段时间...截至今天，当前的 EC2 Offers JSON 文件 ( https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json ) 为 173MB，因此检索和解析都需要一段时间.当前结果是 99 个不同的实例类型。
现在有 boto3.client('ec2').describe_instance_types() 和相应的 aws-cli 命令 aws ec2 describe-instance-types:
'''EC2 describe_instance_types usage example'''
import boto3
def ec2_instance_types(region_name):
'''Yield all available EC2 instance types in region <region_name>'''
ec2 = boto3.client('ec2', region_name=region_name)
describe_args = {}
while True:
describe_result = ec2.describe_instance_types(**describe_args)
yield from [i['InstanceType'] for i in describe_result['InstanceTypes']]
if 'NextToken' not in describe_result:
break
describe_args['NextToken'] = describe_result['NextToken']
for ec2_type in ec2_instance_types('us-east-1'):
print(ec2_type)
预计 运行 时间约为 3 秒。