diff --git a/detail_pages_opensearch.py b/detail_pages_opensearch.py index 0e30a8d..a13d9d5 100644 --- a/detail_pages_opensearch.py +++ b/detail_pages_opensearch.py @@ -40,18 +40,14 @@ def description(id, defaults): ) -def unavailable_instances(itype, instance_details): - data_file = "meta/regions_aws.yaml" - +def unavailable_instances(instance_details, all_regions): denylist = [] - with open(data_file, "r") as f: - aws_regions = yaml.safe_load(f) - instance_regions = instance_details["Pricing"].keys() + instance_regions = instance_details["Pricing"].keys() - # If there is no price for a region and os, then it is unavailable - for r in aws_regions: - if r not in instance_regions: - denylist.append([aws_regions[r], r, "All", "*"]) + # If there is no price for a region and os, then it is unavailable + for r in all_regions: + if r not in instance_regions: + denylist.append([all_regions[r], r, "All", "*"]) return denylist @@ -140,6 +136,7 @@ def prices(pricing): def load_service_attributes(): special_attrs = [ "pricing", + "regions", ] data_file = "meta/service_attributes_opensearch.csv" @@ -215,6 +212,7 @@ def map_cache_attributes(i, imap): # Nested attributes in instances.json that we handle differently special_attributes = [ "pricing", + "regions", ] instance_details = {} @@ -245,7 +243,7 @@ def map_cache_attributes(i, imap): return instance_details -def build_detail_pages_opensearch(instances, destination_file): +def build_detail_pages_opensearch(instances, all_regions): subdir = os.path.join("www", "aws", "opensearch") ifam, fam_lookup, variants = assemble_the_families(instances) @@ -267,7 +265,7 @@ def build_detail_pages_opensearch(instances, destination_file): instance_details["Pricing"] = prices(i["pricing"]) fam = fam_lookup[instance_type] fam_members = ifam[fam] - denylist = unavailable_instances(instance_type, instance_details) + denylist = unavailable_instances(instance_details, all_regions) defaults = initial_prices(instance_details, instance_type) idescription = description(instance_details, defaults) @@ -282,6 +280,7 @@ def build_detail_pages_opensearch(instances, destination_file): unavailable=denylist, defaults=defaults, variants=variants[instance_type[0:2]], + regions=all_regions, ) ) sitemap.append(instance_page) diff --git a/detail_pages_redshift.py b/detail_pages_redshift.py index 2c2d55e..e8e6123 100644 --- a/detail_pages_redshift.py +++ b/detail_pages_redshift.py @@ -40,18 +40,14 @@ def description(id, defaults): ) -def unavailable_instances(itype, instance_details): - data_file = "meta/regions_aws.yaml" - +def unavailable_instances(instance_details, all_regions): denylist = [] - with open(data_file, "r") as f: - aws_regions = yaml.safe_load(f) - instance_regions = instance_details["Pricing"].keys() + instance_regions = instance_details["Pricing"].keys() - # If there is no price for a region and os, then it is unavailable - for r in aws_regions: - if r not in instance_regions: - denylist.append([aws_regions[r], r, "All", "*"]) + # If there is no price for a region and os, then it is unavailable + for r in all_regions: + if r not in instance_regions: + denylist.append([all_regions[r], r, "All", "*"]) return denylist @@ -140,6 +136,7 @@ def prices(pricing): def load_service_attributes(): special_attrs = [ "pricing", + "regions", ] data_file = "meta/service_attributes_redshift.csv" @@ -215,6 +212,7 @@ def map_cache_attributes(i, imap): # Nested attributes in instances.json that we handle differently special_attributes = [ "pricing", + "regions", ] instance_details = {} @@ -245,7 +243,7 @@ def map_cache_attributes(i, imap): return instance_details -def build_detail_pages_redshift(instances, destination_file): +def build_detail_pages_redshift(instances, all_regions): subdir = os.path.join("www", "aws", "redshift") ifam, fam_lookup, variants = assemble_the_families(instances) @@ -267,7 +265,7 @@ def build_detail_pages_redshift(instances, destination_file): instance_details["Pricing"] = prices(i["pricing"]) fam = fam_lookup[instance_type] fam_members = ifam[fam] - denylist = unavailable_instances(instance_type, instance_details) + denylist = unavailable_instances(instance_details, all_regions) defaults = initial_prices(instance_details, instance_type) idescription = description(instance_details, defaults) @@ -282,6 +280,7 @@ def build_detail_pages_redshift(instances, destination_file): unavailable=denylist, defaults=defaults, variants=variants[instance_type[0:2]], + regions=all_regions, ) ) sitemap.append(instance_page) diff --git a/in/instance-type-opensearch.html.mako b/in/instance-type-opensearch.html.mako index 0f699fc..5eb51ab 100644 --- a/in/instance-type-opensearch.html.mako +++ b/in/instance-type-opensearch.html.mako @@ -85,36 +85,13 @@
diff --git a/in/instance-type-redshift.html.mako b/in/instance-type-redshift.html.mako index 66f8ab8..9cc777c 100644 --- a/in/instance-type-redshift.html.mako +++ b/in/instance-type-redshift.html.mako @@ -85,36 +85,13 @@
diff --git a/in/opensearch.html.mako b/in/opensearch.html.mako index 329fa4f..1cf8201 100644 --- a/in/opensearch.html.mako +++ b/in/opensearch.html.mako @@ -25,35 +25,17 @@
diff --git a/in/redshift.html.mako b/in/redshift.html.mako index 88e29d5..163216c 100644 --- a/in/redshift.html.mako +++ b/in/redshift.html.mako @@ -25,35 +25,17 @@
diff --git a/opensearch.py b/opensearch.py index 1708c8e..9cedc62 100755 --- a/opensearch.py +++ b/opensearch.py @@ -117,16 +117,18 @@ def scrape(output_file, input_file=None): # map the region location = ec2.canonicalize_location(attributes["location"]) instance_type = attributes["instanceType"] - try: + if location == "Any": + region = "us-east-1" + elif location == "Asia Pacific (Osaka-Local)": + # at one point this region was local but was upgraded to a standard region + # however some SKUs still reference the old region + region = "ap-northeast-3" + regions[location] = region + elif location not in regions.values(): + region = attributes["regionCode"] + regions[location] = region + else: region = regions[location] - except KeyError as e: - if location == "Any": - region = "us-east-1" - else: - print( - f"WARNING: No region data for location={location}. Ignoring instance with sku={sku}, type={instance_type}" - ) - continue # set the attributes in line with the ec2 index attributes["region"] = region @@ -149,6 +151,7 @@ def scrape(output_file, input_file=None): new_attributes.pop("region", None) new_attributes.pop("usagetype", None) new_attributes["pricing"] = attributes["pricing"] + new_attributes["regions"] = {} instances[instance_type] = new_attributes @@ -185,6 +188,15 @@ def scrape(output_file, input_file=None): "ondemand": float(dimension["pricePerUnit"]["USD"]) } + # build the list of regions where each instance is available + # we have to do a reverse lookup from the regions list + l = "" + for l, r in regions.items(): + if instance["region"] == r: + location = l + break + instances[instance["instance_type"]]["regions"][instance["region"]] = l + reserved_mapping = { "1yr All Upfront": "yrTerm1.allUpfront", "1yr Partial Upfront": "yrTerm1.partialUpfront", diff --git a/rds.py b/rds.py index 290f504..720240e 100644 --- a/rds.py +++ b/rds.py @@ -214,17 +214,6 @@ def scrape(output_file, input_file=None): else: region = regions[location] - # try: - # region = regions[location] - # except KeyError as e: - # if location == "Any": - # region = "us-east-1" - # else: - # print( - # f"WARNING: No region data for location={location}. Ignoring instance with sku={sku}, type={instance_type}" - # ) - # continue - # set the attributes in line with the ec2 index attributes["region"] = region attributes["memory"] = attributes["memory"].split(" ")[0] diff --git a/redshift.py b/redshift.py index 006602c..283f8fe 100755 --- a/redshift.py +++ b/redshift.py @@ -105,16 +105,19 @@ def scrape(output_file, input_file=None): # map the region location = ec2.canonicalize_location(attributes["location"]) instance_type = attributes["instanceType"] - try: + + if location == "Any": + region = "us-east-1" + elif location == "Asia Pacific (Osaka-Local)": + # at one point this region was local but was upgraded to a standard region + # however some SKUs still reference the old region + region = "ap-northeast-3" + regions[location] = region + elif location not in regions.values(): + region = attributes["regionCode"] + regions[location] = region + else: region = regions[location] - except KeyError as e: - if location == "Any": - region = "us-east-1" - else: - print( - f"WARNING: No region data for location={location}. Ignoring instance with sku={sku}, type={instance_type}" - ) - continue # set the attributes in line with the ec2 index attributes["region"] = region @@ -137,6 +140,7 @@ def scrape(output_file, input_file=None): new_attributes.pop("region", None) new_attributes.pop("usagetype", None) new_attributes["pricing"] = attributes["pricing"] + new_attributes["regions"] = {} instances[instance_type] = new_attributes @@ -173,6 +177,15 @@ def scrape(output_file, input_file=None): "ondemand": float(dimension["pricePerUnit"]["USD"]) } + # build the list of regions where each instance is available + # we have to do a reverse lookup from the regions list + l = "" + for l, r in regions.items(): + if instance["region"] == r: + location = l + break + instances[instance["instance_type"]]["regions"][instance["region"]] = l + reserved_mapping = { "1yr All Upfront": "yrTerm1.allUpfront", "1yr Partial Upfront": "yrTerm1.partialUpfront", diff --git a/render.py b/render.py index 65637fc..4c3ffce 100644 --- a/render.py +++ b/render.py @@ -256,11 +256,11 @@ def render(data_file, template_file, destination_file, detail_pages=True): elif data_file == "www/opensearch/instances.json": all_regions = regions["main"].copy() if detail_pages: - sitemap.extend(build_detail_pages_opensearch(instances, destination_file)) + sitemap.extend(build_detail_pages_opensearch(instances, all_regions)) elif data_file == "www/redshift/instances.json": all_regions = regions["main"].copy() if detail_pages: - sitemap.extend(build_detail_pages_redshift(instances, destination_file)) + sitemap.extend(build_detail_pages_redshift(instances, all_regions)) generated_at = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") pricing_json, instance_azs_json = per_region_pricing( @@ -300,9 +300,6 @@ def render(data_file, template_file, destination_file, detail_pages=True): "www/cache/index.html", ) ) - import sys - - sys.exit(0) sitemap.extend( render( "www/redshift/instances.json",