Skip to content

Commit

Permalink
elasticache automatic regions
Browse files Browse the repository at this point in the history
  • Loading branch information
@EverettBerry
EverettBerry committed Aug 17, 2023
1 parent d458b54 commit c01ed03
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 87 deletions.
31 changes: 22 additions & 9 deletions cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,16 +85,19 @@ def scrape(output_file, input_file=None):
# map the region
location = ec2.canonicalize_location(attributes["location"])
instance_type = attributes["instanceType"]
try:

if location == "Any":
region = "us-east-1"
elif location == "Asia Pacific (Osaka-Local)":
# at one point this region was local but was upgraded to a standard region
# however some SKUs still reference the old region
region = "ap-northeast-3"
regions[location] = region
elif location not in regions.values():
region = attributes["regionCode"]
regions[location] = region
else:
region = regions[location]
except KeyError as e:
if location == "Any":
region = "us-east-1"
else:
print(
f"WARNING: No region data for location={location}. Ignoring instance with sku={sku}, type={instance_type}"
)
continue

# Fix https://github.com/vantage-sh/ec2instances.info/issues/644 - Outpost pricing overwriting reserved
loctype = attributes["locationType"]
Expand Down Expand Up @@ -130,6 +133,7 @@ def scrape(output_file, input_file=None):
new_attributes.pop("region", None)
new_attributes.pop("usagetype", None)
new_attributes["pricing"] = attributes["pricing"]
new_attributes["regions"] = {}

instances[instance_type] = new_attributes

Expand Down Expand Up @@ -168,6 +172,15 @@ def scrape(output_file, input_file=None):
"ondemand": float(dimension["pricePerUnit"]["USD"])
}

# build the list of regions where each instance is available
# we have to do a reverse lookup from the regions list
l = ""
for l, r in regions.items():
if instance["region"] == r:
location = l
break
instances[instance["instance_type"]]["regions"][instance["region"]] = l

reserved_mapping = {
"1yr All Upfront": "yrTerm1.allUpfront",
"1yr Partial Upfront": "yrTerm1.partialUpfront",
Expand Down
33 changes: 16 additions & 17 deletions detail_pages_cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,23 +57,19 @@ def description(id, defaults):
)


def unavailable_instances(itype, instance_details):
data_file = "meta/regions_aws.yaml"

def unavailable_instances(instance_details, all_regions):
denylist = []
with open(data_file, "r") as f:
aws_regions = yaml.safe_load(f)
instance_regions = instance_details["Pricing"].keys()
instance_regions = instance_details["Pricing"].keys()

# If there is no price for a region and os, then it is unavailable
for r in aws_regions:
if r not in instance_regions:
denylist.append([aws_regions[r], r, "All", "*"])
else:
instance_regions_oss = instance_details["Pricing"][r].keys()
for os in cache_engine_mapping.values():
if os not in instance_regions_oss:
denylist.append([aws_regions[r], r, os, os])
# If there is no price for a region and os, then it is unavailable
for r in all_regions:
if r not in instance_regions:
denylist.append([all_regions[r], r, "All", "*"])
else:
instance_regions_oss = instance_details["Pricing"][r].keys()
for os in cache_engine_mapping.values():
if os not in instance_regions_oss:
denylist.append([all_regions[r], r, os, os])
return denylist


Expand Down Expand Up @@ -166,6 +162,7 @@ def load_service_attributes():
special_attrs = [
"pricing",
"cache_parameters",
"regions",
]
data_file = "meta/service_attributes_cache.csv"

Expand Down Expand Up @@ -241,6 +238,7 @@ def map_cache_attributes(i, imap):
# Nested attributes in instances.json that we handle differently
special_attributes = [
"pricing",
"regions",
]

instance_details = {}
Expand Down Expand Up @@ -271,7 +269,7 @@ def map_cache_attributes(i, imap):
return instance_details


def build_detail_pages_cache(instances, destination_file):
def build_detail_pages_cache(instances, all_regions):
subdir = os.path.join("www", "aws", "elasticache")

ifam, fam_lookup, variants = assemble_the_families(instances)
Expand All @@ -293,7 +291,7 @@ def build_detail_pages_cache(instances, destination_file):
instance_details["Pricing"] = prices(i["pricing"])
fam = fam_lookup[instance_type]
fam_members = ifam[fam]
denylist = unavailable_instances(instance_type, instance_details)
denylist = unavailable_instances(instance_details, all_regions)
defaults = initial_prices(instance_details, instance_type)
idescription = description(instance_details, defaults)

Expand All @@ -308,6 +306,7 @@ def build_detail_pages_cache(instances, destination_file):
unavailable=denylist,
defaults=defaults,
variants=variants[instance_type[6:8]],
regions=all_regions,
)
)
sitemap.append(instance_page)
Expand Down
40 changes: 11 additions & 29 deletions in/cache.html.mako
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,35 +25,17 @@
<span class="caret"></span>
</a>
<ul class="dropdown-menu" role="menu">
<li><a class="dropdown-item" href="javascript:;" data-region='af-south-1'>Africa (Cape Town)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-east-1'>Asia-Pacific (Hong Kong)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-south-1'>Asia-Pacific (Mumbai)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-south-2'>Asia-Pacific (Hyderabad)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-northeast-3'>Asia Pacific (Osaka-Local)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-northeast-2'>Asia-Pacific (Seoul)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-southeast-1'>Asia-Pacific (Singapore)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-southeast-2'>Asia-Pacific (Sydney)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-southeast-3'>Asia-Pacific (Jakarta)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-southeast-4'>Asia-Pacific (Melbourne)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ap-northeast-1'>Asia-Pacific (Tokyo)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='ca-central-1'>Canada (Central)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-central-1'>Europe (Frankfurt)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-central-2'>Europe (Zurich)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-west-1'>Europe (Ireland)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-west-2'>Europe (London)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-west-3'>Europe (Paris)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-north-1'>Europe (Stockholm)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-south-1'>Europe (Milan)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='eu-south-2'>Europe (Spain)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='me-south-1'>Middle East (Bahrain)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='me-central-1'>Middle East (UAE)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='sa-east-1'>South America (S&atilde;o Paulo)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='us-east-1'>US East (N. Virginia)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='us-east-2'>US East (Ohio)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='us-west-1'>US West (Northern California)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='us-west-2'>US West (Oregon)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='us-gov-west-1'>AWS GovCloud (US-West)</a></li>
<li><a class="dropdown-item" href="javascript:;" data-region='us-gov-east-1'>AWS GovCloud (US-East)</a></li>
<li>
<input type="text" id="dropdown-search" class="ms-2 mb-2 form-control dropdown-search" placeholder="Search" />
</li>
% for region, region_name in regions["main"].items():
<li>
<a class="dropdown-item" href="javascript:;" data-region='${region}'>
<span>${region_name}</span>
<span class="dropdown-region">${region}</span>
</a>
</li>
% endfor
</ul>
</div>

Expand Down
34 changes: 6 additions & 28 deletions in/instance-type-cache.html.mako
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,34 +87,12 @@
<select class="form-select form-select-sm" id="region">
<!-- TODO: Localize default option order -->
<option value='us-east-1'>US East (N. Virginia)</option>
<option value='af-south-1'>Africa (Cape Town)</option>
<option value='ap-east-1'>Asia-Pacific (Hong Kong)</option>
<option value='ap-south-1'>Asia-Pacific (Mumbai)</option>
<option value='ap-south-2'>Asia-Pacific (Hyderabad)</option>
<option value='ap-northeast-3'>Asia-Pacific (Osaka)</option>
<option value='ap-northeast-2'>Asia-Pacific (Seoul)</option>
<option value='ap-southeast-1'>Asia-Pacific (Singapore)</option>
<option value='ap-southeast-2'>Asia-Pacific (Sydney)</option>
<option value='ap-southeast-3'>Asia-Pacific (Jakarta)</option>
<option value='ap-southeast-4'>Asia-Pacific (Melbourne)</option>
<option value='ap-northeast-1'>Asia-Pacific (Tokyo)</option>
<option value='ca-central-1'>Canada (Central)</option>
<option value='eu-central-1'>Europe (Frankfurt)</option>
<option value='eu-central-2'>Europe (Zurich)</option>
<option value='eu-west-1'>Europe (Ireland)</option>
<option value='eu-west-2'>Europe (London)</option>
<option value='eu-west-3'>Europe (Paris)</option>
<option value='eu-north-1'>Europe (Stockholm)</option>
<option value='eu-south-1'>Europe (Milan)</option>
<option value='eu-south-2'>Europe (Spain)</option>
<option value='me-south-1'>Middle East (Bahrain)</option>
<option value='me-central-1'>Middle East (UAE)</option>
<option value='sa-east-1'>South America (S&atilde;o Paulo)</option>
<option value='us-east-2'>US East (Ohio)</option>
<option value='us-west-1'>US West (California)</option>
<option value='us-west-2'>US West (Oregon)</option>
<option value='us-gov-west-1'>AWS GovCloud (US-West)</option>
<option value='us-gov-east-1'>AWS GovCloud (US-East)</option>
% for api_name, region in regions.items():
% if api_name == 'us-east-1':
<% continue %>
% endif
<option value='${api_name}'>${region}</option>
% endfor
</select>
</div>
<div class="col-6 mb-2">
Expand Down
11 changes: 7 additions & 4 deletions render.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,12 +250,15 @@ def render(data_file, template_file, destination_file, detail_pages=True):
if detail_pages:
sitemap.extend(build_detail_pages_rds(instances, all_regions))
elif data_file == "www/cache/instances.json":
all_regions = regions["main"].copy()
if detail_pages:
sitemap.extend(build_detail_pages_cache(instances, destination_file))
sitemap.extend(build_detail_pages_cache(instances, all_regions))
elif data_file == "www/opensearch/instances.json":
all_regions = regions["main"].copy()
if detail_pages:
sitemap.extend(build_detail_pages_opensearch(instances, destination_file))
elif data_file == "www/redshift/instances.json":
all_regions = regions["main"].copy()
if detail_pages:
sitemap.extend(build_detail_pages_redshift(instances, destination_file))

Expand Down Expand Up @@ -290,16 +293,16 @@ def render(data_file, template_file, destination_file, detail_pages=True):
sitemap.extend(
render("www/rds/instances.json", "in/rds.html.mako", "www/rds/index.html")
)
import sys

sys.exit(0)
sitemap.extend(
render(
"www/cache/instances.json",
"in/cache.html.mako",
"www/cache/index.html",
)
)
import sys

sys.exit(0)
sitemap.extend(
render(
"www/redshift/instances.json",
Expand Down

0 comments on commit c01ed03

Please sign in to comment.