Merge pull request #1215 from GlobalNOC/2.0.11-dev

2.0.11

.gitignore

@@ -11,6 +11,7 @@ perl-lib/OESS/Makefile
 perl-lib/OESS/MANIFEST.bak
 perl-lib/OESS/t/conf/SNMP/*
 frontend/www/.htpasswd
+frontend/.htpasswd
 nox/**/Makefile
 nox/**/.deps/
 nox/**/*.la
@@ -48,3 +49,4 @@ perl-lib/OESS/tap/
 perl-lib/OESS/lib/OESS/pod2htmd.tmp
 perl-lib/OESS/lib/OESS/pod2htmi.tmp
 perl-lib/OESS/lib/OESS/t/conf/database.xml
+frontend/.htpasswd

app/Makefile

@@ -1,5 +1,5 @@
 NAME= oess-core
-VERSION = 2.0.10
+VERSION = 2.0.11
 
 rpm:	dist
 	rpmbuild -ta dist/$(NAME)-$(VERSION).tar.gz

app/oess-core.spec

@@ -1,6 +1,6 @@
 Name:		oess-core
-Version:	2.0.10
-Release:	2%{?dist}
+Version:	2.0.11
+Release:	1%{?dist}
 Summary:	The core OESS service providers
 
 Group:		Network
@@ -20,7 +20,7 @@ Requires: /bin/bash
 Requires: /usr/bin/perl
 Requires: perl(base), perl(constant), perl(strict), perl(warnings)
 
-Requires: perl-OESS >= 2.0.10
+Requires: perl-OESS >= 2.0.11
 
 Requires: perl(AnyEvent), perl(AnyEvent::DBus), perl(AnyEvent::RabbitMQ)
 Requires: perl(CPAN), perl(CPAN::Shell)

app/oess_setup.pl

@@ -145,7 +145,7 @@ sub main{
     my $tsds_password = required_parameter("TSDS Password: ");
     ReadMode('normal');
     my $grafana_url = optional_parameter("Grafana URL", "https://localhost/grafana");
-
+    my $third_party_mgmt = yes_or_no_parameter("Are you using third party User/Workgroup management? ");
     #put all of this into a config file
     print "Creating Configuration file (/etc/oess/database.xml)\n";
     open(FILE, "> /etc/oess/database.xml");
@@ -170,6 +170,7 @@ END
     print FILE "  <process name='fvd' status='disabled' />\n";
     print FILE "  <process name='watchdog' status='disabled' />\n";
     print FILE "  <rabbitMQ user='guest' pass='guest' host='localhost' port='5672' vhost='/' />\n";
+    print FILE "  <third_party_mgmt>$third_party_management</third_party_management>";
     print FILE "</config>\n";
     close(FILE);
 

docs/README.md

@@ -18,3 +18,9 @@ and is used to generate the navigation for each section.
 The links on each section's main page is **not** auto-generated and
 must be maintained manually. This allows for entering generic info
 about each section.
+
+## Run Documentation Server Locally
+
+```bash
+bundle exec jekyll serve --host 0.0.0.0 --port 1234
+```

docs/_data/api/admin-admin/add_user.json

@@ -1 +1 @@
-{"output_type":"application/json","input_params":{"email_address":{"allow_null":0,"name":"email_address","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter email_address only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null},"status":{"allow_null":1,"name":"status","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter status only accepts printable characters and spaces, including newlines. This excludes control characters.","required":0,"multiple":0,"max_length":null},"type":{"allow_null":1,"name":"type","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter type only accepts printable characters and spaces, including newlines. This excludes control characters.","required":0,"multiple":0,"max_length":null},"family_name":{"allow_null":0,"name":"family_name","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter family_name only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null},"first_name":{"allow_null":0,"name":"first_name","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter first_name only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null},"auth_name":{"allow_null":0,"name":"auth_name","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter auth_name only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null}},"name":"add_user","description":"","default_input_validators":[],"expires":"-1d"}
+{"output_type":"application/json","input_params":{"email_address":{"allow_null":0,"name":"email_address","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter email_address only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null},"status":{"allow_null":1,"name":"status","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter status only accepts printable characters and spaces, including newlines. This excludes control characters.","required":0,"multiple":0,"max_length":null},"family_name":{"allow_null":0,"name":"family_name","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter family_name only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null},"first_name":{"allow_null":0,"name":"first_name","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter first_name only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null},"auth_name":{"allow_null":0,"name":"auth_name","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter auth_name only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null}},"name":"add_user","description":"","default_input_validators":[],"expires":"-1d"}

docs/_data/api/admin-admin/add_user_to_workgroup.json

@@ -1 +1 @@
-{"output_type":"application/json","input_params":{"workgroup_id":{"allow_null":1,"name":"workgroup_id","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter workgroup_id only accepts integer numbers.","required":0,"multiple":0,"max_length":null},"user_id":{"allow_null":0,"name":"user_id","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter user_id only accepts integer numbers.","required":1,"multiple":0,"max_length":null}},"name":"add_user_to_workgroup","description":"","default_input_validators":[],"expires":"-1d"}
+{"output_type":"application/json","input_params":{"workgroup_id":{"allow_null":1,"name":"workgroup_id","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter workgroup_id only accepts integer numbers.","required":0,"multiple":0,"max_length":null},"user_id":{"allow_null":0,"name":"user_id","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter user_id only accepts integer numbers.","required":1,"multiple":0,"max_length":null},"role":{"allow_null":0,"name":"role","ignore_default_input_validators":0,"min_length":null,"description":"","pattern":"^([[:print:][:space:]]+)$","input_validators":[],"validation_error_text":"Parameter role only accepts printable characters and spaces, including newlines. This excludes control characters.","required":1,"multiple":0,"max_length":null}},"name":"add_user_to_workgroup","description":"","default_input_validators":[],"expires":"-1d"}

docs/_data/api/monitoring/get_oess_status.json

@@ -0,0 +1 @@
+{"output_type":"application/json","input_params":{},"name":"get_oess_status","description":"Returns a hash of three main services and their status as up or down","default_input_validators":[],"expires":"-1d"}

docs/_data/user-documentation/navigation.yml

@@ -7,5 +7,5 @@ pages:
     url: /user-documentation/workgroups.html
   - title: Network Entities
     url: /user-documentation/network-entities.html
-  - title: Cloud Providers
-    url: /user-documentation/cloud-providers.html
+  - title: Cloud Connections
+    url: /user-documentation/cloud-connections.html

docs/_layouts/cgi.html

@@ -19,7 +19,7 @@
           <div class="nav-wrapper-content">
 
             {%- if page.title -%}
-              <h1 class="post-title">{{ page.title | escape }}</h1>
+              <h1 class="post-title cgi-module">{{ page.title | escape }}</h1>
             {%- endif -%}
             <hr/>
             <a href="{{ "/" | relative_url }}">OESS</a> | <a href="{{ "/the-basics" | relative_url }}">Introduction</a> | <a href="{{ "/the-basics/oess-api.html" | relative_url }}">OESS API</a> | <a href="{{ page.url | relative_url }}">{{ page.title }}</a>
@@ -39,12 +39,19 @@ <h1 class="post-title">{{ page.title | escape }}</h1>
                   <hr/>
                   <br/>
 
-                  <h2>{{ method.name }}</h2>
+                  <h2 class="cgi-method">{{ method.name }}</h2>
                   <p>{{ method.description }}</p>
 
-                  {% for input_param in method.input_params %}
-                  {% assign param = input_param[1] %}
-                  <h4>{{ param.name }} {% if param.required == 1 %}<i style="font-size: .75em">required</i>{% endif %}</h4>
+                  {% capture param_names %}
+                  {% for input_param in method.input_params %},{{ input_param[0] }},{% endfor %}
+                  {% endcapture %}
+
+                  {% assign sorted_p = param_names | split:"," | sort %}
+
+                  {% for name in sorted_p %}
+                  {% assign param = method.input_params[name] %}
+
+                  <h4 class="cgi-param">{{ param.name }} {% if param.required == 1 %}<span class="cgi-required">Required</span>{% endif %}</h4>
                   <p>{{ param.description }}</p>
                   {% endfor %}
                   {% endfor %}

docs/_sass/minima/_custom.scss

@@ -18,3 +18,27 @@
     list-style-type: none;
     margin: 0;
 }
+
+/**
+ * CGI Documentation styles
+ */
+
+.cgi-module {
+    font-weight: bold;
+}
+
+.cgi-method {
+    font-weight: bold;
+}
+
+h4.cgi-param {
+    font-family: "Lucida Console", Monaco, monospace;
+    font-size: 1.1em;
+    font-weight: bold;
+    margin-bottom: 8px;
+}
+
+.cgi-required {
+    font-size: .75em;
+    color: #777;
+}

docs/generate-webservice-docs.pl

@@ -29,7 +29,15 @@
     if ($help !~ m/(\{.*\})|(\[.*\])/gm) {
         next;
     }
-    my $methods = decode_json($2);
+
+    my $methods;
+    eval {
+        $methods = decode_json($2);
+    };
+    if ($@) {
+        warn "Error while decoding $module->{name}: $@";
+        next;
+    }
 
     my $title = $module->{source};
     $title =~ s/\.\.\/frontend\/webservice//g;
@@ -49,7 +57,7 @@
     }
 
     # Make directory in case it doesn't already exist.
-    `mkdir _data/api/$module->{name}/`;
+    `mkdir -p _data/api/$module->{name}/`;
 
     print "Loading method data for $module->{source}\n";
     foreach my $method (@$methods) {

docs/operations-manual/software-configuration.md

@@ -3,3 +3,108 @@ layout: operations-manual
 title: Software Configuration
 name: software-configuration
 ---
+
+## Third Party Access Control
+
+OESS currently supports two access control backends.
+
+- Built-in user and workgroup database
+- Grouper access management system
+
+### Configuration
+
+To configure an Access Control system other than the Built-In
+database, add the following configuration to
+`/etc/oess/database.xml`. By default `third_party_mgmt` will be set to
+`n`.
+
+#### **Example**
+
+```xml
+<third_party_mgmt>y</third_party_management>
+```
+
+### Grouper Requirements
+
+Grouper requires a little setup before it may be used with
+OESS. Within the OESS stem create the three following Attribute Names
+and Attribute Definitions.
+
+**Attribute Name**
+- workgroup-external-id
+- workgroup-id
+- workgroup-type
+
+**Attribute Definition**
+- workgroup-external-id: Single value string
+- workgroup-id: Single value string
+- workgroup-type: Single value string
+
+**Grouper Layout**
+
+The OESS Grouper layout is composed of a stem for each OESS Workgroup
+and three Groups within each Workgroup stem. A `users` group within
+the OESS stem is created to identify all users who may access OESS.
+
+```
+oess/
+  admin/
+    admin
+    normal
+    read-only
+  alpha/
+    admin
+    normal
+    read-only
+  users
+```
+
+## Maximum Allowed Bandwidth for Cloud Provider Connections
+
+Each connection to a Cloud Provider will have some bandwidth
+restrictions. On the cloud provider side, these restrictions are based
+on the speeds defined by the Cloud Provider's web API. On the OESS
+side these will likely be set based on some set of business
+requirements.
+
+Consider the case where OESS is connected to a Cloud Provider via a
+10Gb interconnect. While the Cloud Provider might allow a single 10Gb
+connection to the interconnect, because no other connection could be
+provisioned without oversubscription, OESS administators may wish to
+prevent this.
+
+### Configuration
+
+To configure which speeds may be used on an Endpoint terminating on a
+Cloud Provider's interconnect edit
+`/etc/oess/interface-speed-config.xml`.
+
+This configuration file contains a list of `interface-selector`s. Each
+`interface-selector` is used to classify an Endpoint's Interface.
+
+If the Interface's configured speed is within `min_bandwidth` and
+`max_bandwidth`, and is of the same configured
+`cloud_interconnect_type`, the Max Bandwidth allowed for an Endpoint
+will be restricted to the `speeds` within the `interface-selector`.
+
+#### **Example**
+
+In this example, Azure ExpressRoute Interfaces have been configured to
+allow different Endpoint bandwidths based on the underlying Physical
+Interface's speed.
+
+- Interfaces between `100Mb` and `1Gb` may only be used to provision `50Mb` Endpoints.
+- Interfaces between `10Gb` adn `100Gb` may be used to provision `50Mb`, `500Mb`, and `1Gb` Endpoints.
+
+```xml
+<config>
+  <interface-selector min_bandwidth="100" max_bandwidth="1000" cloud_interconnect_type="azure-express-route">
+    <speed rate="50" />
+  </interface-selector>
+  <interface-selector min_bandwidth="10000" max_bandwidth="100000" cloud_interconnect_type="azure-express-route">
+    <speed rate="50" />
+    <speed rate="500" />
+    <speed rate="1000" />
+  </interface-selector>
+</config>
+```

docs/user-documentation.md

@@ -16,4 +16,4 @@ permalink: /user-documentation/
 
 ## <a href="{{ "/user-documentation/network-entities.html" | relative_url }}">Creating and Managing Network Entities</a>
 
-## <a href="{{ "/user-documentation/cloud-providers.html" | relative_url }}">Cloud Providers</a>
+## <a href="{{ "/user-documentation/cloud-connections.html" | relative_url }}">Cloud Connections</a>

docs/user-documentation/cloud-connections.md

@@ -0,0 +1,61 @@
+---
+layout: user-documentation
+title: Cloud Connections
+name: cloud-connections
+---
+
+OESS supports provisiong connections to AWS, Microsoft Azure, and
+Google Cloud Platform. As provisioning for each cloud is slightly
+different, use the videos in this section to setup your cloud enabled
+endpoints.
+
+## Amazon Web Services: Hosted Connections
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/OhE-Rclp6Pg" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
+### Workflow Overview
+
+1. Create a new Connection in OESS.
+0. Create an Endpoint to AWS using your AWS Customer Id.
+0. Create an Endpoint to your network.
+0. Create a peer for each created Endpoint.
+0. Save the Connection.
+0. Under the Connections pane in the AWS Direct Connect console, select the newly created Hosted Connection and choose View details. Select the confirmation check box and choose Accept connection.
+0. Create a Virtual Interface to use on the Hosted Connection, and configure the Hosted Connection's Endpoint in OESS to enabling peering between AWS and OESS.
+
+---
+
+## Google Cloud Platform: Partner Interconnects
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/iMYEIIGQwAw" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
+### Workflow Overview
+
+1. Create a Partner Interconnect via the GCP web portal. Note the generated Pairing Keys; These are used to create Endpoints within OESS.
+0. Create a new Connection in OESS.
+0. Create an Endpoint to GCP using each Pairing Key mentioned in Step 1.
+0. Create an Endpoint to your network.
+0. Create a peer for each created Endpoint.
+0. Save the Connection.
+0. Return to the GCP web portal and approve the Interconnect.
+
+---
+
+## Microsoft Azure: ExpressRoutes
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/mH2CTFw3qdQ" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
+### Workflow Overview
+
+1. Create an ExpressRoute via the Azure web portal. Note the generated Service Key; This is ued to create Endpoints within OESS.
+0. Create a new Connection in OESS.
+0. Create two Endpoints to Azure using the Service Key mentioned in Step 1.
+0. Create an Endpoint to your network.
+0. Create a peer for each created Endpoint.
+0. Save the Connection.
+
+**Note:** If manually configuring peer addresses, Azure expects a
+`/30` for both the primary and secondary Endpoints. The first address
+will be used by the peer and the second will be used by Azure. For
+example, if `192.168.100.248/30` is used, `192.168.100.249/30` will be
+used by the peer and `192.168.100.250/30` will be used by Azure.