From 7e4399eac23422d360cc2556d2e015b9ad3dcf5e Mon Sep 17 00:00:00 2001 From: Zach LeBlanc Date: Mon, 11 Sep 2023 15:00:17 -0500 Subject: [PATCH 1/2] Patch EC2 Workflow (#75) Co-authored-by: zjleblanc Co-authored-by: willtome --- cloud/README.md | 11 +- cloud/restore_ec2.yml | 10 ++ cloud/setup.yml | 138 ++++++++++++++++++ cloud/snapshot_ec2.yml | 10 ++ .../demo/cloud/roles/aws/defaults/main.yml | 1 + .../demo/cloud/roles/aws/tasks/restore_vm.yml | 62 ++++++++ .../cloud/roles/aws/tasks/snapshot_vm.yml | 43 ++++++ .../demo/cloud/roles/aws/vars/snapshot_vm.yml | 10 ++ linux/patching.yml | 13 ++ 9 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 cloud/restore_ec2.yml create mode 100644 cloud/snapshot_ec2.yml create mode 100644 collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml create mode 100644 collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml create mode 100644 collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml diff --git a/cloud/README.md b/cloud/README.md index 15cb1078c..7ef5e410d 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -10,7 +10,7 @@ - [Configure Credentials](#configure-credentials) - [Add Workshop Credential Password](#add-workshop-credential-password) - [Remove Inventory Variables](#remove-inventory-variables) - - [Getting your Puiblic Key for Create Infra Job](#getting-your-puiblic-key-for-create-infra-job) + - [Getting your Puiblic Key for Create Keypair Job](#getting-your-puiblic-key-for-create-keypair-job) - [Suggested Usage](#suggested-usage) - [Known Issues](#known-issues) @@ -20,8 +20,11 @@ This category of demos shows examples of multi-cloud provisioning and management ### Jobs - [**Cloud / Create Infra**](create_infra.yml) - Creates a VPC with required routing and firewall rules for provisioning VMs +- [**Cloud / Create Keypair**](aws_key.yml) - Creates a keypair for connecting to EC2 instances - [**Cloud / Create VM**](create_vm.yml) - Create a VM based on a [blueprint](blueprints/) in the selected cloud provider - [**Cloud / Destroy VM**](destroy_vm.yml) - Destroy a VM that has been created in a cloud provider. VM must be imported into dynamic inventory to be deleted. +- [**Cloud / Snapshot EC2**](snapshot_ec2.yml) - Snapshot a VM that has been created in a cloud provider. VM must be imported into dynamic inventory to be snapshot. +- [**Cloud / Restore EC2 from Snapshot**](snapshot_ec2.yml) - Restore a VM that has been created in a cloud provider. By default, volumes will be restored from their latest snapshot. VM must be imported into dynamic inventory to be patched. ### Inventory @@ -46,7 +49,7 @@ After running the setup job template, there are a few steps required to make the 1) Remove Workshop Inventory variables on the Details page of the inventory. Required until [RFE](https://github.com/ansible/workshops/issues/1597]) is complete -### Getting your Puiblic Key for Create Infra Job +### Getting your Puiblic Key for Create Keypair Job 1) Connect to the command line of your Controller server. This is easiest to do by opening the VS Code Web Editor from the landing page where you found the Controller login details. 2) Open a Terminal Window in the VS Code Web Editor. @@ -56,9 +59,11 @@ After running the setup job template, there are a few steps required to make the ## Suggested Usage -**Cloud / Create Infra** -The Create Infra job builds cloud infrastructure based on the provider definition in the included `demo.cloud` collection. +**Cloud / Create Keypair** - The Create Keypair job creates an EC2 keypair which can be used when creating EC2 instances to enable SSH access. **Cloud / Create VM** - The Create VM job builds a VM in the given provider based on the included `demo.cloud` collection. VM [blueprints](blueprints/) define variables for each provider that override the defaults in the collection. When creating VMs it is recommended to follow naming conventions that can be used as host patterns. (eg. VM names: `win1`, `win2`, `win3`. Host Pattern: `win*` ) +**Cloud / AWS / Patch EC2 Workflow** - Create a VPC and one or more linux VM(s) in AWS using the `Cloud / Create VPC` and `Cloud / Create VM` templates. Run the workflow and observe the instance snapshots followed by patching operation. Optionally, use the survey to force a patch failure in order to demonstrate the restore path. At this time, the workflow does not support patching Windows instances. + ## Known Issues Azure does not work without a custom execution environment that includes the Azure dependencies. \ No newline at end of file diff --git a/cloud/restore_ec2.yml b/cloud/restore_ec2.yml new file mode 100644 index 000000000..93890ec01 --- /dev/null +++ b/cloud/restore_ec2.yml @@ -0,0 +1,10 @@ +--- +- name: Restore ec2 instance from snapshot + hosts: "{{ _hosts | default(omit) }}" + gather_facts: false + + tasks: + - name: Include restore from snapshot role + ansible.builtin.include_role: + name: "demo.cloud.aws" + tasks_from: restore_vm diff --git a/cloud/setup.yml b/cloud/setup.yml index 64ece260b..3161f878d 100644 --- a/cloud/setup.yml +++ b/cloud/setup.yml @@ -367,6 +367,91 @@ controller_templates: variable: aws_keypair_owner required: true + - name: Cloud / AWS / Snapshot EC2 + job_type: run + organization: Default + credentials: + - AWS + project: Ansible official demo project + playbook: cloud/snapshot_ec2.yml + inventory: Demo Inventory + notification_templates_started: Telemetry + notification_templates_success: Telemetry + notification_templates_error: Telemetry + survey_enabled: true + survey: + name: '' + description: '' + spec: + - question_name: AWS Region + type: multiplechoice + variable: aws_region + required: true + default: us-east-1 + choices: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - question_name: Specify target hosts + type: text + variable: _hosts + required: false + + - name: Cloud / AWS / Restore EC2 from Snapshot + job_type: run + organization: Default + credentials: + - AWS + project: Ansible official demo project + playbook: cloud/restore_ec2.yml + inventory: Demo Inventory + notification_templates_started: Telemetry + notification_templates_success: Telemetry + notification_templates_error: Telemetry + survey_enabled: true + survey: + name: '' + description: '' + spec: + - question_name: AWS Region + type: multiplechoice + variable: aws_region + required: true + default: us-east-1 + choices: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - question_name: Specify target hosts + type: text + variable: _hosts + required: false + + - name: "LINUX / Patching" + job_type: check + inventory: "Demo Inventory" + project: "Ansible official demo project" + playbook: "linux/patching.yml" + execution_environment: Default execution environment + notification_templates_started: Telemetry + notification_templates_success: Telemetry + notification_templates_error: Telemetry + use_fact_cache: true + ask_job_type_on_launch: true + credentials: + - "Demo Credential" + survey_enabled: true + survey: + name: '' + description: '' + spec: + - question_name: Server Name or Pattern + type: text + variable: _hosts + required: true + controller_workflows: - name: Deploy Cloud Stack in AWS description: A workflow to deploy a cloud stack @@ -475,3 +560,56 @@ controller_workflows: feedback: Failed to create AWS instance - identifier: Tag Report unified_job_template: Cloud / AWS / Tags Report + + - name: Cloud / AWS / Patch EC2 Workflow + description: A workflow to patch ec2 instances with snapshot and restore on failure. + organization: Default + notification_templates_started: Telemetry + notification_templates_success: Telemetry + notification_templates_error: Telemetry + survey_enabled: true + survey: + name: '' + description: '' + spec: + - question_name: AWS Region + type: multiplechoice + variable: aws_region + required: true + default: us-east-1 + choices: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - question_name: Specify target hosts + type: text + variable: _hosts + required: true + default: os_linux + simplified_workflow_nodes: + - identifier: Project Sync + unified_job_template: Ansible official demo project + success_nodes: + - Take Snapshot + - identifier: Inventory Sync + unified_job_template: AWS Inventory + success_nodes: + - Take Snapshot + - identifier: Take Snapshot + unified_job_template: Cloud / AWS / Snapshot EC2 + success_nodes: + - Patch Instance + - identifier: Patch Instance + unified_job_template: LINUX / Patching + job_type: run + failure_nodes: + - Restore from Snapshot + - identifier: Restore from Snapshot + unified_job_template: Cloud / AWS / Restore EC2 from Snapshot + failure_nodes: + - Ticket - Restore Failed + - identifier: Ticket - Restore Failed + unified_job_template: 'SUBMIT FEEDBACK' + extra_data: + feedback: Cloud / AWS / Patch EC2 Workflow | Failed to restore ec2 from snapshot \ No newline at end of file diff --git a/cloud/snapshot_ec2.yml b/cloud/snapshot_ec2.yml new file mode 100644 index 000000000..3be63bf93 --- /dev/null +++ b/cloud/snapshot_ec2.yml @@ -0,0 +1,10 @@ +--- +- name: Snapshot ec2 instance + hosts: "{{ _hosts | default(omit) }}" + gather_facts: false + + tasks: + - name: Include snapshot role + ansible.builtin.include_role: + name: "demo.cloud.aws" + tasks_from: snapshot_vm diff --git a/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml b/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml index b152f7ed5..b985bccc4 100644 --- a/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml +++ b/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml @@ -21,3 +21,4 @@ aws_env_tag: prod aws_purpose_tag: ansible_demo aws_ansiblegroup_tag: cloud aws_ec2_wait: true +aws_snapshots: {} \ No newline at end of file diff --git a/collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml b/collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml new file mode 100644 index 000000000..7d6297578 --- /dev/null +++ b/collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml @@ -0,0 +1,62 @@ +--- +- name: AWS | RESTORE VM + delegate_to: localhost + block: + - name: AWS | RESTORE VM | stop vm + amazon.aws.ec2_instance: + region: "{{ aws_region }}" + instance_ids: "{{ instance_id }}" + state: stopped + wait: true + + - name: AWS | RESTORE VM | get volumes + register: r_vol_info + amazon.aws.ec2_vol_info: + region: "{{ aws_region }}" + filters: + attachment.instance-id: "{{ instance_id }}" + + - name: AWS | RESTORE VM | detach volumes + loop: "{{ r_vol_info.volumes }}" + loop_control: + loop_var: volume + label: "{{ volume.id }}" + amazon.aws.ec2_vol: + region: "{{ aws_region }}" + id: "{{ volume.id }}" + instance: None + + - name: AWS | RESTORE VM | attach snapshots from stat + when: inventory_hostname in aws_snapshots + loop: "{{ aws_snapshots[inventory_hostname] }}" + loop_control: + loop_var: snap + label: "{{ snap.snapshot_id }}" + amazon.aws.ec2_vol: + region: "{{ aws_region }}" + instance: "{{ instance_id }}" + snapshot: "{{ snap.snapshot_id }}" + device_name: "{{ snap.device }}" + + - name: AWS | RESTORE VM | get all snapshots + when: inventory_hostname not in aws_snapshots + register: r_snapshots + amazon.aws.ec2_snapshot_info: + region: "{{ aws_region }}" + filters: + "tag:Name": "{{ inventory_hostname }}" + + - name: AWS | RESTORE VM | create volume from latest snapshot + when: inventory_hostname not in aws_snapshots + amazon.aws.ec2_vol: + region: "{{ aws_region }}" + instance: "{{ instance_id }}" + snapshot: "{{ r_snapshots.snapshots[0].snapshot_id }}" + device_name: "/dev/sda1" + + - name: AWS | RESTORE VM | start vm + amazon.aws.ec2_instance: + region: "{{ aws_region }}" + instance_ids: "{{ instance_id }}" + state: started + wait: true \ No newline at end of file diff --git a/collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml b/collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml new file mode 100644 index 000000000..a82e6c1bd --- /dev/null +++ b/collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml @@ -0,0 +1,43 @@ +--- +- name: AWS | SNAPSHOT VM + delegate_to: localhost + block: + - name: AWS | SNAPSHOT VM | assert id + ansible.builtin.assert: + that: instance_id is defined + fail_msg: "instance_id is required for snapshot operations" + + - name: AWS | SNAPSHOT VM | include vars + ansible.builtin.include_vars: + file: snapshot_vm.yml + + - name: AWS | SNAPSHOT VM | get volumes + register: r_vol_info + amazon.aws.ec2_vol_info: + region: "{{ aws_region }}" + filters: + attachment.instance-id: "{{ instance_id }}" + + - name: AWS | SNAPSHOT VM | take snapshots + loop: "{{ r_vol_info.volumes }}" + loop_control: + loop_var: volume + label: "{{ volume.id }}" + register: r_snapshots + amazon.aws.ec2_snapshot: + region: "{{ aws_region }}" + volume_id: "{{ volume.id }}" + description: "Snapshot taken by Red Hat Product demos" + snapshot_tags: "{{ tags }}" + +- name: AWS | SNAPSHOT VM | format snapshot stat + ansible.builtin.set_fact: + snapshot_stat: + - key: "{{ inventory_hostname }}" + value: "{{ r_snapshots.results | json_query(aws_ec2_snapshot_query) }}" + +- name: AWS | SNAPSHOT VM | record snapshot with host key + ansible.builtin.set_stats: + data: + aws_snapshots: "{{ snapshot_stat | items2dict }}" + diff --git a/collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml b/collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml new file mode 100644 index 000000000..9ad07a7c7 --- /dev/null +++ b/collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml @@ -0,0 +1,10 @@ +# Set stat_snapshots with model: +# [ +# { +# "snapshot_id": "snap-0e981f05704e19ffd", +# "vol_id": "vol-0bd55f313bb7bcdd8", +# "device": "/dev/sda1" +# }, +# ... +# ] +aws_ec2_snapshot_query: "[].{snapshot_id: snapshot_id, vol_id: volume.id, device: volume.attachment_set[?instance_id=='{{ instance_id }}'].device | [0]}" \ No newline at end of file diff --git a/linux/patching.yml b/linux/patching.yml index 23b91eba1..d567292eb 100644 --- a/linux/patching.yml +++ b/linux/patching.yml @@ -29,9 +29,22 @@ - ansible_local.insights.system_id is defined - name: Deploy report server + when: not ansible_check_mode delegate_to: "{{ report_server }}" run_once: true # noqa: run-once[task] block: + - name: Install firewall dependencies + ansible.builtin.dnf: + name: + - firewalld + - python3-firewall + state: present + + - name: Start firewalld + ansible.builtin.service: + name: firewalld + state: started + - name: Build report server ansible.builtin.include_role: name: "{{ item }}" From 2cd3ec6f724e230ba96fa56640cf80f3129594a8 Mon Sep 17 00:00:00 2001 From: Matthew Fernandez Date: Wed, 13 Sep 2023 08:09:34 -0600 Subject: [PATCH 2/2] Extend create vm job template (#97) --- cloud/setup.yml | 10 +++++++++- .../demo/cloud/roles/aws/defaults/main.yml | 2 +- .../demo/cloud/roles/aws/tasks/restore_vm.yml | 2 +- .../demo/cloud/roles/aws/tasks/snapshot_vm.yml | 1 - .../demo/cloud/roles/aws/vars/snapshot_vm.yml | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cloud/setup.yml b/cloud/setup.yml index 3161f878d..cc7bc1db1 100644 --- a/cloud/setup.yml +++ b/cloud/setup.yml @@ -249,6 +249,14 @@ controller_templates: variable: create_vm_aws_keypair_name required: true default: aws-test-key + - question_name: AWS Instance Type (defaults to blueprint value) + type: text + variable: create_vm_aws_instance_size + required: false + - question_name: AWS Image Filter (defaults to blueprint value) + type: text + variable: create_vm_aws_image_filter + required: false - name: Cloud / AWS / Delete VM job_type: run @@ -612,4 +620,4 @@ controller_workflows: - identifier: Ticket - Restore Failed unified_job_template: 'SUBMIT FEEDBACK' extra_data: - feedback: Cloud / AWS / Patch EC2 Workflow | Failed to restore ec2 from snapshot \ No newline at end of file + feedback: Cloud / AWS / Patch EC2 Workflow | Failed to restore ec2 from snapshot diff --git a/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml b/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml index b985bccc4..4f06b1f4e 100644 --- a/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml +++ b/collections/ansible_collections/demo/cloud/roles/aws/defaults/main.yml @@ -21,4 +21,4 @@ aws_env_tag: prod aws_purpose_tag: ansible_demo aws_ansiblegroup_tag: cloud aws_ec2_wait: true -aws_snapshots: {} \ No newline at end of file +aws_snapshots: {} diff --git a/collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml b/collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml index 7d6297578..730ecadb8 100644 --- a/collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml +++ b/collections/ansible_collections/demo/cloud/roles/aws/tasks/restore_vm.yml @@ -59,4 +59,4 @@ region: "{{ aws_region }}" instance_ids: "{{ instance_id }}" state: started - wait: true \ No newline at end of file + wait: true diff --git a/collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml b/collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml index a82e6c1bd..0826f8cc3 100644 --- a/collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml +++ b/collections/ansible_collections/demo/cloud/roles/aws/tasks/snapshot_vm.yml @@ -40,4 +40,3 @@ ansible.builtin.set_stats: data: aws_snapshots: "{{ snapshot_stat | items2dict }}" - diff --git a/collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml b/collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml index 9ad07a7c7..e1be6a492 100644 --- a/collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml +++ b/collections/ansible_collections/demo/cloud/roles/aws/vars/snapshot_vm.yml @@ -7,4 +7,4 @@ # }, # ... # ] -aws_ec2_snapshot_query: "[].{snapshot_id: snapshot_id, vol_id: volume.id, device: volume.attachment_set[?instance_id=='{{ instance_id }}'].device | [0]}" \ No newline at end of file +aws_ec2_snapshot_query: "[].{snapshot_id: snapshot_id, vol_id: volume.id, device: volume.attachment_set[?instance_id=='{{ instance_id }}'].device | [0]}"