Merge pull request #2 from kameshsampath/main

(fix!): Refactor Enable GH Action

.github/workflows/cleanup.yml

@@ -0,0 +1,50 @@
+name: Clean up
+on:
+    workflow_dispatch:
+env:
+    SNOWFLAKE_DEFAULT_CONNECTION_NAME: "workflow"
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_USER: ${{ secrets.SNOWFLAKE_USER }}
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_PRIVATE_KEY_PASSPHRASE:
+        ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_PRIVATE_KEY_RAW:
+        ${{ secrets.PRIVATE_KEY_RAW }}
+    GIT_REPO_DB: MY_GIT_REPOS
+    GIT_REPO_SCHEMA: GITHUB
+    GIT_REPO_NAME: snow_cli_demo
+    TODO_APP_NAME: todos_app
+    TODO_APP_WH: TODO_APP_WH
+    TODO_APP_DB: TODO_APP_DB
+    TODO_APP_SCHEMA: APPS
+    TODOS_DATA_SCHEMA: DATA
+jobs:
+    Deploy:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  # todo update to pick the right branch
+                  ref: ${{ env.GITHUB_REF_NAME  }}
+            - uses: Snowflake-Labs/snowflake-cli-action@v1.5
+              with:
+                  cli-version: "latest"
+                  default-config-file-path:
+                      ${{ github.workspace }}/config/config.toml"
+            - name: Check Version and Verify Connection
+              env:
+                  PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+              run: |
+                  snow --version
+                  snow connection test
+                  echo "Using branch $GITHUB_REF_NAME"
+
+            - name: Setup Todos App Database Objects
+              env:
+                  PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+                  GIT_REPO_FQN:
+                      ${{ env.GIT_REPO_DB }}.${{ env.GIT_REPO_SCHEMA }}.${{
+                      env.GIT_REPO_NAME }}
+              run: |
+                  snow git execute "@$GIT_REPO_FQN/branches/$GITHUB_REF_NAME/cleanup.sql" \
+                    --variable "db_name='$TODO_APP_DB'" \
+                    --variable "git_repo_name='$GIT_REPO_FQN'"

.github/workflows/snow.yml

@@ -0,0 +1,86 @@
+name: Deploy
+on:
+    workflow_dispatch:
+    release:
+        types: [published]
+    push:
+        paths:
+            - "de/**"
+            - "data/*.csv"
+            - "app/**"
+env:
+    SNOWFLAKE_DEFAULT_CONNECTION_NAME: "workflow"
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_USER: ${{ secrets.SNOWFLAKE_USER }}
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_PRIVATE_KEY_PASSPHRASE:
+        ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+    SNOWFLAKE_CONNECTIONS_WORKFLOW_PRIVATE_KEY_RAW:
+        ${{ secrets.PRIVATE_KEY_RAW }}
+    GIT_REPO_DB: MY_GIT_REPOS
+    GIT_REPO_SCHEMA: GITHUB
+    GIT_REPO_NAME: snow_cli_demo
+    TODO_APP_NAME: todos_app
+    TODO_APP_WH: TODO_APP_WH
+    TODO_APP_DB: TODO_APP_DB
+    TODO_APP_SCHEMA: APPS
+    TODOS_DATA_SCHEMA: DATA
+jobs:
+    Deploy:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  # todo update to pick the right branch
+                  ref: ${{ env.GITHUB_REF_NAME  }}
+            - uses: Snowflake-Labs/snowflake-cli-action@v1.5
+              with:
+                  cli-version: "latest"
+                  default-config-file-path:
+                      ${{ github.workspace }}/config/config.toml
+            - name: Check Version and Verify Connection
+              env:
+                  PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+              run: |
+                  snow --version
+                  snow connection test
+                  echo "Using branch $GITHUB_REF_NAME"
+
+            - name: Refresh Repo
+              env:
+                  PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+                  GIT_REPO_FQN:
+                      ${{ env.GIT_REPO_DB }}.${{ env.GIT_REPO_SCHEMA }}.${{
+                      env.GIT_REPO_NAME }}
+              run: |
+                  snow git fetch "$GIT_REPO_FQN"
+
+            - name: Setup Todos App Database Objects
+              env:
+                  PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+                  GIT_REPO_FQN:
+                      ${{ env.GIT_REPO_DB }}.${{ env.GIT_REPO_SCHEMA }}.${{
+                      env.GIT_REPO_NAME }}
+              run: |
+                  snow git execute @"${GIT_REPO_FQN}/branches/${GITHUB_REF_NAME}/de/" \
+                    --variable "db_name='$TODO_APP_DB'" \
+                    --variable "schema_name='$TODOS_DATA_SCHEMA'" \
+                    --variable "wh_name='$TODO_APP_WH'" \
+                    --variable "git_repo_name='$GIT_REPO_FQN'" \
+                    --variable "git_branch='$GIT_BRANCH'" \
+                    --database $GIT_REPO_DB --schema $GIT_REPO_SCHEMA
+
+            - name: Deploy TODOs Streamlit App
+              env:
+                  PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+              run: |
+                  snow streamlit deploy --replace \
+                  --database $TODO_APP_DB --schema $TODO_APP_SCHEMA
+              working-directory: app
+
+            - name: Get App URL
+              env:
+                  PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+              run: |
+                  snow streamlit get-url $TODO_APP_NAME \
+                    --database $TODO_APP_DB --schema $TODO_APP_SCHEMA
+              working-directory: app

.gitignore

@@ -0,0 +1 @@
+work

.tool-versions

@@ -0,0 +1 @@
+python 3.11.9

README.md

@@ -8,30 +8,54 @@ As part of this quick demo we will see how to use `snow` for,
 - [x] Deploying [Streamlit in Snowflake](https://docs.snowflake.com/en/developer-guide/streamlit/about-streamlit)
 - [ ] Setup CI/CD using [Snowflake CLI](https://github.com/Snowflake-Labs/snowflake-cli-action) GitHub Action
 
+> [!WARNING]
+> Currently the demo works only with SNOW CLI 3.1+. This demo was last tested with 3.1RC4
+> ```shell
+> pip install -U git+https://github.com/snowflakedb/snowflake-cli.git@v3.1.0-rc4
+> ```
+
 ## Pre-requisites
 
 - Snowflake [Trial Account](https://signup.snowflake.com/)
 - [GitHub](https://github.com) Account
 - [SNOW CLI](https://docs.snowflake.com/en/developer-guide/snowflake-cli/index)
 
+If you need to use GitHub Action set the following GitHub Environment Secrets to your fork,
+
+- `SNOWFLAKE_ACCOUNT` - the Snowflake Accouunt ID
+- `SNOWFLAKE_USER` - The user to perform all operations, for simple checks try with `ACCOUNTADMIN` before trying granular roles
+- `PRIVATE_KEY_PASSPHRASE` - The Passphrase to be used to decrypt the Private Key 
+- `PRIVATE_KEY_RAW`- The **ENCRYPTED** private key to be used when connecting via the GH Action.
+
 ### Snowflake Environment
 
 Create a database, schema and a warehouse to use to hold all the git repos and related objects
 
+We will set them as our default datbase, schema and warehouse for the rest of the demo,
+
 ```shell
-snow sql --stdin <<EOF
-CREATE DATABASE IF NOT EXISTS MY_GIT_REPOS;
-CREATE SCHEMA IF NOT EXISTS GITHUB;
-CREATE WAREHOUSE IF NOT EXISTS MY_GIT_WH;
-EOF
+export GIT_REPO_DB='MY_GIT_REPOS'
+export GIT_REPO_SCHEMA='GITHUB'
+export GIT_REPO_WAREHOUSE='MY_GIT_WH'
 ```
 
-We will set them as our default datbase, schema and warehouse for the rest of the demo,
+```shell
+snow object create database \
+  name="$GIT_REPOS_DATABASE" \
+  comment='Database to hold all my Git repositories and related objects'
+```
 
 ```shell
-export SNOWFLAKE_CONNECTIONS_TRIAL_DATABASE='MY_GIT_REPOS'
-export SNOWFLAKE_CONNECTIONS_TRIAL_SCHEMA='GITHUB'
-export SNOWFLAKE_CONNECTIONS_TRIAL_WAREHOUSE='MY_GIT_WH'
+snow object create schema \
+  name="$GITHUB_REPOS_SCHEMA" \
+  comment='Schema to hold all GitHub repositories' \
+  --database="$GIT_REPOS_DATABASE"
+```
+
+```shell
+snow object create warehouse \
+  name="$GIT_REPO_WAREHOUSE" \
+  comment='Warehouse that will be used in this demo.'
 ```
 
 ### Create Git Repos
@@ -40,42 +64,47 @@ Set few variables that we can interploate later in the script,
 
 ```shell
 export GIT_REPO_NAME='snow_cli_demo'
-export GIT_BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+export GITHUB_REF_NAME="$(git rev-parse --abbrev-ref HEAD)"
 ```
 
 #### Git Integration Demo
 
 ```shell
 #https://github.com/Snowflake-Labs/snow-cli-demo.git
-snow git setup "$GIT_REPO_NAME"
+snow git setup "$GIT_REPO_NAME" \
+  --database="$GIT_REPO_DB" \
+  --schema="$GIT_REPO_SCHEMA"
 ```
 
 * `Repo URL`: https://github.com/Snowflake-Labs/snow-cli-demo.git
 * Select `N` to secret as this`sf-git-integration-demo` for public repo or provide the secret usually the [GitHub PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens)
 * Default to create an git API integration
 
+```shell
+export GIT_REPO_FQN="$GIT_REPO_DB.$GIT_REPO_NAME.$GITHUB_REPOS_SCHEMA"
+```
 Fetch all branches, tags and commits,
 
 ```shell
-snow git fetch "$GIT_REPO_NAME"
+snow git fetch "$GIT_REPO_FQN"
 ```
 
 List all branches
 
 ```shell
-snow git list-branches "$GIT_REPO_NAME"
+snow git list-branches "$GIT_REPO_FQN"
 ```
 
 List all tags
 
 ```shell
-snow git list-tags "$GIT_REPO_NAME"
+snow git list-tags "$GIT_REPO_FQN"
 ```
 
 List all files on branch snow-cli,
 
 ```shell
-snow git list-files "@$GIT_REPO_NAME/branches/$GIT_BRANCH/" --pattern '.*\.sql'
+snow git list-files "@$GIT_REPO_FQN/branches/$GITHUB_REF_NAME/" --pattern '.*\.sql'
 ```
 
 ## Run SQL from Git Repo
@@ -85,29 +114,34 @@ We will create a simple `TODOS` table and load the data from `todos.sql`.
 To make the setup and tear down easy and customizable, let use the following env
 
 ```shell
-export TODO_WH='TODO_APP_WH'
-export TODO_DB_NAME='TODO_APP_DB'
-export TODO_SCHEMA_NAME='DATA'
+export TODO_APP_WH='TODO_APP_WH'
+export TODO_APP_DB='TODO_APP_DB'
+export TODO_APP_SCHEMA='APPS'
+export TODOS_DATA_SCHEMA='DATA'
 ```
 
 Setup TODO APP warehouse, database, schemas and tables,
 
 ```shell
-snow git execute "@$GIT_REPO_NAME/branches/$GIT_BRANCH/todos.sql" \
-  --variable "db_name='$TODO_DB_NAME'" \
-  --variable "schema_name='$TODO_SCHEMA_NAME'" \
-  --variable "wh_name='$TODO_WH'" \
-  --variable "git_repo_name='$SNOWFLAKE_CONNECTIONS_TRIAL_DATABASE.$SNOWFLAKE_CONNECTIONS_TRIAL_SCHEMA.$GIT_REPO_NAME'" \
-  --variable "git_branch='$GIT_BRANCH'"
+snow git execute "@$GIT_REPO_FQN/branches/$GITHUB_REF_NAME/de" \
+        --variable "db_name='$TODO_APP_DB'" \
+        --variable "schema_name='$TODOS_DATA_SCHEMA'" \
+        --variable "wh_name='$TODO_APP_WH'" \
+        --variable "git_repo_name='$GIT_REPO_FQN'" \
+        --variable "git_branch='$GITHUB_REF_NAME'" \
+        --database $GIT_REPO_DB --schema $GIT_REPO_SCHEMA
 ```
 
 Verify and check if objects has been created,
 
 ```shell
 snow sql -q "select * from todos" \
-  --dbname="$TODO_DB_NAME" \
-  --schema="$TODO_SCHEMA_NAME"
+  --database="$TODO_APP_DB" \
+  --schema="$TODOS_DATA_SCHEMA"
 ```
+> [!NOTE]
+> **ONLY IF GH Action is Enabled**
+> Update anything under app, de or data folders. Do a commit push to see GH Action trigger and updating your Snowflake data and app
 
 ## Deploy Streamlit Application
 
@@ -117,35 +151,33 @@ There is simple Streamlit application that is available under [app](./app) direc
 cd app
 ```
 
-Let us create a schema to deploy the Todo APP,
-
-```shell
-snow sql -q 'create schema if not exists apps' --dbname="$TODO_DB_NAME"
-```
-
 Deploy Streamlit app,
 
 ```shell
-snow streamlit deploy --dbname="$TODO_DB_NAME" --schema="apps"
+snow streamlit deploy --replace \
+  --database $TODO_APP_DB --schema $TODO_APP_SCHEMA
 ```
 
 You can use the URL from the output of the successful deployment to access the application.
 
 >[!TIP]
 > You can also get the URL of the application anytime using the command
 >```shell
-> snow streamlit get-url todo_app --dbname=--dbname="$TODO_DB_NAME" --schema="apps"
+> snow streamlit get-url todo_app
 > ```
 > You can find the app name in [snowflake.yml](./app/snowflake.yml)
 
 ## Cleanup
 
 ```shell
-snow git execute "@$GIT_REPO_NAME/branches/$GIT_BRANCH/cleanup.sql" \
-  --variable "db_name='$TODO_DB_NAME'"
+snow git execute "@$GIT_REPO_FQN/branches/$GIT_BRANCH/cleanup.sql" \
+  --variable "db_name='$TODO_APP_DB'" \
+  --variable "git_repo_name='$GIT_REPO_FQN'"
 ```
 
-Verify clean up and the `$TODO_DB_NAME` should be listed,
+It should delete the `$TODO_APP_DB` and the `MY_GIT_REPOS.GITHUB.SNOW_CLI_DEMO` Git repository.
+
+Verify clean up and the `$TODO_APP_DB` should not be listed,
 
 ```shell
 snow sql -q "SHOW DATABASES"

app/.gitignore

@@ -2,3 +2,4 @@
 .venv/
 app.zip
 __pycache__
+TODO.md

app/snowflake.yml

@@ -1,13 +1,20 @@
 definition_version: "2"
+env:
+    TODO_APP_NAME: todos_app
+    TODO_APP_DB: TODO_APP_DB
+    TODO_APP_SCHEMA: APPS
+    TODO_APP_WH: TODO_APP_WH
 entities:
     todos_app:
         type: streamlit
         identifier:
-            name: todos_app
+            name: <% ctx.env.TODO_APP_NAME %>
+            database: <% ctx.env.TODO_APP_DB %>
+            schema: <% ctx.env.TODO_APP_SCHEMA %>
         main_file: streamlit_app.py
         pages_dir: pages
-        query_warehouse: todo_app_wh
-        stage: todos_app
+        query_warehouse: <% ctx.env.TODO_APP_WH %>
+        stage: <% ctx.env.TODO_APP_NAME %>
         artifacts:
             - streamlit_app.py
             - environment.yml

cleanup.sql

@@ -1,4 +1,6 @@
 --!jinja
 USE ROLE ACCOUNTADMIN;
 
-DROP DATABASE IF EXISTS {{db_name}};
+DROP DATABASE IF EXISTS {{db_name}};
+
+DROP GIT REPOSITORY IF EXISTS {{git_repo_name}};

config/config.toml

@@ -0,0 +1,5 @@
+default_connection_name = "workflow"
+
+[connections]
+[connections.workflow]
+authenticator="SNOWFLAKE_JWT"