diff --git a/models/_samples/incremental/example_incremental_model.sql b/models/_samples/incremental/example_incremental_model.sql deleted file mode 100644 index cf8cb7b..0000000 --- a/models/_samples/incremental/example_incremental_model.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config( - materialized='incremental', - on_schema_change='sync_all_columns' - ) -}} - -with source as ( - select * from {{ ref('example_source_for_incremental') }} - {% if is_incremental() %} - -- this filter will only be applied on an incremental run - where _etl_loaded_at > (select max(_etl_loaded_at) from {{ this }}) - {% endif %} -) - -select * -from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_no_timestamp.sql b/models/_samples/incremental/example_incremental_model_no_timestamp.sql deleted file mode 100644 index 9e0bdf7..0000000 --- a/models/_samples/incremental/example_incremental_model_no_timestamp.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config( - materialized='incremental', - on_schema_change='sync_all_columns' - ) -}} - -with source as ( - select * from {{ ref('example_source_for_incremental') }} - {% if is_incremental() %} - -- this filter will only be applied on an incremental run - where event_id not in (select event_id from {{ this }}) - {% endif %} -) - -select * -from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_relative_start_and_end_dates.sql b/models/_samples/incremental/example_incremental_model_relative_start_and_end_dates.sql deleted file mode 100644 index a108873..0000000 --- a/models/_samples/incremental/example_incremental_model_relative_start_and_end_dates.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='event_id', - on_schema_change='sync_all_columns' - ) -}} - - - -with source as ( - select * from {{ ref('example_source_for_incremental') }} - {% if is_incremental() %} - -- this filter will only be applied on an incremental run - where _etl_loaded_at >= dateadd(day, -2, current_date) - and _etl_loaded_at <= dateadd(day, -1, current_date) - - {% endif %} - - -) - -select * -from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_using_project_variable_lookback_window.sql b/models/_samples/incremental/example_incremental_model_using_project_variable_lookback_window.sql deleted file mode 100644 index 532c878..0000000 --- a/models/_samples/incremental/example_incremental_model_using_project_variable_lookback_window.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='event_id', - on_schema_change='sync_all_columns' - ) -}} - -with source as ( - select * from {{ ref('example_source_for_incremental') }} - {% if is_incremental() %} - -- this filter will only be applied on an incremental run - where _etl_loaded_at > ( - - select {{ dbt.dateadd( - var('incremental_lookback_period'), - -var('incremental_lookback_value'), - "max(_etl_loaded_at)") }} - - from {{ this }} - ) - - - {% endif %} - - -) - -select * -from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_with_unique_key.sql b/models/_samples/incremental/example_incremental_model_with_unique_key.sql deleted file mode 100644 index c359fac..0000000 --- a/models/_samples/incremental/example_incremental_model_with_unique_key.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='event_id', - on_schema_change='sync_all_columns' - ) -}} - -with source as ( - select * from {{ ref('example_source_for_incremental') }} - {% if is_incremental() %} - -- this filter will only be applied on an incremental run - where _etl_loaded_at > (select {{ dbt.dateadd("hour", -3, "max(_etl_loaded_at)") }} from {{ this }}) - - - {% endif %} -) - -select * -from source \ No newline at end of file diff --git a/models/_samples/incremental/example_source_for_incremental.sql b/models/_samples/incremental/example_source_for_incremental.sql deleted file mode 100644 index 1cbe674..0000000 --- a/models/_samples/incremental/example_source_for_incremental.sql +++ /dev/null @@ -1,95 +0,0 @@ - -{{ - config( - materialized='view' - ) -}} - -{% set country_options=['US', 'Mexico', 'Canada', 'Germany', 'England', 'France', 'Ireland'] %} -{% set page_url_options=['https://www.getdbt.com/', 'https://docs.getdbt.com/', 'https://docs.getdbt.com/quickstarts'] %} -{% set page_url_title_options=['dbt Labs | Transform Data in Your Warehouse', 'What is dbt? | dbt Developer Hub', 'Quickstarts | dbt Developer Hub'] %} - -with series as ( - -{{ dbt_utils.generate_series(3000000) }} - -), - -seconds_this_month as ( - -select - generated_number as event_id, - - -- events occur every second in the month - {{ dbt.dateadd("second", "event_id", dbt.date_trunc('month', 'current_timestamp')) }} as event_timestamp - - -from series - -where event_timestamp <= current_timestamp - -), - -final as ( - - select - event_id, - round(1 + date_part('second', event_timestamp)::int / 10.0) as user_id, - - case - when date_part('second', event_timestamp)::int % 3 < 2 - then 'page_view' - else 'page_ping' - end as event, - - event_timestamp, - - case - when date_part('second', event_timestamp)::int % 3 = 0 - then '{{ page_url_options[0] }}' - when date_part('second', event_timestamp)::int % 3 = 1 - then '{{ page_url_options[1] }}' - else '{{ page_url_options[2] }}' - end as page_url, - - case - when date_part('second', event_timestamp)::int % 3 = 0 - then '{{ page_url_title_options[0] }}' - when date_part('second', event_timestamp)::int % 3 = 1 - then '{{ page_url_title_options[1] }}' - else '{{ page_url_title_options[2] }}' - end as page_title, - - case - when round(1 + date_part('second', event_timestamp)::int / 10.0) % 7 < 4 - then 'computer' - else 'mobile' - end as device_type, - - case - when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 1 - then '{{ country_options[0] }}' - when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 2 - then '{{ country_options[1] }}' - when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 3 - then '{{ country_options[2] }}' - when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 4 - then '{{ country_options[3] }}' - when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 5 - then '{{ country_options[4] }}' - when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 6 - then '{{ country_options[5] }}' - else '{{ country_options[6] }}' - end as geo, - - -- rounded event timestamp to mimic data being loaded every 5 seconds - {{ dbt.dateadd("second", "- ((event_id % 60) % 5)", "event_timestamp" ) }} as _etl_loaded_at - - - - from seconds_this_month - -) - -select * -from final diff --git a/models/_samples/marts/dim_jaffle_shop_customers.sql b/models/_samples/marts/dim_jaffle_shop_customers.sql deleted file mode 100644 index 3137755..0000000 --- a/models/_samples/marts/dim_jaffle_shop_customers.sql +++ /dev/null @@ -1,33 +0,0 @@ -with stg_customers as ( - select * from {{ ref('stg_jaffle_shop__customers') }} -), - -alphabet_grouping as ( - select * from {{ ref('alphabet_grouping') }} -), - -stg_customers_tansformed as ( - select - customer_id, - first_name, - last_name, - lower(left(last_name, 1)) as first_character_last_name - - from stg_customers -), - -final as ( - -select - stg_customers_tansformed.customer_id, - stg_customers_tansformed.first_name, - stg_customers_tansformed.last_name, - alphabet_grouping.letter_grouping - -from stg_customers_tansformed -join alphabet_grouping -on stg_customers_tansformed.first_character_last_name = alphabet_grouping.letter - -) - -select * from final \ No newline at end of file diff --git a/models/_samples/model_governance/_model_governance__models.yml b/models/_samples/model_governance/_model_governance__models.yml deleted file mode 100644 index 6c83505..0000000 --- a/models/_samples/model_governance/_model_governance__models.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: 2 - -models: - - name: example_private_finance_model - access: private - group: finance - latest_version: 2 - config: - contract: - enforced: true - columns: - - name: order_id - data_type: int - constraints: - # not null constraint is enforced in snowflake - - type: not_null - # unique constraint is NOT enforced in snowflake - - type: unique - warn_unenforced: false - tests: - - unique - - name: order_date - data_type: date - - name: customer_id - data_type: int - - name: order_status_code - data_type: varchar - - name: priority_code - data_type: varchar - - name: clerk_name - data_type: varchar - - name: ship_priority - data_type: int - - name: order_count - data_type: int - - name: gross_item_sales_amount - data_type: number - - name: item_discount_amount - data_type: number - - name: item_tax_amount - data_type: number - - name: net_item_sales_amount - data_type: number - versions: - - v: 2 - config: - alias: example_private_finance_model - columns: - - include: all - exclude: - - priority_code - - v: 1 - - - name: example_selecting_from_private_model - # change this to marketing if you want to see the error from not having access to the private finance model - group: finance - - - name: example_selecting_from_old_version_of_private_model - group: finance \ No newline at end of file diff --git a/models/_samples/model_governance/example_private_finance_model_v1.sql b/models/_samples/model_governance/example_private_finance_model_v1.sql deleted file mode 100644 index 39a619b..0000000 --- a/models/_samples/model_governance/example_private_finance_model_v1.sql +++ /dev/null @@ -1,2 +0,0 @@ -select * -from {{ ref('fct_orders') }} diff --git a/models/_samples/model_governance/example_private_finance_model_v2.sql b/models/_samples/model_governance/example_private_finance_model_v2.sql deleted file mode 100644 index e069595..0000000 --- a/models/_samples/model_governance/example_private_finance_model_v2.sql +++ /dev/null @@ -1,3 +0,0 @@ -select - * exclude priority_code -from {{ ref('fct_orders') }} diff --git a/models/_samples/model_governance/example_selecting_from_old_version_of_private_model.sql b/models/_samples/model_governance/example_selecting_from_old_version_of_private_model.sql deleted file mode 100644 index e9f49ce..0000000 --- a/models/_samples/model_governance/example_selecting_from_old_version_of_private_model.sql +++ /dev/null @@ -1,7 +0,0 @@ - -select * - --- specifying old version of model -from {{ ref('example_private_finance_model', v=1) }} - - diff --git a/models/_samples/model_governance/example_selecting_from_private_model.sql b/models/_samples/model_governance/example_selecting_from_private_model.sql deleted file mode 100644 index 6f6d84d..0000000 --- a/models/_samples/model_governance/example_selecting_from_private_model.sql +++ /dev/null @@ -1,7 +0,0 @@ - -select * - --- defaults to the latest version because didnt specify the version in the ref -from {{ ref('example_private_finance_model') }} - - diff --git a/models/_samples/python/_python__models.yml b/models/_samples/python/_python__models.yml deleted file mode 100644 index 9331595..0000000 --- a/models/_samples/python/_python__models.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: 2 - -models: - - name: py01__python_builtins__describe - # Document within the same codebase - description: My transformation written in Python - # Configure in ways that feel intuitive and familiar - config: - materialized: table - tags: ['python'] - - - name: py02__use_variables__customers_limit_10 - config: - materialized: table - tags: ['python'] - target_name: "{{ target.name }}" - description: Customer dimensions table limited - columns: - - name: customer_id - description: Primary id on the customers table - tests: - - unique - - not_null - - # You can choose _not_ to config your Python files, though we think you should - # If you choose not to, simply ensure you have added a config to your - # Python files that materializes them as a table, otherwise they will fail. \ No newline at end of file diff --git a/models/_samples/python/py01__python_builtins__describe.py b/models/_samples/python/py01__python_builtins__describe.py deleted file mode 100644 index 3275f0e..0000000 --- a/models/_samples/python/py01__python_builtins__describe.py +++ /dev/null @@ -1,12 +0,0 @@ -# The required model() function must return a single DataFrame -def model(dbt, session): - - # bring in reference model as dataframe - customers_df = dbt.ref("dim_customers") - - # do transformation - apply "describe" function - described_df = customers_df.describe() - - # return dataframe - return described_df - \ No newline at end of file diff --git a/models/_samples/python/py02__use_variables__customers_limit_10.py b/models/_samples/python/py02__use_variables__customers_limit_10.py deleted file mode 100644 index 866a862..0000000 --- a/models/_samples/python/py02__use_variables__customers_limit_10.py +++ /dev/null @@ -1,16 +0,0 @@ -# The required model() function must return a single DataFrame -def model(dbt, session): - # bring in variable from config.yml - target_name = dbt.config.get("target_name") - - # bring in reference model as dataframe - customers_df = dbt.ref("dim_customers") - - # limit data in dev, using if statement and variable from config.yml, - # IF your project target is set to "dev" (and not left as "default") - if target_name == "dev": - customers_df = customers_df.limit(10) - - # return dataframe - return customers_df - \ No newline at end of file diff --git a/models/_samples/python/py03__define_function__payment_glitch.py b/models/_samples/python/py03__define_function__payment_glitch.py deleted file mode 100644 index 52476d6..0000000 --- a/models/_samples/python/py03__define_function__payment_glitch.py +++ /dev/null @@ -1,15 +0,0 @@ -# define a temporary function -def times_two(x): - return x * 2 - -# The required model() function must return a single DataFrame -def model(dbt, session): - dbt.config(materialized="table") - - # bring in reference model as dataframe - payments_glitch = dbt.ref("fct_orders") - - # apply infinite money glitch by using function - df = payments_glitch.withColumn("amount__glitch", times_two(payments_glitch["gross_item_sales_amount"])) - return df - \ No newline at end of file diff --git a/models/_samples/python/py04__add_udf_function__payment_glitch.py b/models/_samples/python/py04__add_udf_function__payment_glitch.py deleted file mode 100644 index 9e0b260..0000000 --- a/models/_samples/python/py04__add_udf_function__payment_glitch.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -UDFs - You can use the @udf decorator or udf function to define an "anonymous" function - and call it within your model function's DataFrame transformation. -""" -# import Python packages -import snowflake.snowpark.types as T -import snowflake.snowpark.functions as F -import numpy - -# define a temporary function, using a temporary snowpark UDF -def register_udf_add_random(): - add_random = F.udf( - # use 'lambda' syntax, for simple functional behavior - lambda x: x + numpy.random.randint(1,99), - return_type=T.FloatType(), - input_types=[T.FloatType()] - ) - return add_random - -# The required model() function must return a single DataFrame -def model(dbt, session): - - dbt.config( - materialized = "table", - packages = ["numpy"] - ) - - # bring in reference Python model as dataframe - payments_glitch = dbt.ref("py03__define_function__payment_glitch") - - add_random = register_udf_add_random() - - # add money, who knows by how much, by calling function - df = payments_glitch.withColumn("amount_plus_random", add_random("gross_item_sales_amount")) - - return df diff --git a/models/_samples/snapshot/_snapshot__models.yml b/models/_samples/snapshot/_snapshot__models.yml deleted file mode 100644 index a6786f4..0000000 --- a/models/_samples/snapshot/_snapshot__models.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: 2 - -models: - - name: example_join_snapshots - columns: - - name: surrogate_key - tests: - - unique - - not_null \ No newline at end of file diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql deleted file mode 100644 index 0281922..0000000 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ /dev/null @@ -1,49 +0,0 @@ -with order_snapshot as ( - select - * exclude dbt_valid_to, - coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to - from {{ ref('example_orders_snapshot') }} -), - -orders_line_items_snapshot as ( - select - * exclude dbt_valid_to, - coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to - from {{ ref('example_orders_line_items_snapshot') }} -), - -joined as ( - -select - orders_line_items_snapshot.id as line_item_id, - order_snapshot.order_id, - order_snapshot.payment_method, - order_snapshot.status, - orders_line_items_snapshot.amount, - greatest(order_snapshot.dbt_valid_from, - orders_line_items_snapshot.dbt_valid_from) as valid_from, - least(order_snapshot.dbt_valid_to, - orders_line_items_snapshot.dbt_valid_to) as valid_to, - min(least(order_snapshot.order_created_at, orders_line_items_snapshot.order_created_at)) as order_created_at, - max(least(order_snapshot.order_updated_at, orders_line_items_snapshot.order_updated_at)) as order_updated_at - - -from order_snapshot - -left join orders_line_items_snapshot -on order_snapshot.order_id = orders_line_items_snapshot.order_id -and order_snapshot.dbt_valid_from <= orders_line_items_snapshot.dbt_valid_to -and order_snapshot.dbt_valid_to >= orders_line_items_snapshot.dbt_valid_from - -group by 1,2,3,4,5,6,7 - -), - -final as ( - select - {{ dbt_utils.generate_surrogate_key(['line_item_id', 'valid_from', 'valid_to']) }} as surrogate_key, - * - from joined -) - -select * from final \ No newline at end of file diff --git a/models/_samples/snapshot/example_orders_line_items_source_for_snapshot.sql b/models/_samples/snapshot/example_orders_line_items_source_for_snapshot.sql deleted file mode 100644 index 75c4889..0000000 --- a/models/_samples/snapshot/example_orders_line_items_source_for_snapshot.sql +++ /dev/null @@ -1,108 +0,0 @@ -{{ - config( - materialized='view' - ) -}} - - -with id_series as ( - -{{ dbt_utils.generate_series(100000) }} - -), - -adding_order_id as ( - - -select - generated_number as id, - - -- wacky way to get some semblance of randomness for which ids are tied to which orders - dense_rank() over ( - order by - case - when id % 5 = 0 - then id - 1 - when id % 5 = 1 - then id - 2 - when id % 9 = 0 - then id - 1 - else id - end - ) - as order_id - - -from id_series - -), - -distinct_order_ids as ( - select - distinct order_id - - from adding_order_id -), - -adding_order_created_at as ( - select - order_id, - {{ dbt.dateadd( - 'month', - -1, - dbt.dateadd("second", "order_id", dbt.date_trunc('day', 'current_timestamp'))) - }} as order_created_at - - from distinct_order_ids -), - -adding_order_details as ( - select - order_id, - -- every 5th order gets an update - order_id % 5 = 0 as needs_update, - - order_created_at, - - case - -- every 5th order gets randomly updated - when order_id % 5 = 0 - then current_timestamp - else order_created_at - end as order_updated_at - - - from adding_order_created_at -), - -final as ( - - select - adding_order_id.id, - adding_order_id.order_id, - - case - when adding_order_details.needs_update - then uniform(5, 35, random())::float - .01 - when id % 3 = 0 - then 9.99 - when id % 3 = 1 - then 5.99 - else 19.99 - end as amount, - - adding_order_details.order_created_at, - - adding_order_details.order_updated_at - - - from adding_order_id - - join adding_order_details - using (order_id) - -) - -select * -from final -order by 1, 2 \ No newline at end of file diff --git a/models/_samples/snapshot/example_orders_source_for_snapshot.sql b/models/_samples/snapshot/example_orders_source_for_snapshot.sql deleted file mode 100644 index 385d419..0000000 --- a/models/_samples/snapshot/example_orders_source_for_snapshot.sql +++ /dev/null @@ -1,120 +0,0 @@ -{{ - config( - materialized='view' - ) -}} - - -{% set statuses=['returned', 'completed', 'return_pending', 'shipped', 'placed'] %} - - -with id_series as ( - -{{ dbt_utils.generate_series(100000) }} - -), - -adding_order_id as ( - - -select - generated_number as id, - - -- wacky way to get some semblance of randomness for which ids are tied to which orders - dense_rank() over ( - order by - case - when id % 5 = 0 - then id - 1 - when id % 5 = 1 - then id - 2 - when id % 9 = 0 - then id - 1 - else id - end - ) - as order_id - - -from id_series - -), - -distinct_order_ids as ( - select - distinct order_id - - from adding_order_id -), - -adding_order_created_at as ( - select - order_id, - {{ dbt.dateadd( - 'month', - -1, - dbt.dateadd("second", "order_id", dbt.date_trunc('day', 'current_timestamp'))) - }} as order_created_at - - from distinct_order_ids -), - -final as ( - select - order_id, - -- every 5th order gets an update - order_id % 5 = 0 as needs_update, - - case - when adding_order_created_at.order_id % 10 <= 6 - then 'credit' - when adding_order_created_at.order_id % 10 <= 8 - then 'debit' - else 'cash' - end as payment_method, - - case - -- every 5th order gets randomly updated - when order_id % 5 = 0 - then - case - when uniform(0, 5, random()) = 0 - then '{{ statuses[0] }}' - when uniform(0, 5, random()) = 1 - then '{{ statuses[1] }}' - when uniform(0, 5, random()) = 2 - then '{{ statuses[2] }}' - when uniform(0, 5, random()) = 3 - then '{{ statuses[3] }}' - else '{{ statuses[4] }}' - end - when order_id % 3 = 0 - then 'placed' - when order_id % 3 = 1 - then 'shipped' - else 'returned' - end as status, - - order_created_at, - - case - -- every 5th order gets randomly updated - when order_id % 5 = 0 - then current_timestamp - else order_created_at - end as order_updated_at - - - from adding_order_created_at -) - -select - order_id, - status, - payment_method, - order_created_at, - order_updated_at - -from final - -order by 1 \ No newline at end of file diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml deleted file mode 100644 index a745c31..0000000 --- a/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: 2 - -models: - - - name: stg_jaffle_shop__orders - description: > - multi line description - of my staging orders model from - jaffle shop - tests: - - dbt_utils.recency: - datepart: day - field: _etl_loaded_at - interval: 7 - columns: - - name: order_id - tests: - - not_null: - config: - where: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) - severity: error - error_if: ">20" - warn_if: ">10" - limit: 25 - store_failures: true - - filtered_unique: - records_to_check: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) - - name: order_date - description: | - multi line description - of my order date column from my - jaffle shop orders model - tests: - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: date - - - name: stg_jaffle_shop__customers - description: | - multi line description - of my staging - jaffle shop customers model - tests: - - not_empty - - dbt_utils.equal_rowcount: - compare_model: source('jaffle_shop', 'customers') - - dbt_utils.equality: - compare_model: source('jaffle_shop', 'customers') - compare_columns: - - first_name - - last_name - - dbt_utils.expression_is_true: - expression: "len(first_name) > 1" - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - - columns: - - name: customer_id - description: > - multi line description - of my customer id column from my - jaffle shop customers model - tests: - - unique - - not_null - - unique_and_not_null - - name: first_name - tests: - - dbt_utils.at_least_one - - dbt_utils.not_constant - - dbt_utils.not_empty_string - - dbt_utils.not_accepted_values: - values: ['test'] diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml deleted file mode 100644 index be514df..0000000 --- a/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: 2 - -sources: - - name: jaffle_shop - description: our famous jaffle shop customers and orders - - # example to show how to use environment variables within the database property for a source - database: "{{ 'raw' if env_var('DBT_ENV_NAME') == 'dev' else 'raw' }}" - - # example to show how to use a long chunk of custom logic in the schema property. also shows off target - schema: | - {%- if target.name == "default" or target.name == "dev" -%} jaffle_shop - {%- elif target.name == "qa"-%} jaffle_shop - {%- elif target.name == "prod"-%} jaffle_shop - {%- else -%} jaffle_shop - {%- endif -%} - - loader: fivetran - freshness: - warn_after: - count: 7 - period: day - error_after: - count: 14 - period: day - loaded_at_field: _etl_loaded_at - tables: - - name: customers - identifier: customers - freshness: null - columns: - - name: id - tests: - - not_null - - name: orders - identifier: orders - quoting: - database: false - schema: false - identifier: false - columns: - - name: id - tests: - - not_null - \ No newline at end of file diff --git a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql b/models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql deleted file mode 100644 index 0e10767..0000000 --- a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql +++ /dev/null @@ -1,53 +0,0 @@ -{#- -Explaining Configurations - - grants: shows how to grant a specific privilege (in this case select) on the object to a role in Snowflake (in this case transformer) - - post_hook: shows you can run a statement after your model is built - - alias: sets the name of the object that will be created in the warehouse and overrides the file name. in this case, the model - will b created as staging_jaffle_shop_customers - - materialized: overwrites the materialization strategy from the dbt_project.yml file to be table instead of view - - persist_docs: persists the documentation into the database - - schema: adjusting the schema that the model gets built in to be a concatenation of the target schema and "jaffle_shop" - - database: example of how you can overwrite the target database by setting the database in the config. in this example, - we're overriding to the analytics database (even though that is the same as the target database, just an example) - --#} - -{{ - config( - grants = {'select': ['transformer']}, - post_hook = 'select * from {{ this }} limit 1', - alias = 'staging_jaffle_shop_customers', - materialized = 'table', - persist_docs = {"relation": true, "columns": true}, - tags = ['finance'], - schema = 'jaffle_shop', - database = 'analytics' - ) -}} - -{{ - config( - materialized='table' - ) -}} - - - -with source as ( - - select * from {{ source('jaffle_shop', 'customers') }} - -), - -renamed as ( - - select - id as customer_id, - first_name, - last_name - - from source - -) - -select * from renamed diff --git a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql b/models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql deleted file mode 100644 index afa5cf8..0000000 --- a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql +++ /dev/null @@ -1,48 +0,0 @@ -{#- -Explaining Configurations - - grants: shows how to grant a specific privilege (in this case select) on the object to a role in Snowflake (in this case transformer) - - post_hook: shows you can run a statement after your model is built - - alias: sets the name of the object that will be created in the warehouse and overrides the file name. in this case, the model - will b created as staging_jaffle_shop_orders - - materialized: overwrites the materialization strategy from the dbt_project.yml file to be table instead of view - - persist_docs: persists the documentation into the database - - schema: adjusting the schema that the model gets built in to be a concatenation of the target schema and "jaffle_shop" - - database: example of how you can overwrite the target database by setting the database in the config. in this example, - we're overriding to the analytics database (even though that is the same as the target database, just an example) - --#} - -{{ - config( - grants = {'select': ['transformer']}, - post_hook = 'select * from {{ this }} limit 1', - alias = 'staging_jaffle_shop_orders', - materialized = 'table', - persist_docs = {"relation": true, "columns": true}, - tags = ['finance', 'orders'], - schema = 'jaffle_shop', - database = 'analytics' - ) -}} - - -with source as ( - - select * from {{ source('jaffle_shop', 'orders') }} - -), - -renamed as ( - - select - id as order_id, - user_id as customer_id, - order_date, - status, - _etl_loaded_at - - from source - -) - -select * from renamed diff --git a/models/_samples/staging/my_old_unorganized_model.sql b/models/_samples/staging/my_old_unorganized_model.sql deleted file mode 100644 index 5fba9ab..0000000 --- a/models/_samples/staging/my_old_unorganized_model.sql +++ /dev/null @@ -1,8 +0,0 @@ - -select - customers.id as customer_id, - orders.id as order_id, - orders.status as order_status -from {{ source('jaffle_shop', 'customers') }} customers -join {{ source('jaffle_shop', 'orders') }} orders -on customers.id = orders.user_id diff --git a/models/_samples/staging/my_unorganized_model.sql b/models/_samples/staging/my_unorganized_model.sql deleted file mode 100644 index 5fba9ab..0000000 --- a/models/_samples/staging/my_unorganized_model.sql +++ /dev/null @@ -1,8 +0,0 @@ - -select - customers.id as customer_id, - orders.id as order_id, - orders.status as order_status -from {{ source('jaffle_shop', 'customers') }} customers -join {{ source('jaffle_shop', 'orders') }} orders -on customers.id = orders.user_id diff --git a/models/aggregates/_agg__exposures.yml b/models/aggregates/_agg__exposures.yml deleted file mode 100644 index 21346fa..0000000 --- a/models/aggregates/_agg__exposures.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: 2 - -exposures: - - name: yearly_part_rollup - description: | - # An h1 header - ============ - - Paragraphs are separated by a blank line. - - 2nd paragraph. *Italic*, **bold**, and `monospace` - Itemized lists look like: - * this one - * that one - * the other one - - # type could be {dashboard, notebook, analysis, ml, application} - type: dashboard - - # this is just a link to the thing itself for click through from documentation - url: https://example.com - - # convenience feature - relative scale of {high, medium, low} - maturity: high - - # documentation purposes for point of contact if stuff breaks - owner: - name: DBT User Bob - email: dbtuserbob@getdbt.com - - # IMPORTANT: determines the lineage relationship of the exposure construct to the rest of your DAG - depends_on: - - ref('agg_ship_modes_dynamic_pivot') \ No newline at end of file diff --git a/models/aggregates/_agg__models.yml b/models/aggregates/_agg__models.yml deleted file mode 100644 index 8bd5f47..0000000 --- a/models/aggregates/_agg__models.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: 2 - -models: - - name: agg_customer_orders__all_time - description: Aggregated orders data by customer - columns: - - name: customer_id - description: Primary key for customers - tests: - - unique - - not_null - - name: total_sales - description: Sum of gross items sales amount by customer - - - name: agg_regions_segments - description: Aggregated model by region and segment. - columns: - - name: region - description: One of the five global regions. - tests: - - accepted_values: - values: ['MIDDLE EAST','AFRICA','EUROPE','ASIA','AMERICA'] - - - name: market_segment - description: One of the five market segments. - tests: - - accepted_values: - values: ['HOUSEHOLD','AUTOMOBILE','FURNITURE','BUILDING','MACHINERY'] - - - name: total_sales - description: Total sales for the region and market segment. - tests: - - not_null - - # These two models are just different ways of doing the same thing (pivot over categories) using jinja and the PIVOT operation in Snowflake - - name: agg_ship_modes_dynamic_pivot - description: Example of creating a pivot table with hard-coded columns based on a query of the ship modes that are in the system - columns: - - name: order_year - description: year of the order - - - name: agg_ship_modes_hardcoded_pivot - description: Example of creating a pivot table with dynamic columns based on the ship modes that are in the system - columns: - - name: order_year - description: year of the order diff --git a/models/aggregates/agg_customer_orders__all_time.sql b/models/aggregates/agg_customer_orders__all_time.sql deleted file mode 100644 index 31c18d6..0000000 --- a/models/aggregates/agg_customer_orders__all_time.sql +++ /dev/null @@ -1,9 +0,0 @@ -select - * -from {{ - metrics.calculate( - metric('total_revenue'), - dimensions=['customer_id'] - ) -}} -order by 2 desc \ No newline at end of file diff --git a/models/aggregates/agg_regions_segments.sql b/models/aggregates/agg_regions_segments.sql deleted file mode 100644 index c159b7a..0000000 --- a/models/aggregates/agg_regions_segments.sql +++ /dev/null @@ -1,29 +0,0 @@ -with - -customers as ( - select * from {{ ref('dim_customers') }} -), - -orders as ( - select * from {{ ref('fct_orders') }} -), - -agg_orders as ( - select - customer_id, - sum(net_item_sales_amount) as total_sales - from orders - group by 1 -), - -final as ( - select - customers.region, - customers.market_segment, - round(sum(agg_orders.total_sales),2) as total_sales - from customers - left join agg_orders on customers.customer_id = agg_orders.customer_id - group by 1, 2 -) - -select * from final \ No newline at end of file diff --git a/models/aggregates/agg_ship_modes_dynamic_pivot.sql b/models/aggregates/agg_ship_modes_dynamic_pivot.sql deleted file mode 100644 index c702b89..0000000 --- a/models/aggregates/agg_ship_modes_dynamic_pivot.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* Create a pivot table with dynamic columns based on the ship modes that are in the system */ - -{%- call statement('result', fetch_result=True) -%} - - {# this pulls the unique ship modes from the fct_order_items table #} - select ship_mode from {{ ref('fct_order_items') }} group by 1 - -{%- endcall %} - -{% set ship_modes = load_result('result').table.columns[0].values() %} - -select - date_part('year', order_date) as order_year, - - {# Loop over ship_modes array from above, and sum based on whether the record matches the ship mode #} - {%- for ship_mode in ship_modes -%} - sum(case when ship_mode = '{{ship_mode}}' then gross_item_sales_amount end) as "{{ship_mode|replace(' ', '_')}}_amount" - {%- if not loop.last -%},{% endif %} - {% endfor %} - -from {{ ref('fct_order_items') }} -group by 1 \ No newline at end of file diff --git a/models/aggregates/agg_ship_modes_hardcoded_pivot.sql b/models/aggregates/agg_ship_modes_hardcoded_pivot.sql deleted file mode 100644 index c702b89..0000000 --- a/models/aggregates/agg_ship_modes_hardcoded_pivot.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* Create a pivot table with dynamic columns based on the ship modes that are in the system */ - -{%- call statement('result', fetch_result=True) -%} - - {# this pulls the unique ship modes from the fct_order_items table #} - select ship_mode from {{ ref('fct_order_items') }} group by 1 - -{%- endcall %} - -{% set ship_modes = load_result('result').table.columns[0].values() %} - -select - date_part('year', order_date) as order_year, - - {# Loop over ship_modes array from above, and sum based on whether the record matches the ship mode #} - {%- for ship_mode in ship_modes -%} - sum(case when ship_mode = '{{ship_mode}}' then gross_item_sales_amount end) as "{{ship_mode|replace(' ', '_')}}_amount" - {%- if not loop.last -%},{% endif %} - {% endfor %} - -from {{ ref('fct_order_items') }} -group by 1 \ No newline at end of file diff --git a/models/intermediate/finance/_int_finance__docs.md b/models/intermediate/finance/_int_finance__docs.md deleted file mode 100644 index 2eb8d1d..0000000 --- a/models/intermediate/finance/_int_finance__docs.md +++ /dev/null @@ -1,18 +0,0 @@ -# the intent of this .md is to remove redundancy in the documentation - -# the below are descriptions from order_items -{% docs base_price %} since extended_price is the line item total, we back out the price per item {% enddocs %} - -{% docs discounted_price %} factoring in the discount_percentage, the line item discount total {% enddocs %} - -{% docs tax_rate %} tax rate of the order item {% enddocs %} - -{% docs gross_item_sales_amount %} same as extended_price {% enddocs %} - -{% docs discounted_item_sales_amount %} line item (includes quantity) discount amount{% enddocs %} - -{% docs item_discount_amount %} item level discount amount. this is always a negative number {% enddocs %} - -{% docs item_tax_amount %} item level tax total {% enddocs %} - -{% docs net_item_sales_amount %} the net total which factors in discount and tax {% enddocs %} \ No newline at end of file diff --git a/models/intermediate/finance/_int_finance__models.yml b/models/intermediate/finance/_int_finance__models.yml deleted file mode 100644 index 480cc11..0000000 --- a/models/intermediate/finance/_int_finance__models.yml +++ /dev/null @@ -1,159 +0,0 @@ -version: 2 - -models: - - name: int_line_items_amounts_calculated - description: Intermediate model where we calculate item price, discounts and tax. This model is at the order item level. - columns: - - name: order_item_id - description: '{{ doc("order_item_id") }}' - tests: - - unique - - not_null - - name: order_id - description: foreign id for orders - - name: part_id - description: foreign id for part - - name: supplier_id - description: foreign id for suppliers - - name: return_flag - description: '{{ doc("return_flag") }}' - - name: line_number - description: '{{ doc("line_number") }}' - - name: order_item_status_code - description: status of the order item - - name: ship_date - description: '{{ doc("ship_date") }}' - - name: commit_date - description: '{{ doc("commit_date") }}' - - name: receipt_date - description: '{{ doc("receipt_date") }}' - - name: ship_mode - description: '{{ doc("ship_mode") }}' - - name: extended_price - description: '{{ doc("extended_price") }}' - - name: quantity - description: total units - - name: base_price - description: '{{ doc("base_price") }}' - - name: discount_percentage - description: '{{ doc("discount_percentage") }}' - - name: discounted_price - description: '{{ doc("discounted_price") }}' - - name: gross_item_sales_amount - description: '{{ doc("gross_item_sales_amount") }}' - - name: discounted_item_sales_amount - description: '{{ doc("discounted_item_sales_amount") }}' - - name: item_discount_amount - description: '{{ doc("item_discount_amount") }}' - - name: tax_rate - description: '{{ doc("tax_rate") }}' - - name: item_tax_amount - description: item level tax total - - name: net_item_sales_amount - description: '{{ doc("net_item_sales_amount") }}' - - - name: int_order_items_joined - description: Intermediate model joining orders and line items. This model is at the order item level. - tags: - - output_table_type|orders - columns: - - name: order_item_id - description: '{{ doc("order_item_id") }}' - tests: - - unique - - not_null - - name: order_id - description: foreign id for orders - - name: customer_id - description: foreign id for customers - - name: part_id - description: foreign id for part - - name: supplier_id - description: foreign id for suppliers - - name: order_date - description: date of the order - - name: order_status_code - description: status of the order - - name: priority_code - description: code associated with the order - - name: clerk_name - description: id of the clerk - - name: ship_priority - description: numeric representation of the shipping priority, zero being the default - - name: return_flag - description: '{{ doc("return_flag") }}' - - name: line_number - description: '{{ doc("line_number") }}' - - name: order_item_status_code - description: status of the order item - - name: ship_date - description: '{{ doc("ship_date") }}' - - name: commit_date - description: '{{ doc("commit_date") }}' - - name: receipt_date - description: '{{ doc("receipt_date") }}' - - name: ship_mode - description: '{{ doc("ship_mode") }}' - - name: extended_price - description: '{{ doc("extended_price") }}' - - name: quantity - description: total units - - name: base_price - description: '{{ doc("base_price") }}' - - name: discount_percentage - description: '{{ doc("discount_percentage") }}' - - name: discounted_price - description: '{{ doc("discounted_price") }}' - - name: gross_item_sales_amount - description: '{{ doc("gross_item_sales_amount") }}' - - name: discounted_item_sales_amount - description: '{{ doc("discounted_item_sales_amount") }}' - - name: item_discount_amount - description: '{{ doc("item_discount_amount") }}' - - name: tax_rate - description: '{{ doc("tax_rate") }}' - - name: item_tax_amount - description: item level tax total - - name: net_item_sales_amount - description: '{{ doc("net_item_sales_amount") }}' - - - name: int_part_suppliers_joined - description: Intermediate model where we join part, supplier and part_supplier. This model is at the part supplier level. - columns: - - name: part_supplier_id - description: primary id of the models - tests: - - unique - - not_null - - name: part_id - description: foreign id for part - - name: part_name - description: name of the part - - name: manufacturer - description: manufacturer of the part - - name: brand - description: brand of the part - - name: part_type - description: type of part including material - - name: part_size - description: size of the part - - name: container - description: container of the part - - name: retail_price - description: '{{ doc("retail_price") }}' - - name: supplier_id - description: foreign id for supplier - - name: supplier_name - description: '{{ doc("supplier_name") }}' - - name: supplier_address - description: '{{ doc("supplier_address") }}' - - name: phone_number - description: '{{ doc("phone_number") }}' - - name: account_balance - description: '{{ doc("account_balance") }}' - - name: nation_id - description: foreign id for nation - - name: available_quantity - description: '{{ doc("available_quantity") }}' - - name: cost - description: '{{ doc("cost") }}' \ No newline at end of file diff --git a/models/intermediate/finance/int_line_items_amounts_calculated.sql b/models/intermediate/finance/int_line_items_amounts_calculated.sql deleted file mode 100644 index e1ef5fe..0000000 --- a/models/intermediate/finance/int_line_items_amounts_calculated.sql +++ /dev/null @@ -1,42 +0,0 @@ -with line_item as ( - - select * from {{ ref('stg_tpch__line_items') }} - -) -select - - order_item_id, - order_id, - part_id, - supplier_id, - - return_flag, - line_number, - order_item_status_code, - ship_date, - commit_date, - receipt_date, - ship_mode, - extended_price, - quantity, - - -- extended_price is actually the line item total, - -- so we back out the extended price per item - (extended_price/nullif(quantity, 0)){{ money() }} as base_price, - discount_percentage, - (base_price * (1 - discount_percentage)){{ money() }} as discounted_price, - - extended_price as gross_item_sales_amount, - (extended_price * (1 - discount_percentage)){{ money() }} as discounted_item_sales_amount, - -- We model discounts as negative amounts - (-1 * extended_price * discount_percentage){{ money() }} as item_discount_amount, - tax_rate, - ((gross_item_sales_amount + item_discount_amount) * tax_rate){{ money() }} as item_tax_amount, - ( - gross_item_sales_amount + - item_discount_amount + - item_tax_amount - ){{ money() }} as net_item_sales_amount - -from - line_item diff --git a/models/intermediate/finance/int_order_items_joined.sql b/models/intermediate/finance/int_order_items_joined.sql deleted file mode 100644 index 0f77b7d..0000000 --- a/models/intermediate/finance/int_order_items_joined.sql +++ /dev/null @@ -1,51 +0,0 @@ -with orders as ( - - select * from {{ ref('stg_tpch__orders') }} - -), - -line_item as ( - - select * from {{ ref('int_line_items_amounts_calculated') }} - -) -select - - line_item.order_item_id, - orders.order_id, - orders.customer_id, - line_item.part_id, - line_item.supplier_id, - - orders.order_date, - orders.order_status_code, - orders.priority_code, - orders.clerk_name, - orders.ship_priority, - - line_item.return_flag, - line_item.line_number, - line_item.order_item_status_code, - line_item.ship_date, - line_item.commit_date, - line_item.receipt_date, - line_item.ship_mode, - line_item.extended_price, - line_item.quantity, - - line_item.base_price, - line_item.discount_percentage, - line_item.discounted_price, - line_item.gross_item_sales_amount, - line_item.discounted_item_sales_amount, - line_item.item_discount_amount, - line_item.tax_rate, - line_item.item_tax_amount, - line_item.net_item_sales_amount - -from - orders -inner join line_item - on orders.order_id = line_item.order_id -order by - orders.order_date diff --git a/models/intermediate/finance/int_part_suppliers_joined.sql b/models/intermediate/finance/int_part_suppliers_joined.sql deleted file mode 100644 index 3d65fee..0000000 --- a/models/intermediate/finance/int_part_suppliers_joined.sql +++ /dev/null @@ -1,53 +0,0 @@ -with part as ( - - select * from {{ ref('stg_tpch__parts') }} - -), - -supplier as ( - - select * from {{ ref('stg_tpch__suppliers') }} - -), - -part_supplier as ( - - select * from {{ ref('stg_tpch__part_suppliers') }} - -), - -final as ( - select - - part_supplier.part_supplier_id, - part.part_id, - part.name as part_name, - part.manufacturer, - part.brand, - part.type as part_type, - part.size as part_size, - part.container, - part.retail_price, - - supplier.supplier_id, - supplier.supplier_name, - supplier.supplier_address, - supplier.phone_number, - supplier.account_balance, - supplier.nation_id, - - part_supplier.available_quantity, - part_supplier.cost -from - part -inner join - part_supplier - on part.part_id = part_supplier.part_id -inner join - supplier - on part_supplier.supplier_id = supplier.supplier_id -order by - part.part_id -) - -select * from final diff --git a/models/marts/finance/_finance__models.yml b/models/marts/finance/_finance__models.yml deleted file mode 100644 index 25d6f0d..0000000 --- a/models/marts/finance/_finance__models.yml +++ /dev/null @@ -1,101 +0,0 @@ -version: 2 - -groups: - - name: finance - owner: - email: finance@jaffleshop.com - name: Finance Person - -models: - - name: fct_order_items - description: order items fact table - columns: - - name: order_item_id - description: '{{ doc("order_item_id") }}' - tests: - - unique - - not_null - - name: order_id - description: foreign id for orders - - name: order_date - description: date of the order - - name: customer_id - description: foreign id for customers - - name: part_id - description: foreign id for part - - name: supplier_id - description: foreign id for suppliers - - name: order_item_status_code - description: status of the order item - - name: return_flag - description: '{{ doc("return_flag") }}' - - name: line_number - description: '{{ doc("line_number") }}' - - name: ship_date - description: '{{ doc("ship_date") }}' - - name: commit_date - description: '{{ doc("commit_date") }}' - - name: receipt_date - description: '{{ doc("receipt_date") }}' - - name: ship_mode - description: '{{ doc("ship_mode") }}' - - name: supplier_cost - description: '{{ doc("cost") }}' - - name: base_price - description: '{{ doc("base_price") }}' - - name: discount_percentage - description: '{{ doc("discount_percentage") }}' - - name: discounted_price - description: '{{ doc("discounted_price") }}' - - name: tax_rate - description: '{{ doc("tax_rate") }}' - - name: order_item_count - description: count of order items - - name: quantity - description: total units - - name: gross_item_sales_amount - description: '{{ doc("gross_item_sales_amount") }}' - - name: discounted_item_sales_amount - description: '{{ doc("discounted_item_sales_amount") }}' - - name: item_discount_amount - description: '{{ doc("item_discount_amount") }}' - - name: item_tax_amount - description: '{{ doc("item_tax_amount") }}' - - name: net_item_sales_amount - description: '{{ doc("net_item_sales_amount") }}' - - - name: fct_orders - description: orders fact table - columns: - - name: order_id - description: primary id of the model - tests: - - unique - - not_null - - name: customer_id - description: foreign id for customers - tests: - - relationships: - to: ref('dim_customers') - field: customer_id - severity: error - - name: order_date - description: date of the order - - name: status_code - description: status of the order - - name: priority_code - description: code associated with the order - - name: clerk_name - description: id of the clerk - - name: ship_priority - description: numeric representation of the shipping priority, zero being the default - - name: order_count - description: count of order - - name: gross_item_sales_amount - description: '{{ doc("gross_item_sales_amount") }}' - - name: item_discount_amount - description: '{{ doc("item_discount_amount") }}' - - name: item_tax_amount - description: '{{ doc("item_tax_amount") }}' - - name: net_item_sales_amount - description: '{{ doc("net_item_sales_amount") }}' diff --git a/models/marts/finance/_finance_metrics.yml b/models/marts/finance/_finance_metrics.yml deleted file mode 100644 index 0799f1c..0000000 --- a/models/marts/finance/_finance_metrics.yml +++ /dev/null @@ -1,51 +0,0 @@ -version: 2 - -metrics: - - name: total_revenue - label: Total Jaffle Shop Gross Revenue - description: sum of gross revenue at the Jaffle Shop - model: ref('fct_orders') - - calculation_method: sum - expression: gross_item_sales_amount - - timestamp: order_date - time_grains: [day, week, month, quarter, year, all_time] - - dimensions: - - customer_id - - priority_code - - clerk_name - - - - name: total_orders - label: Total Jaffle Orders Placed - description: count of orders placed at the Jaffle Shop - model: ref('fct_orders') - - calculation_method: count - expression: order_id - - timestamp: order_date - time_grains: [day, week, month, quarter, year, all_time] - - dimensions: - - customer_id - - priority_code - - clerk_name - - - - name: revenue_per_order - label: Average Revenue per Order in Period - description: Average Revenue per Order in Period - - calculation_method: derived - expression: " {{ metric('total_revenue') }} / {{ metric('total_orders') }} " - - timestamp: order_date - time_grains: [day, week, month, quarter, year, all_time] - - dimensions: - - customer_id - - priority_code - - clerk_name \ No newline at end of file diff --git a/models/marts/finance/fct_order_items.sql b/models/marts/finance/fct_order_items.sql deleted file mode 100644 index 67e5b22..0000000 --- a/models/marts/finance/fct_order_items.sql +++ /dev/null @@ -1,58 +0,0 @@ -{{ - config( - tags = ['finance'] - ) -}} - -with order_item as ( - - select * from {{ ref('int_order_items_joined') }} - -), -part_supplier as ( - - select * from {{ ref('int_part_suppliers_joined') }} - -), -final as ( - select - order_item.order_item_id, - order_item.order_id, - order_item.order_date, - order_item.customer_id, - order_item.part_id, - order_item.supplier_id, - order_item.order_item_status_code, - order_item.return_flag, - order_item.line_number, - order_item.ship_date, - order_item.commit_date, - order_item.receipt_date, - order_item.ship_mode, - part_supplier.cost as supplier_cost, - {# ps.retail_price, #} - order_item.base_price, - order_item.discount_percentage, - order_item.discounted_price, - order_item.tax_rate, - - 1 as order_item_count, - order_item.quantity, - order_item.gross_item_sales_amount, - order_item.discounted_item_sales_amount, - order_item.item_discount_amount, - order_item.item_tax_amount, - order_item.net_item_sales_amount - - from - order_item - inner join part_supplier - on order_item.part_id = part_supplier.part_id and - order_item.supplier_id = part_supplier.supplier_id -) -select - * -from - final -order by - order_date \ No newline at end of file diff --git a/models/marts/finance/fct_orders.sql b/models/marts/finance/fct_orders.sql deleted file mode 100644 index 3fcc62a..0000000 --- a/models/marts/finance/fct_orders.sql +++ /dev/null @@ -1,37 +0,0 @@ -{{ - config( - tags = ['finance'] - ) -}} - -with order_item as ( - - select * from {{ ref('int_order_items_joined') }} - -), - -final as ( - - select - - order_id, - order_date, - customer_id, - order_status_code, - priority_code, - clerk_name, - ship_priority, - - 1 as order_count, - sum(gross_item_sales_amount) as gross_item_sales_amount, - sum(item_discount_amount) as item_discount_amount, - sum(item_tax_amount) as item_tax_amount, - sum(net_item_sales_amount) as net_item_sales_amount - - from order_item - {{ dbt_utils.group_by(n = 8) }} - -) - -select * from final -order by order_date \ No newline at end of file diff --git a/models/marts/marketing/_marketing__models.yml b/models/marts/marketing/_marketing__models.yml deleted file mode 100644 index a0d3eda..0000000 --- a/models/marts/marketing/_marketing__models.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -groups: - - name: marketing - owner: - email: marketing@jaffleshop.com - name: Marketing Person - -models: - - name: dim_customers - description: Customer dimensions table - columns: - - name: customer_id - description: Primary id on the customers table - tests: - - unique - - not_null - - name: region - description: region name - tests: - - accepted_values: - values: ['AFRICA','MIDDLE EAST','ASIA','EUROPE','AMERICA'] - severity: warn - - name: name - description: customer id - - name: address - description: address of the customer - - name: nation - description: nation name - - name: phone_number - description: phone number of the customer - - name: account_balance - description: '{{ doc("account_balance") }}' - - name: market_segment - description: market segment of the customer \ No newline at end of file diff --git a/models/marts/marketing/dim_customers.sql b/models/marts/marketing/dim_customers.sql deleted file mode 100644 index 3c9415d..0000000 --- a/models/marts/marketing/dim_customers.sql +++ /dev/null @@ -1,46 +0,0 @@ -{{ - config( - transient = false - ) -}} - -with customer as ( - - select * from {{ ref('stg_tpch__customers') }} - -), -nation as ( - - select * from {{ ref('stg_tpch__nations') }} - -), -region as ( - - select * from {{ ref('stg_tpch__regions') }} - -), -final as ( - select - customer.customer_id, - customer.name, - customer.address, - {# nation.nation_id as nation_id, #} - nation.name as nation, - {# region.region_id as region_id, #} - region.name as region, - customer.phone_number, - customer.account_balance, - customer.market_segment - from - customer - inner join nation - on customer.nation_id = nation.nation_id - inner join region - on nation.region_id = region.region_id -) -select - * -from - final -order by - customer_id diff --git a/models/marts/operations/_operations__models.yml b/models/marts/operations/_operations__models.yml deleted file mode 100644 index 1d46d11..0000000 --- a/models/marts/operations/_operations__models.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: 2 - -groups: - - name: operations - owner: - email: operations@jaffleshop.com - name: Operations Person - -models: - - name: dim_parts - description: Parts dimensions table - columns: - - name: part_id - description: primary id of the model - tests: - - unique - - not_null - - name: manufacturer - description: manufacturer of the part - - name: name - description: name of the part - - name: brand - description: brand of the part - - name: type - description: type of part including material - - name: size - description: size of the part - - name: container - description: container of the part - - name: retail_price - description: '{{ doc("retail_price") }}' - - - name: dim_suppliers - description: Suppliers dimensions table - columns: - - name: supplier_id - description: primary id of the model - tests: - - unique - - not_null - - name: supplier_name - description: '{{ doc("supplier_name") }}' - - name: supplier_address - description: '{{ doc("supplier_address") }}' - - name: nation - description: nation name - - name: region - description: region name - - name: phone_number - description: '{{ doc("phone_number") }}' - - name: account_balance - description: '{{ doc("account_balance") }}' diff --git a/models/marts/operations/dim_parts.sql b/models/marts/operations/dim_parts.sql deleted file mode 100644 index 61b5645..0000000 --- a/models/marts/operations/dim_parts.sql +++ /dev/null @@ -1,22 +0,0 @@ -with part as ( - - select * from {{ref('stg_tpch__parts')}} - -), - -final as ( - select - part_id, - manufacturer, - name, - brand, - type, - size, - container, - retail_price - from - part -) -select * -from final -order by part_id diff --git a/models/marts/operations/dim_suppliers.sql b/models/marts/operations/dim_suppliers.sql deleted file mode 100644 index aa7202a..0000000 --- a/models/marts/operations/dim_suppliers.sql +++ /dev/null @@ -1,33 +0,0 @@ -with supplier as ( - - select * from {{ ref('stg_tpch__suppliers') }} - -), -nation as ( - - select * from {{ ref('stg_tpch__nations') }} -), -region as ( - - select * from {{ ref('stg_tpch__regions') }} - -), -final as ( - - select - supplier.supplier_id, - supplier.supplier_name, - supplier.supplier_address, - nation.name as nation, - region.name as region, - supplier.phone_number, - supplier.account_balance - from - supplier - inner join nation - on supplier.nation_id = nation.nation_id - inner join region - on nation.region_id = region.region_id -) - -select * from final diff --git a/models/staging/stripe/stg_stripe__payments.sql b/models/staging/stripe/stg_stripe__payments.sql index 61ede9e..668ac2b 100644 --- a/models/staging/stripe/stg_stripe__payments.sql +++ b/models/staging/stripe/stg_stripe__payments.sql @@ -13,6 +13,12 @@ select -- datetimes created as created_at -from {{ ref('snapshot_stg_payments') }} --- pull only the most recent update for each unique record -where dbt_valid_to is null +from {{ source('stripe', 'payment') }} + +{% if target.name == 'CI' %} + + where created_at >= dateadd('day',-3,current_date()) + +{% endif %} + + diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..15929fc --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,8 @@ +packages: +- package: dbt-labs/codegen + version: 0.12.0 +- package: dbt-labs/dbt_utils + version: 1.1.1 +- package: dbt-labs/audit_helper + version: 0.9.0 +sha1_hash: 9870bb20bb92c7ac21919c6a77ce1e3a49d76c76 diff --git a/packages.yml b/packages.yml index 1b69596..4d9830c 100644 --- a/packages.yml +++ b/packages.yml @@ -1,15 +1,7 @@ packages: - package: dbt-labs/codegen - version: 0.9.0 + version: 0.12.0 - package: dbt-labs/dbt_utils - version: 1.1.0 + version: 1.1.1 - package: dbt-labs/audit_helper - version: 0.9.0 - - package: dbt-labs/metrics - version: 1.5.0 - - package: brooklyn-data/dbt_artifacts - version: 2.4.2 - - package: dbt-labs/dbt_project_evaluator - version: 0.6.0 - - package: calogica/dbt_expectations - version: 0.8.5 + version: 0.9.0 \ No newline at end of file diff --git a/seeds/samples/_seeds.yml b/seeds/samples/_seeds.yml deleted file mode 100644 index 4d03df3..0000000 --- a/seeds/samples/_seeds.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -seeds: - - name: alphabet_grouping - description: A mapping of letters to a random group - columns: - - name: letter - tests: - - unique - - not_null - - name: letter_grouping - description: just as assignment for each letter to some number from 1-10 \ No newline at end of file diff --git a/seeds/samples/alphabet_grouping.csv b/seeds/samples/alphabet_grouping.csv deleted file mode 100644 index 5dd554f..0000000 --- a/seeds/samples/alphabet_grouping.csv +++ /dev/null @@ -1,27 +0,0 @@ -letter,letter_grouping -a,1 -b,1 -c,2 -d,2 -e,3 -f,3 -g,3 -h,4 -i,4 -j,4 -k,5 -l,5 -m,6 -n,6 -o,7 -p,7 -q,7 -r,8 -s,8 -t,8 -u,9 -v,9 -w,9 -x,10 -y,10 -z,10 \ No newline at end of file diff --git a/snapshots/_samples/_example__snapshots.yml b/snapshots/_samples/_example__snapshots.yml deleted file mode 100644 index 0b03612..0000000 --- a/snapshots/_samples/_example__snapshots.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 2 - -snapshots: - - name: example_orders_line_items_snapshot - columns: - - name: id - tests: - - relationships: - to: ref('example_orders_line_items_source_for_snapshot') - field: id - - name: dbt_scd_id - tests: - - unique - - not_null - - name: example_check_snapshot - columns: - - name: id - tests: - - relationships: - to: ref('example_orders_line_items_source_for_snapshot') - field: id - - name: dbt_scd_id - tests: - - unique - - not_null \ No newline at end of file diff --git a/snapshots/_samples/example_check_snapshot.sql b/snapshots/_samples/example_check_snapshot.sql deleted file mode 100644 index 7fd0573..0000000 --- a/snapshots/_samples/example_check_snapshot.sql +++ /dev/null @@ -1,20 +0,0 @@ -{% snapshot example_check_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='id', - strategy='check', - check_cols=['check_cols'], - invalidate_hard_deletes=True - ) - }} - - {% set column_names = dbt_utils.get_filtered_columns_in_relation(from=ref('example_orders_line_items_source_for_snapshot'), except=["id"]) %} - - - select - *, - {{ dbt_utils.generate_surrogate_key(column_names) }} as check_cols - from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_generate_schema_snapshot.sql b/snapshots/_samples/example_generate_schema_snapshot.sql deleted file mode 100644 index 2174d7b..0000000 --- a/snapshots/_samples/example_generate_schema_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_generate_schema_snapshot %} - {{ - config( - target_database='analytics', - target_schema=generate_schema_name('snapshots'), - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_line_items_snapshot.sql b/snapshots/_samples/example_orders_line_items_snapshot.sql deleted file mode 100644 index 63603f5..0000000 --- a/snapshots/_samples/example_orders_line_items_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_line_items_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_snapshot.sql b/snapshots/_samples/example_orders_snapshot.sql deleted file mode 100644 index e4e7ad6..0000000 --- a/snapshots/_samples/example_orders_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='order_id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/snapshot_int_line_items.sql b/snapshots/snapshot_int_line_items.sql deleted file mode 100644 index 90bc550..0000000 --- a/snapshots/snapshot_int_line_items.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% snapshot snapshot_int_line_items %} - --- Example showing check columns strategy and using custom schemas (this is rare) - --- if you don't want to write to the same schema in development and prod, --- use environment-aware variable from dbt_project.yml - --- if you do not have a reliable updated at timestamp, checek columns for changes. - -{{ - config( - target_database='analytics', - target_schema=var('example_target_snapshot_schema'), - unique_key='order_item_id', - strategy='check', - check_cols=['gross_item_sales_amount', 'net_item_sales_amount'], - ) -}} - -select * from {{ ref('int_line_items_amounts_calculated') }} - -{% endsnapshot %} \ No newline at end of file diff --git a/snapshots/snapshot_stg_payments.sql b/snapshots/snapshot_stg_payments.sql deleted file mode 100644 index adc61b2..0000000 --- a/snapshots/snapshot_stg_payments.sql +++ /dev/null @@ -1,16 +0,0 @@ -{% snapshot snapshot_stg_payments %} - -{{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='ID', - - strategy='timestamp', - updated_at='_BATCHED_AT', - ) -}} - -select * from {{ source('stripe', 'payment') }} - -{% endsnapshot %} \ No newline at end of file diff --git a/tests/_samples/audit_helper_compare_all_columns__stg_tpch__orders.sql b/tests/_samples/audit_helper_compare_all_columns__stg_tpch__orders.sql deleted file mode 100644 index f294976..0000000 --- a/tests/_samples/audit_helper_compare_all_columns__stg_tpch__orders.sql +++ /dev/null @@ -1,17 +0,0 @@ -{# https://github.com/dbt-labs/dbt-audit-helper#compare_all_columns-source #} - -{{ - config( - severity = 'warn' - ) -}} - -{{ - audit_helper.compare_all_columns( - a_relation=ref('stg_tpch__orders'), - b_relation=api.Relation.create(database='analytics', schema='analytics', identifier='stg_tpch__orders'), - - primary_key='order_id' - ) -}} -where not perfect_match \ No newline at end of file diff --git a/tests/_samples/example_singular_test_multiple_models.sql b/tests/_samples/example_singular_test_multiple_models.sql deleted file mode 100644 index 3a3d082..0000000 --- a/tests/_samples/example_singular_test_multiple_models.sql +++ /dev/null @@ -1,7 +0,0 @@ - -select count(*) -from {{ ref('stg_jaffle_shop__customers') }} -join {{ ref('stg_jaffle_shop__orders') }} -using (customer_id) - -having count(*) = 0 \ No newline at end of file diff --git a/tests/generic/filtered_unique.sql b/tests/generic/filtered_unique.sql deleted file mode 100644 index 19f3ed1..0000000 --- a/tests/generic/filtered_unique.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% test filtered_unique(model, column_name, records_to_check=True) %} - - with all_records as ( - select * - from {{ model }} - ), - - new_records as ( - - select {{ column_name }} - from all_records - where {{ records_to_check }} - - ) - - select new_records.{{ column_name }} - from new_records - join all_records - using ({{ column_name }}) - group by 1 having count(*) > 1 - -{% endtest %} diff --git a/tests/generic/not_empty.sql b/tests/generic/not_empty.sql deleted file mode 100644 index 318cd29..0000000 --- a/tests/generic/not_empty.sql +++ /dev/null @@ -1,12 +0,0 @@ -{% test not_empty(model) %} - - with num_records as ( - select count(*) as total_rows - from {{ model }} - ) - - select * - from num_records - where total_rows = 0 - -{% endtest %} \ No newline at end of file diff --git a/tests/generic/unique_and_not_null.sql b/tests/generic/unique_and_not_null.sql deleted file mode 100644 index 3f04f32..0000000 --- a/tests/generic/unique_and_not_null.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% test unique_and_not_null(model, column_name) %} - - with null_records as ( - select {{ column_name }} - from {{ model }} - where {{ column_name }} is null - ), - - duplicates as ( - select {{ column_name }} - from {{ model }} - group by 1 - having count(*) > 1 - ) - - select * - from null_records - union all - select * - from duplicates - -{% endtest %} \ No newline at end of file diff --git a/tests/stg_tpch_orders_assert_positive_price.sql b/tests/stg_tpch_orders_assert_positive_price.sql deleted file mode 100644 index 392d3ae..0000000 --- a/tests/stg_tpch_orders_assert_positive_price.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ - config( - enabled=true, - severity='error', - tags = ['finance'] - ) -}} - -with orders as ( - select * from {{ ref('stg_tpch__orders') }} -) - -select * -from orders -where total_price < 0 -