From 3db279b79c821f6d33f9be236ac8b7896342526e Mon Sep 17 00:00:00 2001 From: Big Boss Date: Sun, 28 Jul 2024 15:51:12 -0500 Subject: [PATCH] bb/supabase cleanup (#634) * supabase cleanup drop duplicate send_account_created index enable pg_cron create cronjob for deleting send account transfers with no send account created row * add delete send account transfers with no send account * go back to indexing all USDC an SEND token transfers * just filter the rows actually * fix checkout specs to be less flaky --- ...account-sendtag-checkout.onboarded.spec.ts | 9 +-- .../tests/fixtures/checkout/page.ts | 5 ++ packages/shovel/etc/config.json | 25 ++++---- .../integrations/send-account-transfers.ts | 15 +++-- .../test/__snapshots__/index.test.ts.snap | 19 +++--- ...ate_send_account_created_account_index.sql | 1 + ...unt_transfers_filter_non_send_accounts.sql | 29 +++++++++ .../tests/send_account_transfers_test.sql | 62 +++++++++++++++++-- 8 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 supabase/migrations/20240728180824_delete_duplicate_send_account_created_account_index.sql create mode 100644 supabase/migrations/202407281808278_send_account_transfers_filter_non_send_accounts.sql diff --git a/packages/playwright/tests/account-sendtag-checkout.onboarded.spec.ts b/packages/playwright/tests/account-sendtag-checkout.onboarded.spec.ts index f798d2fbb..2a627853f 100644 --- a/packages/playwright/tests/account-sendtag-checkout.onboarded.spec.ts +++ b/packages/playwright/tests/account-sendtag-checkout.onboarded.spec.ts @@ -112,7 +112,8 @@ const verifyReferralReward = async ( return rewardAmount } -const generateTagName = () => faker.string.alphanumeric({ length: { min: 1, max: 20 } }) +// tag names are limited to 1-20 characters +const generateTagName = () => faker.string.alphanumeric({ length: { min: 6, max: 20 } }) const checkReferralCodeVisibility = async ( checkoutPage: CheckoutPage, @@ -207,7 +208,7 @@ test('can refer a tag', async ({ const tagsToRegister = Array.from( { length: Math.floor(Math.random() * 5) + 1 }, () => `${faker.lorem.word()}_${test.info().parallelIndex}` - ).sort() + ).sort((a, b) => a.localeCompare(b)) for (const tagName of tagsToRegister) { await addPendingTag(checkoutPage, tagName) } @@ -283,7 +284,7 @@ test('can refer multiple tags in separate transactions', async ({ // First transaction with up to 2 tags const firstTags = Array.from({ length: Math.floor(Math.random() * 2) + 1 }, () => generateTagName() - ).sort() + ).sort((a, b) => a.localeCompare(b)) for (const tagName of firstTags) { await addPendingTag(checkoutPage, tagName) } @@ -315,7 +316,7 @@ test('can refer multiple tags in separate transactions', async ({ // Second transaction with up to 3 tags const secondTags = Array.from({ length: Math.floor(Math.random() * 3) + 1 }, () => generateTagName() - ).sort() + ).sort((a, b) => a.localeCompare(b)) for (const tagName of secondTags) { await addPendingTag(checkoutPage, tagName) } diff --git a/packages/playwright/tests/fixtures/checkout/page.ts b/packages/playwright/tests/fixtures/checkout/page.ts index 63ce16024..15e7c809c 100644 --- a/packages/playwright/tests/fixtures/checkout/page.ts +++ b/packages/playwright/tests/fixtures/checkout/page.ts @@ -104,6 +104,11 @@ export class CheckoutPage { const signTransactionButton = this.page.getByRole('button', { name: 'Confirm' }) expect?.(signTransactionButton).toBeEnabled({ timeout: 10_000 }) // blockchain stuff is a bit slow await this.page.bringToFront() + const errorButton = this.page.getByRole('button', { name: 'Error' }) + expect?.(errorButton).toBeHidden() + await this.page + .getByRole('button', { name: 'Loading' }) + .waitFor({ state: 'detached', timeout: 10_000 }) await signTransactionButton.click() await confirmTagsRequest await confirmTagsResponse diff --git a/packages/shovel/etc/config.json b/packages/shovel/etc/config.json index 877f6265a..3db64afd7 100644 --- a/packages/shovel/etc/config.json +++ b/packages/shovel/etc/config.json @@ -152,7 +152,16 @@ }, { "name": "log_addr", - "column": "log_addr" + "column": "log_addr", + "filter_op": "contains", + "filter_arg": [ + "0x036CbD53842c5426634e7929541eC2318f3dCF7e", + "0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238", + "0x3f14920c99BEB920Afa163031c4e47a3e03B3e4A", + "0x7cEfbe54c37a35dCdaD29b86373ca8353a2F4680", + "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", + "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + ] } ], "event": { @@ -164,23 +173,13 @@ "indexed": true, "name": "from", "type": "address", - "column": "f", - "filter_op": "contains", - "filter_ref": { - "integration": "send_account_created", - "column": "account" - } + "column": "f" }, { "indexed": true, "name": "to", "type": "address", - "column": "t", - "filter_op": "contains", - "filter_ref": { - "integration": "send_account_created", - "column": "account" - } + "column": "t" }, { "indexed": false, diff --git a/packages/shovel/src/integrations/send-account-transfers.ts b/packages/shovel/src/integrations/send-account-transfers.ts index 9ddd331a3..83667d591 100644 --- a/packages/shovel/src/integrations/send-account-transfers.ts +++ b/packages/shovel/src/integrations/send-account-transfers.ts @@ -36,6 +36,11 @@ export const integration: Omit = { { name: 'log_addr', column: 'log_addr', + filter_op: 'contains', + filter_arg: [ + ...new Set(Object.values(sendTokenAddress)), + ...new Set(Object.values(usdcAddress)), + ].sort(), }, ] as BlockData[], event: { @@ -48,16 +53,18 @@ export const integration: Omit = { name: 'from', type: 'address', column: 'f', - filter_op: 'contains', - filter_ref: sendAccountFactorySenderFilterRef, + // wait for this https://github.com/orgs/indexsupply/discussions/268 + // filter_op: 'contains', + // filter_ref: sendAccountFactorySenderFilterRef, }, { indexed: true, name: 'to', type: 'address', column: 't', - filter_op: 'contains', - filter_ref: sendAccountFactorySenderFilterRef, + // wait for this https://github.com/orgs/indexsupply/discussions/268 + // filter_op: 'contains', + // filter_ref: sendAccountFactorySenderFilterRef, }, { indexed: false, diff --git a/packages/shovel/test/__snapshots__/index.test.ts.snap b/packages/shovel/test/__snapshots__/index.test.ts.snap index 074b67cac..4f6d7e14c 100644 --- a/packages/shovel/test/__snapshots__/index.test.ts.snap +++ b/packages/shovel/test/__snapshots__/index.test.ts.snap @@ -97,6 +97,15 @@ exports[`shovel config 1`] = ` }, { "column": "log_addr", + "filter_arg": [ + "0x036CbD53842c5426634e7929541eC2318f3dCF7e", + "0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238", + "0x3f14920c99BEB920Afa163031c4e47a3e03B3e4A", + "0x7cEfbe54c37a35dCdaD29b86373ca8353a2F4680", + "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", + "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + ], + "filter_op": "contains", "name": "log_addr", }, ], @@ -106,22 +115,12 @@ exports[`shovel config 1`] = ` "inputs": [ { "column": "f", - "filter_op": "contains", - "filter_ref": { - "column": "account", - "integration": "send_account_created", - }, "indexed": true, "name": "from", "type": "address", }, { "column": "t", - "filter_op": "contains", - "filter_ref": { - "column": "account", - "integration": "send_account_created", - }, "indexed": true, "name": "to", "type": "address", diff --git a/supabase/migrations/20240728180824_delete_duplicate_send_account_created_account_index.sql b/supabase/migrations/20240728180824_delete_duplicate_send_account_created_account_index.sql new file mode 100644 index 000000000..3e8c090d5 --- /dev/null +++ b/supabase/migrations/20240728180824_delete_duplicate_send_account_created_account_index.sql @@ -0,0 +1 @@ +drop index if exists "public"."send_account_created_account"; diff --git a/supabase/migrations/202407281808278_send_account_transfers_filter_non_send_accounts.sql b/supabase/migrations/202407281808278_send_account_transfers_filter_non_send_accounts.sql new file mode 100644 index 000000000..4d5d7b235 --- /dev/null +++ b/supabase/migrations/202407281808278_send_account_transfers_filter_non_send_accounts.sql @@ -0,0 +1,29 @@ +set check_function_bodies = off; + +-- create trigger function for filtering send_account_transfers with no send_account_created +create or replace function private.filter_send_account_transfers_with_no_send_account_created() + returns trigger + language plpgsql + security definer + as $$ +begin +-- Deletes send_account_transfers with no send_account_created. +-- This is due to performance issues in our shovel indexer and using filter_ref to limit indexing to only +-- send_account_transfers with send_account_created. +-- For now, we index all USDC and SEND token transfers, and use this function filter any send_account_transfers with no send_account_created. +-- See https://github.com/orgs/indexsupply/discussions/268 + if exists ( select 1 from send_account_created where account = new.f ) + or exists ( select 1 from send_account_created where account = new.t ) + then + return new; + else + return null; + end if; +end; +$$; + +-- create trigger on send_account_transfers table +create trigger filter_send_account_transfers_with_no_send_account_created +before insert on public.send_account_transfers +for each row +execute function private.filter_send_account_transfers_with_no_send_account_created(); diff --git a/supabase/tests/send_account_transfers_test.sql b/supabase/tests/send_account_transfers_test.sql index 964a39364..03f90ba15 100644 --- a/supabase/tests/send_account_transfers_test.sql +++ b/supabase/tests/send_account_transfers_test.sql @@ -1,5 +1,5 @@ BEGIN; -SELECT plan(3); +SELECT plan(5); -- Create the necessary extensions CREATE EXTENSION "basejump-supabase_test_helpers"; -- noqa: RF05 @@ -8,6 +8,10 @@ CREATE EXTENSION "basejump-supabase_test_helpers"; -- noqa: RF05 SELECT tests.create_supabase_user('test_user_from'); SELECT tests.create_supabase_user('test_user_to'); +insert into send_account_created (chain_id, log_addr, block_time, user_op_hash, tx_hash, account, ig_name, src_name, block_num, tx_idx, log_idx) +values (8453, '\xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266', floor(extract(EPOCH FROM timestamptz '2013-07-01 12:00:00')), '\x1234', '\x1234', '\x1234567890ABCDEF1234567890ABCDEF12345678', 'send_account_created', 'send_account_created', 1, 0, 0), + (8453, '\xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266', floor(extract(EPOCH FROM timestamptz '2013-07-01 12:00:00')), '\x1234', '\x1234', '\xB0B7D5E8A4B6D534B3F608E9D27871F85A4E98DA', 'send_account_transfers', 'send_account_transfers', 1, 0, 0); + INSERT INTO send_accounts (user_id, address, chain_id, init_code) VALUES ( tests.get_supabase_uid('test_user_from'), @@ -57,7 +61,7 @@ VALUES ( -- Test if the trigger function populated the additional columns correctly SELECT results_eq( $$ - select + select (data->>'log_addr')::citext as log_addr, (data->>'f')::citext as f, (data->>'t')::citext as t, @@ -102,8 +106,8 @@ WHERE id = ( -- Test if the delete was successful SELECT is_empty( $$ - SELECT id - FROM send_account_transfers + SELECT id + FROM send_account_transfers WHERE f = '\x1234567890ABCDEF1234567890ABCDEF12345678'::bytea AND t = '\xB0B7D5E8A4B6D534B3F608E9D27871F85A4E98DA'::bytea $$, 'Test if the trigger function removes the activity row' @@ -120,5 +124,55 @@ SELECT is_empty( 'Test if the trigger function removes the activity row' ); +-- Test filter_non_send_account_transfers_before_insert removes transactions when the sender is not a send account +delete from send_account_transfers; +INSERT INTO send_account_transfers ( + f, + t, + v, + block_time, + ig_name, + src_name, + tx_hash, + block_num, + tx_idx, + log_idx, + abi_idx, + chain_id, + log_addr +) +VALUES ( + '\xa71CE00000000000000000000000000000000000', + '\xb055000000000000000000000000000000000000', + 100, + floor(extract(EPOCH FROM timestamptz '2013-07-01 12:00:00')), + 'send_account_transfers', + 'send_account_transfers', + '\x1234', + 1, + 0, + 0, + 0, + 8453, + '\xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266' +); + +SELECT is_empty( + $$ + select 1 + from send_account_transfers + where f = '\xa71CE00000000000000000000000000000000000' + $$, + 'Test if the trigger function filters send_account_transfers with no send_account_created' +); + +select is_empty( + $$ + select 1 + from send_account_transfers + $$, + 'Test if the trigger function filters send_account_transfers with no send_account_created' +); + SELECT * FROM finish(); ROLLBACK;