bb/supabase cleanup (#634)

* supabase cleanup drop duplicate send_account_created index enable pg_cron create cronjob for deleting send account transfers with no send account created row * add delete send account transfers with no send account * go back to indexing all USDC an SEND token transfers * just filter the rows actually * fix checkout specs to be less flaky
0xsend · Jul 28, 2024 · 3db279b · 3db279b
1 parent 8a07da4
commit 3db279b
Show file tree

Hide file tree

Showing 8 changed files with 130 additions and 35 deletions.
diff --git a/packages/playwright/tests/account-sendtag-checkout.onboarded.spec.ts b/packages/playwright/tests/account-sendtag-checkout.onboarded.spec.ts
@@ -112,7 +112,8 @@ const verifyReferralReward = async (
   return rewardAmount
 }
 
-const generateTagName = () => faker.string.alphanumeric({ length: { min: 1, max: 20 } })
+// tag names are limited to 1-20 characters
+const generateTagName = () => faker.string.alphanumeric({ length: { min: 6, max: 20 } })
 
 const checkReferralCodeVisibility = async (
   checkoutPage: CheckoutPage,
@@ -207,7 +208,7 @@ test('can refer a tag', async ({
   const tagsToRegister = Array.from(
     { length: Math.floor(Math.random() * 5) + 1 },
     () => `${faker.lorem.word()}_${test.info().parallelIndex}`
-  ).sort()
+  ).sort((a, b) => a.localeCompare(b))
   for (const tagName of tagsToRegister) {
     await addPendingTag(checkoutPage, tagName)
   }
@@ -283,7 +284,7 @@ test('can refer multiple tags in separate transactions', async ({
   // First transaction with up to 2 tags
   const firstTags = Array.from({ length: Math.floor(Math.random() * 2) + 1 }, () =>
     generateTagName()
-  ).sort()
+  ).sort((a, b) => a.localeCompare(b))
   for (const tagName of firstTags) {
     await addPendingTag(checkoutPage, tagName)
   }
@@ -315,7 +316,7 @@ test('can refer multiple tags in separate transactions', async ({
   // Second transaction with up to 3 tags
   const secondTags = Array.from({ length: Math.floor(Math.random() * 3) + 1 }, () =>
     generateTagName()
-  ).sort()
+  ).sort((a, b) => a.localeCompare(b))
   for (const tagName of secondTags) {
     await addPendingTag(checkoutPage, tagName)
   }

diff --git a/packages/playwright/tests/fixtures/checkout/page.ts b/packages/playwright/tests/fixtures/checkout/page.ts
@@ -104,6 +104,11 @@ export class CheckoutPage {
     const signTransactionButton = this.page.getByRole('button', { name: 'Confirm' })
     expect?.(signTransactionButton).toBeEnabled({ timeout: 10_000 }) // blockchain stuff is a bit slow
     await this.page.bringToFront()
+    const errorButton = this.page.getByRole('button', { name: 'Error' })
+    expect?.(errorButton).toBeHidden()
+    await this.page
+      .getByRole('button', { name: 'Loading' })
+      .waitFor({ state: 'detached', timeout: 10_000 })
     await signTransactionButton.click()
     await confirmTagsRequest
     await confirmTagsResponse

diff --git a/packages/shovel/etc/config.json b/packages/shovel/etc/config.json
@@ -152,7 +152,16 @@
         },
         {
           "name": "log_addr",
-          "column": "log_addr"
+          "column": "log_addr",
+          "filter_op": "contains",
+          "filter_arg": [
+            "0x036CbD53842c5426634e7929541eC2318f3dCF7e",
+            "0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238",
+            "0x3f14920c99BEB920Afa163031c4e47a3e03B3e4A",
+            "0x7cEfbe54c37a35dCdaD29b86373ca8353a2F4680",
+            "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
+            "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"
+          ]
         }
       ],
       "event": {
@@ -164,23 +173,13 @@
             "indexed": true,
             "name": "from",
             "type": "address",
-            "column": "f",
-            "filter_op": "contains",
-            "filter_ref": {
-              "integration": "send_account_created",
-              "column": "account"
-            }
+            "column": "f"
           },
           {
             "indexed": true,
             "name": "to",
             "type": "address",
-            "column": "t",
-            "filter_op": "contains",
-            "filter_ref": {
-              "integration": "send_account_created",
-              "column": "account"
-            }
+            "column": "t"
           },
           {
             "indexed": false,

diff --git a/packages/shovel/src/integrations/send-account-transfers.ts b/packages/shovel/src/integrations/send-account-transfers.ts
@@ -36,6 +36,11 @@ export const integration: Omit<Integration, 'sources'> = {
     {
       name: 'log_addr',
       column: 'log_addr',
+      filter_op: 'contains',
+      filter_arg: [
+        ...new Set(Object.values(sendTokenAddress)),
+        ...new Set(Object.values(usdcAddress)),
+      ].sort(),
     },
   ] as BlockData[],
   event: {
@@ -48,16 +53,18 @@ export const integration: Omit<Integration, 'sources'> = {
         name: 'from',
         type: 'address',
         column: 'f',
-        filter_op: 'contains',
-        filter_ref: sendAccountFactorySenderFilterRef,
+        // wait for this https://github.com/orgs/indexsupply/discussions/268
+        // filter_op: 'contains',
+        // filter_ref: sendAccountFactorySenderFilterRef,
       },
       {
         indexed: true,
         name: 'to',
         type: 'address',
         column: 't',
-        filter_op: 'contains',
-        filter_ref: sendAccountFactorySenderFilterRef,
+        // wait for this https://github.com/orgs/indexsupply/discussions/268
+        // filter_op: 'contains',
+        // filter_ref: sendAccountFactorySenderFilterRef,
       },
       {
         indexed: false,

diff --git a/packages/shovel/test/__snapshots__/index.test.ts.snap b/packages/shovel/test/__snapshots__/index.test.ts.snap
@@ -97,6 +97,15 @@ exports[`shovel config 1`] = `
         },
         {
           "column": "log_addr",
+          "filter_arg": [
+            "0x036CbD53842c5426634e7929541eC2318f3dCF7e",
+            "0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238",
+            "0x3f14920c99BEB920Afa163031c4e47a3e03B3e4A",
+            "0x7cEfbe54c37a35dCdaD29b86373ca8353a2F4680",
+            "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
+            "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48",
+          ],
+          "filter_op": "contains",
           "name": "log_addr",
         },
       ],
@@ -106,22 +115,12 @@ exports[`shovel config 1`] = `
         "inputs": [
           {
             "column": "f",
-            "filter_op": "contains",
-            "filter_ref": {
-              "column": "account",
-              "integration": "send_account_created",
-            },
             "indexed": true,
             "name": "from",
             "type": "address",
           },
           {
             "column": "t",
-            "filter_op": "contains",
-            "filter_ref": {
-              "column": "account",
-              "integration": "send_account_created",
-            },
             "indexed": true,
             "name": "to",
             "type": "address",

diff --git a/supabase/migrations/20240728180824_delete_duplicate_send_account_created_account_index.sql b/supabase/migrations/20240728180824_delete_duplicate_send_account_created_account_index.sql
@@ -0,0 +1 @@
+drop index if exists "public"."send_account_created_account";
diff --git a/supabase/migrations/202407281808278_send_account_transfers_filter_non_send_accounts.sql b/supabase/migrations/202407281808278_send_account_transfers_filter_non_send_accounts.sql
@@ -0,0 +1,29 @@
+set check_function_bodies = off;
+
+-- create trigger function for filtering send_account_transfers with no send_account_created
+create or replace function private.filter_send_account_transfers_with_no_send_account_created()
+ returns trigger
+ language plpgsql
+ security definer
+ as $$
+begin
+-- Deletes send_account_transfers with no send_account_created.
+-- This is due to performance issues in our shovel indexer and using filter_ref to limit indexing to only
+-- send_account_transfers with send_account_created.
+-- For now, we index all USDC and SEND token transfers, and use this function filter any send_account_transfers with no send_account_created.
+-- See https://github.com/orgs/indexsupply/discussions/268
+  if exists ( select 1 from send_account_created where account = new.f )
+    or exists ( select 1 from send_account_created where account = new.t )
+  then
+    return new;
+  else
+    return null;
+  end if;
+end;
+$$;
+
+-- create trigger on send_account_transfers table
+create trigger filter_send_account_transfers_with_no_send_account_created
+before insert on public.send_account_transfers
+for each row
+execute function private.filter_send_account_transfers_with_no_send_account_created();
diff --git a/supabase/tests/send_account_transfers_test.sql b/supabase/tests/send_account_transfers_test.sql
@@ -1,5 +1,5 @@
 BEGIN;
-SELECT plan(3);
+SELECT plan(5);
 
 -- Create the necessary extensions
 CREATE EXTENSION "basejump-supabase_test_helpers"; -- noqa: RF05
@@ -8,6 +8,10 @@ CREATE EXTENSION "basejump-supabase_test_helpers"; -- noqa: RF05
 SELECT tests.create_supabase_user('test_user_from');
 SELECT tests.create_supabase_user('test_user_to');
 
+insert into send_account_created (chain_id, log_addr, block_time, user_op_hash, tx_hash, account, ig_name, src_name, block_num, tx_idx, log_idx)
+values (8453, '\xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266', floor(extract(EPOCH FROM timestamptz '2013-07-01 12:00:00')), '\x1234', '\x1234', '\x1234567890ABCDEF1234567890ABCDEF12345678', 'send_account_created', 'send_account_created', 1, 0, 0),
+       (8453, '\xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266', floor(extract(EPOCH FROM timestamptz '2013-07-01 12:00:00')), '\x1234', '\x1234', '\xB0B7D5E8A4B6D534B3F608E9D27871F85A4E98DA', 'send_account_transfers', 'send_account_transfers', 1, 0, 0);
+
 INSERT INTO send_accounts (user_id, address, chain_id, init_code)
 VALUES (
     tests.get_supabase_uid('test_user_from'),
@@ -57,7 +61,7 @@ VALUES (
 -- Test if the trigger function populated the additional columns correctly
 SELECT results_eq(
     $$
-        select 
+        select
             (data->>'log_addr')::citext as log_addr,
             (data->>'f')::citext as f,
             (data->>'t')::citext as t,
@@ -102,8 +106,8 @@ WHERE id = (
 -- Test if the delete was successful
 SELECT is_empty(
     $$
-        SELECT id 
-        FROM send_account_transfers 
+        SELECT id
+        FROM send_account_transfers
         WHERE f = '\x1234567890ABCDEF1234567890ABCDEF12345678'::bytea AND t = '\xB0B7D5E8A4B6D534B3F608E9D27871F85A4E98DA'::bytea
     $$,
     'Test if the trigger function removes the activity row'
@@ -120,5 +124,55 @@ SELECT is_empty(
     'Test if the trigger function removes the activity row'
 );
 
+-- Test filter_non_send_account_transfers_before_insert removes transactions when the sender is not a send account
+delete from send_account_transfers;
+INSERT INTO send_account_transfers (
+    f,
+    t,
+    v,
+    block_time,
+    ig_name,
+    src_name,
+    tx_hash,
+    block_num,
+    tx_idx,
+    log_idx,
+    abi_idx,
+    chain_id,
+    log_addr
+)
+VALUES (
+    '\xa71CE00000000000000000000000000000000000',
+    '\xb055000000000000000000000000000000000000',
+    100,
+    floor(extract(EPOCH FROM timestamptz '2013-07-01 12:00:00')),
+    'send_account_transfers',
+    'send_account_transfers',
+    '\x1234',
+    1,
+    0,
+    0,
+    0,
+    8453,
+    '\xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266'
+);
+
+SELECT is_empty(
+    $$
+        select 1
+        from send_account_transfers
+        where f = '\xa71CE00000000000000000000000000000000000'
+    $$,
+    'Test if the trigger function filters send_account_transfers with no send_account_created'
+);
+
+select is_empty(
+    $$
+        select 1
+        from send_account_transfers
+    $$,
+    'Test if the trigger function filters send_account_transfers with no send_account_created'
+);
+
 SELECT * FROM finish();
 ROLLBACK;