From 446e2fa17bac542f84f83dc0b4bbd1c17cfeb132 Mon Sep 17 00:00:00 2001 From: youngkidwarrior Date: Wed, 9 Oct 2024 23:32:30 -0700 Subject: [PATCH] DistributorV2 --- apps/distributor/src/app.ts | 25 ++++ apps/distributor/src/distributorv2.ts | 189 +++++++++----------------- apps/distributor/src/weights.ts | 13 ++ 3 files changed, 103 insertions(+), 124 deletions(-) diff --git a/apps/distributor/src/app.ts b/apps/distributor/src/app.ts index ef9dc4377..348d21e8f 100644 --- a/apps/distributor/src/app.ts +++ b/apps/distributor/src/app.ts @@ -4,6 +4,7 @@ import { DistributorV1Worker } from './distributor' import { StandardMerkleTree } from '@openzeppelin/merkle-tree' import { selectAll } from 'app/utils/supabase/selectAll' import { supabaseAdmin } from './supabase' +import { DistributorV2Worker } from './distributorv2' const logger = pino({ level: process.env.LOG_LEVEL || 'info', @@ -12,6 +13,7 @@ const logger = pino({ // Initialize DistributorWorker const distributorV1Worker = new DistributorV1Worker(logger) +const distributorV2Worker = new DistributorV2Worker(logger) // Initialize Express app const app = express() @@ -32,6 +34,13 @@ distributorRouter.get('/v1', async (req: Request, res: Response) => { }) }) +distributorRouter.get('/v2', async (req: Request, res: Response) => { + res.json({ + distributor: true, + ...distributorV2Worker.toJSON(), + }) +}) + // Middleware for checking authorization const checkAuthorization = (req: Request, res: Response, next: () => void) => { if (!req.headers.authorization?.includes(process.env.SUPABASE_SERVICE_ROLE as string)) { @@ -117,6 +126,22 @@ distributorRouter.post('/v1', checkAuthorization, async (req, res) => { }) }) +distributorRouter.post('/v2', checkAuthorization, async (req, res) => { + const { id } = req.body as { id: string } + logger.info({ id }, 'Received request to calculate distribution') + try { + await distributorV2Worker.calculateDistribution(id) + } catch (err) { + logger.error(err, 'Error while calculating distribution') + throw err + } + + res.json({ + distributor: true, + id: id, + }) +}) + app.use('/distributor', distributorRouter) export default app diff --git a/apps/distributor/src/distributorv2.ts b/apps/distributor/src/distributorv2.ts index 480136250..18f8f684a 100644 --- a/apps/distributor/src/distributorv2.ts +++ b/apps/distributor/src/distributorv2.ts @@ -9,7 +9,7 @@ import { supabaseAdmin, } from './supabase' import { fetchAllBalances, isMerkleDropActive } from './wagmi' -import { calculateWeights, calculatePercentageWithBips, PERC_DENOM } from './weights' +import { calculateWeights, calculateMultiplier, PERC_DENOM } from './weights' const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) @@ -47,9 +47,6 @@ export class DistributorV2Worker { } } - /** - * Calculates distribution shares for distributions in qualification period. - */ private async calculateDistributions() { this.log.info('Calculating distributions') @@ -57,7 +54,7 @@ export class DistributorV2Worker { .from('distributions') .select( `*, - distribution_verification_values (*)`, + distribution_verification_values (*)`, { count: 'exact' } ) .lte('qualification_start', new Date().toISOString()) @@ -103,9 +100,6 @@ export class DistributorV2Worker { } } - /** - * Calculates distribution shares for a single distribution. - */ private async _calculateDistributionShares( distribution: Tables<'distributions'> & { distribution_verification_values: Tables<'distribution_verification_values'>[] @@ -113,7 +107,6 @@ export class DistributorV2Worker { ): Promise { const log = this.log.child({ distribution_id: distribution.id }) - // verify tranche is not created when in production if (await isMerkleDropActive(distribution)) { throw new Error('Tranche is active. Cannot calculate distribution shares.') } @@ -140,7 +133,6 @@ export class DistributorV2Worker { } log.info(`Found ${verifications.length} verifications.`) - // log.debug({ verifications }) if (log.isLevelEnabled('debug')) { await Bun.write( 'dist/verifications.json', @@ -155,12 +147,21 @@ export class DistributorV2Worker { acc[verification.type] = { fixedValue: BigInt(verification.fixed_value), bipsValue: BigInt(verification.bips_value), + multiplier_min: verification.multiplier_min, + multiplier_max: verification.multiplier_max, + multiplier_step: verification.multiplier_step, } return acc }, {} as Record< Database['public']['Enums']['verification_type'], - { fixedValue?: bigint; bipsValue?: bigint } + { + fixedValue?: bigint + bipsValue?: bigint + multiplier_min: number + multiplier_max: number + multiplier_step: number + } > ) const verificationsByUserId = verifications.reduce( @@ -173,7 +174,6 @@ export class DistributorV2Worker { ) log.info(`Found ${Object.keys(verificationsByUserId).length} users with verifications.`) - // log.debug({ verificationsByUserId }) if (log.isLevelEnabled('debug')) { await Bun.write( 'dist/verificationsByUserId.json', @@ -211,7 +211,6 @@ export class DistributorV2Worker { ) log.info(`Found ${hodlerAddresses.length} addresses.`) - // log.debug({ hodlerAddresses }) if (log.isLevelEnabled('debug')) { await Bun.write( 'dist/hodlerAddresses.json', @@ -221,7 +220,6 @@ export class DistributorV2Worker { }) } - // lookup balances of all hodler addresses in qualification period const batches = inBatches(hodlerAddresses).flatMap(async (addresses) => { return await Promise.all( fetchAllBalances({ @@ -237,9 +235,7 @@ export class DistributorV2Worker { } log.info(`Found ${minBalanceAddresses.length} balances.`) - // log.debug({ balances }) - // Filter out hodler with not enough send token balance minBalanceAddresses = minBalanceAddresses.filter( ({ balance }) => BigInt(balance) >= BigInt(distribution.hodler_min_balance) ) @@ -247,7 +243,6 @@ export class DistributorV2Worker { log.info( `Found ${minBalanceAddresses.length} balances after filtering hodler_min_balance of ${distribution.hodler_min_balance}` ) - // log.debug({ balances }) if (log.isLevelEnabled('debug')) { await Bun.write( @@ -258,12 +253,9 @@ export class DistributorV2Worker { }) } - // Calculate hodler pool share weights const distAmt = BigInt(distribution.amount) - const hodlerPoolBips = BigInt(distribution.hodler_pool_bips) - const fixedPoolBips = BigInt(distribution.fixed_pool_bips) - const bonusPoolBips = BigInt(distribution.bonus_pool_bips) - const hodlerPoolAvailableAmount = calculatePercentageWithBips(distAmt, hodlerPoolBips) + const fixedPoolAvailableAmount = distAmt + const minBalanceByAddress: Record = minBalanceAddresses.reduce( (acc, balance) => { acc[balance.address] = BigInt(balance.balance) @@ -271,77 +263,58 @@ export class DistributorV2Worker { }, {} as Record ) - const { totalWeight, weightPerSend, poolWeights, weightedShares } = calculateWeights( - minBalanceAddresses, - hodlerPoolAvailableAmount - ) - log.info( - { totalWeight, hodlerPoolAvailableAmount, weightPerSend }, - `Calculated ${Object.keys(poolWeights).length} weights.` - ) - // log.debug({ poolWeights }) - if (log.isLevelEnabled('debug')) { - await Bun.write('dist/poolWeights.json', JSON.stringify(poolWeights, jsonBigint, 2)).catch( - (e) => { - log.error(e, 'Error writing poolWeights.json') - } - ) - } - - if (totalWeight === 0n) { - log.warn('Total weight is 0. Skipping distribution.') - return - } - - const fixedPoolAvailableAmount = calculatePercentageWithBips(distAmt, fixedPoolBips) let fixedPoolAllocatedAmount = 0n const fixedPoolAmountsByAddress: Record = {} - const bonusPoolBipsByAddress: Record = {} - const maxBonusPoolBips = (bonusPoolBips * PERC_DENOM) / hodlerPoolBips // 3500*10000/6500 = 5384.615384615385% 1.53X for (const [userId, verifications] of Object.entries(verificationsByUserId)) { const hodler = hodlerAddressesByUserId[userId] - if (!hodler || !hodler.address) { - continue - } + if (!hodler || !hodler.address) continue const { address } = hodler - if (!minBalanceByAddress[address]) { - continue - } + if (!minBalanceByAddress[address]) continue + + let totalReferrals = 0n + let userFixedAmount = 0n + for (const verification of verifications) { - const { fixedValue, bipsValue } = verificationValues[verification.type] - if (fixedValue && fixedPoolAllocatedAmount + fixedValue <= fixedPoolAvailableAmount) { - if (fixedPoolAmountsByAddress[address] === undefined) { - fixedPoolAmountsByAddress[address] = 0n - } - fixedPoolAmountsByAddress[address] += fixedValue - fixedPoolAllocatedAmount += fixedValue - } - if (bipsValue) { - bonusPoolBipsByAddress[address] = (bonusPoolBipsByAddress[address] || 0n) as bigint - bonusPoolBipsByAddress[address] += bipsValue - bonusPoolBipsByAddress[address] = - (bonusPoolBipsByAddress[address] as bigint) > maxBonusPoolBips - ? maxBonusPoolBips - : (bonusPoolBipsByAddress[address] as bigint) // cap at max bonus pool bips + const verificationValue = verificationValues[verification.type] + if (verification.type === 'total_tag_referral') { + // @ts-expect-error this is json + totalReferrals = BigInt(verification.metadata?.total_referrals ?? 0) + } else if (verificationValue.fixedValue) { + userFixedAmount += verificationValue.fixedValue } } + + if (totalReferrals > 0n) { + const totalTagReferralValue = verificationValues.total_tag_referral + const multiplier = calculateMultiplier(totalReferrals, totalTagReferralValue) + userFixedAmount = (userFixedAmount * multiplier) / PERC_DENOM + } + + if ( + userFixedAmount > 0n && + fixedPoolAllocatedAmount + userFixedAmount <= fixedPoolAvailableAmount + ) { + fixedPoolAmountsByAddress[address] = + (fixedPoolAmountsByAddress[address] || 0n) + userFixedAmount + fixedPoolAllocatedAmount += userFixedAmount + } } - const hodlerShares = Object.values(weightedShares) - let totalAmount = 0n - let totalHodlerPoolAmount = 0n - let totalBonusPoolAmount = 0n - let totalFixedPoolAmount = 0n + const hodlerPoolAvailableAmount = distAmt - fixedPoolAllocatedAmount + + let hodlerShares: { address: string; amount: bigint }[] = [] + if (hodlerPoolAvailableAmount > 0n) { + const { weightedShares } = calculateWeights(minBalanceAddresses, hodlerPoolAvailableAmount) + hodlerShares = Object.values(weightedShares) + } + + const totalAmount = 0n + const totalHodlerPoolAmount = 0n + const totalBonusPoolAmount = 0n + const totalFixedPoolAmount = 0n - log.info( - { - maxBonusPoolBips, - }, - 'Calculated fixed & bonus pool amounts.' - ) - // log.debug({ hodlerShares, fixedPoolAmountsByAddress, bonusPoolBipsByAddress }) if (log.isLevelEnabled('debug')) { await Bun.write('dist/hodlerShares.json', JSON.stringify(hodlerShares, jsonBigint, 2)).catch( (e) => { @@ -354,53 +327,24 @@ export class DistributorV2Worker { ).catch((e) => { log.error(e, 'Error writing fixedPoolAmountsByAddress.json') }) - await Bun.write( - 'dist/bonusPoolBipsByAddress.json', - JSON.stringify(bonusPoolBipsByAddress, jsonBigint, 2) - ).catch((e) => { - log.error(e, 'Error writing bonusPoolBipsByAddress.json') - }) } - const shares = hodlerShares - .map((share) => { - const userId = hodlerUserIdByAddress[share.address] - const bonusBips = bonusPoolBipsByAddress[share.address] || 0n - const hodlerPoolAmount = share.amount - const bonusPoolAmount = calculatePercentageWithBips(hodlerPoolAmount, bonusBips) - const fixedPoolAmount = fixedPoolAmountsByAddress[share.address] || 0n - const amount = hodlerPoolAmount + bonusPoolAmount + fixedPoolAmount - totalAmount += amount - totalHodlerPoolAmount += hodlerPoolAmount - totalBonusPoolAmount += bonusPoolAmount - totalFixedPoolAmount += fixedPoolAmount - - if (!userId) { - log.debug({ share }, 'Hodler not found for address. Skipping share.') - return null - } - // log.debug( - // { - // address: share.address, - // balance: balancesByAddress[share.address], - // amount: amount, - // bonusBips, - // hodlerPoolAmount, - // bonusPoolAmount, - // fixedPoolAmount, - // }, - // 'Calculated share.' - // ) + const shares = Object.entries(fixedPoolAmountsByAddress) + .map(([address, fixedAmount]) => { + const hodlerShare = hodlerShares.find((share) => share.address === address) + const hodlerAmount = hodlerShare ? hodlerShare.amount : 0n + const totalAmount = fixedAmount + hodlerAmount + const userId = hodlerUserIdByAddress[address] // @ts-expect-error supabase-js does not support bigint return { - address: share.address, + address, distribution_id: distribution.id, user_id: userId, - amount: amount.toString(), - bonus_pool_amount: bonusPoolAmount.toString(), - fixed_pool_amount: fixedPoolAmount.toString(), - hodler_pool_amount: hodlerPoolAmount.toString(), + amount: totalAmount.toString(), + fixed_pool_amount: fixedAmount.toString(), + hodler_pool_amount: hodlerAmount.toString(), + bonus_pool_amount: '0', } as Tables<'distribution_shares'> }) .filter(Boolean) as Tables<'distribution_shares'>[] @@ -414,14 +358,12 @@ export class DistributorV2Worker { totalFixedPoolAmount, fixedPoolAllocatedAmount, fixedPoolAvailableAmount, - maxBonusPoolBips, name: distribution.name, shares: shares.length, }, 'Distribution totals' ) log.info(`Calculated ${shares.length} shares.`) - // log.debug({ shares }) if (log.isLevelEnabled('debug')) { await Bun.write('dist/shares.json', JSON.stringify(shares, jsonBigint, 2)).catch((e) => { log.error(e, 'Error writing shares.json') @@ -434,7 +376,6 @@ export class DistributorV2Worker { ) } - // ensure share amounts do not exceed the total distribution amount, ideally this should be done in the database const totalShareAmounts = shares.reduce((acc, share) => acc + BigInt(share.amount), 0n) if (totalShareAmounts > distAmt) { throw new Error('Share amounts exceed total distribution amount') @@ -456,7 +397,7 @@ export class DistributorV2Worker { } catch (error) { this.log.error(error, `Error processing block. ${(error as Error).message}`) } - await sleep(60_000) // sleep for 1 minute + await sleep(60_000) } this.log.info('Distributor stopped.') diff --git a/apps/distributor/src/weights.ts b/apps/distributor/src/weights.ts index 0b3b18ef1..685ac9b08 100644 --- a/apps/distributor/src/weights.ts +++ b/apps/distributor/src/weights.ts @@ -33,6 +33,19 @@ export function calculatePercentageWithBips(value: bigint, bips: bigint) { return percentage / PERC_DENOM } +export function calculateMultiplier( + totalReferrals: bigint, + verificationValue: { multiplier_min: number; multiplier_max: number; multiplier_step: number } +): bigint { + const { multiplier_min, multiplier_max, multiplier_step } = verificationValue + const steps = Math.min( + Number(totalReferrals), + Math.floor((multiplier_max - multiplier_min) / multiplier_step) + ) + const multiplier = multiplier_min + steps * multiplier_step + return BigInt(Math.floor(multiplier * Number(PERC_DENOM))) +} + /** * Given a list of balances and a distribution amount, calculate the distribution weights and share amounts. */