From d3612980393237ba72d7e540e236d1860575761d Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 6 Nov 2020 07:34:23 -0800 Subject: [PATCH 1/5] logs/history speed optimizations --- src/balancer/OceanPool.ts | 17 ++++++++++------- src/balancer/Pool.ts | 5 ++--- src/balancer/PoolFactory.ts | 4 +--- src/exchange/FixedRateExchange.ts | 6 +++++- src/models/Config.ts | 5 +++++ src/ocean/Ocean.ts | 6 ++++-- src/utils/ConfigHelper.ts | 12 ++++++++---- test/unit/balancer/Balancer.test.ts | 3 ++- 8 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/balancer/OceanPool.ts b/src/balancer/OceanPool.ts index c587ec95..398e5a55 100644 --- a/src/balancer/OceanPool.ts +++ b/src/balancer/OceanPool.ts @@ -54,6 +54,7 @@ export enum PoolCreateProgressStep { export class OceanPool extends Pool { public oceanAddress: string = null public dtAddress: string = null + public startBlock: number constructor( web3: Web3, @@ -62,12 +63,14 @@ export class OceanPool extends Pool { poolABI: AbiItem | AbiItem[] = null, factoryAddress: string = null, oceanAddress: string = null, - gaslimit?: number + startBlock?: number ) { - super(web3, logger, factoryABI, poolABI, factoryAddress, gaslimit) + super(web3, logger, factoryABI, poolABI, factoryAddress) if (oceanAddress) { this.oceanAddress = oceanAddress } + if (startBlock) this.startBlock = startBlock + else this.startBlock = 0 } /** @@ -855,7 +858,7 @@ export class OceanPool extends Pool { const factory = new this.web3.eth.Contract(this.factoryABI, this.factoryAddress) const events = await factory.getPastEvents('BPoolRegistered', { filter: {}, - fromBlock: BPFACTORY_DEPLOY_BLOCK, + fromBlock: this.startBlock, toBlock: 'latest' }) events.sort((a, b) => (a.blockNumber > b.blockNumber ? 1 : -1)) @@ -920,7 +923,7 @@ export class OceanPool extends Pool { const events = await factory.getPastEvents('BPoolRegistered', { filter: account ? { registeredBy: account } : {}, - fromBlock: BPFACTORY_DEPLOY_BLOCK, + fromBlock: this.startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { @@ -941,7 +944,7 @@ export class OceanPool extends Pool { const events = await factory.getPastEvents('BPoolRegistered', { filter: {}, - fromBlock: BPFACTORY_DEPLOY_BLOCK, + fromBlock: this.startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { @@ -988,7 +991,7 @@ export class OceanPool extends Pool { const dtAddress = await this.getDTAddress(poolAddress) const filter: Filter = account ? { caller: account } : {} let events: EventData[] - + if (startBlock === 0) startBlock = this.startBlock events = await pool.getPastEvents('LOG_SWAP', { filter, fromBlock: startBlock, @@ -1034,7 +1037,7 @@ export class OceanPool extends Pool { const factory = new this.web3.eth.Contract(this.factoryABI, this.factoryAddress) const events = await factory.getPastEvents('BPoolRegistered', { filter: {}, - fromBlock: BPFACTORY_DEPLOY_BLOCK, + fromBlock: this.startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { diff --git a/src/balancer/Pool.ts b/src/balancer/Pool.ts index 26ef8256..e27dc5ed 100644 --- a/src/balancer/Pool.ts +++ b/src/balancer/Pool.ts @@ -26,10 +26,9 @@ export class Pool extends PoolFactory { logger: Logger, factoryABI: AbiItem | AbiItem[] = null, poolABI: AbiItem | AbiItem[] = null, - factoryAddress: string = null, - gaslimit?: number + factoryAddress: string = null ) { - super(web3, logger, factoryABI, factoryAddress, gaslimit) + super(web3, logger, factoryABI, factoryAddress) if (poolABI) this.poolABI = poolABI else this.poolABI = jsonpoolABI.abi as AbiItem[] } diff --git a/src/balancer/PoolFactory.ts b/src/balancer/PoolFactory.ts index c6811e69..953176e4 100644 --- a/src/balancer/PoolFactory.ts +++ b/src/balancer/PoolFactory.ts @@ -15,8 +15,7 @@ export class PoolFactory { web3: Web3, logger: Logger, factoryABI: AbiItem | AbiItem[] = null, - factoryAddress: string = null, - gaslimit?: number + factoryAddress: string = null ) { this.web3 = web3 @@ -25,7 +24,6 @@ export class PoolFactory { if (factoryAddress) { this.factoryAddress = factoryAddress } - if (gaslimit) this.GASLIMIT_DEFAULT = gaslimit this.logger = logger } diff --git a/src/exchange/FixedRateExchange.ts b/src/exchange/FixedRateExchange.ts index af7c614f..1e748a1b 100644 --- a/src/exchange/FixedRateExchange.ts +++ b/src/exchange/FixedRateExchange.ts @@ -39,6 +39,7 @@ export class OceanFixedRateExchange { public contract: Contract = null private logger: Logger public datatokens: DataTokens + public startBlock: number /** * Instantiate FixedRateExchange @@ -53,10 +54,13 @@ export class OceanFixedRateExchange { fixedRateExchangeAddress: string = null, fixedRateExchangeABI: AbiItem | AbiItem[] = null, oceanAddress: string = null, - datatokens: DataTokens + datatokens: DataTokens, + startBlock?: number ) { this.web3 = web3 this.fixedRateExchangeAddress = fixedRateExchangeAddress + if (startBlock) this.startBlock = startBlock + else this.startBlock = 0 this.fixedRateExchangeABI = fixedRateExchangeABI || (defaultFixedRateExchangeABI.abi as AbiItem[]) this.oceanAddress = oceanAddress diff --git a/src/models/Config.ts b/src/models/Config.ts index d1cf7d57..e7a3e924 100644 --- a/src/models/Config.ts +++ b/src/models/Config.ts @@ -96,6 +96,11 @@ export class Config { * @type {any} */ public metadataContractABI?: AbiItem | AbiItem[] + /** + * block number of the deployment + * @type {number} + */ + public startBlock?: number /** * Log level. * @type {boolean | LogLevel} diff --git a/src/ocean/Ocean.ts b/src/ocean/Ocean.ts index 4318e0db..9c933ac9 100644 --- a/src/ocean/Ocean.ts +++ b/src/ocean/Ocean.ts @@ -59,7 +59,8 @@ export class Ocean extends Instantiable { instanceConfig.config.poolFactoryABI, instanceConfig.config.poolABI, instanceConfig.config.poolFactoryAddress, - instanceConfig.config.oceanTokenAddress + instanceConfig.config.oceanTokenAddress, + instanceConfig.config.startBlock ) instance.fixedRateExchange = new OceanFixedRateExchange( instanceConfig.config.web3Provider, @@ -67,7 +68,8 @@ export class Ocean extends Instantiable { instanceConfig.config.fixedRateExchangeAddress, instanceConfig.config.fixedRateExchangeAddressABI, instanceConfig.config.oceanTokenAddress, - instance.datatokens + instance.datatokens, + instanceConfig.config.startBlock ) instance.OnChainMetadataCache = new OnChainMetadataCache( instanceConfig.config.web3Provider, diff --git a/src/utils/ConfigHelper.ts b/src/utils/ConfigHelper.ts index 2da3651e..cde9930c 100644 --- a/src/utils/ConfigHelper.ts +++ b/src/utils/ConfigHelper.ts @@ -27,7 +27,8 @@ const configs: ConfigHelperConfig[] = [ factoryAddress: '0x1234', poolFactoryAddress: null, fixedRateExchangeAddress: null, - metadataContractAddress: null + metadataContractAddress: null, + startBlock: 0 }, { // barge @@ -40,7 +41,8 @@ const configs: ConfigHelperConfig[] = [ factoryAddress: null, poolFactoryAddress: null, fixedRateExchangeAddress: null, - metadataContractAddress: null + metadataContractAddress: null, + startBlock: 0 }, { networkId: 4, @@ -52,7 +54,8 @@ const configs: ConfigHelperConfig[] = [ factoryAddress: null, poolFactoryAddress: null, fixedRateExchangeAddress: null, - metadataContractAddress: null + metadataContractAddress: null, + startBlock: 7294090 }, { networkId: 1, @@ -64,7 +67,8 @@ const configs: ConfigHelperConfig[] = [ factoryAddress: null, poolFactoryAddress: null, fixedRateExchangeAddress: null, - metadataContractAddress: null + metadataContractAddress: null, + startBlock: 11105522 } ] diff --git a/test/unit/balancer/Balancer.test.ts b/test/unit/balancer/Balancer.test.ts index cf92e5ba..d990048f 100644 --- a/test/unit/balancer/Balancer.test.ts +++ b/test/unit/balancer/Balancer.test.ts @@ -109,7 +109,8 @@ describe('Balancer flow', () => { OceanPoolFactory.abi as AbiItem[], OceanSPool.abi as AbiItem[], OceanPoolFactoryAddress, - oceanTokenAddress + oceanTokenAddress, + 0 ) assert(Pool !== null) }) From 31f8c38ca8c90a719edbe5d5d4c6d9eebb02682f Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 6 Nov 2020 07:37:50 -0800 Subject: [PATCH 2/5] speed up fixedrate --- src/exchange/FixedRateExchange.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/exchange/FixedRateExchange.ts b/src/exchange/FixedRateExchange.ts index 1e748a1b..9948f54d 100644 --- a/src/exchange/FixedRateExchange.ts +++ b/src/exchange/FixedRateExchange.ts @@ -363,7 +363,7 @@ export class OceanFixedRateExchange { const result: FixedPriceExchange[] = [] const events = await this.contract.getPastEvents('ExchangeCreated', { filter: { datatoken: dataTokenAddress.toLowerCase() }, - fromBlock: 0, + fromBlock: this.startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { @@ -392,7 +392,7 @@ export class OceanFixedRateExchange { const result: FixedPriceExchange[] = [] const events = await this.contract.getPastEvents('ExchangeCreated', { filter: {}, - fromBlock: 0, + fromBlock: this.startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { @@ -415,7 +415,7 @@ export class OceanFixedRateExchange { const result: FixedPriceSwap[] = [] const events = await this.contract.getPastEvents('Swapped', { filter: { exchangeId: exchangeId }, - fromBlock: 0, + fromBlock: this.startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { @@ -434,7 +434,7 @@ export class OceanFixedRateExchange { const result: FixedPriceSwap[] = [] const events = await this.contract.getPastEvents('ExchangeCreated', { filter: {}, - fromBlock: 0, + fromBlock: this.startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { From 1d717cb3b03b150d16bb684bd13a1d7bdafa55a6 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 6 Nov 2020 23:49:08 -0800 Subject: [PATCH 3/5] complete refactor getPoolLogs & getEventData --- src/balancer/OceanPool.ts | 78 ++++++++++++++++++--------------------- src/balancer/Pool.ts | 39 ++++++++++++++++++++ 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/src/balancer/OceanPool.ts b/src/balancer/OceanPool.ts index 398e5a55..69803fc7 100644 --- a/src/balancer/OceanPool.ts +++ b/src/balancer/OceanPool.ts @@ -1,6 +1,6 @@ import Web3 from 'web3' import { AbiItem } from 'web3-utils/types' -import { TransactionReceipt } from 'web3-core' +import { TransactionReceipt, Log } from 'web3-core' import { Pool } from './Pool' import { EventData, Filter } from 'web3-eth-contract' import BigNumber from 'bignumber.js' @@ -987,43 +987,34 @@ export class OceanPool extends Pool { account?: string ): Promise { const results: PoolTransaction[] = [] - const pool = new this.web3.eth.Contract(this.poolABI, poolAddress) const dtAddress = await this.getDTAddress(poolAddress) - const filter: Filter = account ? { caller: account } : {} - let events: EventData[] if (startBlock === 0) startBlock = this.startBlock - events = await pool.getPastEvents('LOG_SWAP', { - filter, - fromBlock: startBlock, - toBlock: 'latest' - }) - - for (let i = 0; i < events.length; i++) { - if (!account || events[i].returnValues[0].toLowerCase() === account.toLowerCase()) - results.push(await this.getEventData('swap', poolAddress, dtAddress, events[i])) - } - - events = await pool.getPastEvents('LOG_JOIN', { - filter, - fromBlock: startBlock, - toBlock: 'latest' - }) - - for (let i = 0; i < events.length; i++) { - if (!account || events[i].returnValues[0].toLowerCase() === account.toLowerCase()) - results.push(await this.getEventData('join', poolAddress, dtAddress, events[i])) - } - - events = await pool.getPastEvents('LOG_EXIT', { - filter, + const swapTopic = super.getSwapEventSignature() + const joinTopic = super.getJoinEventSignature() + const exitTopic = super.getExitEventSignature() + let addressTopic + if (account) + addressTopic = '0x000000000000000000000000' + account.substring(2).toLowerCase() + else addressTopic = null + const events = await this.web3.eth.getPastLogs({ + address: poolAddress, + topics: [[swapTopic, joinTopic, exitTopic], addressTopic], fromBlock: startBlock, toBlock: 'latest' }) for (let i = 0; i < events.length; i++) { - if (!account || events[i].returnValues[0].toLowerCase() === account.toLowerCase()) - results.push(await this.getEventData('exit', poolAddress, dtAddress, events[i])) + switch (events[i].topics[0]) { + case swapTopic: + results.push(await this.getEventData('swap', poolAddress, dtAddress, events[i])) + break + case joinTopic: + results.push(await this.getEventData('join', poolAddress, dtAddress, events[i])) + break + case exitTopic: + results.push(await this.getEventData('exit', poolAddress, dtAddress, events[i])) + break + } } - return results } @@ -1055,41 +1046,44 @@ export class OceanPool extends Pool { type: PoolTransactionType, poolAddress: string, dtAddress: string, - data: EventData + data: Log ): Promise { const blockDetails = await this.web3.eth.getBlock(data.blockNumber) let result: PoolTransaction = { poolAddress, dtAddress, - caller: data.returnValues[0], + caller: data.topics[1], transactionHash: data.transactionHash, blockNumber: data.blockNumber, timestamp: parseInt(String(blockDetails.timestamp)), type } - + let params switch (type) { case 'swap': + params = this.web3.eth.abi.decodeParameters(['uint256', 'uint256'], data.data) result = { ...result, - tokenIn: data.returnValues[1], - tokenOut: data.returnValues[2], - tokenAmountIn: this.web3.utils.fromWei(data.returnValues[3]), - tokenAmountOut: this.web3.utils.fromWei(data.returnValues[4]) + tokenIn: '0x' + data.topics[2].substring(data.topics[2].length - 40), + tokenOut: '0x' + data.topics[2].substring(data.topics[3].length - 40), + tokenAmountIn: this.web3.utils.fromWei(params[0]), + tokenAmountOut: this.web3.utils.fromWei(params[1]) } break case 'join': + params = this.web3.eth.abi.decodeParameters(['uint256'], data.data) result = { ...result, - tokenIn: data.returnValues[1], - tokenAmountIn: this.web3.utils.fromWei(data.returnValues[2]) + tokenIn: '0x' + data.topics[2].substring(data.topics[2].length - 40), + tokenAmountIn: this.web3.utils.fromWei(params[0]) } break case 'exit': + params = this.web3.eth.abi.decodeParameters(['uint256'], data.data) result = { ...result, - tokenOut: data.returnValues[1], - tokenAmountOut: this.web3.utils.fromWei(data.returnValues[2]) + tokenOut: '0x' + data.topics[2].substring(data.topics[2].length - 40), + tokenAmountOut: this.web3.utils.fromWei(params[0]) } break } diff --git a/src/balancer/Pool.ts b/src/balancer/Pool.ts index e27dc5ed..5d068b4a 100644 --- a/src/balancer/Pool.ts +++ b/src/balancer/Pool.ts @@ -1149,4 +1149,43 @@ export class Pool extends PoolFactory { } return amount } + + /** + * Get LOG_SWAP encoded topic + * @return {String} + */ + public getSwapEventSignature(): string { + const abi = this.poolABI as AbiItem[] + const eventdata = abi.find(function (o) { + if (o.name === 'LOG_SWAP' && o.type === 'event') return o + }) + const topic = this.web3.eth.abi.encodeEventSignature(eventdata as any) + return topic + } + + /** + * Get LOG_JOIN encoded topic + * @return {String} + */ + public getJoinEventSignature(): string { + const abi = this.poolABI as AbiItem[] + const eventdata = abi.find(function (o) { + if (o.name === 'LOG_JOIN' && o.type === 'event') return o + }) + const topic = this.web3.eth.abi.encodeEventSignature(eventdata as any) + return topic + } + + /** + * Get LOG_EXIT encoded topic + * @return {String} + */ + public getExitEventSignature(): string { + const abi = this.poolABI as AbiItem[] + const eventdata = abi.find(function (o) { + if (o.name === 'LOG_EXIT' && o.type === 'event') return o + }) + const topic = this.web3.eth.abi.encodeEventSignature(eventdata as any) + return topic + } } From 33bcb47d455cfbbbd9d8c26961f4127e9c514a82 Mon Sep 17 00:00:00 2001 From: mihaisc Date: Tue, 10 Nov 2020 16:49:59 +0200 Subject: [PATCH 4/5] parallelized get shares and tx Signed-off-by: mihaisc --- src/balancer/OceanPool.ts | 116 +++++++++++++++++++++++++------------- 1 file changed, 77 insertions(+), 39 deletions(-) diff --git a/src/balancer/OceanPool.ts b/src/balancer/OceanPool.ts index 69803fc7..0c615285 100644 --- a/src/balancer/OceanPool.ts +++ b/src/balancer/OceanPool.ts @@ -171,10 +171,11 @@ export class OceanPool extends Pool { const tokens = await this.getCurrentTokens(poolAddress) let token: string - for (token of tokens) { - // TODO: Potential timing attack, left side: true - if (token !== this.oceanAddress) this.dtAddress = token - } + if (tokens != null) + for (token of tokens) { + // TODO: Potential timing attack, left side: true + if (token !== this.oceanAddress) this.dtAddress = token + } return this.dtAddress } @@ -886,7 +887,9 @@ export class OceanPool extends Pool { * @return {String[]} - amount of ocean tokens received */ public async getOceanReceived(poolAddress: string, dtAmount: string): Promise { + console.log('poolAddress', poolAddress) const dtAddress = await this.getDTAddress(poolAddress) + console.log('dtAddress', dtAddress) return this.calcOutGivenIn(poolAddress, dtAddress, this.oceanAddress, dtAmount) } @@ -933,6 +936,20 @@ export class OceanPool extends Pool { return result } + private async getResult(account: string, event: EventData): Promise { + const shares = await super.sharesBalance(account, event.returnValues[0]) + if (parseFloat(shares) > 0) { + const dtAddress = await this.getDTAddress(event.returnValues[0]) + if (dtAddress) { + const onePool: PoolShare = { + shares, + poolAddress: event.returnValues[0], + did: didPrefixed(didNoZeroX(dtAddress)) + } + return onePool + } + } + } /** * Search all pools in which a user has shares * @param {String} account @@ -947,20 +964,17 @@ export class OceanPool extends Pool { fromBlock: this.startBlock, toBlock: 'latest' }) - for (let i = 0; i < events.length; i++) { - const shares = await super.sharesBalance(account, events[i].returnValues[0]) - if (parseFloat(shares) > 0) { - const dtAddress = await this.getDTAddress(events[i].returnValues[0]) - if (dtAddress) { - const onePool: PoolShare = { - shares, - poolAddress: events[i].returnValues[0], - did: didPrefixed(didNoZeroX(dtAddress)) - } - result.push(onePool) - } - } - } + let results = await Promise.all( + events.map((event) => { + return this.getResult(account, event) + }) + ) + results = results.filter((share) => { + return share !== undefined + }) + + result.push(...results) + return result } @@ -1002,19 +1016,40 @@ export class OceanPool extends Pool { fromBlock: startBlock, toBlock: 'latest' }) - for (let i = 0; i < events.length; i++) { - switch (events[i].topics[0]) { - case swapTopic: - results.push(await this.getEventData('swap', poolAddress, dtAddress, events[i])) - break - case joinTopic: - results.push(await this.getEventData('join', poolAddress, dtAddress, events[i])) - break - case exitTopic: - results.push(await this.getEventData('exit', poolAddress, dtAddress, events[i])) - break - } - } + + let eventResults = await Promise.all( + events.map((event) => { + switch (event.topics[0]) { + case swapTopic: + return this.getEventData('swap', poolAddress, dtAddress, event) + break + case joinTopic: + return this.getEventData('join', poolAddress, dtAddress, event) + break + case exitTopic: + return this.getEventData('exit', poolAddress, dtAddress, event) + break + } + }) + ) + eventResults = eventResults.filter((share) => { + return share !== undefined + }) + results.push(...eventResults) + + // for (let i = 0; i < events.length; i++) { + // switch (events[i].topics[0]) { + // case swapTopic: + // results.push(await this.getEventData('swap', poolAddress, dtAddress, events[i])) + // break + // case joinTopic: + // results.push(await this.getEventData('join', poolAddress, dtAddress, events[i])) + // break + // case exitTopic: + // results.push(await this.getEventData('exit', poolAddress, dtAddress, events[i])) + // break + // } + // } return results } @@ -1031,14 +1066,17 @@ export class OceanPool extends Pool { fromBlock: this.startBlock, toBlock: 'latest' }) - for (let i = 0; i < events.length; i++) { - const logs = await this.getPoolLogs( - events[i].returnValues[0], - events[i].blockNumber, - account - ) - for (let j = 0; j < logs.length; j++) results.push(logs[j]) - } + + let eventsResults = await Promise.all( + events.map((event) => { + return this.getPoolLogs(event.returnValues[0], event.blockNumber, account) + }) + ) + + const eventResults = eventsResults.reduce((elem1, elem2) => elem1.concat(elem2)) + console.log(eventResults) + results.push(...eventResults) + return results } From 4e16fcc20e72805fdada36860008f5b15bce82c5 Mon Sep 17 00:00:00 2001 From: mihaisc Date: Tue, 10 Nov 2020 16:58:36 +0200 Subject: [PATCH 5/5] remove console, fix travis Signed-off-by: mihaisc --- src/balancer/OceanPool.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/balancer/OceanPool.ts b/src/balancer/OceanPool.ts index 0c615285..c2dcbdde 100644 --- a/src/balancer/OceanPool.ts +++ b/src/balancer/OceanPool.ts @@ -887,9 +887,7 @@ export class OceanPool extends Pool { * @return {String[]} - amount of ocean tokens received */ public async getOceanReceived(poolAddress: string, dtAmount: string): Promise { - console.log('poolAddress', poolAddress) const dtAddress = await this.getDTAddress(poolAddress) - console.log('dtAddress', dtAddress) return this.calcOutGivenIn(poolAddress, dtAddress, this.oceanAddress, dtAmount) } @@ -950,6 +948,7 @@ export class OceanPool extends Pool { } } } + /** * Search all pools in which a user has shares * @param {String} account @@ -1067,14 +1066,13 @@ export class OceanPool extends Pool { toBlock: 'latest' }) - let eventsResults = await Promise.all( + const eventsResults = await Promise.all( events.map((event) => { return this.getPoolLogs(event.returnValues[0], event.blockNumber, account) }) ) const eventResults = eventsResults.reduce((elem1, elem2) => elem1.concat(elem2)) - console.log(eventResults) results.push(...eventResults) return results