Merge pull request #441 from oceanprotocol/feature/pool_logs_history_speed_optimisation

logs/history speed optimizations

src/balancer/OceanPool.ts

@@ -1,6 +1,6 @@
 import Web3 from 'web3'
 import { AbiItem } from 'web3-utils/types'
-import { TransactionReceipt } from 'web3-core'
+import { TransactionReceipt, Log } from 'web3-core'
 import { Pool } from './Pool'
 import { EventData, Filter } from 'web3-eth-contract'
 import BigNumber from 'bignumber.js'
@@ -54,6 +54,7 @@ export enum PoolCreateProgressStep {
 export class OceanPool extends Pool {
   public oceanAddress: string = null
   public dtAddress: string = null
+  public startBlock: number
 
   constructor(
     web3: Web3,
@@ -62,12 +63,14 @@ export class OceanPool extends Pool {
     poolABI: AbiItem | AbiItem[] = null,
     factoryAddress: string = null,
     oceanAddress: string = null,
-    gaslimit?: number
+    startBlock?: number
   ) {
-    super(web3, logger, factoryABI, poolABI, factoryAddress, gaslimit)
+    super(web3, logger, factoryABI, poolABI, factoryAddress)
     if (oceanAddress) {
       this.oceanAddress = oceanAddress
     }
+    if (startBlock) this.startBlock = startBlock
+    else this.startBlock = 0
   }
 
   /**
@@ -168,6 +171,7 @@ export class OceanPool extends Pool {
     const tokens = await this.getCurrentTokens(poolAddress)
     let token: string
 
+    if (tokens != null)
       for (token of tokens) {
         // TODO: Potential timing attack, left side: true
         if (token !== this.oceanAddress) this.dtAddress = token
@@ -855,7 +859,7 @@ export class OceanPool extends Pool {
     const factory = new this.web3.eth.Contract(this.factoryABI, this.factoryAddress)
     const events = await factory.getPastEvents('BPoolRegistered', {
       filter: {},
-      fromBlock: BPFACTORY_DEPLOY_BLOCK,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
     events.sort((a, b) => (a.blockNumber > b.blockNumber ? 1 : -1))
@@ -920,7 +924,7 @@ export class OceanPool extends Pool {
 
     const events = await factory.getPastEvents('BPoolRegistered', {
       filter: account ? { registeredBy: account } : {},
-      fromBlock: BPFACTORY_DEPLOY_BLOCK,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
     for (let i = 0; i < events.length; i++) {
@@ -930,6 +934,21 @@ export class OceanPool extends Pool {
     return result
   }
 
+  private async getResult(account: string, event: EventData): Promise<PoolShare> {
+    const shares = await super.sharesBalance(account, event.returnValues[0])
+    if (parseFloat(shares) > 0) {
+      const dtAddress = await this.getDTAddress(event.returnValues[0])
+      if (dtAddress) {
+        const onePool: PoolShare = {
+          shares,
+          poolAddress: event.returnValues[0],
+          did: didPrefixed(didNoZeroX(dtAddress))
+        }
+        return onePool
+      }
+    }
+  }
+
   /**
    * Search all pools in which a user has shares
    * @param {String} account
@@ -941,23 +960,20 @@ export class OceanPool extends Pool {
 
     const events = await factory.getPastEvents('BPoolRegistered', {
       filter: {},
-      fromBlock: BPFACTORY_DEPLOY_BLOCK,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
-    for (let i = 0; i < events.length; i++) {
+    let results = await Promise.all(
-      const shares = await super.sharesBalance(account, events[i].returnValues[0])
+      events.map((event) => {
-      if (parseFloat(shares) > 0) {
+        return this.getResult(account, event)
-        const dtAddress = await this.getDTAddress(events[i].returnValues[0])
+      })
-        if (dtAddress) {
+    )
-          const onePool: PoolShare = {
+    results = results.filter((share) => {
-            shares,
+      return share !== undefined
-            poolAddress: events[i].returnValues[0],
+    })
-            did: didPrefixed(didNoZeroX(dtAddress))
+
-          }
+    result.push(...results)
-          result.push(onePool)
+
-        }
-      }
-    }
     return result
   }
 
@@ -984,43 +1000,55 @@ export class OceanPool extends Pool {
     account?: string
   ): Promise<PoolTransaction[]> {
     const results: PoolTransaction[] = []
-    const pool = new this.web3.eth.Contract(this.poolABI, poolAddress)
     const dtAddress = await this.getDTAddress(poolAddress)
-    const filter: Filter = account ? { caller: account } : {}
+    if (startBlock === 0) startBlock = this.startBlock
-    let events: EventData[]
+    const swapTopic = super.getSwapEventSignature()
-
+    const joinTopic = super.getJoinEventSignature()
-    events = await pool.getPastEvents('LOG_SWAP', {
+    const exitTopic = super.getExitEventSignature()
-      filter,
+    let addressTopic
+    if (account)
+      addressTopic = '0x000000000000000000000000' + account.substring(2).toLowerCase()
+    else addressTopic = null
+    const events = await this.web3.eth.getPastLogs({
+      address: poolAddress,
+      topics: [[swapTopic, joinTopic, exitTopic], addressTopic],
       fromBlock: startBlock,
       toBlock: 'latest'
     })
 
-    for (let i = 0; i < events.length; i++) {
+    let eventResults = await Promise.all(
-      if (!account || events[i].returnValues[0].toLowerCase() === account.toLowerCase())
+      events.map((event) => {
-        results.push(await this.getEventData('swap', poolAddress, dtAddress, events[i]))
+        switch (event.topics[0]) {
+          case swapTopic:
+            return this.getEventData('swap', poolAddress, dtAddress, event)
+            break
+          case joinTopic:
+            return this.getEventData('join', poolAddress, dtAddress, event)
+            break
+          case exitTopic:
+            return this.getEventData('exit', poolAddress, dtAddress, event)
+            break
         }
-
-    events = await pool.getPastEvents('LOG_JOIN', {
-      filter,
-      fromBlock: startBlock,
-      toBlock: 'latest'
       })
-
+    )
-    for (let i = 0; i < events.length; i++) {
+    eventResults = eventResults.filter((share) => {
-      if (!account || events[i].returnValues[0].toLowerCase() === account.toLowerCase())
+      return share !== undefined
-        results.push(await this.getEventData('join', poolAddress, dtAddress, events[i]))
-    }
-
-    events = await pool.getPastEvents('LOG_EXIT', {
-      filter,
-      fromBlock: startBlock,
-      toBlock: 'latest'
     })
-    for (let i = 0; i < events.length; i++) {
+    results.push(...eventResults)
-      if (!account || events[i].returnValues[0].toLowerCase() === account.toLowerCase())
-        results.push(await this.getEventData('exit', poolAddress, dtAddress, events[i]))
-    }
 
+    // for (let i = 0; i < events.length; i++) {
+    //   switch (events[i].topics[0]) {
+    //     case swapTopic:
+    //       results.push(await this.getEventData('swap', poolAddress, dtAddress, events[i]))
+    //       break
+    //     case joinTopic:
+    //       results.push(await this.getEventData('join', poolAddress, dtAddress, events[i]))
+    //       break
+    //     case exitTopic:
+    //       results.push(await this.getEventData('exit', poolAddress, dtAddress, events[i]))
+    //       break
+    //   }
+    // }
     return results
   }
 
@@ -1034,17 +1062,19 @@ export class OceanPool extends Pool {
     const factory = new this.web3.eth.Contract(this.factoryABI, this.factoryAddress)
     const events = await factory.getPastEvents('BPoolRegistered', {
       filter: {},
-      fromBlock: BPFACTORY_DEPLOY_BLOCK,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
-    for (let i = 0; i < events.length; i++) {
+
-      const logs = await this.getPoolLogs(
+    const eventsResults = await Promise.all(
-        events[i].returnValues[0],
+      events.map((event) => {
-        events[i].blockNumber,
+        return this.getPoolLogs(event.returnValues[0], event.blockNumber, account)
-        account
+      })
     )
-      for (let j = 0; j < logs.length; j++) results.push(logs[j])
+
-    }
+    const eventResults = eventsResults.reduce((elem1, elem2) => elem1.concat(elem2))
+    results.push(...eventResults)
+
     return results
   }
 
@@ -1052,41 +1082,44 @@ export class OceanPool extends Pool {
     type: PoolTransactionType,
     poolAddress: string,
     dtAddress: string,
-    data: EventData
+    data: Log
   ): Promise<PoolTransaction> {
     const blockDetails = await this.web3.eth.getBlock(data.blockNumber)
     let result: PoolTransaction = {
       poolAddress,
       dtAddress,
-      caller: data.returnValues[0],
+      caller: data.topics[1],
       transactionHash: data.transactionHash,
       blockNumber: data.blockNumber,
       timestamp: parseInt(String(blockDetails.timestamp)),
       type
     }
-
+    let params
     switch (type) {
       case 'swap':
+        params = this.web3.eth.abi.decodeParameters(['uint256', 'uint256'], data.data)
         result = {
           ...result,
-          tokenIn: data.returnValues[1],
+          tokenIn: '0x' + data.topics[2].substring(data.topics[2].length - 40),
-          tokenOut: data.returnValues[2],
+          tokenOut: '0x' + data.topics[2].substring(data.topics[3].length - 40),
-          tokenAmountIn: this.web3.utils.fromWei(data.returnValues[3]),
+          tokenAmountIn: this.web3.utils.fromWei(params[0]),
-          tokenAmountOut: this.web3.utils.fromWei(data.returnValues[4])
+          tokenAmountOut: this.web3.utils.fromWei(params[1])
         }
         break
       case 'join':
+        params = this.web3.eth.abi.decodeParameters(['uint256'], data.data)
         result = {
           ...result,
-          tokenIn: data.returnValues[1],
+          tokenIn: '0x' + data.topics[2].substring(data.topics[2].length - 40),
-          tokenAmountIn: this.web3.utils.fromWei(data.returnValues[2])
+          tokenAmountIn: this.web3.utils.fromWei(params[0])
         }
         break
       case 'exit':
+        params = this.web3.eth.abi.decodeParameters(['uint256'], data.data)
         result = {
           ...result,
-          tokenOut: data.returnValues[1],
+          tokenOut: '0x' + data.topics[2].substring(data.topics[2].length - 40),
-          tokenAmountOut: this.web3.utils.fromWei(data.returnValues[2])
+          tokenAmountOut: this.web3.utils.fromWei(params[0])
         }
         break
     }

src/balancer/Pool.ts

@@ -26,10 +26,9 @@ export class Pool extends PoolFactory {
     logger: Logger,
     factoryABI: AbiItem | AbiItem[] = null,
     poolABI: AbiItem | AbiItem[] = null,
-    factoryAddress: string = null,
+    factoryAddress: string = null
-    gaslimit?: number
   ) {
-    super(web3, logger, factoryABI, factoryAddress, gaslimit)
+    super(web3, logger, factoryABI, factoryAddress)
     if (poolABI) this.poolABI = poolABI
     else this.poolABI = jsonpoolABI.abi as AbiItem[]
   }
@@ -1150,4 +1149,43 @@ export class Pool extends PoolFactory {
     }
     return amount
   }
+
+  /**
+   * Get LOG_SWAP encoded topic
+   * @return {String}
+   */
+  public getSwapEventSignature(): string {
+    const abi = this.poolABI as AbiItem[]
+    const eventdata = abi.find(function (o) {
+      if (o.name === 'LOG_SWAP' && o.type === 'event') return o
+    })
+    const topic = this.web3.eth.abi.encodeEventSignature(eventdata as any)
+    return topic
+  }
+
+  /**
+   * Get LOG_JOIN encoded topic
+   * @return {String}
+   */
+  public getJoinEventSignature(): string {
+    const abi = this.poolABI as AbiItem[]
+    const eventdata = abi.find(function (o) {
+      if (o.name === 'LOG_JOIN' && o.type === 'event') return o
+    })
+    const topic = this.web3.eth.abi.encodeEventSignature(eventdata as any)
+    return topic
+  }
+
+  /**
+   * Get LOG_EXIT encoded topic
+   * @return {String}
+   */
+  public getExitEventSignature(): string {
+    const abi = this.poolABI as AbiItem[]
+    const eventdata = abi.find(function (o) {
+      if (o.name === 'LOG_EXIT' && o.type === 'event') return o
+    })
+    const topic = this.web3.eth.abi.encodeEventSignature(eventdata as any)
+    return topic
+  }
 }

src/balancer/PoolFactory.ts

@@ -15,8 +15,7 @@ export class PoolFactory {
     web3: Web3,
     logger: Logger,
     factoryABI: AbiItem | AbiItem[] = null,
-    factoryAddress: string = null,
+    factoryAddress: string = null
-    gaslimit?: number
   ) {
     this.web3 = web3
 
@@ -25,7 +24,6 @@ export class PoolFactory {
     if (factoryAddress) {
       this.factoryAddress = factoryAddress
     }
-    if (gaslimit) this.GASLIMIT_DEFAULT = gaslimit
     this.logger = logger
   }
 

src/exchange/FixedRateExchange.ts

@@ -39,6 +39,7 @@ export class OceanFixedRateExchange {
   public contract: Contract = null
   private logger: Logger
   public datatokens: DataTokens
+  public startBlock: number
 
   /**
    * Instantiate FixedRateExchange
@@ -53,10 +54,13 @@ export class OceanFixedRateExchange {
     fixedRateExchangeAddress: string = null,
     fixedRateExchangeABI: AbiItem | AbiItem[] = null,
     oceanAddress: string = null,
-    datatokens: DataTokens
+    datatokens: DataTokens,
+    startBlock?: number
   ) {
     this.web3 = web3
     this.fixedRateExchangeAddress = fixedRateExchangeAddress
+    if (startBlock) this.startBlock = startBlock
+    else this.startBlock = 0
     this.fixedRateExchangeABI =
       fixedRateExchangeABI || (defaultFixedRateExchangeABI.abi as AbiItem[])
     this.oceanAddress = oceanAddress
@@ -359,7 +363,7 @@ export class OceanFixedRateExchange {
     const result: FixedPriceExchange[] = []
     const events = await this.contract.getPastEvents('ExchangeCreated', {
       filter: { datatoken: dataTokenAddress.toLowerCase() },
-      fromBlock: 0,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
     for (let i = 0; i < events.length; i++) {
@@ -388,7 +392,7 @@ export class OceanFixedRateExchange {
     const result: FixedPriceExchange[] = []
     const events = await this.contract.getPastEvents('ExchangeCreated', {
       filter: {},
-      fromBlock: 0,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
     for (let i = 0; i < events.length; i++) {
@@ -411,7 +415,7 @@ export class OceanFixedRateExchange {
     const result: FixedPriceSwap[] = []
     const events = await this.contract.getPastEvents('Swapped', {
       filter: { exchangeId: exchangeId },
-      fromBlock: 0,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
     for (let i = 0; i < events.length; i++) {
@@ -430,7 +434,7 @@ export class OceanFixedRateExchange {
     const result: FixedPriceSwap[] = []
     const events = await this.contract.getPastEvents('ExchangeCreated', {
       filter: {},
-      fromBlock: 0,
+      fromBlock: this.startBlock,
       toBlock: 'latest'
     })
     for (let i = 0; i < events.length; i++) {

src/models/Config.ts

@@ -96,6 +96,11 @@ export class Config {
    * @type {any}
    */
   public metadataContractABI?: AbiItem | AbiItem[]
+  /**
+   * block number of the deployment
+   * @type {number}
+   */
+  public startBlock?: number
   /**
    * Log level.
    * @type {boolean | LogLevel}

src/ocean/Ocean.ts

@@ -59,7 +59,8 @@ export class Ocean extends Instantiable {
       instanceConfig.config.poolFactoryABI,
       instanceConfig.config.poolABI,
       instanceConfig.config.poolFactoryAddress,
-      instanceConfig.config.oceanTokenAddress
+      instanceConfig.config.oceanTokenAddress,
+      instanceConfig.config.startBlock
     )
     instance.fixedRateExchange = new OceanFixedRateExchange(
       instanceConfig.config.web3Provider,
@@ -67,7 +68,8 @@ export class Ocean extends Instantiable {
       instanceConfig.config.fixedRateExchangeAddress,
       instanceConfig.config.fixedRateExchangeAddressABI,
       instanceConfig.config.oceanTokenAddress,
-      instance.datatokens
+      instance.datatokens,
+      instanceConfig.config.startBlock
     )
     instance.OnChainMetadataCache = new OnChainMetadataCache(
       instanceConfig.config.web3Provider,

src/utils/ConfigHelper.ts

@@ -27,7 +27,8 @@ const configs: ConfigHelperConfig[] = [
     factoryAddress: '0x1234',
     poolFactoryAddress: null,
     fixedRateExchangeAddress: null,
-    metadataContractAddress: null
+    metadataContractAddress: null,
+    startBlock: 0
   },
   {
     // barge
@@ -40,7 +41,8 @@ const configs: ConfigHelperConfig[] = [
     factoryAddress: null,
     poolFactoryAddress: null,
     fixedRateExchangeAddress: null,
-    metadataContractAddress: null
+    metadataContractAddress: null,
+    startBlock: 0
   },
   {
     networkId: 4,
@@ -52,7 +54,8 @@ const configs: ConfigHelperConfig[] = [
     factoryAddress: null,
     poolFactoryAddress: null,
     fixedRateExchangeAddress: null,
-    metadataContractAddress: null
+    metadataContractAddress: null,
+    startBlock: 7294090
   },
   {
     networkId: 1,
@@ -64,7 +67,8 @@ const configs: ConfigHelperConfig[] = [
     factoryAddress: null,
     poolFactoryAddress: null,
     fixedRateExchangeAddress: null,
-    metadataContractAddress: null
+    metadataContractAddress: null,
+    startBlock: 11105522
   }
 ]
 

test/unit/balancer/Balancer.test.ts

@@ -109,7 +109,8 @@ describe('Balancer flow', () => {
       OceanPoolFactory.abi as AbiItem[],
       OceanSPool.abi as AbiItem[],
       OceanPoolFactoryAddress,
-      oceanTokenAddress
+      oceanTokenAddress,
+      0
     )
     assert(Pool !== null)
   })