The Graph sync status (#466)

* WIP

* query update

* quick fix

Signed-off-by: mihaisc <mihai.scarlat@smartcontrol.ro>

* get blocks number when no provider, added threshold

* format code

* naming fix

* show graph out of sync message inside announcement banner

* added loader

* moved sync component

* refactor all the things

* new atoms/AnnouncementBanner : banner component reduced to presentation only, where its content is always passed as props

* revised molecules/NetworkBanner: the former AnnouncementBanner now holds all the specific network detection logic, in the end also returns the atoms/AnnouncementBanner

* new hook hooks/useGraphSyncStatus: move all the graph fetching logic in there so we can use its status in multiple places in the app without all this props passing. This also decouples the SyncStatus component in footer from its logic

* in App.tsx, add the graph sync warning banner in another atoms/AnnouncementBanner, getting its values from the hook

* data flow refactor

* .env.example tweak

* race condition fighting

* subgraph loading

* polygon fallback fix

* no interval fetching

* turn around logic for adding infura ID

* removed graphNotSynched

Co-authored-by: mihaisc <mihai.scarlat@smartcontrol.ro>
Co-authored-by: Norbi <katunanorbert@gmai.com>
Co-authored-by: Matthias Kretschmann <m@kretschmann.io>

.env.example

@@ -1,4 +1,5 @@
-# Network, possible values: development, pacific, rinkeby, mainnet
+# Default network, possible values: 
+# "development", "ropsten", "rinkeby", "mainnet", "polygon"
 GATSBY_NETWORK="rinkeby"
 
 #GATSBY_INFURA_PROJECT_ID="xxx"

src/components/App.tsx

@@ -1,14 +1,16 @@
 import React, { ReactElement } from 'react'
+import { graphql, PageProps, useStaticQuery } from 'gatsby'
+import Alert from './atoms/Alert'
 import Footer from './organisms/Footer'
 import Header from './organisms/Header'
 import Styles from '../global/Styles'
-import styles from './App.module.css'
-import { useSiteMetadata } from '../hooks/useSiteMetadata'
-import Alert from './atoms/Alert'
-import { graphql, PageProps, useStaticQuery } from 'gatsby'
-import { useAccountPurgatory } from '../hooks/useAccountPurgatory'
-import AnnouncementBanner from './molecules/AnnouncementBanner'
 import { useWeb3 } from '../providers/Web3'
+import { useSiteMetadata } from '../hooks/useSiteMetadata'
+import { useAccountPurgatory } from '../hooks/useAccountPurgatory'
+import NetworkBanner from './molecules/NetworkBanner'
+import styles from './App.module.css'
+import AnnouncementBanner from './atoms/AnnouncementBanner'
+import { useGraphSyncStatus } from '../hooks/useGraphSyncStatus'
 
 const contentQuery = graphql`
   query AppQuery {
@@ -39,15 +41,25 @@ export default function App({
   const { warning } = useSiteMetadata()
   const { accountId } = useWeb3()
   const { isInPurgatory, purgatoryData } = useAccountPurgatory(accountId)
+  const { isGraphSynced, blockHead, blockGraph } = useGraphSyncStatus()
 
   return (
     <Styles>
       <div className={styles.app}>
-        {!location.pathname.includes('/asset/did') && <AnnouncementBanner />}
+        {!isGraphSynced && (
+          <AnnouncementBanner
+            text={`The data for this network has only synced to Ethereum block ${blockGraph} (out of ${blockHead}). Please check back soon.`}
+            state="error"
+          />
+        )}
+        {!location.pathname.includes('/asset/did') && <NetworkBanner />}
+
         <Header />
+
         {(props as PageProps).uri === '/' && (
           <Alert text={warning.main} state="info" />
         )}
+
         {isInPurgatory && (
           <Alert
             title={purgatory.title}

src/components/atoms/AnnouncementBanner.module.css

@@ -5,6 +5,21 @@
   background-color: var(--background-content);
 }
 
+.banner.error {
+  background-color: var(--brand-alert-red);
+  color: var(--brand-white);
+}
+
+.banner.warning {
+  background-color: var(--brand-alert-yellow);
+  color: var(--brand-white);
+}
+
+.banner.success {
+  background-color: var(--brand-alert-green);
+  color: var(--brand-white);
+}
+
 .banner > div {
   display: inline-block;
 }

src/components/atoms/AnnouncementBanner.tsx

@@ -0,0 +1,44 @@
+import React, { ReactElement } from 'react'
+import classNames from 'classnames/bind'
+import Markdown from '../atoms/Markdown'
+import Button from '../atoms/Button'
+import styles from './AnnouncementBanner.module.css'
+
+const cx = classNames.bind(styles)
+
+export interface AnnouncementAction {
+  name: string
+  style?: string
+  handleAction: () => void
+}
+
+export default function AnnouncementBanner({
+  text,
+  action,
+  state,
+  className
+}: {
+  text: string
+  action?: AnnouncementAction
+  state?: 'success' | 'warning' | 'error'
+  className?: string
+}): ReactElement {
+  const styleClasses = cx({
+    banner: true,
+    error: state === 'error',
+    warning: state === 'warning',
+    success: state === 'success',
+    [className]: className
+  })
+
+  return (
+    <div className={styleClasses}>
+      {text && <Markdown className={styles.text} text={text} />}
+      {action && (
+        <Button style="text" size="small" onClick={action.handleAction}>
+          {action.name}
+        </Button>
+      )}
+    </div>
+  )
+}

src/components/molecules/MarketStats.tsx

@@ -19,7 +19,7 @@ export default function MarketStats(): ReactElement {
   const [totalValueLocked, setTotalValueLocked] = useState<string>()
   const [totalOceanLiquidity, setTotalOceanLiquidity] = useState<string>()
   const [poolCount, setPoolCount] = useState<number>()
-  const { data } = useQuery(getTotalPoolsValues)
+  const { data } = useQuery(getTotalPoolsValues, { pollInterval: 20000 })
 
   useEffect(() => {
     if (!data || !data.poolFactories || data.poolFactories.length === 0) return

src/components/molecules/NetworkBanner.tsx

@@ -1,21 +1,15 @@
 import React, { ReactElement, useEffect, useState } from 'react'
-import styles from './AnnouncementBanner.module.css'
-import Markdown from '../atoms/Markdown'
 import { useWeb3 } from '../../providers/Web3'
 import { addCustomNetwork, NetworkObject } from '../../utils/web3'
 import { getOceanConfig } from '../../utils/ocean'
 import { getProviderInfo } from 'web3modal'
 import { useOcean } from '../../providers/Ocean'
 import { useSiteMetadata } from '../../hooks/useSiteMetadata'
-import Button from '../atoms/Button'
+import AnnouncementBanner, {
+  AnnouncementAction
+} from '../atoms/AnnouncementBanner'
 
-export interface AnnouncementAction {
+const networkMatic: NetworkObject = {
-  name: string
-  style?: string
-  handleAction: () => void
-}
-
-const network: NetworkObject = {
   chainId: 137,
   name: 'Matic Network',
   urlList: [
@@ -24,7 +18,7 @@ const network: NetworkObject = {
   ]
 }
 
-export default function AnnouncementBanner(): ReactElement {
+export default function NetworkBanner(): ReactElement {
   const { web3Provider } = useWeb3()
   const { config, connect } = useOcean()
   const { announcement } = useSiteMetadata()
@@ -34,7 +28,7 @@ export default function AnnouncementBanner(): ReactElement {
 
   const addCustomNetworkAction = {
     name: 'Add custom network',
-    handleAction: () => addCustomNetwork(web3Provider, network)
+    handleAction: () => addCustomNetwork(web3Provider, networkMatic)
   }
   const switchToPolygonAction = {
     name: 'Switch to Polygon',
@@ -88,16 +82,5 @@ export default function AnnouncementBanner(): ReactElement {
     }
   }, [web3Provider, config, announcement])
 
-  return (
+  return <AnnouncementBanner text={text} action={action} />
-    <div className={styles.container}>
-      <div className={styles.banner}>
-        {text && <Markdown className={styles.text} text={text} />}
-        {action && (
-          <Button style="text" size="small" onClick={action.handleAction}>
-            {action.name}
-          </Button>
-        )}
-      </div>
-    </div>
-  )
 }

src/components/molecules/SyncStatus.module.css

@@ -0,0 +1,14 @@
+.sync {
+  display: inline-block;
+  font-size: var(--font-size-mini);
+  cursor: help;
+}
+
+.text {
+  margin-left: calc(var(--spacer) / 12);
+}
+
+.status {
+  transform: scale(0.7);
+  transform-origin: bottom;
+}

src/components/molecules/SyncStatus.tsx

@@ -0,0 +1,23 @@
+import React, { ReactElement } from 'react'
+import Tooltip from '../atoms/Tooltip'
+import Status from '../atoms/Status'
+import { useGraphSyncStatus } from '../../hooks/useGraphSyncStatus'
+import styles from './SyncStatus.module.css'
+
+export default function SyncStatus(): ReactElement {
+  const { isGraphSynced, blockGraph, blockHead } = useGraphSyncStatus()
+
+  return (
+    <div className={styles.sync}>
+      <Tooltip
+        content={`Synced to Ethereum block ${blockGraph} (out of ${blockHead})`}
+      >
+        <Status
+          className={styles.status}
+          state={isGraphSynced ? 'success' : 'error'}
+        />
+        <span className={styles.text}>{blockGraph}</span>
+      </Tooltip>
+    </div>
+  )
+}

src/components/organisms/Footer.tsx

@@ -5,6 +5,7 @@ import { useSiteMetadata } from '../../hooks/useSiteMetadata'
 import { Link } from 'gatsby'
 import MarketStats from '../molecules/MarketStats'
 import BuildId from '../atoms/BuildId'
+import SyncStatus from '../molecules/SyncStatus'
 
 export default function Footer(): ReactElement {
   const { copyright } = useSiteMetadata()
@@ -13,9 +14,8 @@ export default function Footer(): ReactElement {
   return (
     <footer className={styles.footer}>
       <div className={styles.content}>
-        <BuildId />
+        <SyncStatus /> | <BuildId />
         <MarketStats />
-
         <div className={styles.copyright}>
           © {year} <Markdown text={copyright} /> —{' '}
           <Link to="/terms">Terms</Link>

src/hooks/useGraphSyncStatus.ts

@@ -0,0 +1,104 @@
+import { useState, useEffect } from 'react'
+import fetch from 'cross-fetch'
+import { useOcean } from '../providers/Ocean'
+import { useWeb3 } from '../providers/Web3'
+import { Logger } from '@oceanprotocol/lib'
+import Web3 from 'web3'
+import { ConfigHelperConfig } from '@oceanprotocol/lib/dist/node/utils/ConfigHelper'
+
+const blockDifferenceThreshold = 30
+const ethGraphUrl = `https://api.thegraph.com/subgraphs/name/blocklytics/ethereum-blocks`
+const ethGraphQuery =
+  '{"query":"  query Blocks{   blocks(first: 1, skip: 0, orderBy: number, orderDirection: desc, where: {number_gt: 9300000}) { id number timestamp  author  difficulty  gasUsed  gasLimit } }","variables":{},"operationName":"Blocks"}'
+const graphQuery =
+  '{"query":"  query Meta {   _meta {      block {        hash       number     }      deployment      hasIndexingErrors    }  }","variables":{},"operationName":"Meta"}'
+
+export interface UseGraphSyncStatus {
+  isGraphSynced: boolean
+  blockHead: number
+  blockGraph: number
+}
+
+async function fetchGraph(url: string, queryBody: string): Promise<Response> {
+  try {
+    const response = await fetch(url, {
+      method: 'POST',
+      body: queryBody
+    })
+
+    return await response.json()
+  } catch (error) {
+    console.error('Error parsing json: ' + error.message)
+  }
+}
+
+async function getBlockHead(config: ConfigHelperConfig) {
+  // for ETH main, get block from graph fetch
+  if (config.network === 'mainnet') {
+    const response: any = await fetchGraph(ethGraphUrl, ethGraphQuery)
+    return Number(response.data.blocks[0].number)
+  }
+
+  // for everything else, create new web3 instance with infura
+  const web3Instance = new Web3(config.nodeUri)
+  const blockHead = await web3Instance.eth.getBlockNumber()
+  return blockHead
+}
+
+async function getBlockSubgraph(subgraphUri: string) {
+  const response: any = await fetchGraph(
+    `${subgraphUri}/subgraphs/name/oceanprotocol/ocean-subgraph`,
+    graphQuery
+  )
+  const blockNumberGraph = Number(response.data._meta.block.number)
+  return blockNumberGraph
+}
+
+export function useGraphSyncStatus(): UseGraphSyncStatus {
+  const { config } = useOcean()
+  const { block, web3Loading } = useWeb3()
+  const [blockGraph, setBlockGraph] = useState<number>()
+  const [blockHead, setBlockHead] = useState<number>()
+  const [isGraphSynced, setIsGraphSynced] = useState(true)
+  const [subgraphLoading, setSubgraphLoading] = useState(false)
+
+  // Get and set head block
+  useEffect(() => {
+    if (!config || !config.nodeUri || web3Loading) return
+
+    async function initBlockHead() {
+      const blockHead = block || (await getBlockHead(config))
+      setBlockHead(blockHead)
+      Logger.log('[GraphStatus] Head block: ', blockHead)
+    }
+    initBlockHead()
+  }, [web3Loading, block, config.nodeUri])
+
+  // Get and set subgraph block
+  useEffect(() => {
+    if (!config || !config.subgraphUri) return
+    async function initBlockSubgraph() {
+      setSubgraphLoading(true)
+      const blockGraph = await getBlockSubgraph(config.subgraphUri)
+      setBlockGraph(blockGraph)
+      setSubgraphLoading(false)
+      Logger.log('[GraphStatus] Latest block from subgraph: ', blockGraph)
+    }
+    initBlockSubgraph()
+  }, [config.subgraphUri])
+
+  // Set sync status
+  useEffect(() => {
+    if ((!blockGraph && !blockHead) || web3Loading || subgraphLoading) return
+
+    const difference = blockHead - blockGraph
+
+    if (difference > blockDifferenceThreshold) {
+      setIsGraphSynced(false)
+      return
+    }
+    setIsGraphSynced(true)
+  }, [blockGraph, blockHead])
+
+  return { blockHead, blockGraph, isGraphSynced }
+}

src/providers/Web3.tsx

@@ -28,7 +28,9 @@ interface Web3ProviderValue {
   networkId: number
   networkDisplayName: string
   networkData: EthereumListsChain
+  block: number
   isTestnet: boolean
+  web3Loading: boolean
   connect: () => Promise<void>
   logout: () => Promise<void>
 }
@@ -107,13 +109,16 @@ function Web3Provider({ children }: { children: ReactNode }): ReactElement {
   const [networkId, setNetworkId] = useState<number>()
   const [networkDisplayName, setNetworkDisplayName] = useState<string>()
   const [networkData, setNetworkData] = useState<EthereumListsChain>()
+  const [block, setBlock] = useState<number>()
   const [isTestnet, setIsTestnet] = useState<boolean>()
   const [accountId, setAccountId] = useState<string>()
+  const [web3Loading, setWeb3Loading] = useState<boolean>()
 
   const connect = useCallback(async () => {
     if (!web3Modal) return
 
     try {
+      setWeb3Loading(true)
       Logger.log('[web3] Connecting Web3...')
 
       const provider = await web3Modal?.connect()
@@ -132,6 +137,8 @@ function Web3Provider({ children }: { children: ReactNode }): ReactElement {
       Logger.log('[web3] account id', accountId)
     } catch (error) {
       Logger.error('[web3] Error: ', error.message)
+    } finally {
+      setWeb3Loading(false)
     }
   }, [web3Modal])
 
@@ -186,6 +193,20 @@ function Web3Provider({ children }: { children: ReactNode }): ReactElement {
     Logger.log(`[web3] Network display name set to: ${networkDisplayName}`)
   }, [networkId, networksList])
 
+  // -----------------------------------
+  // Get and set latest head block
+  // -----------------------------------
+  useEffect(() => {
+    if (!web3) return
+
+    async function getBlock() {
+      const block = await web3.eth.getBlockNumber()
+      setBlock(block)
+      Logger.log('[web3] Head block: ', block)
+    }
+    getBlock()
+  }, [web3, networkId])
+
   // -----------------------------------
   // Logout helper
   // -----------------------------------
@@ -236,7 +257,9 @@ function Web3Provider({ children }: { children: ReactNode }): ReactElement {
         networkId,
         networkDisplayName,
         networkData,
+        block,
         isTestnet,
+        web3Loading,
         connect,
         logout
       }}

src/utils/ocean.ts

@@ -13,7 +13,9 @@ export function getOceanConfig(
 ): ConfigHelperConfig {
   return new ConfigHelper().getConfig(
     network,
-    process.env.GATSBY_INFURA_PROJECT_ID
+    network === 'polygon' || network === 137
+      ? undefined
+      : process.env.GATSBY_INFURA_PROJECT_ID
   ) as ConfigHelperConfig
 }
 

src/utils/web3.ts

@@ -56,6 +56,7 @@ export function addCustomNetwork(
 ): void {
   const newNewtworkData = {
     chainId: `0x${network.chainId.toString(16)}`,
+    chainName: network.name,
     rpcUrls: network.urlList
   }
   web3Provider.request(