Merge pull request #15 from oceanprotocol/feature/discord

Refactor and discord members

.editorconfig

@@ -1,13 +0,0 @@
-
-# EditorConfig is awesome: http://EditorConfig.org
-
-[*]
-indent_style = space
-indent_size = 4
-end_of_line = lf
-insert_final_newline = true
-charset = utf-8
-trim_trailing_whitespace = true
-
-[*.json]
-indent_size = 2

.eslintrc

@@ -1,3 +1,3 @@
 {
-    "extends": "oceanprotocol"
+  "extends": ["oceanprotocol", "prettier"]
 }

.github/dependabot.yml

@@ -1,8 +1,8 @@
 version: 2
 updates:
-- package-ecosystem: npm
+  - package-ecosystem: npm
-  directory: "/"
+    directory: '/'
-  schedule:
+    schedule:
-    interval: daily
+      interval: daily
-    time: '03:00'
+      time: '03:00'
-    timezone: Europe/Berlin
+      timezone: Europe/Berlin

.gitignore

@@ -1,4 +1,4 @@
 node_modules
 npm-debug.log
 yarn.lock
-package-lock.json
+.vercel

.nowignore

@@ -1,3 +0,0 @@
-media/
-test/
-package-lock.json

.prettierrc

@@ -0,0 +1,6 @@
+{
+  "semi": false,
+  "singleQuote": true,
+  "trailingComma": "none",
+  "tabWidth": 2
+}

.travis.yml

@@ -2,4 +2,4 @@ language: node_js
 node_js: node
 
 notifications:
-    email: false
+  email: false

README.md

@@ -21,25 +21,27 @@ Endpoint: [`https://oceanprotocol-community.now.sh`](https://oceanprotocol-commu
 
 ```json
 {
-    "github": {
+  "github": {
-        "stars": 1000,
+    "stars": 1000,
-        "repos": 1000
+    "repos": 1000
-    },
+  },
-    "medium": {
+  "medium": {
-        "followers": 1000
+    "followers": 1000
-    },
+  },
-    "bounties": {
+  "bounties": {
-        "gitcoin": 1000,
+    "open": 1000,
-        "bountiesNetwork": 1000,
+    "total": 1000
-        "total": 1000
+  },
-    },
+  "twitter": {
-    "twitter": {
+    "followers": 1000
-        "followers": 1000
+  },
-    },
+  "telegram": {
-    "telegram": {
+    "community": 1000,
-        "community": 1000,
+    "news": 1000
-        "news": 1000
+  },
-    }
+  "discord": {
+    "members": 1000
+  }
 }
 ```
 
@@ -48,7 +50,7 @@ Endpoint: [`https://oceanprotocol-community.now.sh`](https://oceanprotocol-commu
 Install dependencies:
 
 ```bash
-npm install -g now
+npm install -g vercel
 npm install
 ```
 

api/index.js

@@ -0,0 +1,64 @@
+import { logError } from './utils'
+
+import fetchGitHubRepos from './networks/github'
+import fetchBounties from './networks/bounties'
+import fetchMedium from './networks/medium'
+import fetchTwitter from './networks/twitter'
+import fetchTelegram from './networks/telegram'
+import fetchDiscord from './networks/discord'
+
+//
+// Create the response
+//
+export default async (req, res) => {
+  res.setHeader('Access-Control-Allow-Origin', '*')
+  res.setHeader('Access-Control-Allow-Methods', 'GET')
+
+  let response = {}
+
+  try {
+    const github = await fetchGitHubRepos()
+    response = { ...response, github }
+  } catch (error) {
+    logError(error.message)
+  }
+
+  try {
+    const bounties = await fetchBounties()
+    response = { ...response, bounties }
+  } catch (error) {
+    logError(error.message)
+  }
+
+  try {
+    const medium = await fetchMedium()
+    response = { ...response, medium }
+  } catch (error) {
+    logError(error.message)
+  }
+
+  try {
+    const twitter = await fetchTwitter()
+    response = { ...response, twitter }
+  } catch (error) {
+    logError(error.message)
+  }
+
+  try {
+    const telegram = await fetchTelegram()
+    response = { ...response, telegram }
+  } catch (error) {
+    logError(error.message)
+  }
+
+  try {
+    const discord = await fetchDiscord()
+    response = { ...response, discord }
+  } catch (error) {
+    // fake fallback response cause puppeteer fails a lot
+    response = { ...response, discord: { members: '240' } }
+    logError(error.message)
+  }
+
+  res.json(response)
+}

api/networks/bounties.js

@@ -0,0 +1,31 @@
+import fetch from 'node-fetch'
+import { log, logError } from '../utils'
+
+const getGitcoin = async () => {
+  const response = await fetch(
+    'https://gitcoin.co/api/v0.1/bounties/?&org=oceanprotocol'
+  )
+
+  if (response.status !== 200) {
+    logError(`Non-200 response code from Gitcoin: ${response.status}`)
+    return null
+  }
+
+  const total = await response.json()
+  const open = total.filter((item) => item.is_open === true)
+
+  return { total: total.length, open: open.length }
+}
+
+export default async function fetchBounties() {
+  const start = Date.now()
+  const { total, open } = await getGitcoin()
+
+  log(
+    '✓ bounties. ' +
+      `Total: ${total} bounties. Open: ${open} bounties. ` +
+      `Elapsed: ${new Date() - start}ms`
+  )
+
+  return { open, total }
+}

api/networks/discord.js

@@ -0,0 +1,43 @@
+import chrome from 'chrome-aws-lambda'
+import { log } from '../utils'
+
+const isDev = process.env.VERCEL_URL === undefined
+const puppeteer = isDev ? require('puppeteer') : require('puppeteer-core')
+
+export default async function fetchDiscord() {
+  const url = 'https://discord.com/invite/TnXjkR5'
+  const start = Date.now()
+
+  const config = {
+    ignoreHTTPSErrors: true,
+    ...(isDev
+      ? { headless: true }
+      : {
+          args: chrome.args,
+          executablePath: await chrome.executablePath,
+          headless: chrome.headless
+        })
+  }
+
+  const browser = await puppeteer.launch(config)
+  const page = await browser.newPage()
+  await page.goto(url)
+
+  const members = await page.evaluate(() => {
+    // get the activity count element
+    const membersElement = document.querySelector(
+      '[class*="activityCount"] > div:last-child span'
+    )
+    const membersElementText = membersElement.innerText
+    const number = membersElementText.replace(' Members', '')
+    return number
+  })
+
+  log(
+    '✓ Discord. ' +
+      `Total: ${members} members. ` +
+      `Elapsed: ${new Date() - start}ms`
+  )
+
+  return { members: Number(members) }
+}

api/networks/github.js

@@ -0,0 +1,48 @@
+import fetch from 'node-fetch'
+import { log, logError, arrSum } from '../utils'
+
+// Request options for all fetch calls
+const options = {
+  headers: {
+    // For getting topics, see note on https://developer.github.com/v3/search/
+    // Accept: 'application/vnd.github.mercy-preview+json'
+    Accept: 'application/vnd.github.preview'
+  }
+}
+
+//
+// Fetch all public GitHub repos
+//
+export default async function fetchGitHubRepos() {
+  const url =
+    'https://api.github.com/orgs/oceanprotocol/repos?type=public&per_page=200'
+  const start = Date.now()
+  const response = await fetch(url, options)
+
+  if (response.status !== 200) {
+    logError(`Non-200 response code from GitHub: ${response.status}`)
+    return null
+  }
+
+  const json = await response.json()
+
+  const numbers = []
+
+  json.map((item) => {
+    if (item.stargazers_count) {
+      return numbers.push(item.stargazers_count)
+    }
+    return null
+  })
+
+  const stars = arrSum(numbers)
+  const repositories = json.length
+
+  log(
+    '✓ GitHub. ' +
+      `Total: ${repositories} public projects with a total of ${stars} stargazers. ` +
+      `Elapsed: ${new Date() - start}ms`
+  )
+
+  return { stars, repositories }
+}

api/networks/medium.js

@@ -0,0 +1,27 @@
+import fetch from 'node-fetch'
+import { log, logError } from '../utils'
+
+export default async function fetchMedium() {
+  const url = 'https://medium.com/oceanprotocol?format=json'
+  const start = Date.now()
+  const response = await fetch(url)
+
+  if (response.status !== 200) {
+    logError(`Non-200 response code from Medium: ${response.status}`)
+    return null
+  }
+
+  const responseText = await response.text()
+  const json = await JSON.parse(responseText.replace('])}while(1);</x>', ''))
+  const { collection } = json.payload
+
+  const followers = collection.metadata.followerCount
+
+  log(
+    '✓ Medium. ' +
+      `Total: ${followers} followers. ` +
+      `Elapsed: ${new Date() - start}ms`
+  )
+
+  return { followers }
+}

api/networks/telegram.js

@@ -0,0 +1,51 @@
+import fetch from 'node-fetch'
+import { load } from 'cheerio'
+import { log, logError } from '../utils'
+
+export default async function fetchTelegram() {
+  const urlCommunity = 'https://t.me/oceanprotocol_community/?pagehidden=false'
+  const start = Date.now()
+  const responseCommunity = await fetch(urlCommunity)
+
+  if (responseCommunity.status !== 200) {
+    logError(`Non-200 response code from Telegram: ${responseCommunity.status}`)
+    return null
+  }
+  const bodyCommunity = await responseCommunity.text()
+  const dataCommunity = await load(bodyCommunity, { normalizeWhitespace: true })
+
+  let infoCommunity = dataCommunity('.tgme_page_extra').text()
+  infoCommunity = infoCommunity
+    .replace(' members', '')
+    .replace(' ', '')
+    .replace(' ', '')
+  const community = parseInt(infoCommunity)
+
+  log(
+    '✓ Telegram. ' +
+      `Total: ${community} oceanprotocol_community members. ` +
+      `Elapsed: ${new Date() - start}ms`
+  )
+
+  const urlNews = 'https://t.me/oceanprotocol/?pagehidden=false'
+  const responseNews = await fetch(urlNews)
+
+  if (responseNews.status !== 200) {
+    logError(`Non-200 response code from Telegram: ${responseNews.status}`)
+    return null
+  }
+  const bodyNews = await responseNews.text()
+  const dataNews = await load(bodyNews, { normalizeWhitespace: true })
+
+  let infoNews = dataNews('.tgme_page_extra').text()
+  infoNews = infoNews.replace(' members', '').replace(' ', '').replace(' ', '')
+  const news = parseInt(infoNews)
+
+  log(
+    '✓ Telegram. ' +
+      `Total: ${news} oceanprotocol members. ` +
+      `Elapsed: ${new Date() - start}ms`
+  )
+
+  return { community, news }
+}

api/networks/twitter.js

@@ -0,0 +1,25 @@
+import fetch from 'node-fetch'
+import { log, logError } from '../utils'
+
+export default async function fetchTwitter() {
+  const url =
+    'https://cdn.syndication.twimg.com/widgets/followbutton/info.json?screen_names=oceanprotocol'
+  const start = Date.now()
+  const response = await fetch(url)
+
+  if (response.status !== 200) {
+    logError(`Non-200 response code from Twitter: ${response.status}`)
+    return null
+  }
+
+  const json = await response.json()
+  const followers = json[0].followers_count
+
+  log(
+    '✓ Twitter. ' +
+      `Total: ${followers} followers. ` +
+      `Elapsed: ${new Date() - start}ms`
+  )
+
+  return { followers }
+}

api/utils.js

@@ -0,0 +1,9 @@
+/* eslint-disable no-console */
+const chalk = require('chalk')
+
+const log = (text) => console.log(text)
+const logError = (text) => console.error(chalk.bold.red(text))
+
+const arrSum = (arr) => arr.reduce((a, b) => a + b, 0)
+
+module.exports = { log, logError, arrSum }

index.js

@@ -1,56 +0,0 @@
-const ms = require('ms')
-const { logError } = require('./utils')
-
-const fetchGitHubRepos = require('./networks/github')
-const fetchBounties = require('./networks/bounties')
-const fetchMedium = require('./networks/medium')
-const fetchTwitter = require('./networks/twitter')
-const fetchTelegram = require('./networks/telegram')
-
-let cacheGithub = null
-let cacheBounties = null
-let cacheMedium = null
-let cacheTwitter = null
-let cacheTelegram = null
-
-//
-// Create the response
-//
-module.exports = async (req, res) => {
-    res.setHeader('Access-Control-Allow-Origin', '*')
-    res.setHeader('Access-Control-Allow-Methods', 'GET')
-
-    try {
-        /* eslint-disable require-atomic-updates */
-        if (!cacheGithub || Date.now() - cacheGithub.lastUpdate > ms('5m')) {
-            cacheGithub = await fetchGitHubRepos()
-        }
-
-        if (!cacheBounties || Date.now() - cacheBounties.lastUpdate > ms('5m')) {
-            cacheBounties = await fetchBounties()
-        }
-
-        if (!cacheMedium || Date.now() - cacheMedium.lastUpdate > ms('5m')) {
-            cacheMedium = await fetchMedium()
-        }
-
-        if (!cacheTwitter || Date.now() - cacheTwitter.lastUpdate > ms('5m')) {
-            cacheTwitter = await fetchTwitter()
-        }
-
-        if (!cacheTelegram || Date.now() - cacheTelegram.lastUpdate > ms('5m')) {
-            cacheTelegram = await fetchTelegram()
-        }
-        /* eslint-enable require-atomic-updates */
-    } catch (error) {
-        logError(error.message)
-    }
-
-    res.send({
-        github: cacheGithub,
-        bounties: cacheBounties,
-        medium: cacheMedium,
-        twitter: cacheTwitter,
-        telegram: cacheTelegram
-    })
-}

networks/bounties.js

@@ -1,62 +0,0 @@
-const fetch = require('node-fetch')
-const { log, logError } = require('../utils')
-
-const getGitcoin = async () => {
-    const response = await fetch('https://gitcoin.co/api/v0.1/bounties/?&org=oceanprotocol&is_open=true')
-
-    if (response.status !== 200) {
-        logError(`Non-200 response code from Gitcoin: ${response.status}`)
-        return null
-    }
-
-    const gitcoin = await response.json() // returns only open bounties by default
-
-    return gitcoin.length
-}
-
-const getBountiesNetwork = async () => {
-    const response = await fetch('https://api.bounties.network/bounty/?search=ocean%20protocol&bountyStage=1&platform=bounties-network')
-
-    if (response.status !== 200) {
-        logError(`Non-200 response code from Bounties Network: ${response.status}`)
-        return null
-    }
-
-    const bountiesNetwork = await response.json()
-
-    return bountiesNetwork.results.length
-}
-
-const getTotal = async () => {
-    const response = await fetch('https://api.bounties.network/bounty/?search=ocean%20protocol')
-
-    if (response.status !== 200) {
-        logError(`Non-200 response code from Bounties Network: ${response.status}`)
-        return null
-    }
-
-    const allBounties = await response.json()
-
-    return allBounties.count
-}
-
-const fetchBounties = async () => {
-    const start = Date.now()
-    const gitcoin = await getGitcoin()
-    const bountiesNetwork = await getBountiesNetwork()
-    const total = await getTotal()
-
-    log(
-        'Re-built bounties cache. ' +
-        `Total: ${total} bounties. ` +
-        `Elapsed: ${new Date() - start}ms`
-    )
-
-    return {
-        gitcoin,
-        bountiesNetwork,
-        total
-    }
-}
-
-module.exports = fetchBounties

networks/github.js

@@ -1,49 +0,0 @@
-const fetch = require('node-fetch')
-const { log, logError, arrSum } = require('../utils')
-
-// Request options for all fetch calls
-const options = {
-    headers: {
-        // For getting topics, see note on https://developer.github.com/v3/search/
-        // Accept: 'application/vnd.github.mercy-preview+json'
-        Accept: 'application/vnd.github.preview'
-    }
-}
-
-//
-// Fetch all public GitHub repos
-//
-const fetchGitHubRepos = async () => {
-    const url = 'https://api.github.com/orgs/oceanprotocol/repos?type=public&per_page=200'
-    const start = Date.now()
-    const response = await fetch(url, options)
-
-    if (response.status !== 200) {
-        logError(`Non-200 response code from GitHub: ${response.status}`)
-        return null
-    }
-
-    const json = await response.json()
-
-    const numbers = []
-
-    json.map(item => {
-        if (item.stargazers_count) {
-            return numbers.push(item.stargazers_count)
-        }
-        return null
-    })
-
-    const stars = arrSum(numbers)
-    const repositories = json.length
-
-    log(
-        'Re-built github cache. ' +
-        `Total: ${repositories} public projects with a total of ${stars} stargazers. ` +
-        `Elapsed: ${new Date() - start}ms`
-    )
-
-    return { stars, repositories }
-}
-
-module.exports = fetchGitHubRepos

networks/medium.js

@@ -1,29 +0,0 @@
-const fetch = require('node-fetch')
-const { log, logError } = require('../utils')
-
-const fetchMedium = async () => {
-    const url = 'https://medium.com/oceanprotocol?format=json'
-    const start = Date.now()
-    const response = await fetch(url)
-
-    if (response.status !== 200) {
-        logError(`Non-200 response code from Medium: ${response.status}`)
-        return null
-    }
-
-    const responseText = await response.text()
-    const json = await JSON.parse(responseText.replace('])}while(1);</x>', ''))
-    const { collection } = json.payload
-
-    const followers = collection.metadata.followerCount
-
-    log(
-        'Re-built medium cache. ' +
-        `Total: ${followers} followers. ` +
-        `Elapsed: ${new Date() - start}ms`
-    )
-
-    return { followers }
-}
-
-module.exports = fetchMedium

networks/telegram.js

@@ -1,50 +0,0 @@
-const fetch = require('node-fetch')
-const cheerio = require('cheerio')
-const { log, logError } = require('../utils')
-
-const fetchTelegram = async () => {
-    const urlCommunity = 'https://t.me/oceanprotocol_community/?pagehidden=false'
-    const start = Date.now()
-    const responseCommunity = await fetch(urlCommunity)
-
-    if (responseCommunity.status !== 200) {
-        logError(`Non-200 response code from Telegram: ${responseCommunity.status}`)
-        return null
-    }
-    const bodyCommunity = await responseCommunity.text()
-    const dataCommunity = await cheerio.load(bodyCommunity, { normalizeWhitespace: true })
-
-    let infoCommunity = dataCommunity('.tgme_page_extra').text()
-    infoCommunity = infoCommunity.replace(' members', '').replace(' ', '').replace(' ', '')
-    const membersCommunity = parseInt(infoCommunity)
-
-    log(
-        'Re-built telegram cache. ' +
-        `Total: ${membersCommunity} oceanprotocol_community members. ` +
-        `Elapsed: ${new Date() - start}ms`
-    )
-
-    const urlNews = 'https://t.me/oceanprotocol/?pagehidden=false'
-    const responseNews = await fetch(urlNews)
-
-    if (responseNews.status !== 200) {
-        logError(`Non-200 response code from Telegram: ${responseNews.status}`)
-        return null
-    }
-    const bodyNews = await responseNews.text()
-    const dataNews = await cheerio.load(bodyNews, { normalizeWhitespace: true })
-
-    let infoNews = dataNews('.tgme_page_extra').text()
-    infoNews = infoNews.replace(' members', '').replace(' ', '').replace(' ', '')
-    const membersNews = parseInt(infoNews)
-
-    log(
-        'Re-built telegram cache. ' +
-        `Total: ${membersCommunity} oceanprotocol_community members. ` +
-        `Elapsed: ${new Date() - start}ms`
-    )
-
-    return { community: membersCommunity, news: membersNews }
-}
-
-module.exports = fetchTelegram

networks/twitter.js

@@ -1,26 +0,0 @@
-const fetch = require('node-fetch')
-const { log, logError } = require('../utils')
-
-const fetchTwitter = async () => {
-    const url = 'https://cdn.syndication.twimg.com/widgets/followbutton/info.json?screen_names=oceanprotocol'
-    const start = Date.now()
-    const response = await fetch(url)
-
-    if (response.status !== 200) {
-        logError(`Non-200 response code from Twitter: ${response.status}`)
-        return null
-    }
-
-    const json = await response.json()
-    const followers = json[0].followers_count
-
-    log(
-        'Re-built twitter cache. ' +
-        `Total: ${followers} followers. ` +
-        `Elapsed: ${new Date() - start}ms`
-    )
-
-    return { followers }
-}
-
-module.exports = fetchTwitter

now.json

@@ -1,14 +0,0 @@
-{
-  "version": 2,
-  "name": "oceanprotocol-community",
-  "alias": "oceanprotocol-community.now.sh",
-  "env": {
-    "NODE_ENV": "production"
-  },
-  "builds": [
-    {
-      "src": "index.js",
-      "use": "@now/node"
-    }
-  ]
-}

package.json

@@ -1,21 +1,24 @@
 {
   "name": "@oceanprotocol/community-numbers",
-  "private": true,
   "version": "1.0.0",
-  "main": "index.js",
+  "main": "api/index.js",
   "scripts": {
-    "start": "now dev",
+    "start": "vercel dev",
     "test": "eslint --ignore-path .gitignore --ext .js,.jsx ."
   },
   "dependencies": {
-    "chalk": "4.1.0",
+    "chalk": "^4.1.0",
     "cheerio": "^1.0.0-rc.3",
-    "ms": "^2.1.1",
+    "chrome-aws-lambda": "^3.1.1",
-    "node-fetch": "2.6.0"
+    "chrome-finder": "^1.0.7",
+    "node-fetch": "^2.6.0",
+    "puppeteer": "^4.0.0",
+    "puppeteer-core": "^4.0.0"
   },
   "devDependencies": {
-    "eslint": "^7.0.0",
+    "eslint": "^7.2.0",
     "eslint-config-oceanprotocol": "^1.5.0",
+    "eslint-config-prettier": "^6.11.0",
     "eslint-plugin-node": "^11.1.0"
   }
 }

utils.js

@@ -1,9 +0,0 @@
-/* eslint-disable no-console */
-const chalk = require('chalk')
-
-const log = text => console.log(text)
-const logError = text => console.error(chalk.bold.red(text))
-
-const arrSum = arr => arr.reduce((a, b) => a + b, 0)
-
-module.exports = { log, logError, arrSum }

vercel.json

@@ -0,0 +1,3 @@
+{
+  "routes": [{ "src": "/", "dest": "/api" }]
+}