Add benchmark script (#7869)

The script `benchmark.js` will collect page load metrics from the
extension, and print them to a file or the console. A method for
collecting metrics was added to the web driver to help with this.

This script will calculate the min, max, average, and standard
deviation for four metrics: 'firstPaint', 'domContentLoaded', 'load',
and 'domInteractive'. The variation between samples is sometimes high,
with the results varying between samples if only 3 were taken. However,
all tests I've done locally with 5 samples have produced results within
one standard deviation of each other. The default number of samples has
been set to 10, which should be more than enough to produce consistent
results.

The benchmark can be run with the npm script `benchmark:chrome` or
`benchmark:firefox`, e.g. `yarn benchmark:chrome`.

package.json

@@ -8,6 +8,8 @@
     "doc": "jsdoc -c development/.jsdoc.json",
     "publish-docs": "gh-pages -d docs/jsdocs",
     "start:test": "gulp dev:test",
+    "benchmark:chrome": "SELENIUM_BROWSER=chrome node test/e2e/benchmark.js",
+    "benchmark:firefox": "SELENIUM_BROWSER=firefox node test/e2e/benchmark.js",
     "build:test": "gulp build:test",
     "test": "yarn test:unit && yarn lint",
     "dapp": "node development/static-server.js test/e2e/contract-test --port 8080",

test/e2e/benchmark.js

@@ -0,0 +1,166 @@
+#!/usr/bin/env node
+
+const path = require('path')
+const { promises: fs, constants: fsConstants } = require('fs')
+const { By, Key } = require('selenium-webdriver')
+const { withFixtures } = require('./helpers')
+const { PAGES } = require('./webdriver/driver')
+
+const DEFAULT_NUM_SAMPLES = 10
+const ALL_PAGES = Object.values(PAGES)
+
+async function measurePage (pageName) {
+  let metrics
+  await withFixtures({ fixtures: 'imported-account' }, async ({ driver }) => {
+    const passwordField = await driver.findElement(By.css('#password'))
+    await passwordField.sendKeys('correct horse battery staple')
+    await passwordField.sendKeys(Key.ENTER)
+    await driver.navigate(pageName)
+    await driver.delay(1000)
+    metrics = await driver.collectMetrics()
+  })
+  return metrics
+}
+
+function calculateResult (calc) {
+  return (result) => {
+    const calculatedResult = {}
+    for (const key of Object.keys(result)) {
+      calculatedResult[key] = calc(result[key])
+    }
+    return calculatedResult
+  }
+}
+const calculateSum = (array) => array.reduce((sum, val) => sum + val)
+const calculateAverage = (array) => calculateSum(array) / array.length
+const minResult = calculateResult((array) => Math.min(...array))
+const maxResult = calculateResult((array) => Math.max(...array))
+const averageResult = calculateResult(array => calculateAverage(array))
+const standardDeviationResult = calculateResult((array) => {
+  const average = calculateAverage(array)
+  const squareDiffs = array.map(value => Math.pow(value - average, 2))
+  return Math.sqrt(calculateAverage(squareDiffs))
+})
+
+async function profilePageLoad (pages, numSamples) {
+  const results = {}
+  for (const pageName of pages) {
+    const runResults = []
+    for (let i = 0; i < numSamples; i += 1) {
+      runResults.push(await measurePage(pageName))
+    }
+
+    if (runResults.some(result => result.navigation.lenth > 1)) {
+      throw new Error(`Multiple navigations not supported`)
+    } else if (runResults.some(result => result.navigation[0].type !== 'navigate')) {
+      throw new Error(`Navigation type ${runResults.find(result => result.navigation[0].type !== 'navigate').navigation[0].type} not supported`)
+    }
+
+    const result = {
+      firstPaint: runResults.map(result => result.paint['first-paint']),
+      domContentLoaded: runResults.map(result => result.navigation[0] && result.navigation[0].domContentLoaded),
+      load: runResults.map(result => result.navigation[0] && result.navigation[0].load),
+      domInteractive: runResults.map(result => result.navigation[0] && result.navigation[0].domInteractive),
+    }
+
+    results[pageName] = {
+      min: minResult(result),
+      max: maxResult(result),
+      average: averageResult(result),
+      standardDeviation: standardDeviationResult(result),
+    }
+  }
+  return results
+}
+
+async function isWritable (directory) {
+  try {
+    await fs.access(directory, fsConstants.W_OK)
+    return true
+  } catch (error) {
+    if (error.code !== 'EACCES') {
+      throw error
+    }
+    return false
+  }
+}
+
+async function getFirstParentDirectoryThatExists (directory) {
+  while (true) {
+    try {
+      await fs.access(directory, fsConstants.F_OK)
+      return directory
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      } else if (directory === path.dirname(directory)) {
+        throw new Error('Failed to find parent directory that exists')
+      }
+      directory = path.dirname(directory)
+    }
+  }
+}
+
+async function main () {
+  const args = process.argv.slice(2)
+
+  let pages = ['notification']
+  let numSamples = DEFAULT_NUM_SAMPLES
+  let outputPath
+  let outputDirectory
+  let existingParentDirectory
+
+  while (args.length) {
+    if (/^(--pages|-p)$/i.test(args[0])) {
+      if (args[1] === undefined) {
+        throw new Error('Missing pages argument')
+      }
+      pages = args[1].split(',')
+      for (const page of pages) {
+        if (!ALL_PAGES.includes(page)) {
+          throw new Error(`Invalid page: '${page}`)
+        }
+      }
+      args.splice(0, 2)
+    } else if (/^(--samples|-s)$/i.test(args[0])) {
+      if (args[1] === undefined) {
+        throw new Error('Missing number of samples')
+      }
+      numSamples = parseInt(args[1], 10)
+      if (isNaN(numSamples)) {
+        throw new Error(`Invalid 'samples' argument given: '${args[1]}'`)
+      }
+      args.splice(0, 2)
+    } else if (/^(--out|-o)$/i.test(args[0])) {
+      if (args[1] === undefined) {
+        throw new Error('Missing output filename')
+      }
+      outputPath = path.resolve(args[1])
+      outputDirectory = path.dirname(outputPath)
+      existingParentDirectory = await getFirstParentDirectoryThatExists(outputDirectory)
+      if (!await isWritable(existingParentDirectory)) {
+        throw new Error(`Specified directory is not writable: '${args[1]}'`)
+      }
+      args.splice(0, 2)
+    } else {
+      throw new Error(`Unrecognized argument: '${args[0]}'`)
+    }
+  }
+
+  const results = await profilePageLoad(pages, numSamples)
+
+  if (outputPath) {
+    if (outputDirectory !== existingParentDirectory) {
+      await fs.mkdir(outputDirectory, { recursive: true })
+    }
+    await fs.writeFile(outputPath, JSON.stringify(results, null, 2))
+  } else {
+    console.log(JSON.stringify(results, null, 2))
+  }
+}
+
+main()
+  .catch(e => {
+    console.error(e)
+    process.exit(1)
+  })

test/e2e/webdriver/driver.js

@@ -91,6 +91,12 @@ class Driver {
     return await this.driver.get(`${this.extensionUrl}/${page}.html`)
   }
 
+  // Metrics
+
+  async collectMetrics () {
+    return await this.driver.executeScript(collectMetrics)
+  }
+
   // Window management
 
   async openNewPage (url) {
@@ -180,6 +186,29 @@ class Driver {
   }
 }
 
+function collectMetrics () {
+  const results = {
+    paint: {},
+    navigation: [],
+  }
+
+  performance.getEntriesByType('paint').forEach((paintEntry) => {
+    results.paint[paintEntry.name] = paintEntry.startTime
+  })
+
+  performance.getEntriesByType('navigation').forEach((navigationEntry) => {
+    results.navigation.push({
+      domContentLoaded: navigationEntry.domContentLoadedEventEnd,
+      load: navigationEntry.loadEventEnd,
+      domInteractive: navigationEntry.domInteractive,
+      redirectCount: navigationEntry.redirectCount,
+      type: navigationEntry.type,
+    })
+  })
+
+  return results
+}
+
 Driver.PAGES = {
   HOME: 'home',
   NOTIFICATION: 'notification',