metamask-extension/test/e2e/benchmark.js

#!/usr/bin/env node

const path = require('path')
const { promises: fs, constants: fsConstants } = require('fs')
const ttest = require('ttest')
const { By, Key } = require('selenium-webdriver')
const { withFixtures } = require('./helpers')
const { PAGES } = require('./webdriver/driver')

const DEFAULT_NUM_SAMPLES = 20
const ALL_PAGES = Object.values(PAGES)

async function measurePage(pageName) {
  let metrics
  await withFixtures({ fixtures: 'imported-account' }, async ({ driver }) => {
    await driver.navigate()
    const passwordField = await driver.findElement(By.css('#password'))
    await passwordField.sendKeys('correct horse battery staple')
    await passwordField.sendKeys(Key.ENTER)
    await driver.findElement(By.css('.selected-account__name'))
    await driver.navigate(pageName)
    await driver.delay(1000)
    metrics = await driver.collectMetrics()
  })
  return metrics
}

function calculateResult(calc) {
  return (result) => {
    const calculatedResult = {}
    for (const key of Object.keys(result)) {
      calculatedResult[key] = calc(result[key])
    }
    return calculatedResult
  }
}
const calculateSum = (array) => array.reduce((sum, val) => sum + val)
const calculateAverage = (array) => calculateSum(array) / array.length
const minResult = calculateResult((array) => Math.min(...array))
const maxResult = calculateResult((array) => Math.max(...array))
const averageResult = calculateResult((array) => calculateAverage(array))
const standardDeviationResult = calculateResult((array) => {
  const average = calculateAverage(array)
  const squareDiffs = array.map((value) => Math.pow(value - average, 2))
  return Math.sqrt(calculateAverage(squareDiffs))
})
// 95% margin of error calculated using Student's t-distribution
const calculateMarginOfError = (array) =>
  ttest(array).confidence()[1] - calculateAverage(array)
const marginOfErrorResult = calculateResult((array) =>
  calculateMarginOfError(array),
)

async function profilePageLoad(pages, numSamples) {
  const results = {}
  for (const pageName of pages) {
    const runResults = []
    for (let i = 0; i < numSamples; i += 1) {
      runResults.push(await measurePage(pageName))
    }

    if (runResults.some((result) => result.navigation.lenth > 1)) {
      throw new Error(`Multiple navigations not supported`)
    } else if (
      runResults.some((result) => result.navigation[0].type !== 'navigate')
    ) {
      throw new Error(
        `Navigation type ${
          runResults.find((result) => result.navigation[0].type !== 'navigate')
            .navigation[0].type
        } not supported`,
      )
    }

    const result = {
      firstPaint: runResults.map((metrics) => metrics.paint['first-paint']),
      domContentLoaded: runResults.map(
        (metrics) =>
          metrics.navigation[0] && metrics.navigation[0].domContentLoaded,
      ),
      load: runResults.map(
        (metrics) => metrics.navigation[0] && metrics.navigation[0].load,
      ),
      domInteractive: runResults.map(
        (metrics) =>
          metrics.navigation[0] && metrics.navigation[0].domInteractive,
      ),
    }

    results[pageName] = {
      min: minResult(result),
      max: maxResult(result),
      average: averageResult(result),
      standardDeviation: standardDeviationResult(result),
      marginOfError: marginOfErrorResult(result),
    }
  }
  return results
}

async function isWritable(directory) {
  try {
    await fs.access(directory, fsConstants.W_OK)
    return true
  } catch (error) {
    if (error.code !== 'EACCES') {
      throw error
    }
    return false
  }
}

async function getFirstParentDirectoryThatExists(directory) {
  let nextDirectory = directory
  for (;;) {
    try {
      await fs.access(nextDirectory, fsConstants.F_OK)
      return nextDirectory
    } catch (error) {
      if (error.code !== 'ENOENT') {
        throw error
      } else if (nextDirectory === path.dirname(nextDirectory)) {
        throw new Error('Failed to find parent directory that exists')
      }
      nextDirectory = path.dirname(nextDirectory)
    }
  }
}

async function main() {
  const args = process.argv.slice(2)

  let pages = ['home']
  let numSamples = DEFAULT_NUM_SAMPLES
  let outputPath
  let outputDirectory
  let existingParentDirectory

  while (args.length) {
    if (/^(--pages|-p)$/u.test(args[0])) {
      if (args[1] === undefined) {
        throw new Error('Missing pages argument')
      }
      pages = args[1].split(',')
      for (const page of pages) {
        if (!ALL_PAGES.includes(page)) {
          throw new Error(`Invalid page: '${page}`)
        }
      }
      args.splice(0, 2)
    } else if (/^(--samples|-s)$/u.test(args[0])) {
      if (args[1] === undefined) {
        throw new Error('Missing number of samples')
      }
      numSamples = parseInt(args[1], 10)
      if (isNaN(numSamples)) {
        throw new Error(`Invalid 'samples' argument given: '${args[1]}'`)
      }
      args.splice(0, 2)
    } else if (/^(--out|-o)$/u.test(args[0])) {
      if (args[1] === undefined) {
        throw new Error('Missing output filename')
      }
      outputPath = path.resolve(args[1])
      outputDirectory = path.dirname(outputPath)
      existingParentDirectory = await getFirstParentDirectoryThatExists(
        outputDirectory,
      )
      if (!(await isWritable(existingParentDirectory))) {
        throw new Error(`Specified directory is not writable: '${args[1]}'`)
      }
      args.splice(0, 2)
    } else {
      throw new Error(`Unrecognized argument: '${args[0]}'`)
    }
  }

  const results = await profilePageLoad(pages, numSamples)

  if (outputPath) {
    if (outputDirectory !== existingParentDirectory) {
      await fs.mkdir(outputDirectory, { recursive: true })
    }
    await fs.writeFile(outputPath, JSON.stringify(results, null, 2))
  } else {
    console.log(JSON.stringify(results, null, 2))
  }
}

main().catch((e) => {
  console.error(e)
  process.exit(1)
})