mirror of
https://github.com/kremalicious/metamask-extension.git
synced 2024-12-23 09:52:26 +01:00
cc90fca2f6
The benchmark script can now be set to retry upon failure, like the E2E tests do. The default is zero, just as with the E2E tests. A retry of 2 has been set in CI to match the E2E tests as well. The `retry` module had to be adjusted to throw an error in the case of failure. Previously it just set the exit code, but that only worked because it was the last thing called before the process ended. That is no longer the case.
200 lines
6.0 KiB
JavaScript
200 lines
6.0 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
const path = require('path');
|
|
const { promises: fs, constants: fsConstants } = require('fs');
|
|
const yargs = require('yargs/yargs');
|
|
const { hideBin } = require('yargs/helpers');
|
|
const ttest = require('ttest');
|
|
const { retry } = require('../../development/lib/retry');
|
|
const { exitWithError } = require('../../development/lib/exit-with-error');
|
|
const { withFixtures } = require('./helpers');
|
|
const { PAGES } = require('./webdriver/driver');
|
|
|
|
const DEFAULT_NUM_SAMPLES = 20;
|
|
const ALL_PAGES = Object.values(PAGES);
|
|
|
|
async function measurePage(pageName) {
|
|
let metrics;
|
|
await withFixtures({ fixtures: 'imported-account' }, async ({ driver }) => {
|
|
await driver.navigate();
|
|
await driver.fill('#password', 'correct horse battery staple');
|
|
await driver.press('#password', driver.Key.ENTER);
|
|
await driver.findElement('.selected-account__name');
|
|
await driver.navigate(pageName);
|
|
await driver.delay(1000);
|
|
metrics = await driver.collectMetrics();
|
|
});
|
|
return metrics;
|
|
}
|
|
|
|
function calculateResult(calc) {
|
|
return (result) => {
|
|
const calculatedResult = {};
|
|
for (const key of Object.keys(result)) {
|
|
calculatedResult[key] = calc(result[key]);
|
|
}
|
|
return calculatedResult;
|
|
};
|
|
}
|
|
const calculateSum = (array) => array.reduce((sum, val) => sum + val);
|
|
const calculateAverage = (array) => calculateSum(array) / array.length;
|
|
const minResult = calculateResult((array) => Math.min(...array));
|
|
const maxResult = calculateResult((array) => Math.max(...array));
|
|
const averageResult = calculateResult((array) => calculateAverage(array));
|
|
const standardDeviationResult = calculateResult((array) => {
|
|
if (array.length === 1) {
|
|
return 0;
|
|
}
|
|
const average = calculateAverage(array);
|
|
const squareDiffs = array.map((value) => Math.pow(value - average, 2));
|
|
return Math.sqrt(calculateAverage(squareDiffs));
|
|
});
|
|
// 95% margin of error calculated using Student's t-distribution
|
|
const calculateMarginOfError = (array) =>
|
|
ttest(array).confidence()[1] - calculateAverage(array);
|
|
const marginOfErrorResult = calculateResult((array) =>
|
|
array.length === 1 ? 0 : calculateMarginOfError(array),
|
|
);
|
|
|
|
async function profilePageLoad(pages, numSamples, retries) {
|
|
const results = {};
|
|
for (const pageName of pages) {
|
|
const runResults = [];
|
|
for (let i = 0; i < numSamples; i += 1) {
|
|
let result;
|
|
await retry(retries, async () => {
|
|
result = await measurePage(pageName);
|
|
});
|
|
runResults.push(result);
|
|
}
|
|
|
|
if (runResults.some((result) => result.navigation.lenth > 1)) {
|
|
throw new Error(`Multiple navigations not supported`);
|
|
} else if (
|
|
runResults.some((result) => result.navigation[0].type !== 'navigate')
|
|
) {
|
|
throw new Error(
|
|
`Navigation type ${
|
|
runResults.find((result) => result.navigation[0].type !== 'navigate')
|
|
.navigation[0].type
|
|
} not supported`,
|
|
);
|
|
}
|
|
|
|
const result = {
|
|
firstPaint: runResults.map((metrics) => metrics.paint['first-paint']),
|
|
domContentLoaded: runResults.map(
|
|
(metrics) =>
|
|
metrics.navigation[0] && metrics.navigation[0].domContentLoaded,
|
|
),
|
|
load: runResults.map(
|
|
(metrics) => metrics.navigation[0] && metrics.navigation[0].load,
|
|
),
|
|
domInteractive: runResults.map(
|
|
(metrics) =>
|
|
metrics.navigation[0] && metrics.navigation[0].domInteractive,
|
|
),
|
|
};
|
|
|
|
results[pageName] = {
|
|
min: minResult(result),
|
|
max: maxResult(result),
|
|
average: averageResult(result),
|
|
standardDeviation: standardDeviationResult(result),
|
|
marginOfError: marginOfErrorResult(result),
|
|
};
|
|
}
|
|
return results;
|
|
}
|
|
|
|
async function isWritable(directory) {
|
|
try {
|
|
await fs.access(directory, fsConstants.W_OK);
|
|
return true;
|
|
} catch (error) {
|
|
if (error.code !== 'EACCES') {
|
|
throw error;
|
|
}
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function getFirstParentDirectoryThatExists(directory) {
|
|
let nextDirectory = directory;
|
|
for (;;) {
|
|
try {
|
|
await fs.access(nextDirectory, fsConstants.F_OK);
|
|
return nextDirectory;
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT') {
|
|
throw error;
|
|
} else if (nextDirectory === path.dirname(nextDirectory)) {
|
|
throw new Error('Failed to find parent directory that exists');
|
|
}
|
|
nextDirectory = path.dirname(nextDirectory);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const { argv } = yargs(hideBin(process.argv)).usage(
|
|
'$0 [options]',
|
|
'Run a page load benchmark',
|
|
(_yargs) =>
|
|
_yargs
|
|
.option('pages', {
|
|
array: true,
|
|
default: ['home'],
|
|
description:
|
|
'Set the page(s) to be benchmarked. This flag can accept multiple values (space-separated).',
|
|
choices: ALL_PAGES,
|
|
})
|
|
.option('samples', {
|
|
default: DEFAULT_NUM_SAMPLES,
|
|
description: 'The number of times the benchmark should be run.',
|
|
type: 'number',
|
|
})
|
|
.option('out', {
|
|
description:
|
|
'Output filename. Output printed to STDOUT of this is omitted.',
|
|
type: 'string',
|
|
normalize: true,
|
|
})
|
|
.option('retries', {
|
|
default: 0,
|
|
description:
|
|
'Set how many times each benchmark sample should be retried upon failure.',
|
|
type: 'number',
|
|
}),
|
|
);
|
|
|
|
const { pages, samples, out, retries } = argv;
|
|
|
|
let outputDirectory;
|
|
let existingParentDirectory;
|
|
if (out) {
|
|
outputDirectory = path.dirname(out);
|
|
existingParentDirectory = await getFirstParentDirectoryThatExists(
|
|
outputDirectory,
|
|
);
|
|
if (!(await isWritable(existingParentDirectory))) {
|
|
throw new Error('Specified output file directory is not writable');
|
|
}
|
|
}
|
|
|
|
const results = await profilePageLoad(pages, samples, retries);
|
|
|
|
if (out) {
|
|
if (outputDirectory !== existingParentDirectory) {
|
|
await fs.mkdir(outputDirectory, { recursive: true });
|
|
}
|
|
await fs.writeFile(out, JSON.stringify(results, null, 2));
|
|
} else {
|
|
console.log(JSON.stringify(results, null, 2));
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
exitWithError(error);
|
|
});
|