mirror of
https://github.com/kremalicious/metamask-extension.git
synced 2024-12-22 17:33:23 +01:00
Add run-e2e-test.js
script (#11301)
This script makes it easier to run an individual E2E test. In the past I've run individual scripts by editing `run-all.sh` manually, but now that can be done more easily with this script. It also allows setting the number of retries to use and the browser to use from the CLI. This script has been added as an npm script as well, called 'test:e2e:single'. The `run-all.sh` script was rewritten in JavaScript to make it easier to pass through a `--retries` argument. The default number of retries has been set to zero to make local testing easier. It has been set to 2 on CI. This was mainly done to consolidate the code used to run an E2E test in one place, to make later improvements easier.
This commit is contained in:
parent
f7c37cab51
commit
7535d63466
@ -343,7 +343,7 @@ jobs:
|
||||
command: |
|
||||
if .circleci/scripts/test-run-e2e.sh
|
||||
then
|
||||
yarn test:e2e:chrome
|
||||
yarn test:e2e:chrome --retries 2
|
||||
fi
|
||||
no_output_timeout: 20m
|
||||
- store_artifacts:
|
||||
@ -370,7 +370,7 @@ jobs:
|
||||
command: |
|
||||
if .circleci/scripts/test-run-e2e.sh
|
||||
then
|
||||
yarn test:e2e:chrome:metrics
|
||||
yarn test:e2e:chrome:metrics --retries 2
|
||||
fi
|
||||
no_output_timeout: 20m
|
||||
- store_artifacts:
|
||||
@ -397,7 +397,7 @@ jobs:
|
||||
command: |
|
||||
if .circleci/scripts/test-run-e2e.sh
|
||||
then
|
||||
yarn test:e2e:firefox
|
||||
yarn test:e2e:firefox --retries 2
|
||||
fi
|
||||
no_output_timeout: 20m
|
||||
- store_artifacts:
|
||||
@ -424,7 +424,7 @@ jobs:
|
||||
command: |
|
||||
if .circleci/scripts/test-run-e2e.sh
|
||||
then
|
||||
yarn test:e2e:firefox:metrics
|
||||
yarn test:e2e:firefox:metrics --retries 2
|
||||
fi
|
||||
no_output_timeout: 20m
|
||||
- store_artifacts:
|
||||
|
16
development/lib/exit-with-error.js
Normal file
16
development/lib/exit-with-error.js
Normal file
@ -0,0 +1,16 @@
|
||||
/**
|
||||
* Exit the process with an error message.
|
||||
*
|
||||
* Note that this should be called before the process ends, but it will not
|
||||
* itself end the process. This is because the Node.js documentation strongly
|
||||
* advises against calling `process.exit` directly.
|
||||
*
|
||||
* @param {string} errorMessage - The error message that is causing the non-
|
||||
* zero exit code.
|
||||
*/
|
||||
function exitWithError(errorMessage) {
|
||||
console.error(errorMessage);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
module.exports = { exitWithError };
|
25
development/lib/retry.js
Normal file
25
development/lib/retry.js
Normal file
@ -0,0 +1,25 @@
|
||||
const { exitWithError } = require('./exit-with-error');
|
||||
|
||||
/**
|
||||
* Run the given function, retrying it upon failure until reaching the
|
||||
* specified number of retries.
|
||||
*
|
||||
* @param {number} retries - The number of retries upon failure to attempt.
|
||||
* @param {function} functionToRetry - The function that will be retried upon failure.
|
||||
*/
|
||||
async function retry(retries, functionToRetry) {
|
||||
let attempts = 0;
|
||||
while (attempts <= retries) {
|
||||
try {
|
||||
await functionToRetry();
|
||||
return;
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
} finally {
|
||||
attempts += 1;
|
||||
}
|
||||
}
|
||||
exitWithError('Retry limit reached');
|
||||
}
|
||||
|
||||
module.exports = { retry };
|
12
package.json
12
package.json
@ -32,11 +32,12 @@
|
||||
"test:unit:lax": "mocha --exit --require test/env.js --require test/setup.js --ignore './app/scripts/controllers/permissions/*.test.js' --recursive './{app,shared}/**/*.test.js'",
|
||||
"test:unit:strict": "mocha --exit --require test/env.js --require test/setup.js --recursive './app/scripts/controllers/permissions/*.test.js'",
|
||||
"test:unit:path": "mocha --exit --require test/env.js --require test/setup.js --recursive",
|
||||
"test:e2e:chrome": "SELENIUM_BROWSER=chrome test/e2e/run-all.sh",
|
||||
"test:e2e:chrome:metrics": "SELENIUM_BROWSER=chrome mocha test/e2e/metrics.spec.js",
|
||||
"test:e2e:firefox": "SELENIUM_BROWSER=firefox test/e2e/run-all.sh",
|
||||
"test:e2e:firefox:metrics": "SELENIUM_BROWSER=firefox mocha test/e2e/metrics.spec.js",
|
||||
"test:e2e:chrome": "SELENIUM_BROWSER=chrome node test/e2e/run-all.js",
|
||||
"test:e2e:chrome:metrics": "SELENIUM_BROWSER=chrome node test/e2e/run-e2e-test.js test/e2e/metrics.spec.js",
|
||||
"test:e2e:firefox": "SELENIUM_BROWSER=firefox node test/e2e/run-all.js",
|
||||
"test:e2e:firefox:metrics": "SELENIUM_BROWSER=firefox node test/e2e/run-e2e-test.js test/e2e/metrics.spec.js",
|
||||
"test:coverage": "nyc --silent --check-coverage yarn test:unit:strict && nyc --silent --no-clean yarn test:unit:lax && nyc report --reporter=text --reporter=html",
|
||||
"test:e2e:single": "node test/e2e/run-e2e-test.js",
|
||||
"test:coverage:jest": "jest --coverage --maxWorkers=2",
|
||||
"test:coverage:strict": "nyc --check-coverage yarn test:unit:strict",
|
||||
"test:coverage:path": "nyc --check-coverage yarn test:unit:path",
|
||||
@ -312,7 +313,8 @@
|
||||
"vinyl-buffer": "^1.0.1",
|
||||
"vinyl-source-stream": "^2.0.0",
|
||||
"watchify": "^3.11.1",
|
||||
"webpack": "^4.41.6"
|
||||
"webpack": "^4.41.6",
|
||||
"yargs": "^17.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^14.15.1",
|
||||
|
57
test/e2e/run-all.js
Normal file
57
test/e2e/run-all.js
Normal file
@ -0,0 +1,57 @@
|
||||
const path = require('path');
|
||||
const { promises: fs } = require('fs');
|
||||
const yargs = require('yargs/yargs');
|
||||
const { hideBin } = require('yargs/helpers');
|
||||
const { runInShell } = require('../../development/lib/run-command');
|
||||
const { exitWithError } = require('../../development/lib/exit-with-error');
|
||||
|
||||
async function main() {
|
||||
const { argv } = yargs(hideBin(process.argv))
|
||||
.usage(
|
||||
'$0 [options]',
|
||||
'Run all E2E tests, with a variable number of retries.',
|
||||
(_yargs) =>
|
||||
_yargs
|
||||
.option('browser', {
|
||||
description: `Set the browser used; either 'chrome' or 'firefox'.`,
|
||||
type: 'string',
|
||||
choices: ['chrome', 'firefox'],
|
||||
})
|
||||
.option('retries', {
|
||||
description:
|
||||
'Set how many times the test should be retried upon failure.',
|
||||
type: 'number',
|
||||
}),
|
||||
)
|
||||
.strict()
|
||||
.help('help');
|
||||
|
||||
const { browser, retries } = argv;
|
||||
|
||||
const testDir = path.join(__dirname, 'tests');
|
||||
const metamaskUiTest = path.join(__dirname, 'metamask-ui.spec.js');
|
||||
|
||||
const testFilenames = await fs.readdir(testDir);
|
||||
const testPaths = testFilenames.map((filename) =>
|
||||
path.join(testDir, filename),
|
||||
);
|
||||
const allE2eTestPaths = [...testPaths, metamaskUiTest];
|
||||
|
||||
const runE2eTestPath = path.join(__dirname, 'run-e2e-test.js');
|
||||
|
||||
const args = [runE2eTestPath];
|
||||
if (browser) {
|
||||
args.push('--browser', browser);
|
||||
}
|
||||
if (retries) {
|
||||
args.push('--retries', retries);
|
||||
}
|
||||
|
||||
for (const testPath of allE2eTestPaths) {
|
||||
await runInShell('node', [...args, testPath]);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
exitWithError(error);
|
||||
});
|
@ -5,27 +5,11 @@ set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
retry () {
|
||||
retry=0
|
||||
limit="${METAMASK_E2E_RETRY_LIMIT:-3}"
|
||||
while [[ $retry -lt $limit ]]
|
||||
do
|
||||
"$@" && break
|
||||
retry=$(( retry + 1 ))
|
||||
sleep 1
|
||||
done
|
||||
readonly __DIR__=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
|
||||
|
||||
if [[ $retry == "$limit" ]]
|
||||
then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
export PATH="$PATH:./node_modules/.bin"
|
||||
|
||||
for spec in test/e2e/tests/*.spec.js
|
||||
for spec in "${__DIR__}"/tests/*.spec.js
|
||||
do
|
||||
retry mocha --no-timeouts "${spec}"
|
||||
node "${__DIR__}/run-e2e-test.js" "${spec}"
|
||||
done
|
||||
|
||||
retry mocha --no-timeouts test/e2e/metamask-ui.spec
|
||||
node "${__DIR__}/run-e2e-test.js" "${__DIR__}/metamask-ui.spec.js"
|
||||
|
73
test/e2e/run-e2e-test.js
Normal file
73
test/e2e/run-e2e-test.js
Normal file
@ -0,0 +1,73 @@
|
||||
const { promises: fs } = require('fs');
|
||||
const yargs = require('yargs/yargs');
|
||||
const { hideBin } = require('yargs/helpers');
|
||||
const { runInShell } = require('../../development/lib/run-command');
|
||||
const { exitWithError } = require('../../development/lib/exit-with-error');
|
||||
const { retry } = require('../../development/lib/retry');
|
||||
|
||||
async function main() {
|
||||
const { argv } = yargs(hideBin(process.argv))
|
||||
.usage(
|
||||
'$0 [options] <e2e-test-path>',
|
||||
'Run a single E2E test, with a variable number of retries.',
|
||||
(_yargs) =>
|
||||
_yargs
|
||||
.option('browser', {
|
||||
default: process.env.SELENIUM_BROWSER,
|
||||
description: `Set the browser used; either 'chrome' or 'firefox'.`,
|
||||
type: 'string',
|
||||
choices: ['chrome', 'firefox'],
|
||||
})
|
||||
.option('retries', {
|
||||
default: 0,
|
||||
description:
|
||||
'Set how many times the test should be retried upon failure.',
|
||||
type: 'number',
|
||||
})
|
||||
.positional('e2e-test-path', {
|
||||
describe: 'The path for the E2E test to run.',
|
||||
type: 'string',
|
||||
normalize: true,
|
||||
}),
|
||||
)
|
||||
.strict()
|
||||
.help('help');
|
||||
|
||||
const { browser, e2eTestPath, retries } = argv;
|
||||
|
||||
if (!browser) {
|
||||
exitWithError(
|
||||
`"The browser must be set, via the '--browser' flag or the SELENIUM_BROWSER environment variable`,
|
||||
);
|
||||
return;
|
||||
} else if (browser !== process.env.SELENIUM_BROWSER) {
|
||||
process.env.SELENIUM_BROWSER = browser;
|
||||
}
|
||||
|
||||
try {
|
||||
const stat = await fs.stat(e2eTestPath);
|
||||
if (!stat.isFile()) {
|
||||
exitWithError('Test path must be a file');
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
exitWithError('Test path specified does not exist');
|
||||
return;
|
||||
} else if (error.code === 'EACCES') {
|
||||
exitWithError(
|
||||
'Access to test path is forbidden by file access permissions',
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
await retry(retries, async () => {
|
||||
await runInShell('yarn', ['mocha', '--no-timeouts', e2eTestPath]);
|
||||
});
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
exitWithError(error);
|
||||
});
|
Loading…
Reference in New Issue
Block a user