Add run-e2e-test.js script (#11301)

This script makes it easier to run an individual E2E test. In the past
I've run individual scripts by editing `run-all.sh` manually, but now
that can be done more easily with this script. It also allows setting
the number of retries to use and the browser to use from the CLI.

This script has been added as an npm script as well, called
'test:e2e:single'.

The `run-all.sh` script was rewritten in JavaScript to make it easier
to pass through a `--retries` argument.

The default number of retries has been set to zero to make local
testing easier. It has been set to 2 on CI.

This was mainly done to consolidate the code used to run an E2E test in
one place, to make later improvements easier.

.circleci/config.yml

@@ -343,7 +343,7 @@ jobs:
           command: |
             if .circleci/scripts/test-run-e2e.sh
             then
-              yarn test:e2e:chrome
+              yarn test:e2e:chrome --retries 2
             fi
           no_output_timeout: 20m
       - store_artifacts:
@@ -370,7 +370,7 @@ jobs:
           command: |
             if .circleci/scripts/test-run-e2e.sh
             then
-              yarn test:e2e:chrome:metrics
+              yarn test:e2e:chrome:metrics --retries 2
             fi
           no_output_timeout: 20m
       - store_artifacts:
@@ -397,7 +397,7 @@ jobs:
           command: |
             if .circleci/scripts/test-run-e2e.sh
             then
-              yarn test:e2e:firefox
+              yarn test:e2e:firefox --retries 2
             fi
           no_output_timeout: 20m
       - store_artifacts:
@@ -424,7 +424,7 @@ jobs:
           command: |
             if .circleci/scripts/test-run-e2e.sh
             then
-              yarn test:e2e:firefox:metrics
+              yarn test:e2e:firefox:metrics --retries 2
             fi
           no_output_timeout: 20m
       - store_artifacts:

development/lib/exit-with-error.js

@@ -0,0 +1,16 @@
+/**
+ * Exit the process with an error message.
+ *
+ * Note that this should be called before the process ends, but it will not
+ * itself end the process. This is because the Node.js documentation strongly
+ * advises against calling `process.exit` directly.
+ *
+ * @param {string} errorMessage - The error message that is causing the non-
+ *   zero exit code.
+ */
+function exitWithError(errorMessage) {
+  console.error(errorMessage);
+  process.exitCode = 1;
+}
+
+module.exports = { exitWithError };

development/lib/retry.js

@@ -0,0 +1,25 @@
+const { exitWithError } = require('./exit-with-error');
+
+/**
+ * Run the given function, retrying it upon failure until reaching the
+ * specified number of retries.
+ *
+ * @param {number} retries - The number of retries upon failure to attempt.
+ * @param {function} functionToRetry - The function that will be retried upon failure.
+ */
+async function retry(retries, functionToRetry) {
+  let attempts = 0;
+  while (attempts <= retries) {
+    try {
+      await functionToRetry();
+      return;
+    } catch (error) {
+      console.error(error);
+    } finally {
+      attempts += 1;
+    }
+  }
+  exitWithError('Retry limit reached');
+}
+
+module.exports = { retry };

package.json

@@ -32,11 +32,12 @@
     "test:unit:lax": "mocha --exit --require test/env.js --require test/setup.js --ignore './app/scripts/controllers/permissions/*.test.js' --recursive './{app,shared}/**/*.test.js'",
     "test:unit:strict": "mocha --exit --require test/env.js --require test/setup.js --recursive './app/scripts/controllers/permissions/*.test.js'",
     "test:unit:path": "mocha --exit --require test/env.js --require test/setup.js --recursive",
-    "test:e2e:chrome": "SELENIUM_BROWSER=chrome test/e2e/run-all.sh",
-    "test:e2e:chrome:metrics": "SELENIUM_BROWSER=chrome mocha test/e2e/metrics.spec.js",
-    "test:e2e:firefox": "SELENIUM_BROWSER=firefox test/e2e/run-all.sh",
-    "test:e2e:firefox:metrics": "SELENIUM_BROWSER=firefox mocha test/e2e/metrics.spec.js",
+    "test:e2e:chrome": "SELENIUM_BROWSER=chrome node test/e2e/run-all.js",
+    "test:e2e:chrome:metrics": "SELENIUM_BROWSER=chrome node test/e2e/run-e2e-test.js test/e2e/metrics.spec.js",
+    "test:e2e:firefox": "SELENIUM_BROWSER=firefox node test/e2e/run-all.js",
+    "test:e2e:firefox:metrics": "SELENIUM_BROWSER=firefox node test/e2e/run-e2e-test.js test/e2e/metrics.spec.js",
     "test:coverage": "nyc --silent --check-coverage yarn test:unit:strict && nyc --silent --no-clean yarn test:unit:lax && nyc report --reporter=text --reporter=html",
+    "test:e2e:single": "node test/e2e/run-e2e-test.js",
     "test:coverage:jest": "jest --coverage --maxWorkers=2",
     "test:coverage:strict": "nyc --check-coverage yarn test:unit:strict",
     "test:coverage:path": "nyc --check-coverage yarn test:unit:path",
@@ -312,7 +313,8 @@
     "vinyl-buffer": "^1.0.1",
     "vinyl-source-stream": "^2.0.0",
     "watchify": "^3.11.1",
-    "webpack": "^4.41.6"
+    "webpack": "^4.41.6",
+    "yargs": "^17.0.1"
   },
   "engines": {
     "node": "^14.15.1",

test/e2e/run-all.js

@@ -0,0 +1,57 @@
+const path = require('path');
+const { promises: fs } = require('fs');
+const yargs = require('yargs/yargs');
+const { hideBin } = require('yargs/helpers');
+const { runInShell } = require('../../development/lib/run-command');
+const { exitWithError } = require('../../development/lib/exit-with-error');
+
+async function main() {
+  const { argv } = yargs(hideBin(process.argv))
+    .usage(
+      '$0 [options]',
+      'Run all E2E tests, with a variable number of retries.',
+      (_yargs) =>
+        _yargs
+          .option('browser', {
+            description: `Set the browser used; either 'chrome' or 'firefox'.`,
+            type: 'string',
+            choices: ['chrome', 'firefox'],
+          })
+          .option('retries', {
+            description:
+              'Set how many times the test should be retried upon failure.',
+            type: 'number',
+          }),
+    )
+    .strict()
+    .help('help');
+
+  const { browser, retries } = argv;
+
+  const testDir = path.join(__dirname, 'tests');
+  const metamaskUiTest = path.join(__dirname, 'metamask-ui.spec.js');
+
+  const testFilenames = await fs.readdir(testDir);
+  const testPaths = testFilenames.map((filename) =>
+    path.join(testDir, filename),
+  );
+  const allE2eTestPaths = [...testPaths, metamaskUiTest];
+
+  const runE2eTestPath = path.join(__dirname, 'run-e2e-test.js');
+
+  const args = [runE2eTestPath];
+  if (browser) {
+    args.push('--browser', browser);
+  }
+  if (retries) {
+    args.push('--retries', retries);
+  }
+
+  for (const testPath of allE2eTestPaths) {
+    await runInShell('node', [...args, testPath]);
+  }
+}
+
+main().catch((error) => {
+  exitWithError(error);
+});

test/e2e/run-all.sh

@@ -5,27 +5,11 @@ set -e
 set -u
 set -o pipefail
 
-retry () {
-  retry=0
-  limit="${METAMASK_E2E_RETRY_LIMIT:-3}"
-  while [[ $retry -lt $limit ]]
+readonly __DIR__=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
+
+for spec in "${__DIR__}"/tests/*.spec.js
 do
-    "$@" && break
-    retry=$(( retry + 1 ))
-    sleep 1
+  node "${__DIR__}/run-e2e-test.js" "${spec}"
 done
 
-  if [[ $retry == "$limit" ]]
-  then
-    exit 1
-  fi
-}
-
-export PATH="$PATH:./node_modules/.bin"
-
-for spec in test/e2e/tests/*.spec.js
-do
-  retry mocha --no-timeouts "${spec}"
-done
-
-retry mocha --no-timeouts test/e2e/metamask-ui.spec
+node "${__DIR__}/run-e2e-test.js" "${__DIR__}/metamask-ui.spec.js"

test/e2e/run-e2e-test.js

@@ -0,0 +1,73 @@
+const { promises: fs } = require('fs');
+const yargs = require('yargs/yargs');
+const { hideBin } = require('yargs/helpers');
+const { runInShell } = require('../../development/lib/run-command');
+const { exitWithError } = require('../../development/lib/exit-with-error');
+const { retry } = require('../../development/lib/retry');
+
+async function main() {
+  const { argv } = yargs(hideBin(process.argv))
+    .usage(
+      '$0 [options] <e2e-test-path>',
+      'Run a single E2E test, with a variable number of retries.',
+      (_yargs) =>
+        _yargs
+          .option('browser', {
+            default: process.env.SELENIUM_BROWSER,
+            description: `Set the browser used; either 'chrome' or 'firefox'.`,
+            type: 'string',
+            choices: ['chrome', 'firefox'],
+          })
+          .option('retries', {
+            default: 0,
+            description:
+              'Set how many times the test should be retried upon failure.',
+            type: 'number',
+          })
+          .positional('e2e-test-path', {
+            describe: 'The path for the E2E test to run.',
+            type: 'string',
+            normalize: true,
+          }),
+    )
+    .strict()
+    .help('help');
+
+  const { browser, e2eTestPath, retries } = argv;
+
+  if (!browser) {
+    exitWithError(
+      `"The browser must be set, via the '--browser' flag or the SELENIUM_BROWSER environment variable`,
+    );
+    return;
+  } else if (browser !== process.env.SELENIUM_BROWSER) {
+    process.env.SELENIUM_BROWSER = browser;
+  }
+
+  try {
+    const stat = await fs.stat(e2eTestPath);
+    if (!stat.isFile()) {
+      exitWithError('Test path must be a file');
+      return;
+    }
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      exitWithError('Test path specified does not exist');
+      return;
+    } else if (error.code === 'EACCES') {
+      exitWithError(
+        'Access to test path is forbidden by file access permissions',
+      );
+      return;
+    }
+    throw error;
+  }
+
+  await retry(retries, async () => {
+    await runInShell('yarn', ['mocha', '--no-timeouts', e2eTestPath]);
+  });
+}
+
+main().catch((error) => {
+  exitWithError(error);
+});