Add --retry-until-failure flag to e2e test script (#19532)

The e2e test script now accepts a `--retry-until-failure` flag that can
be useful for locally reproducing intermittent failures.

Normally the `retries` option will only retry upon failure. But if
you want the test to fail to inspect the failure state, you want it to
keep running until the first failure then stop. This flag accomplishes
that, reversing the retry conditions.

development/lib/retry.js

@@ -11,13 +11,19 @@
  * @param {string} args.rejectionMessage - The message for the rejected promise
  * this function will return in the event of failure. (Default: "Retry limit
  * reached")
+ * @param {string} args.retryUntilFailure - Retries until the function fails.
  * @param {Function} functionToRetry - The function that is run and tested for
  * failure.
  * @returns {Promise<null | Error>} a promise that either resolves to null if
  * the function is successful or is rejected with rejectionMessage otherwise.
  */
 async function retry(
-  { retries, delay = 0, rejectionMessage = 'Retry limit reached' },
+  {
+    retries,
+    delay = 0,
+    rejectionMessage = 'Retry limit reached',
+    retryUntilFailure = false,
+  },
   functionToRetry,
 ) {
   let attempts = 0;
@@ -28,9 +34,14 @@ async function retry(
 
     try {
       await functionToRetry();
-      return;
+      if (!retryUntilFailure) {
+        return;
+      }
     } catch (error) {
       console.error(error);
+      if (retryUntilFailure) {
+        return;
+      }
     } finally {
       attempts += 1;
     }

test/e2e/run-e2e-test.js

@@ -31,6 +31,11 @@ async function main() {
               'Set how many times the test should be retried upon failure.',
             type: 'number',
           })
+          .option('retry-until-failure', {
+            default: false,
+            description: 'Retries until the test fails',
+            type: 'boolean',
+          })
           .option('leave-running', {
             default: false,
             description:
@@ -46,7 +51,14 @@ async function main() {
     .strict()
     .help('help');
 
-  const { browser, debug, e2eTestPath, retries, leaveRunning } = argv;
+  const {
+    browser,
+    debug,
+    e2eTestPath,
+    retries,
+    retryUntilFailure,
+    leaveRunning,
+  } = argv;
 
   if (!browser) {
     exitWithError(
@@ -97,7 +109,7 @@ async function main() {
   const dir = 'test/test-results/e2e';
   fs.mkdir(dir, { recursive: true });
 
-  await retry({ retries }, async () => {
+  await retry({ retries, retryUntilFailure }, async () => {
     await runInShell(
       'yarn',
       [