Add metrics e2e test (#9784)

An e2e test has been added that uses the new mock Segment server to
verify that the three initial page metric events are sent correctly.

Using the mock Segment server requires a special build with this mock
Segment server hostname embedded, so a distinct job for building and
running this test was required. As such, it was left out of the
`run-all.sh` script.

.circleci/config.yml

@@ -18,6 +18,9 @@ workflows:
       - prep-build-test:
           requires:
             - prep-deps
+      - prep-build-test-metrics:
+          requires:
+            - prep-deps
       - prep-build-storybook:
           requires:
             - prep-deps
@@ -34,6 +37,12 @@ workflows:
       - test-e2e-firefox:
           requires:
             - prep-build-test
+      - test-e2e-chrome-metrics:
+          requires:
+            - prep-build-test-metrics
+      - test-e2e-firefox-metrics:
+          requires:
+            - prep-build-test-metrics
       - test-unit:
           requires:
             - prep-deps
@@ -58,6 +67,8 @@ workflows:
             - test-mozilla-lint
             - test-e2e-chrome
             - test-e2e-firefox
+            - test-e2e-chrome-metrics
+            - test-e2e-firefox-metrics
       - benchmark:
           requires:
             - prep-build-test
@@ -162,6 +173,27 @@ jobs:
           paths:
             - dist-test
 
+  prep-build-test-metrics:
+    docker:
+      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
+    resource_class: medium+
+    environment:
+      NODE_OPTIONS: --max_old_space_size=2048
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Build extension for testing metrics
+          command: yarn build:test:metrics
+      - run:
+          name: Move test build to 'dist-test-metrics' to avoid conflict with production build
+          command: mv ./dist ./dist-test-metrics
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist-test-metrics
+
   prep-build-storybook:
     docker:
       - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
@@ -245,6 +277,28 @@ jobs:
           path: test-artifacts
           destination: test-artifacts
 
+  test-e2e-chrome-metrics:
+    docker:
+      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Move test build to dist
+          command: mv ./dist-test-metrics ./dist
+      - run:
+          name: test:e2e:chrome:metrics
+          command: |
+            if .circleci/scripts/test-run-e2e
+            then
+              yarn test:e2e:chrome:metrics
+            fi
+          no_output_timeout: 20m
+      - store_artifacts:
+          path: test-artifacts
+          destination: test-artifacts
+
   test-e2e-firefox:
     docker:
       - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
@@ -270,6 +324,31 @@ jobs:
           path: test-artifacts
           destination: test-artifacts
 
+  test-e2e-firefox-metrics:
+    docker:
+      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
+    steps:
+      - checkout
+      - run:
+          name: Install Firefox
+          command: ./.circleci/scripts/firefox-install
+      - attach_workspace:
+          at: .
+      - run:
+          name: Move test build to dist
+          command: mv ./dist-test-metrics ./dist
+      - run:
+          name: test:e2e:firefox:metrics
+          command: |
+            if .circleci/scripts/test-run-e2e
+            then
+              yarn test:e2e:firefox:metrics
+            fi
+          no_output_timeout: 20m
+      - store_artifacts:
+          path: test-artifacts
+          destination: test-artifacts
+
   benchmark:
     docker:
       - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88

package.json

@@ -10,6 +10,7 @@
     "benchmark:chrome": "SELENIUM_BROWSER=chrome node test/e2e/benchmark.js",
     "benchmark:firefox": "SELENIUM_BROWSER=firefox node test/e2e/benchmark.js",
     "build:test": "yarn build test",
+    "build:test:metrics": "SEGMENT_HOST='http://localhost:9090' SEGMENT_WRITE_KEY='FAKE' SEGMENT_LEGACY_WRITE_KEY='FAKE' yarn build test",
     "test": "yarn test:unit && yarn lint",
     "dapp": "node development/static-server.js node_modules/@metamask/test-dapp/dist --port 8080",
     "dapp-chain": "GANACHE_ARGS='-b 2' concurrently -k -n ganache,dapp -p '[{time}][{name}]' 'yarn ganache:start' 'sleep 5 && yarn dapp'",
@@ -22,7 +23,9 @@
     "test:unit:strict": "mocha --exit --require test/env.js --require test/setup.js --recursive \"test/unit/**/permissions/*.js\"",
     "test:unit:path": "mocha --exit --require test/env.js --require test/setup.js --recursive",
     "test:e2e:chrome": "SELENIUM_BROWSER=chrome test/e2e/run-all.sh",
+    "test:e2e:chrome:metrics": "SELENIUM_BROWSER=chrome mocha test/e2e/metrics.spec.js",
     "test:e2e:firefox": "SELENIUM_BROWSER=firefox test/e2e/run-all.sh",
+    "test:e2e:firefox:metrics": "SELENIUM_BROWSER=firefox mocha test/e2e/metrics.spec.js",
     "test:coverage": "nyc --silent --check-coverage yarn test:unit:strict && nyc --silent --no-clean yarn test:unit:lax && nyc report --reporter=text --reporter=html",
     "test:coverage:strict": "nyc --check-coverage yarn test:unit:strict",
     "test:coverage:path": "nyc --check-coverage yarn test:unit:path",

test/e2e/fixtures/metrics-enabled/state.json

@@ -0,0 +1,168 @@
+{
+  "data": {
+    "AppStateController": {
+      "connectedStatusPopoverHasBeenShown": false,
+      "swapsWelcomeMessageHasBeenShown": true
+    },
+    "CachedBalancesController": {
+      "cachedBalances": {
+        "4": {},
+        "1337": {
+          "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": "0x15af1d78b58c40000"
+        }
+      }
+    },
+    "CurrencyController": {
+      "conversionDate": 1594348502.519,
+      "conversionRate": 240.09,
+      "currentCurrency": "usd",
+      "nativeCurrency": "ETH"
+    },
+    "IncomingTransactionsController": {
+      "incomingTransactions": {},
+      "incomingTxLastFetchedBlocksByNetwork": {
+        "goerli": null,
+        "kovan": null,
+        "mainnet": null,
+        "rinkeby": 5570536,
+        "localhost": 98
+      }
+    },
+    "KeyringController": {
+      "vault": "{\"data\":\"s6TpYjlUNsn7ifhEFTkuDGBUM1GyOlPrim7JSjtfIxgTt8/6MiXgiR/CtFfR4dWW2xhq85/NGIBYEeWrZThGdKGarBzeIqBfLFhw9n509jprzJ0zc2Rf+9HVFGLw+xxC4xPxgCS0IIWeAJQ+XtGcHmn0UZXriXm8Ja4kdlow6SWinB7sr/WM3R0+frYs4WgllkwggDf2/Tv6VHygvLnhtzp6hIJFyTjh+l/KnyJTyZW1TkZhDaNDzX3SCOHT\",\"iv\":\"FbeHDAW5afeWNORfNJBR0Q==\",\"salt\":\"TxZ+WbCW6891C9LK/hbMAoUsSEW1E8pyGLVBU6x5KR8=\"}"
+    },
+    "NetworkController": {
+      "provider": {
+        "nickname": "Localhost 8545",
+        "rpcUrl": "http://localhost:8545",
+        "chainId": "0x539",
+        "ticker": "ETH",
+        "type": "rpc"
+      },
+      "network": "1337"
+    },
+    "OnboardingController": {
+      "onboardingTabs": {},
+      "seedPhraseBackedUp": false
+    },
+    "PermissionsMetadata": {
+      "permissionsLog": [
+        {
+          "id": 1764280960,
+          "method": "eth_requestAccounts",
+          "methodType": "restricted",
+          "origin": "http://127.0.0.1:8080",
+          "request": {
+            "method": "eth_requestAccounts",
+            "jsonrpc": "2.0",
+            "id": 1764280960,
+            "origin": "http://127.0.0.1:8080",
+            "tabId": 2
+          },
+          "requestTime": 1594348329232,
+          "response": {
+            "id": 1764280960,
+            "jsonrpc": "2.0",
+            "result": ["0x5cfe73b6021e818b776b421b1c4db2474086a7e1"]
+          },
+          "responseTime": 1594348332276,
+          "success": true
+        }
+      ],
+      "permissionsHistory": {
+        "http://127.0.0.1:8080": {
+          "eth_accounts": {
+            "accounts": {
+              "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": 1594348332276
+            },
+            "lastApproved": 1594348332276
+          }
+        }
+      },
+      "domainMetadata": {
+        "http://127.0.0.1:8080": {
+          "name": "E2E Test Dapp",
+          "icon": "http://127.0.0.1:8080/metamask-fox.svg",
+          "lastUpdated": 1594348323811,
+          "host": "127.0.0.1:8080"
+        }
+      }
+    },
+    "PreferencesController": {
+      "frequentRpcListDetail": [],
+      "accountTokens": {
+        "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": {
+          "rinkeby": [],
+          "ropsten": []
+        }
+      },
+      "assetImages": {},
+      "tokens": [],
+      "suggestedTokens": {},
+      "useBlockie": false,
+      "useNonceField": false,
+      "usePhishDetect": true,
+      "featureFlags": {
+        "showIncomingTransactions": true,
+        "transactionTime": false
+      },
+      "knownMethodData": {},
+      "participateInMetaMetrics": true,
+      "firstTimeFlowType": "create",
+      "currentLocale": "en",
+      "identities": {
+        "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": {
+          "address": "0x5cfe73b6021e818b776b421b1c4db2474086a7e1",
+          "name": "Account 1"
+        }
+      },
+      "lostIdentities": {},
+      "forgottenPassword": false,
+      "preferences": {
+        "useNativeCurrencyAsPrimaryCurrency": true
+      },
+      "completedOnboarding": true,
+      "metaMetricsId": "fake-metrics-id",
+      "metaMetricsSendCount": 0,
+      "ipfsGateway": "dweb.link",
+      "selectedAddress": "0x5cfe73b6021e818b776b421b1c4db2474086a7e1"
+    },
+    "config": {},
+    "firstTimeInfo": {
+      "date": 1575697234195,
+      "version": "7.7.0"
+    },
+    "PermissionsController": {
+      "permissionsRequests": [],
+      "permissionsDescriptions": {},
+      "domains": {
+        "http://127.0.0.1:8080": {
+          "permissions": [
+            {
+              "@context": ["https://github.com/MetaMask/rpc-cap"],
+              "parentCapability": "eth_accounts",
+              "id": "f55a1c15-ea48-4088-968e-63be474d42fa",
+              "date": 1594348332268,
+              "invoker": "http://127.0.0.1:8080",
+              "caveats": [
+                {
+                  "type": "limitResponseLength",
+                  "value": 1,
+                  "name": "primaryAccountOnly"
+                },
+                {
+                  "type": "filterResponse",
+                  "value": ["0x5cfe73b6021e818b776b421b1c4db2474086a7e1"],
+                  "name": "exposedAccounts"
+                }
+              ]
+            }
+          ]
+        }
+      }
+    }
+  },
+  "meta": {
+    "version": 47
+  }
+}

test/e2e/helpers.js

@@ -1,5 +1,9 @@
 const path = require('path')
+const sinon = require('sinon')
 const createStaticServer = require('../../development/create-static-server')
+const {
+  createSegmentServer,
+} = require('../../development/lib/create-segment-server')
 const Ganache = require('./ganache')
 const FixtureServer = require('./fixture-server')
 const { buildWebDriver } = require('./webdriver')
@@ -11,10 +15,19 @@ const largeDelayMs = regularDelayMs * 2
 const dappPort = 8080
 
 async function withFixtures(options, testSuite) {
-  const { dapp, fixtures, ganacheOptions, driverOptions, title } = options
+  const {
+    dapp,
+    fixtures,
+    ganacheOptions,
+    driverOptions,
+    mockSegment,
+    title,
+  } = options
   const fixtureServer = new FixtureServer()
   const ganacheServer = new Ganache()
   let dappServer
+  let segmentServer
+  let segmentSpy
 
   let webDriver
   try {
@@ -38,11 +51,23 @@ async function withFixtures(options, testSuite) {
         dappServer.on('error', reject)
       })
     }
+    if (mockSegment) {
+      segmentSpy = sinon.spy()
+      segmentServer = createSegmentServer((_request, response, events) => {
+        for (const event of events) {
+          segmentSpy(event)
+        }
+        response.statusCode = 200
+        response.end()
+      })
+      await segmentServer.start(9090)
+    }
     const { driver } = await buildWebDriver(driverOptions)
     webDriver = driver
 
     await testSuite({
       driver,
+      segmentSpy,
     })
 
     if (process.env.SELENIUM_BROWSER === 'chrome') {
@@ -57,7 +82,11 @@ async function withFixtures(options, testSuite) {
     }
   } catch (error) {
     if (webDriver) {
-      await webDriver.verboseReportOnFailure(title)
+      try {
+        await webDriver.verboseReportOnFailure(title)
+      } catch (verboseReportError) {
+        console.error(verboseReportError)
+      }
     }
     throw error
   } finally {
@@ -76,6 +105,9 @@ async function withFixtures(options, testSuite) {
         })
       })
     }
+    if (segmentServer) {
+      await segmentServer.stop()
+    }
   }
 }
 

test/e2e/metrics.spec.js

@@ -0,0 +1,57 @@
+const { strict: assert } = require('assert')
+const { By, Key } = require('selenium-webdriver')
+const { withFixtures } = require('./helpers')
+
+/**
+ * WARNING: These tests must be run using a build created with `yarn build:test:metrics`, so that it has
+ * the correct Segment host and write keys set. Otherwise this test will fail.
+ */
+describe('Segment metrics', function () {
+  this.timeout(0)
+
+  it('should send first three Page metric events upon fullscreen page load', async function () {
+    const ganacheOptions = {
+      accounts: [
+        {
+          secretKey:
+            '0x7C9529A67102755B7E6102D6D950AC5D5863C98713805CEC576B945B15B71EAC',
+          balance: 25000000000000000000,
+        },
+      ],
+    }
+    await withFixtures(
+      {
+        fixtures: 'metrics-enabled',
+        ganacheOptions,
+        title: this.test.title,
+        mockSegment: true,
+      },
+      async ({ driver, segmentSpy }) => {
+        const passwordField = await driver.findElement(By.css('#password'))
+        await passwordField.sendKeys('correct horse battery staple')
+        await passwordField.sendKeys(Key.ENTER)
+
+        // find arbitary element to ensure Home page has loaded
+        await driver.findElement(By.css('[data-testid="eth-overview-send"]'))
+
+        assert.ok(segmentSpy.called, 'Segment should receive metrics')
+        assert.ok(
+          segmentSpy.callCount >= 3,
+          'At least 3 segment events should be sent',
+        )
+
+        const firstSegmentEvent = segmentSpy.getCall(0).args[0]
+        assert.equal(firstSegmentEvent.name, 'Home')
+        assert.equal(firstSegmentEvent.context.page.path, '/')
+
+        const secondSegmentEvent = segmentSpy.getCall(1).args[0]
+        assert.equal(secondSegmentEvent.name, 'Unlock Page')
+        assert.equal(secondSegmentEvent.context.page.path, '/unlock')
+
+        const thirdSegmentEvent = segmentSpy.getCall(2).args[0]
+        assert.equal(thirdSegmentEvent.name, 'Home')
+        assert.equal(thirdSegmentEvent.context.page.path, '/')
+      },
+    )
+  })
+})