Add metrics e2e test (#9784)

An e2e test has been added that uses the new mock Segment server to verify that the three initial page metric events are sent correctly. Using the mock Segment server requires a special build with this mock Segment server hostname embedded, so a distinct job for building and running this test was required. As such, it was left out of the `run-all.sh` script.
2024-10-22 11:22:43 +02:00 · 2020-12-01 17:54:56 -03:30 · 2020-12-01 17:54:56 -03:30 · cc1161a52a
commit cc1161a52a
parent cb44cff168
5 changed files with 341 additions and 2 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -18,6 +18,9 @@ workflows:
      - prep-build-test:
          requires:
            - prep-deps
+      - prep-build-test-metrics:
+          requires:
+            - prep-deps
      - prep-build-storybook:
          requires:
            - prep-deps
@ -34,6 +37,12 @@ workflows:
      - test-e2e-firefox:
          requires:
            - prep-build-test
+      - test-e2e-chrome-metrics:
+          requires:
+            - prep-build-test-metrics
+      - test-e2e-firefox-metrics:
+          requires:
+            - prep-build-test-metrics
      - test-unit:
          requires:
            - prep-deps
@ -58,6 +67,8 @@ workflows:
            - test-mozilla-lint
            - test-e2e-chrome
            - test-e2e-firefox
+            - test-e2e-chrome-metrics
+            - test-e2e-firefox-metrics
      - benchmark:
          requires:
            - prep-build-test
@ -162,6 +173,27 @@ jobs:
          paths:
            - dist-test

+  prep-build-test-metrics:
+    docker:
+      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
+    resource_class: medium+
+    environment:
+      NODE_OPTIONS: --max_old_space_size=2048
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Build extension for testing metrics
+          command: yarn build:test:metrics
+      - run:
+          name: Move test build to 'dist-test-metrics' to avoid conflict with production build
+          command: mv ./dist ./dist-test-metrics
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist-test-metrics
+
  prep-build-storybook:
    docker:
      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
@ -245,6 +277,28 @@ jobs:
          path: test-artifacts
          destination: test-artifacts

+  test-e2e-chrome-metrics:
+    docker:
+      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Move test build to dist
+          command: mv ./dist-test-metrics ./dist
+      - run:
+          name: test:e2e:chrome:metrics
+          command: |
+            if .circleci/scripts/test-run-e2e
+            then
+              yarn test:e2e:chrome:metrics
+            fi
+          no_output_timeout: 20m
+      - store_artifacts:
+          path: test-artifacts
+          destination: test-artifacts
+
  test-e2e-firefox:
    docker:
      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
@ -270,6 +324,31 @@ jobs:
          path: test-artifacts
          destination: test-artifacts

+  test-e2e-firefox-metrics:
+    docker:
+      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
+    steps:
+      - checkout
+      - run:
+          name: Install Firefox
+          command: ./.circleci/scripts/firefox-install
+      - attach_workspace:
+          at: .
+      - run:
+          name: Move test build to dist
+          command: mv ./dist-test-metrics ./dist
+      - run:
+          name: test:e2e:firefox:metrics
+          command: |
+            if .circleci/scripts/test-run-e2e
+            then
+              yarn test:e2e:firefox:metrics
+            fi
+          no_output_timeout: 20m
+      - store_artifacts:
+          path: test-artifacts
+          destination: test-artifacts
+
  benchmark:
    docker:
      - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88
--- a/package.json
+++ b/package.json
@ -10,6 +10,7 @@
    "benchmark:chrome": "SELENIUM_BROWSER=chrome node test/e2e/benchmark.js",
    "benchmark:firefox": "SELENIUM_BROWSER=firefox node test/e2e/benchmark.js",
    "build:test": "yarn build test",
+    "build:test:metrics": "SEGMENT_HOST='http://localhost:9090' SEGMENT_WRITE_KEY='FAKE' SEGMENT_LEGACY_WRITE_KEY='FAKE' yarn build test",
    "test": "yarn test:unit && yarn lint",
    "dapp": "node development/static-server.js node_modules/@metamask/test-dapp/dist --port 8080",
    "dapp-chain": "GANACHE_ARGS='-b 2' concurrently -k -n ganache,dapp -p '[{time}][{name}]' 'yarn ganache:start' 'sleep 5 && yarn dapp'",
@ -22,7 +23,9 @@
    "test:unit:strict": "mocha --exit --require test/env.js --require test/setup.js --recursive \"test/unit/**/permissions/*.js\"",
    "test:unit:path": "mocha --exit --require test/env.js --require test/setup.js --recursive",
    "test:e2e:chrome": "SELENIUM_BROWSER=chrome test/e2e/run-all.sh",
+    "test:e2e:chrome:metrics": "SELENIUM_BROWSER=chrome mocha test/e2e/metrics.spec.js",
    "test:e2e:firefox": "SELENIUM_BROWSER=firefox test/e2e/run-all.sh",
+    "test:e2e:firefox:metrics": "SELENIUM_BROWSER=firefox mocha test/e2e/metrics.spec.js",
    "test:coverage": "nyc --silent --check-coverage yarn test:unit:strict && nyc --silent --no-clean yarn test:unit:lax && nyc report --reporter=text --reporter=html",
    "test:coverage:strict": "nyc --check-coverage yarn test:unit:strict",
    "test:coverage:path": "nyc --check-coverage yarn test:unit:path",
--- a/test/e2e/fixtures/metrics-enabled/state.json
+++ b/test/e2e/fixtures/metrics-enabled/state.json
@ -0,0 +1,168 @@
+{
+  "data": {
+    "AppStateController": {
+      "connectedStatusPopoverHasBeenShown": false,
+      "swapsWelcomeMessageHasBeenShown": true
+    },
+    "CachedBalancesController": {
+      "cachedBalances": {
+        "4": {},
+        "1337": {
+          "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": "0x15af1d78b58c40000"
+        }
+      }
+    },
+    "CurrencyController": {
+      "conversionDate": 1594348502.519,
+      "conversionRate": 240.09,
+      "currentCurrency": "usd",
+      "nativeCurrency": "ETH"
+    },
+    "IncomingTransactionsController": {
+      "incomingTransactions": {},
+      "incomingTxLastFetchedBlocksByNetwork": {
+        "goerli": null,
+        "kovan": null,
+        "mainnet": null,
+        "rinkeby": 5570536,
+        "localhost": 98
+      }
+    },
+    "KeyringController": {
+      "vault": "{\"data\":\"s6TpYjlUNsn7ifhEFTkuDGBUM1GyOlPrim7JSjtfIxgTt8/6MiXgiR/CtFfR4dWW2xhq85/NGIBYEeWrZThGdKGarBzeIqBfLFhw9n509jprzJ0zc2Rf+9HVFGLw+xxC4xPxgCS0IIWeAJQ+XtGcHmn0UZXriXm8Ja4kdlow6SWinB7sr/WM3R0+frYs4WgllkwggDf2/Tv6VHygvLnhtzp6hIJFyTjh+l/KnyJTyZW1TkZhDaNDzX3SCOHT\",\"iv\":\"FbeHDAW5afeWNORfNJBR0Q==\",\"salt\":\"TxZ+WbCW6891C9LK/hbMAoUsSEW1E8pyGLVBU6x5KR8=\"}"
+    },
+    "NetworkController": {
+      "provider": {
+        "nickname": "Localhost 8545",
+        "rpcUrl": "http://localhost:8545",
+        "chainId": "0x539",
+        "ticker": "ETH",
+        "type": "rpc"
+      },
+      "network": "1337"
+    },
+    "OnboardingController": {
+      "onboardingTabs": {},
+      "seedPhraseBackedUp": false
+    },
+    "PermissionsMetadata": {
+      "permissionsLog": [
+        {
+          "id": 1764280960,
+          "method": "eth_requestAccounts",
+          "methodType": "restricted",
+          "origin": "http://127.0.0.1:8080",
+          "request": {
+            "method": "eth_requestAccounts",
+            "jsonrpc": "2.0",
+            "id": 1764280960,
+            "origin": "http://127.0.0.1:8080",
+            "tabId": 2
+          },
+          "requestTime": 1594348329232,
+          "response": {
+            "id": 1764280960,
+            "jsonrpc": "2.0",
+            "result": ["0x5cfe73b6021e818b776b421b1c4db2474086a7e1"]
+          },
+          "responseTime": 1594348332276,
+          "success": true
+        }
+      ],
+      "permissionsHistory": {
+        "http://127.0.0.1:8080": {
+          "eth_accounts": {
+            "accounts": {
+              "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": 1594348332276
+            },
+            "lastApproved": 1594348332276
+          }
+        }
+      },
+      "domainMetadata": {
+        "http://127.0.0.1:8080": {
+          "name": "E2E Test Dapp",
+          "icon": "http://127.0.0.1:8080/metamask-fox.svg",
+          "lastUpdated": 1594348323811,
+          "host": "127.0.0.1:8080"
+        }
+      }
+    },
+    "PreferencesController": {
+      "frequentRpcListDetail": [],
+      "accountTokens": {
+        "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": {
+          "rinkeby": [],
+          "ropsten": []
+        }
+      },
+      "assetImages": {},
+      "tokens": [],
+      "suggestedTokens": {},
+      "useBlockie": false,
+      "useNonceField": false,
+      "usePhishDetect": true,
+      "featureFlags": {
+        "showIncomingTransactions": true,
+        "transactionTime": false
+      },
+      "knownMethodData": {},
+      "participateInMetaMetrics": true,
+      "firstTimeFlowType": "create",
+      "currentLocale": "en",
+      "identities": {
+        "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": {
+          "address": "0x5cfe73b6021e818b776b421b1c4db2474086a7e1",
+          "name": "Account 1"
+        }
+      },
+      "lostIdentities": {},
+      "forgottenPassword": false,
+      "preferences": {
+        "useNativeCurrencyAsPrimaryCurrency": true
+      },
+      "completedOnboarding": true,
+      "metaMetricsId": "fake-metrics-id",
+      "metaMetricsSendCount": 0,
+      "ipfsGateway": "dweb.link",
+      "selectedAddress": "0x5cfe73b6021e818b776b421b1c4db2474086a7e1"
+    },
+    "config": {},
+    "firstTimeInfo": {
+      "date": 1575697234195,
+      "version": "7.7.0"
+    },
+    "PermissionsController": {
+      "permissionsRequests": [],
+      "permissionsDescriptions": {},
+      "domains": {
+        "http://127.0.0.1:8080": {
+          "permissions": [
+            {
+              "@context": ["https://github.com/MetaMask/rpc-cap"],
+              "parentCapability": "eth_accounts",
+              "id": "f55a1c15-ea48-4088-968e-63be474d42fa",
+              "date": 1594348332268,
+              "invoker": "http://127.0.0.1:8080",
+              "caveats": [
+                {
+                  "type": "limitResponseLength",
+                  "value": 1,
+                  "name": "primaryAccountOnly"
+                },
+                {
+                  "type": "filterResponse",
+                  "value": ["0x5cfe73b6021e818b776b421b1c4db2474086a7e1"],
+                  "name": "exposedAccounts"
+                }
+              ]
+            }
+          ]
+        }
+      }
+    }
+  },
+  "meta": {
+    "version": 47
+  }
+}
--- a/test/e2e/helpers.js
+++ b/test/e2e/helpers.js
@ -1,5 +1,9 @@
 const path = require('path')
+const sinon = require('sinon')
 const createStaticServer = require('../../development/create-static-server')
+const {
+  createSegmentServer,
+} = require('../../development/lib/create-segment-server')
 const Ganache = require('./ganache')
 const FixtureServer = require('./fixture-server')
 const { buildWebDriver } = require('./webdriver')
@ -11,10 +15,19 @@ const largeDelayMs = regularDelayMs * 2
 const dappPort = 8080

 async function withFixtures(options, testSuite) {
-  const { dapp, fixtures, ganacheOptions, driverOptions, title } = options
+  const {
+    dapp,
+    fixtures,
+    ganacheOptions,
+    driverOptions,
+    mockSegment,
+    title,
+  } = options
  const fixtureServer = new FixtureServer()
  const ganacheServer = new Ganache()
  let dappServer
+  let segmentServer
+  let segmentSpy

  let webDriver
  try {
@ -38,11 +51,23 @@ async function withFixtures(options, testSuite) {
        dappServer.on('error', reject)
      })
    }
+    if (mockSegment) {
+      segmentSpy = sinon.spy()
+      segmentServer = createSegmentServer((_request, response, events) => {
+        for (const event of events) {
+          segmentSpy(event)
+        }
+        response.statusCode = 200
+        response.end()
+      })
+      await segmentServer.start(9090)
+    }
    const { driver } = await buildWebDriver(driverOptions)
    webDriver = driver

    await testSuite({
      driver,
+      segmentSpy,
    })

    if (process.env.SELENIUM_BROWSER === 'chrome') {
@ -57,7 +82,11 @@ async function withFixtures(options, testSuite) {
    }
  } catch (error) {
    if (webDriver) {
-      await webDriver.verboseReportOnFailure(title)
+      try {
+        await webDriver.verboseReportOnFailure(title)
+      } catch (verboseReportError) {
+        console.error(verboseReportError)
+      }
    }
    throw error
  } finally {
@ -76,6 +105,9 @@ async function withFixtures(options, testSuite) {
        })
      })
    }
+    if (segmentServer) {
+      await segmentServer.stop()
+    }
  }
 }

--- a/test/e2e/metrics.spec.js
+++ b/test/e2e/metrics.spec.js
@ -0,0 +1,57 @@
+const { strict: assert } = require('assert')
+const { By, Key } = require('selenium-webdriver')
+const { withFixtures } = require('./helpers')
+
+/**
+ * WARNING: These tests must be run using a build created with `yarn build:test:metrics`, so that it has
+ * the correct Segment host and write keys set. Otherwise this test will fail.
+ */
+describe('Segment metrics', function () {
+  this.timeout(0)
+
+  it('should send first three Page metric events upon fullscreen page load', async function () {
+    const ganacheOptions = {
+      accounts: [
+        {
+          secretKey:
+            '0x7C9529A67102755B7E6102D6D950AC5D5863C98713805CEC576B945B15B71EAC',
+          balance: 25000000000000000000,
+        },
+      ],
+    }
+    await withFixtures(
+      {
+        fixtures: 'metrics-enabled',
+        ganacheOptions,
+        title: this.test.title,
+        mockSegment: true,
+      },
+      async ({ driver, segmentSpy }) => {
+        const passwordField = await driver.findElement(By.css('#password'))
+        await passwordField.sendKeys('correct horse battery staple')
+        await passwordField.sendKeys(Key.ENTER)
+
+        // find arbitary element to ensure Home page has loaded
+        await driver.findElement(By.css('[data-testid="eth-overview-send"]'))
+
+        assert.ok(segmentSpy.called, 'Segment should receive metrics')
+        assert.ok(
+          segmentSpy.callCount >= 3,
+          'At least 3 segment events should be sent',
+        )
+
+        const firstSegmentEvent = segmentSpy.getCall(0).args[0]
+        assert.equal(firstSegmentEvent.name, 'Home')
+        assert.equal(firstSegmentEvent.context.page.path, '/')
+
+        const secondSegmentEvent = segmentSpy.getCall(1).args[0]
+        assert.equal(secondSegmentEvent.name, 'Unlock Page')
+        assert.equal(secondSegmentEvent.context.page.path, '/unlock')
+
+        const thirdSegmentEvent = segmentSpy.getCall(2).args[0]
+        assert.equal(thirdSegmentEvent.name, 'Home')
+        assert.equal(thirdSegmentEvent.context.page.path, '/')
+      },
+    )
+  })
+})