m/metamask-extension
1
0
Fork 0
mirror of https://github.com/kremalicious/metamask-extension.git synced 2024-11-21 17:37:01 +01:00
Code

Add additional Sentry E2E tests (#20425)

Browse Source 
* Reorganize Sentry error e2e tests

The tests have been reorganized into different describe blocks. Each
describe block is for either before or after initialization, and either
with or without opting into metrics.

This was done to simplify later test additions. The conditions for each
test are now in the describe block, letting us test additional things
in each of these conditions. The conditions were flattened to a single
level to avoid excessive indentation.

* Add error e2e test for background and UI errors

The Sentry e2e tests before initialization only tested background
errors, and the after initialization tests only tested UI errors. Now
both types of errors are tested in both scenarios.

* Add error e2e tests for Sentry error state

E2E tests have been added to test the state object sent along with each
Sentry error.

At the moment this object is empty in some circumstances, but this will
change in later PRs.

* Rename throw test error function

* Only setup debug/test state hooks in dev/test builds

The state hooks used for debugging and testing are now only included in
dev or test builds. The function name was updated and given a JSDoc
description to explain this more clearly as well.

* Add state snapshot assertions

State snapshot assertions have been added to the e2e error tests. These
snapshots will be very useful in reviewing a few PRs that will follow
this one.

We might decide to remove these snapshots after this set of Sentry
refactors, as they might be more work to maintain than they're worth.
But they will be useful at least in the short-term.

The login step has been removed from a few tests because it introduced
indeterminacy (the login process continued asynchronously after the
login, and sometimes was not finished when the error was triggered).

* Ensure login page has rendered during setup

This fixes an intermittent failure on Firefox

* Format snapshots with prettier before writing them

* Use defined set of date fields rather than infering from name

* Remove waits for error screen

The error screen only appears after a long timeout, and it doesn't
affect the next test steps at all.
This commit is contained in:
Mark Stacey 2023-08-16 11:52:25 -02:30 committed by GitHub
parent 9e302ea84e
commit a6ef7bb244
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 610 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/scripts/metamask-controller.js
View File

@ -2771,6 +2771,9 @@ export default class MetamaskController extends EventEmitter {
assetsContractController.getBalancesInSingleCall.bind(
assetsContractController,
),
// E2E testing
throwTestError: this.throwTestError.bind(this),
};
}
@ -4515,6 +4518,21 @@ export default class MetamaskController extends EventEmitter {
return nonceLock.nextNonce;
}
/**
* Throw an artificial error in a timeout handler for testing purposes.
*
* @param message - The error message.
* @deprecated This is only mean to facilitiate E2E testing. We should not
* use this for handling errors.
*/
throwTestError(message) {
setTimeout(() => {
const error = new Error(message);
error.name = 'TestError';
throw error;
});
}
//=============================================================================
// CONFIG
//=============================================================================

466
test/e2e/tests/errors.spec.js
View File

@ -1,8 +1,70 @@
const { resolve } = require('path');
const { promises: fs } = require('fs');
const { strict: assert } = require('assert');
const { get, has, set } = require('lodash');
const { Browser } = require('selenium-webdriver');
const { format } = require('prettier');
const { convertToHexValue, withFixtures } = require('../helpers');
const FixtureBuilder = require('../fixture-builder');
const dateFields = ['metamask.conversionDate'];
/**
* Transform date properties to value types, to ensure that state is
* consistent between test runs.
*
* @param {unknown} data - The data to transform
*/
function transformDates(data) {
for (const field of dateFields) {
if (has(data, field)) {
set(data, field, typeof get(data, field));
}
}
return data;
}
/**
* Check that the data provided matches the snapshot.
*
* @param {object }args - Function arguments.
* @param {any} args.data - The data to compare with the snapshot.
* @param {string} args.snapshot - The name of the snapshot.
* @param {boolean} [args.update] - Whether to update the snapshot if it doesn't match.
*/
async function matchesSnapshot({
data: unprocessedData,
snapshot,
update = process.env.UPDATE_SNAPSHOTS === 'true',
}) {
const data = transformDates(unprocessedData);
const snapshotPath = resolve(__dirname, `./state-snapshots/${snapshot}.json`);
const rawSnapshotData = await fs.readFile(snapshotPath, {
encoding: 'utf-8',
});
const snapshotData = JSON.parse(rawSnapshotData);
try {
assert.deepStrictEqual(data, snapshotData);
} catch (error) {
if (update && error instanceof assert.AssertionError) {
const stringifiedData = JSON.stringify(data);
// filepath specified so that Prettier can infer which parser to use
// from the file extension
const formattedData = format(stringifiedData, {
filepath: 'something.json',
});
await fs.writeFile(snapshotPath, formattedData, {
encoding: 'utf-8',
});
console.log(`Snapshot '${snapshot}' updated`);
return;
}
throw error;
}
}
describe('Sentry errors', function () {
const migrationError =
process.env.SELENIUM_BROWSER === Browser.CHROME
@ -41,8 +103,8 @@ describe('Sentry errors', function () {
],
};
describe('before initialization', function () {
it('should NOT send error events when participateInMetaMetrics is false', async function () {
describe('before initialization, after opting out of metrics', function () {
it('should NOT send error events in the background', async function () {
await withFixtures(
{
fixtures: {
@ -73,7 +135,43 @@ describe('Sentry errors', function () {
},
);
});
it('should send error events', async function () {
it('should NOT send error events in the UI', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: null,
participateInMetaMetrics: false,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Erase `getSentryState` hook, simulating a "before initialization" state
await driver.executeScript(
'window.stateHooks.getSentryState = undefined',
);
// Wait for Sentry request
await driver.delay(3000);
const isPending = await mockedEndpoint.isPending();
assert.ok(
isPending,
'A request to sentry was sent when it should not have been',
);
},
);
});
});
describe('before initialization, after opting into metrics', function () {
it('should send error events in background', async function () {
await withFixtures(
{
fixtures: {
@ -112,40 +210,47 @@ describe('Sentry errors', function () {
},
);
});
});
describe('after initialization', function () {
it('should NOT send error events when participateInMetaMetrics is false', async function () {
it('should capture background application state', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: null,
participateInMetaMetrics: false,
})
.build(),
fixtures: {
...new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: 'fake-metrics-id',
participateInMetaMetrics: true,
})
.build(),
// Intentionally corrupt state to trigger migration error during initialization
meta: undefined,
},
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
testSpecificMock: mockSentryMigratorError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.fill('#password', 'correct horse battery staple');
await driver.press('#password', driver.Key.ENTER);
// Trigger error
driver.executeScript('window.stateHooks.throwTestError()');
driver.delay(3000);
// Wait for Sentry request
const isPending = await mockedEndpoint.isPending();
assert.ok(
isPending,
'A request to sentry was sent when it should not have been',
);
await driver.wait(async () => {
const isPending = await mockedEndpoint.isPending();
return isPending === false;
}, 3000);
const [mockedRequest] = await mockedEndpoint.getSeenRequests();
const mockTextBody = mockedRequest.body.text.split('\n');
const mockJsonBody = JSON.parse(mockTextBody[2]);
const appState = mockJsonBody?.extra?.appState;
await matchesSnapshot({
data: appState,
snapshot: 'errors-before-init-opt-in-background-state',
});
},
);
});
it('should send error events', async function () {
it('should send error events in UI', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
@ -161,15 +266,171 @@ describe('Sentry errors', function () {
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.fill('#password', 'correct horse battery staple');
await driver.press('#password', driver.Key.ENTER);
await driver.findElement('#password');
// Erase `getSentryState` hook, simulating a "before initialization" state
await driver.executeScript(
'window.stateHooks.getSentryState = undefined',
);
// Trigger error
driver.executeScript('window.stateHooks.throwTestError()');
await driver.executeScript('window.stateHooks.throwTestError()');
// Wait for Sentry request
await driver.wait(async () => {
const isPending = await mockedEndpoint.isPending();
return isPending === false;
}, 10000);
}, 3000);
const [mockedRequest] = await mockedEndpoint.getSeenRequests();
const mockTextBody = mockedRequest.body.text.split('\n');
const mockJsonBody = JSON.parse(mockTextBody[2]);
const { level } = mockJsonBody;
const [{ type, value }] = mockJsonBody.exception.values;
// Verify request
assert.equal(type, 'TestError');
assert.equal(value, 'Test Error');
assert.equal(level, 'error');
},
);
});
it('should capture UI application state', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: 'fake-metrics-id',
participateInMetaMetrics: true,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Erase `getSentryState` hook, simulating a "before initialization" state
await driver.executeScript(
'window.stateHooks.getSentryState = undefined',
);
// Trigger error
await driver.executeScript('window.stateHooks.throwTestError()');
// Wait for Sentry request
await driver.wait(async () => {
const isPending = await mockedEndpoint.isPending();
return isPending === false;
}, 3000);
const [mockedRequest] = await mockedEndpoint.getSeenRequests();
const mockTextBody = mockedRequest.body.text.split('\n');
const mockJsonBody = JSON.parse(mockTextBody[2]);
const appState = mockJsonBody?.extra?.appState;
await matchesSnapshot({
data: appState,
snapshot: 'errors-before-init-opt-in-ui-state',
});
},
);
});
});
describe('after initialization, after opting out of metrics', function () {
it('should NOT send error events in the background', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: null,
participateInMetaMetrics: false,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Trigger error
await driver.executeScript(
'window.stateHooks.throwTestBackgroundError()',
);
// Wait for Sentry request
const isPending = await mockedEndpoint.isPending();
assert.ok(
isPending,
'A request to sentry was sent when it should not have been',
);
},
);
});
it('should NOT send error events in the UI', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: null,
participateInMetaMetrics: false,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Trigger error
await driver.executeScript('window.stateHooks.throwTestError()');
// Wait for Sentry request
const isPending = await mockedEndpoint.isPending();
assert.ok(
isPending,
'A request to sentry was sent when it should not have been',
);
},
);
});
});
describe('after initialization, after opting into metrics', function () {
it('should send error events in background', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: 'fake-metrics-id',
participateInMetaMetrics: true,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Trigger error
await driver.executeScript(
'window.stateHooks.throwTestBackgroundError()',
);
// Wait for Sentry request
await driver.wait(async () => {
const isPending = await mockedEndpoint.isPending();
return isPending === false;
}, 3000);
const [mockedRequest] = await mockedEndpoint.getSeenRequests();
const mockTextBody = mockedRequest.body.text.split('\n');
const mockJsonBody = JSON.parse(mockTextBody[2]);
@ -184,5 +445,154 @@ describe('Sentry errors', function () {
},
);
});
it('should capture background application state', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: 'fake-metrics-id',
participateInMetaMetrics: true,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Trigger error
await driver.executeScript(
'window.stateHooks.throwTestBackgroundError()',
);
// Wait for Sentry request
await driver.wait(async () => {
const isPending = await mockedEndpoint.isPending();
return isPending === false;
}, 3000);
const [mockedRequest] = await mockedEndpoint.getSeenRequests();
const mockTextBody = mockedRequest.body.text.split('\n');
const mockJsonBody = JSON.parse(mockTextBody[2]);
const appState = mockJsonBody?.extra?.appState;
assert.deepStrictEqual(Object.keys(appState), [
'browser',
'store',
'version',
]);
assert.ok(
typeof appState?.browser === 'string' &&
appState?.browser.length > 0,
'Invalid browser state',
);
assert.ok(
typeof appState?.version === 'string' &&
appState?.version.length > 0,
'Invalid version state',
);
await matchesSnapshot({
data: appState.store,
snapshot: 'errors-after-init-opt-in-background-state',
});
},
);
});
it('should send error events in UI', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: 'fake-metrics-id',
participateInMetaMetrics: true,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Trigger error
await driver.executeScript('window.stateHooks.throwTestError()');
// Wait for Sentry request
await driver.wait(async () => {
const isPending = await mockedEndpoint.isPending();
return isPending === false;
}, 3000);
const [mockedRequest] = await mockedEndpoint.getSeenRequests();
const mockTextBody = mockedRequest.body.text.split('\n');
const mockJsonBody = JSON.parse(mockTextBody[2]);
const { level, extra } = mockJsonBody;
const [{ type, value }] = mockJsonBody.exception.values;
const { participateInMetaMetrics } = extra.appState.store.metamask;
// Verify request
assert.equal(type, 'TestError');
assert.equal(value, 'Test Error');
assert.equal(level, 'error');
assert.equal(participateInMetaMetrics, true);
},
);
});
it('should capture UI application state', async function () {
await withFixtures(
{
fixtures: new FixtureBuilder()
.withMetaMetricsController({
metaMetricsId: 'fake-metrics-id',
participateInMetaMetrics: true,
})
.build(),
ganacheOptions,
title: this.test.title,
failOnConsoleError: false,
testSpecificMock: mockSentryTestError,
},
async ({ driver, mockedEndpoint }) => {
await driver.navigate();
await driver.findElement('#password');
// Trigger error
await driver.executeScript('window.stateHooks.throwTestError()');
// Wait for Sentry request
await driver.wait(async () => {
const isPending = await mockedEndpoint.isPending();
return isPending === false;
}, 3000);
const [mockedRequest] = await mockedEndpoint.getSeenRequests();
const mockTextBody = mockedRequest.body.text.split('\n');
const mockJsonBody = JSON.parse(mockTextBody[2]);
const appState = mockJsonBody?.extra?.appState;
assert.deepStrictEqual(Object.keys(appState), [
'browser',
'store',
'version',
]);
assert.ok(
typeof appState?.browser === 'string' &&
appState?.browser.length > 0,
'Invalid browser state',
);
assert.ok(
typeof appState?.version === 'string' &&
appState?.version.length > 0,
'Invalid version state',
);
await matchesSnapshot({
data: appState.store,
snapshot: 'errors-after-init-opt-in-ui-state',
});
},
);
});
});
});

50
test/e2e/tests/state-snapshots/errors-after-init-opt-in-background-state.json Normal file
View File

@ -0,0 +1,50 @@
{
"metamask": {
"isInitialized": true,
"connectedStatusPopoverHasBeenShown": true,
"defaultHomeActiveTabName": null,
"networkId": "1337",
"providerConfig": {
"nickname": "Localhost 8545",
"ticker": "ETH",
"type": "rpc"
},
"isUnlocked": false,
"useBlockie": false,
"useNonceField": false,
"usePhishDetect": true,
"featureFlags": { "showIncomingTransactions": true },
"currentLocale": "en",
"forgottenPassword": false,
"preferences": {
"hideZeroBalanceTokens": false,
"showFiatInTestnets": false,
"showTestNetworks": false,
"useNativeCurrencyAsPrimaryCurrency": true
},
"ipfsGateway": "dweb.link",
"participateInMetaMetrics": true,
"metaMetricsId": "fake-metrics-id",
"conversionDate": "number",
"conversionRate": 1700,
"nativeCurrency": "ETH",
"currentCurrency": "usd",
"alertEnabledness": { "unconnectedAccount": true, "web3ShimUsage": true },
"seedPhraseBackedUp": true,
"firstTimeFlowType": "import",
"completedOnboarding": true,
"incomingTxLastFetchedBlockByChainId": {
"0x1": null,
"0xe708": null,
"0x5": null,
"0xaa36a7": null,
"0xe704": null
},
"currentBlockGasLimit": "0x1c9c380",
"unapprovedDecryptMsgCount": 0,
"unapprovedEncryptionPublicKeyMsgCount": 0,
"unapprovedMsgCount": 0,
"unapprovedPersonalMsgCount": 0,
"unapprovedTypedMessagesCount": 0
}
}

57
test/e2e/tests/state-snapshots/errors-after-init-opt-in-ui-state.json Normal file
View File

@ -0,0 +1,57 @@
{
"gas": { "customData": { "price": null, "limit": null } },
"history": { "mostRecentOverviewPage": "/" },
"metamask": {
"isInitialized": true,
"isUnlocked": false,
"isAccountMenuOpen": false,
"customNonceValue": "",
"useBlockie": false,
"featureFlags": { "showIncomingTransactions": true },
"welcomeScreenSeen": false,
"currentLocale": "en",
"currentBlockGasLimit": "",
"preferences": {
"hideZeroBalanceTokens": false,
"showFiatInTestnets": false,
"showTestNetworks": false,
"useNativeCurrencyAsPrimaryCurrency": true
},
"firstTimeFlowType": "import",
"completedOnboarding": true,
"participateInMetaMetrics": true,
"nextNonce": null,
"conversionRate": 1300,
"nativeCurrency": "ETH",
"connectedStatusPopoverHasBeenShown": true,
"defaultHomeActiveTabName": null,
"networkId": "1337",
"providerConfig": {
"nickname": "Localhost 8545",
"ticker": "ETH",
"type": "rpc"
},
"useNonceField": false,
"usePhishDetect": true,
"forgottenPassword": false,
"ipfsGateway": "dweb.link",
"metaMetricsId": "fake-metrics-id",
"conversionDate": "number",
"currentCurrency": "usd",
"alertEnabledness": { "unconnectedAccount": true, "web3ShimUsage": true },
"seedPhraseBackedUp": true,
"incomingTxLastFetchedBlockByChainId": {
"0x1": null,
"0xe708": null,
"0x5": null,
"0xaa36a7": null,
"0xe704": null
},
"unapprovedDecryptMsgCount": 0,
"unapprovedEncryptionPublicKeyMsgCount": 0,
"unapprovedMsgCount": 0,
"unapprovedPersonalMsgCount": 0,
"unapprovedTypedMessagesCount": 0
},
"unconnectedAccount": { "state": "CLOSED" }
}

1
test/e2e/tests/state-snapshots/errors-before-init-opt-in-background-state.json Normal file
View File

@ -0,0 +1 @@
{}

1
test/e2e/tests/state-snapshots/errors-before-init-opt-in-ui-state.json Normal file
View File

@ -0,0 +1 @@
{}

47
ui/index.js
View File

@ -81,7 +81,7 @@ export default function launchMetamaskUi(opts, cb) {
return;
}
startApp(metamaskState, backgroundConnection, opts).then((store) => {
setupDebuggingHelpers(store);
setupStateHooks(store);
cb(
null,
store,
@ -191,18 +191,39 @@ async function startApp(metamaskState, backgroundConnection, opts) {
return store;
}
function setupDebuggingHelpers(store) {
/**
* The following stateHook is a method intended to throw an error, used in
* our E2E test to ensure that errors are attempted to be sent to sentry.
*
* @param {string} [msg] - The error message to throw, defaults to 'Test Error'
*/
window.stateHooks.throwTestError = async function (msg = 'Test Error') {
const error = new Error(msg);
error.name = 'TestError';
throw error;
};
/**
* Setup functions on `window.stateHooks`. Some of these support
* application features, and some are just for debugging or testing.
*
* @param {object} store - The Redux store.
*/
function setupStateHooks(store) {
if (process.env.METAMASK_DEBUG || process.env.IN_TEST) {
/**
* The following stateHook is a method intended to throw an error, used in
* our E2E test to ensure that errors are attempted to be sent to sentry.
*
* @param {string} [msg] - The error message to throw, defaults to 'Test Error'
*/
window.stateHooks.throwTestError = async function (msg = 'Test Error') {
const error = new Error(msg);
error.name = 'TestError';
throw error;
};
/**
* The following stateHook is a method intended to throw an error in the
* background, used in our E2E test to ensure that errors are attempted to be
* sent to sentry.
*
* @param {string} [msg] - The error message to throw, defaults to 'Test Error'
*/
window.stateHooks.throwTestBackgroundError = async function (
msg = 'Test Error',
) {
store.dispatch(actions.throwTestBackgroundError(msg));
};
}
window.stateHooks.getCleanAppState = async function () {
const state = clone(store.getState());
state.version = global.platform.getVersion();

11
ui/store/actions.ts
View File

@ -4489,6 +4489,17 @@ export async function getCurrentNetworkEIP1559Compatibility(): Promise<
return networkEIP1559Compatibility;
}
/**
* Throw an error in the background for testing purposes.
*
* @param message - The error message.
* @deprecated This is only mean to facilitiate E2E testing. We should not use
* this for handling errors.
*/
export async function throwTestBackgroundError(message: string): Promise<void> {
await submitRequestToBackground('throwTestError', [message]);
}
///: BEGIN:ONLY_INCLUDE_IN(snaps)
/**
* Set status of popover warning for the first snap installation.