1
0
mirror of https://github.com/kremalicious/metamask-extension.git synced 2024-12-23 09:52:26 +01:00
metamask-extension/development/ts-migration-dashboard/scripts/build-module-partitions.ts
Elliot Winkler a7d98b695f
Add TypeScript migration dashboard (#13820)
As we convert parts of the codebase to TypeScript, we will want a way to
track progress. This commit adds a dashboard which displays all of the
files that we wish to convert to TypeScript and which files we've
already converted.

The list of all possible files to convert is predetermined by walking
the dependency graph of each entrypoint the build system uses to compile
the extension (the files that the entrypoint imports, the files that the
imports import, etc). The list should not need to be regenerated, but
you can do it by running:

    yarn ts-migration:enumerate

The dashboard is implemented as a separate React app. The CircleCI
configuration has been updated so that when a new commit is pushed, the
React app is built and stored in the CircleCI artifacts. When a PR is
merged, the built files will be pushed to a separate repo whose sole
purpose is to serve the dashboard via GitHub Pages (this is the same
way that the Storybook works). All of the app code and script to build
the app are self-contained under
`development/ts-migration-dashboard`. To build this app yourself, you
can run:

    yarn ts-migration:dashboard:build

or if you want to build automatically as you change files, run:

    yarn ts-migration:dashboard:watch

Then open the following file in your browser (there is no server
component):

    development/ts-migration-dashboard/build/index.html

Finally, although you shouldn't have to do this, to manually deploy the
dashboard once built, you can run:

    git remote add ts-migration-dashboard git@github.com:MetaMask/metamask-extension-ts-migration-dashboard.git
    yarn ts-migration:dashboard:deploy
2022-08-09 14:16:08 -06:00

337 lines
13 KiB
TypeScript

import fs from 'fs';
import path from 'path';
import fg from 'fast-glob';
import madge from 'madge';
import {
BASE_DIRECTORY,
ENTRYPOINT_PATTERNS,
FILES_TO_CONVERT_PATH,
} from './constants';
/**
* Represents a module that has been imported somewhere in the codebase.
*
* @property id - The name of a file or NPM module.
* @property dependents - The modules which are imported by this module.
* @property level - How many modules it takes to import this module (from the
* root of the dependency tree).
* @property isExternal - Whether the module refers to a NPM module.
* @property hasBeenConverted - Whether the module was one of the files we
* wanted to convert to TypeScript and has been converted.
*/
type Module = {
id: string;
dependents: Module[];
level: number;
isExternal: boolean;
hasBeenConverted: boolean;
};
/**
* Represents a set of modules that sit at a certain level within the final
* dependency tree.
*
* @property level - How many modules it takes to import this module (from the
* root of the dependency tree).
* @property children - The modules that share this same level.
* @property children[].name - The name of the item being imported.
* @property children[].hasBeenConverted - Whether or not the module (assuming
* that it is a file in our codebase) has been converted to TypeScript.
*/
export type ModulePartition = {
level: number;
children: {
name: string;
hasBeenConverted: boolean;
}[];
};
/**
* Uses the `madge` package to traverse the dependency graphs assembled from a
* set of entrypoints (a combination of the entrypoints that the build script
* uses to build the extension as well as a manually picked list), builds a
* combined dependency tree, then arranges the nodes of that tree by level,
* which is the number of files it takes to reach a file within the whole tree.
*
* @returns An array of objects which can be used to render the dashboard, where
* each object represents a group of files at a certain level in the dependency
* tree.
*/
export default async function buildModulePartitions(): Promise<
ModulePartition[]
> {
const allowedFilePaths = readFilesToConvert();
const possibleEntryFilePaths = (
await Promise.all(
ENTRYPOINT_PATTERNS.map((entrypointPattern) => {
return fg(
path.resolve(BASE_DIRECTORY, `${entrypointPattern}.{js,ts,tsx}`),
);
}),
)
).flat();
const entryFilePaths = filterFilePaths(
possibleEntryFilePaths.map((possibleEntrypoint) =>
path.relative(BASE_DIRECTORY, possibleEntrypoint),
),
allowedFilePaths,
);
const result = await madge(entryFilePaths, {
baseDir: BASE_DIRECTORY,
tsConfig: path.join(BASE_DIRECTORY, 'tsconfig.json'),
});
const dependenciesByFilePath = result.obj();
const modulesById = buildModulesWithLevels(
dependenciesByFilePath,
entryFilePaths,
allowedFilePaths,
);
return partitionModulesByLevel(modulesById);
}
/**
* Returns the contents of a JSON file that stores the names of the files that
* we plan on converting to TypeScript. All of the dependency information
* will be filtered through this list.
*/
function readFilesToConvert(): string[] {
try {
return JSON.parse(fs.readFileSync(FILES_TO_CONVERT_PATH, 'utf-8'));
} catch (error: unknown) {
throw new Error(
'Could not read or parse list of files to convert. ' +
'Have you tried running the following command?\n\n' +
' yarn ts-migration:enumerate\n\n' +
`Original error: ${error}`,
);
}
}
/**
* Filters the given set of file paths according to the given allow list. As the
* entry file paths could refer to TypeScript files, and the allow list is
* guaranteed to be JavaScript files, the entry file paths are normalized to end
* in `.js` before being filtered.
*
* @param filePaths - A set of file paths.
* @param allowedFilePaths - A set of allowed file paths.
* @returns The filtered file paths.
*/
function filterFilePaths(filePaths: string[], allowedFilePaths: string[]) {
return filePaths.filter((filePath) =>
allowedFilePaths.includes(filePath.replace(/\.tsx?$/u, '.js')),
);
}
/**
* This function takes a flat data structure that represents the dependency
* graph of a system. It traverses the graph, converting it to a tree (i.e.,
* resolving circular dependencies), but where any node of the tree is
* accessible immediately. The "level" of a dependency — how many other
* dependencies it takes to reach that dependency — is also recorded.
*
* @param dependenciesByModuleId - An object that maps a file path in the
* dependency graph to the dependencies that it imports. This information is
* provided by the `madge` package.
* @param entryModuleIds - File paths which will initiate the traversal through
* the dependency graph. These file paths will always be level 0.
* @param allowedFilePaths - The list of original JavaScript files to
* convert to TypeScript; governs which files will end up in the final
* dependency graph.
* @returns A data structure that maps the id of a module in the dependency
* graph to an object that represents that module.
*/
function buildModulesWithLevels(
dependenciesByModuleId: Record<string, string[]>,
entryModuleIds: string[],
allowedFilePaths: string[],
): Record<string, Module> {
// Our overall goal is that we want to partition the graph into different
// sections. We want to find the leaves of the graph — that is, files that
// depend on no other files — then the dependents of the leaves — those files
// that depend on the leaves — then the dependents of the dependents, etc. We
// can derive this information by traversing the graph, and for each module we
// encounter, recording the number of modules it takes to reach that module.
// We call this number the **level**.
//
// We will discuss a couple of optimizations we've made to ensure that graph
// traversal is performant.
//
// Naively, in order to calculate the level of each module, we need to follow
// that module's dependencies, then the dependencies of the dependencies,
// etc., until we hit leaves. Essentially, we need to follow each connection
// in the graph. However, this creates a performance problem, because in a
// large system a file could get imported multiple ways (this is the nature of
// a graph: each node can have multiple incoming connections and multiple
// outgoing connections). For instance:
//
// - `foo.js` could import `bar.js` which could import `baz.js`
// - `foo.js` could also import `baz.js` directly
// - `foo.js` could also import `bar.js` which imports `qux.js` which imports `baz.js`
//
// In this case, even if there are few modules in a system, a subset of those
// modules may be visited multiple times in the course of traversing
// connections between all of them. This is costly and unnecessary.
//
// To address this, as we are traversing the graph, we record modules we've
// visited along with the level when we visited it. If we encounter a module
// again through some other path, and the level this time is less than the
// level we've already recorded, then we know we don't need to revisit that
// module or — crucially — any of its dependencies. Therefore we can skip
// whole sections of the graph.
//
// In addition, in a large enough system, some files could import files that end
// up importing themselves later on:
//
// - `foo.js` could import `bar.js`, which imports `baz.js`, which imports `foo.js`, which...
//
// These are called circular dependencies, and we detect them by tracking the
// files that depend on a file (dependents) in addition to the files on which
// that file depends (dependencies). In the example above, `baz.js` has a
// dependency `foo.js`, and its chain of dependents is `bar.js` -> `foo.js`
// (working backward from `baz.js`). The chain of dependents of `baz.js`
// includes `foo.js`, so we say `foo.js` is a circular dependency of `baz.js`,
// and we don't need to follow it.
const modulesToFill: Module[] = entryModuleIds.map((moduleId) => {
return {
id: moduleId,
dependents: [],
level: 0,
isExternal: false,
hasBeenConverted: /\.tsx?$/u.test(moduleId),
};
});
const modulesById: Record<string, Module> = {};
while (modulesToFill.length > 0) {
const currentModule = modulesToFill.shift() as Module;
const childModulesToFill: Module[] = [];
(dependenciesByModuleId[currentModule.id] ?? []).forEach(
(givenChildModuleId) => {
const npmPackageMatch = givenChildModuleId.match(
/^node_modules\/(?:(@[^/]+)\/)?([^/]+)\/.+$/u,
);
let childModuleId;
if (npmPackageMatch) {
childModuleId =
npmPackageMatch[1] === undefined
? `${npmPackageMatch[2]}`
: `${npmPackageMatch[1]}/${npmPackageMatch[2]}`;
} else {
childModuleId = givenChildModuleId;
}
// Skip circular dependencies
if (
findDirectAndIndirectDependentIdsOf(currentModule).has(childModuleId)
) {
return;
}
// Skip files that weren't on the original list of JavaScript files to
// convert, as we don't want them to show up on the status dashboard
if (
!npmPackageMatch &&
!allowedFilePaths.includes(childModuleId.replace(/\.tsx?$/u, '.js'))
) {
return;
}
const existingChildModule = modulesById[childModuleId];
if (existingChildModule === undefined) {
const childModule: Module = {
id: childModuleId,
dependents: [currentModule],
level: currentModule.level + 1,
isExternal: Boolean(npmPackageMatch),
hasBeenConverted: /\.tsx?$/u.test(childModuleId),
};
childModulesToFill.push(childModule);
} else {
if (currentModule.level + 1 > existingChildModule.level) {
existingChildModule.level = currentModule.level + 1;
// Update descendant modules' levels
childModulesToFill.push(existingChildModule);
} else {
// There is no point in descending into dependencies of this module
// if the new level of the module would be <= the existing level,
// because all of the dependencies from this point on are guaranteed
// to have a higher level and are therefore already at the right
// level.
}
if (!existingChildModule.dependents.includes(currentModule)) {
existingChildModule.dependents.push(currentModule);
}
}
},
);
if (childModulesToFill.length > 0) {
modulesToFill.push(...childModulesToFill);
}
if (!(currentModule.id in modulesById)) {
modulesById[currentModule.id] = currentModule;
}
}
return modulesById;
}
/**
* Given a file in the dependency graph, returns all of the names of the files
* which import that file directly or through some other file.
*
* @param module - A module in the graph.
* @returns The ids of the modules which are incoming connections to
* the module.
*/
function findDirectAndIndirectDependentIdsOf(module: Module): Set<string> {
const modulesToProcess = [module];
const allDependentIds = new Set<string>();
while (modulesToProcess.length > 0) {
const currentModule = modulesToProcess.shift() as Module;
currentModule.dependents.forEach((dependent) => {
if (!allDependentIds.has(dependent.id)) {
allDependentIds.add(dependent.id);
modulesToProcess.push(dependent);
}
});
}
return allDependentIds;
}
/**
* Partitions modules in the dependency graph by level (see {@link buildModulesWithLevels}
* for an explanation). This will be used to render those modules on the
* dashboard in groups.
*
* @param modulesById - An object that maps the id of a module to an object that
* represents that module.
* @returns An array where each item represents a level and is the module ids
* that match that level, sorted by largest level (leaves) to smallest level
* (roots).
*/
function partitionModulesByLevel(
modulesById: Record<string, Module>,
): ModulePartition[] {
const levels = Object.values(modulesById).map((module) => module.level);
const maxLevel = Math.max(...levels);
const modulePartitions: ModulePartition[] = [];
for (let i = 0; i <= maxLevel; i++) {
modulePartitions[i] = { level: i + 1, children: [] };
}
Object.values(modulesById).forEach((module) => {
modulePartitions[module.level].children.push({
name: module.id,
hasBeenConverted: module.hasBeenConverted,
});
});
return modulePartitions.reverse();
}