build - refactor build system for easier configuration (#10718)

* build - refactor build system for easier configuration of before and after bundle

* build - fix dependenciesToBundle option

* build - fix bify external options and other config

* build - refactor for cleanliness

* build - fix minify argument

* build - fix sourcemaps setup

* scripts - refactor setupBundlerDefaults in anticipation of factor bundles

* build - scripts - remove unused pipeline label

* build - scripts - make filepath entry optional

* build - scripts - rename filepath and filename options to entryFilepath and destFilepath

* Update development/build/scripts.js

Co-authored-by: Mark Stacey <markjstacey@gmail.com>

Co-authored-by: Mark Stacey <markjstacey@gmail.com>

development/build/scripts.js

@@ -1,11 +1,9 @@
+const EventEmitter = require('events');
 const gulp = require('gulp');
 const watch = require('gulp-watch');
-const pify = require('pify');
-const pump = pify(require('pump'));
 const source = require('vinyl-source-stream');
 const buffer = require('vinyl-buffer');
 const log = require('fancy-log');
-const { assign } = require('lodash');
 const watchify = require('watchify');
 const browserify = require('browserify');
 const envify = require('loose-envify/custom');
@@ -13,8 +11,11 @@ const sourcemaps = require('gulp-sourcemaps');
 const terser = require('gulp-terser-js');
 const babelify = require('babelify');
 const brfs = require('brfs');
+const pify = require('pify');
+const endOfStream = pify(require('end-of-stream'));
+const labeledStreamSplicer = require('labeled-stream-splicer').obj;
 
-const conf = require('rc')('metamask', {
+const metamaskrc = require('rc')('metamask', {
   INFURA_PROJECT_ID: process.env.INFURA_PROJECT_ID,
   SEGMENT_HOST: process.env.SEGMENT_HOST,
   SEGMENT_WRITE_KEY: process.env.SEGMENT_WRITE_KEY,
@@ -67,10 +68,10 @@ function createScriptTasks({ browserPlatforms, livereload }) {
   };
   const deps = {
     background: createTasksForBuildJsDeps({
-      filename: 'bg-libs',
+      label: 'bg-libs',
       key: 'background',
     }),
-    ui: createTasksForBuildJsDeps({ filename: 'ui-libs', key: 'ui' }),
+    ui: createTasksForBuildJsDeps({ label: 'ui-libs', key: 'ui' }),
   };
 
   // high level tasks
@@ -83,15 +84,15 @@ function createScriptTasks({ browserPlatforms, livereload }) {
 
   return { prod, dev, testDev, test };
 
-  function createTasksForBuildJsDeps({ key, filename }) {
+  function createTasksForBuildJsDeps({ key, label }) {
     return createTask(
       `scripts:deps:${key}`,
-      bundleTask({
+      createNormalBundle({
-        label: filename,
+        label,
-        filename: `${filename}.js`,
+        destFilepath: `${label}.js`,
-        buildLib: true,
+        modulesToExpose: externalDependenciesMap[key],
-        dependenciesToBundle: externalDependenciesMap[key],
         devMode: false,
+        browserPlatforms,
       }),
     );
   }
@@ -104,13 +105,18 @@ function createScriptTasks({ browserPlatforms, livereload }) {
       'initSentry',
     ];
 
-    const standardSubtasks = standardBundles.map((filename) => {
+    const standardSubtasks = standardBundles.map((label) => {
+      let extraEntries;
+      if (devMode && label === 'ui') {
+        extraEntries = ['./development/require-react-devtools.js'];
+      }
       return createTask(
-        `${taskPrefix}:${filename}`,
+        `${taskPrefix}:${label}`,
         createBundleTaskForBuildJsExtensionNormal({
-          filename,
+          label,
           devMode,
           testing,
+          extraEntries,
         }),
       );
     });
@@ -128,7 +134,7 @@ function createScriptTasks({ browserPlatforms, livereload }) {
       createTaskForBuildJsExtensionDisableConsole({ devMode }),
     );
 
-    // task for initiating livereload
+    // task for initiating browser livereload
     const initiateLiveReload = async () => {
       if (devMode) {
         // trigger live reload when the bundles are updated
@@ -156,29 +162,33 @@ function createScriptTasks({ browserPlatforms, livereload }) {
   }
 
   function createBundleTaskForBuildJsExtensionNormal({
-    filename,
+    label,
     devMode,
     testing,
+    extraEntries,
   }) {
-    return bundleTask({
+    return createNormalBundle({
-      label: filename,
+      label,
-      filename: `${filename}.js`,
+      entryFilepath: `./app/scripts/${label}.js`,
-      filepath: `./app/scripts/${filename}.js`,
+      destFilepath: `${label}.js`,
+      extraEntries,
       externalDependencies: devMode
         ? undefined
-        : externalDependenciesMap[filename],
+        : externalDependenciesMap[label],
       devMode,
       testing,
+      browserPlatforms,
     });
   }
 
   function createTaskForBuildJsExtensionDisableConsole({ devMode }) {
-    const filename = 'disable-console';
+    const label = 'disable-console';
-    return bundleTask({
+    return createNormalBundle({
-      label: filename,
+      label,
-      filename: `${filename}.js`,
+      entryFilepath: `./app/scripts/${label}.js`,
-      filepath: `./app/scripts/${filename}.js`,
+      destFilepath: `${label}.js`,
       devMode,
+      browserPlatforms,
     });
   }
 
@@ -186,190 +196,266 @@ function createScriptTasks({ browserPlatforms, livereload }) {
     const inpage = 'inpage';
     const contentscript = 'contentscript';
     return composeSeries(
-      bundleTask({
+      createNormalBundle({
         label: inpage,
-        filename: `${inpage}.js`,
+        entryFilepath: `./app/scripts/${inpage}.js`,
-        filepath: `./app/scripts/${inpage}.js`,
+        destFilepath: `${inpage}.js`,
         externalDependencies: devMode
           ? undefined
           : externalDependenciesMap[inpage],
         devMode,
         testing,
+        browserPlatforms,
       }),
-      bundleTask({
+      createNormalBundle({
         label: contentscript,
-        filename: `${contentscript}.js`,
+        entryFilepath: `./app/scripts/${contentscript}.js`,
-        filepath: `./app/scripts/${contentscript}.js`,
+        destFilepath: `${contentscript}.js`,
         externalDependencies: devMode
           ? undefined
           : externalDependenciesMap[contentscript],
         devMode,
         testing,
+        browserPlatforms,
       }),
     );
   }
-
-  function bundleTask(opts) {
-    let bundler;
-
-    return performBundle;
-
-    async function performBundle() {
-      // initialize bundler if not available yet
-      // dont create bundler until task is actually run
-      if (!bundler) {
-        bundler = generateBundler(opts, performBundle);
-        // output build logs to terminal
-        bundler.on('log', log);
-      }
-
-      const buildPipeline = [
-        bundler.bundle(),
-        // convert bundle stream to gulp vinyl stream
-        source(opts.filename),
-        // Initialize Source Maps
-        buffer(),
-        // loads map from browserify file
-        sourcemaps.init({ loadMaps: true }),
-      ];
-
-      // Minification
-      if (!opts.devMode) {
-        buildPipeline.push(
-          terser({
-            mangle: {
-              reserved: ['MetamaskInpageProvider'],
-            },
-            sourceMap: {
-              content: true,
-            },
-          }),
-        );
-      }
-
-      // Finalize Source Maps
-      if (opts.devMode) {
-        // Use inline source maps for development due to Chrome DevTools bug
-        // https://bugs.chromium.org/p/chromium/issues/detail?id=931675
-        // note: sourcemaps call arity is important
-        buildPipeline.push(sourcemaps.write());
-      } else {
-        buildPipeline.push(
-          sourcemaps.write('../sourcemaps', { addComment: false }),
-        );
-      }
-
-      // write completed bundles
-      browserPlatforms.forEach((platform) => {
-        const dest = `./dist/${platform}`;
-        buildPipeline.push(gulp.dest(dest));
-      });
-
-      // process bundles
-      if (opts.devMode) {
-        try {
-          await pump(buildPipeline);
-        } catch (err) {
-          gracefulError(err);
-        }
-      } else {
-        await pump(buildPipeline);
-      }
-    }
-  }
-
-  function generateBundler(opts, performBundle) {
-    const browserifyOpts = assign({}, watchify.args, {
-      plugin: [],
-      transform: [],
-      debug: true,
-      fullPaths: opts.devMode,
-    });
-
-    if (!opts.buildLib) {
-      if (opts.devMode && opts.filename === 'ui.js') {
-        browserifyOpts.entries = [
-          './development/require-react-devtools.js',
-          opts.filepath,
-        ];
-      } else {
-        browserifyOpts.entries = [opts.filepath];
-      }
-    }
-
-    let bundler = browserify(browserifyOpts)
-      .transform(babelify)
-      .transform(brfs);
-
-    if (opts.buildLib) {
-      bundler = bundler.require(opts.dependenciesToBundle);
-    }
-
-    if (opts.externalDependencies) {
-      bundler = bundler.external(opts.externalDependencies);
-    }
-
-    const environment = getEnvironment({
-      devMode: opts.devMode,
-      test: opts.testing,
-    });
-    if (environment === 'production' && !process.env.SENTRY_DSN) {
-      throw new Error('Missing SENTRY_DSN environment variable');
-    }
-
-    // Inject variables into bundle
-    bundler.transform(
-      envify({
-        METAMASK_DEBUG: opts.devMode,
-        METAMASK_ENVIRONMENT: environment,
-        METAMASK_VERSION: baseManifest.version,
-        NODE_ENV: opts.devMode ? 'development' : 'production',
-        IN_TEST: opts.testing ? 'true' : false,
-        PUBNUB_SUB_KEY: process.env.PUBNUB_SUB_KEY || '',
-        PUBNUB_PUB_KEY: process.env.PUBNUB_PUB_KEY || '',
-        CONF: opts.devMode ? conf : {},
-        SENTRY_DSN: process.env.SENTRY_DSN,
-        INFURA_PROJECT_ID: opts.testing
-          ? '00000000000000000000000000000000'
-          : conf.INFURA_PROJECT_ID,
-        SEGMENT_HOST: conf.SEGMENT_HOST,
-        // When we're in the 'production' environment we will use a specific key only set in CI
-        // Otherwise we'll use the key from .metamaskrc or from the environment variable. If
-        // the value of SEGMENT_WRITE_KEY that we envify is undefined then no events will be tracked
-        // in the build. This is intentional so that developers can contribute to MetaMask without
-        // inflating event volume.
-        SEGMENT_WRITE_KEY:
-          environment === 'production'
-            ? process.env.SEGMENT_PROD_WRITE_KEY
-            : conf.SEGMENT_WRITE_KEY,
-        SEGMENT_LEGACY_WRITE_KEY:
-          environment === 'production'
-            ? process.env.SEGMENT_PROD_LEGACY_WRITE_KEY
-            : conf.SEGMENT_LEGACY_WRITE_KEY,
-      }),
-      {
-        global: true,
-      },
-    );
-
-    // Live reload - minimal rebundle on change
-    if (opts.devMode) {
-      bundler = watchify(bundler);
-      // on any file update, re-runs the bundler
-      bundler.on('update', () => {
-        performBundle();
-      });
-    }
-
-    return bundler;
-  }
 }
 
-function getEnvironment({ devMode, test }) {
+function createNormalBundle({
+  destFilepath,
+  entryFilepath,
+  extraEntries = [],
+  modulesToExpose,
+  externalDependencies,
+  devMode,
+  testing,
+  browserPlatforms,
+}) {
+  return async function () {
+    // create bundler setup and apply defaults
+    const buildConfiguration = createBuildConfiguration();
+    const { bundlerOpts, events } = buildConfiguration;
+
+    const envVars = getEnvironmentVariables({ devMode, testing });
+    setupBundlerDefaults(buildConfiguration, {
+      devMode,
+      envVars,
+    });
+
+    // set bundle entries
+    bundlerOpts.entries = [...extraEntries];
+    if (entryFilepath) {
+      bundlerOpts.entries.push(entryFilepath);
+    }
+
+    if (modulesToExpose) {
+      bundlerOpts.require = bundlerOpts.require.concat(modulesToExpose);
+    }
+
+    if (externalDependencies) {
+      // there doesnt seem to be a standard bify option for this
+      // so we'll put it here but manually call it after bundle
+      bundlerOpts.manualExternal = bundlerOpts.manualExternal.concat(
+        externalDependencies,
+      );
+    }
+
+    // instrument pipeline
+    events.on('configurePipeline', ({ pipeline }) => {
+      // convert bundle stream to gulp vinyl stream
+      // and ensure file contents are buffered
+      pipeline.get('vinyl').push(source(destFilepath));
+      pipeline.get('vinyl').push(buffer());
+      // setup bundle destination
+      browserPlatforms.forEach((platform) => {
+        const dest = `./dist/${platform}/`;
+        pipeline.get('dest').push(gulp.dest(dest));
+      });
+    });
+
+    await bundleIt(buildConfiguration);
+  };
+}
+
+function createBuildConfiguration() {
+  const events = new EventEmitter();
+  const bundlerOpts = {
+    entries: [],
+    transform: [],
+    plugin: [],
+    require: [],
+    // not a standard bify option
+    manualExternal: [],
+  };
+  return { bundlerOpts, events };
+}
+
+function setupBundlerDefaults(buildConfiguration, { devMode, envVars }) {
+  const { bundlerOpts } = buildConfiguration;
+  // devMode options
+  const reloadOnChange = Boolean(devMode);
+  const minify = Boolean(devMode) === false;
+
+  Object.assign(bundlerOpts, {
+    // source transforms
+    transform: [
+      // transpile top-level code
+      babelify,
+      // inline `fs.readFileSync` files
+      brfs,
+    ],
+    // use entryFilepath for moduleIds, easier to determine origin file
+    fullPaths: devMode,
+    // for sourcemaps
+    debug: true,
+  });
+
+  // inject environment variables via node-style `process.env`
+  if (envVars) {
+    bundlerOpts.transform.push([envify(envVars), { global: true }]);
+  }
+
+  // setup reload on change
+  if (reloadOnChange) {
+    setupReloadOnChange(buildConfiguration);
+  }
+
+  if (minify) {
+    setupMinification(buildConfiguration);
+  }
+
+  // setup source maps
+  setupSourcemaps(buildConfiguration, { devMode });
+}
+
+function setupReloadOnChange({ bundlerOpts, events }) {
+  // add plugin to options
+  Object.assign(bundlerOpts, {
+    plugin: [...bundlerOpts.plugin, watchify],
+    // required by watchify
+    cache: {},
+    packageCache: {},
+  });
+  // instrument pipeline
+  events.on('configurePipeline', ({ bundleStream }) => {
+    // handle build error to avoid breaking build process
+    // (eg on syntax error)
+    bundleStream.on('error', (err) => {
+      gracefulError(err);
+    });
+  });
+}
+
+function setupMinification(buildConfiguration) {
+  const { events } = buildConfiguration;
+  events.on('configurePipeline', ({ pipeline }) => {
+    pipeline.get('minify').push(
+      terser({
+        mangle: {
+          reserved: ['MetamaskInpageProvider'],
+        },
+        sourceMap: {
+          content: true,
+        },
+      }),
+    );
+  });
+}
+
+function setupSourcemaps(buildConfiguration, { devMode }) {
+  const { events } = buildConfiguration;
+  events.on('configurePipeline', ({ pipeline }) => {
+    pipeline.get('sourcemaps:init').push(sourcemaps.init({ loadMaps: true }));
+    pipeline
+      .get('sourcemaps:write')
+      // Use inline source maps for development due to Chrome DevTools bug
+      // https://bugs.chromium.org/p/chromium/issues/detail?id=931675
+      .push(
+        devMode
+          ? sourcemaps.write()
+          : sourcemaps.write('../sourcemaps', { addComment: false }),
+      );
+  });
+}
+
+async function bundleIt(buildConfiguration) {
+  const { bundlerOpts, events } = buildConfiguration;
+  const bundler = browserify(bundlerOpts);
+  // manually apply non-standard option
+  bundler.external(bundlerOpts.manualExternal);
+  // output build logs to terminal
+  bundler.on('log', log);
+  // forward update event (used by watchify)
+  bundler.on('update', () => performBundle());
+  await performBundle();
+
+  async function performBundle() {
+    // this pipeline is created for every bundle
+    // the labels are all the steps you can hook into
+    const pipeline = labeledStreamSplicer([
+      'vinyl',
+      [],
+      'sourcemaps:init',
+      [],
+      'minify',
+      [],
+      'sourcemaps:write',
+      [],
+      'dest',
+      [],
+    ]);
+    const bundleStream = bundler.bundle();
+    // trigger build pipeline instrumentations
+    events.emit('configurePipeline', { pipeline, bundleStream });
+    // start bundle, send into pipeline
+    bundleStream.pipe(pipeline);
+    // nothing will consume pipeline, so let it flow
+    pipeline.resume();
+    await endOfStream(pipeline);
+  }
+}
+
+function getEnvironmentVariables({ devMode, testing }) {
+  const environment = getEnvironment({ devMode, testing });
+  if (environment === 'production' && !process.env.SENTRY_DSN) {
+    throw new Error('Missing SENTRY_DSN environment variable');
+  }
+  return {
+    METAMASK_DEBUG: devMode,
+    METAMASK_ENVIRONMENT: environment,
+    METAMASK_VERSION: baseManifest.version,
+    NODE_ENV: devMode ? 'development' : 'production',
+    IN_TEST: testing ? 'true' : false,
+    PUBNUB_SUB_KEY: process.env.PUBNUB_SUB_KEY || '',
+    PUBNUB_PUB_KEY: process.env.PUBNUB_PUB_KEY || '',
+    CONF: devMode ? metamaskrc : {},
+    SENTRY_DSN: process.env.SENTRY_DSN,
+    INFURA_PROJECT_ID: testing
+      ? '00000000000000000000000000000000'
+      : metamaskrc.INFURA_PROJECT_ID,
+    SEGMENT_HOST: metamaskrc.SEGMENT_HOST,
+    // When we're in the 'production' environment we will use a specific key only set in CI
+    // Otherwise we'll use the key from .metamaskrc or from the environment variable. If
+    // the value of SEGMENT_WRITE_KEY that we envify is undefined then no events will be tracked
+    // in the build. This is intentional so that developers can contribute to MetaMask without
+    // inflating event volume.
+    SEGMENT_WRITE_KEY:
+      environment === 'production'
+        ? process.env.SEGMENT_PROD_WRITE_KEY
+        : metamaskrc.SEGMENT_WRITE_KEY,
+    SEGMENT_LEGACY_WRITE_KEY:
+      environment === 'production'
+        ? process.env.SEGMENT_PROD_LEGACY_WRITE_KEY
+        : metamaskrc.SEGMENT_LEGACY_WRITE_KEY,
+  };
+}
+
+function getEnvironment({ devMode, testing }) {
   // get environment slug
   if (devMode) {
     return 'development';
-  } else if (test) {
+  } else if (testing) {
     return 'testing';
   } else if (process.env.CIRCLE_BRANCH === 'master') {
     return 'production';

package.json

@@ -141,6 +141,7 @@
     "json-rpc-engine": "^6.1.0",
     "json-rpc-middleware-stream": "^2.1.1",
     "jsonschema": "^1.2.4",
+    "labeled-stream-splicer": "^2.0.2",
     "localforage": "^1.9.0",
     "lodash": "^4.17.19",
     "loglevel": "^1.4.1",

yarn.lock

@@ -15184,6 +15184,14 @@ labeled-stream-splicer@^2.0.0:
     isarray "^2.0.4"
     stream-splicer "^2.0.0"
 
+labeled-stream-splicer@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/labeled-stream-splicer/-/labeled-stream-splicer-2.0.2.tgz#42a41a16abcd46fd046306cf4f2c3576fffb1c21"
+  integrity sha512-Ca4LSXFFZUjPScRaqOcFxneA0VpKZr4MMYCljyQr4LIewTLb3Y0IUTIsnBBsVubIeEfxeSZpSjSsRM8APEQaAw==
+  dependencies:
+    inherits "^2.0.1"
+    stream-splicer "^2.0.0"
+
 last-run@^1.1.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/last-run/-/last-run-1.1.1.tgz#45b96942c17b1c79c772198259ba943bebf8ca5b"