From 9fd42f1bc2165688fd83ebef1ec8fce4a25680ca Mon Sep 17 00:00:00 2001 From: Mark Stacey Date: Fri, 26 Nov 2021 16:38:23 -0330 Subject: [PATCH] Fix LavaMoat background policy generation (#12844) The LavaMoat policy generation script would sporadically fail because it ran the build concurrently three times, and the build includes steps that delete the `dist` directory and write to it. So if one build process tried to write to the directory after another deleted it, it would fail. This was solved by adding a new `--policy-only` flag to the build script, and a new `scripts:prod` task. The `scripts:prod` task only runs the script tasks for prod, rather than the entire build process. The `--policy-only` flag stops the script tasks once the policy has been written, and stops any other files from being written to disk. This prevents the three concurrent build processes from getting in each others way, and it dramatically speeds up the process. --- development/build/README.md | 3 ++ development/build/index.js | 11 ++++++ development/build/scripts.js | 42 +++++++++++++++++++---- development/build/task.js | 4 ++- development/generate-lavamoat-policies.sh | 6 ++-- 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/development/build/README.md b/development/build/README.md index 1218308c9..7d6cd8d9c 100644 --- a/development/build/README.md +++ b/development/build/README.md @@ -52,6 +52,9 @@ Options: bundle. Setting this to `false` is useful e.g. when linking dependencies that are incompatible with lockdown. [boolean] [default: true] + --policy-only Stops the build after generating the LavaMoat policy, + skipping any writes to disk. + [boolean] [deafult: false] --skip-stats Whether to refrain from logging build progress. Mostly used internally. [boolean] [default: false] diff --git a/development/build/index.js b/development/build/index.js index 7ee92f672..669cf6826 100755 --- a/development/build/index.js +++ b/development/build/index.js @@ -52,6 +52,7 @@ function defineAndRunBuildTasks() { buildType, entryTask, isLavaMoat, + policyOnly, shouldIncludeLockdown, shouldLintFenceFiles, skipStats, @@ -84,6 +85,7 @@ function defineAndRunBuildTasks() { ignoredFiles, isLavaMoat, livereload, + policyOnly, shouldLintFenceFiles, }); @@ -134,6 +136,9 @@ function defineAndRunBuildTasks() { ), ); + // build just production scripts, for LavaMoat policy generation purposes + createTask('scripts:prod', scriptTasks.prod); + // build for CI testing createTask( 'test', @@ -157,6 +162,7 @@ function parseArgv() { BuildType: 'build-type', LintFenceFiles: 'lint-fence-files', Lockdown: 'lockdown', + PolicyOnly: 'policy-only', SkipStats: 'skip-stats', }; @@ -164,6 +170,7 @@ function parseArgv() { boolean: [ NamedArgs.LintFenceFiles, NamedArgs.Lockdown, + NamedArgs.PolicyOnly, NamedArgs.SkipStats, ], string: [NamedArgs.BuildType], @@ -171,6 +178,7 @@ function parseArgv() { [NamedArgs.BuildType]: BuildType.main, [NamedArgs.LintFenceFiles]: true, [NamedArgs.Lockdown]: true, + [NamedArgs.PolicyOnly]: false, [NamedArgs.SkipStats]: false, }, }); @@ -198,10 +206,13 @@ function parseArgv() { ? argv[NamedArgs.LintFenceFiles] : !/dev/iu.test(entryTask); + const policyOnly = argv[NamedArgs.PolicyOnly]; + return { buildType, entryTask, isLavaMoat: process.argv[0].includes('lavamoat'), + policyOnly, shouldIncludeLockdown: argv[NamedArgs.Lockdown], shouldLintFenceFiles, skipStats: argv[NamedArgs.SkipStats], diff --git a/development/build/scripts.js b/development/build/scripts.js index 5fbb95143..f106dbca5 100644 --- a/development/build/scripts.js +++ b/development/build/scripts.js @@ -134,6 +134,10 @@ function getSegmentWriteKey({ buildType, environment }) { throw new Error(`Invalid build type: '${buildType}'`); } +const noopWriteStream = through.obj((_file, _fileEncoding, callback) => + callback(), +); + module.exports = createScriptTasks; function createScriptTasks({ @@ -143,6 +147,7 @@ function createScriptTasks({ isLavaMoat, livereload, shouldLintFenceFiles, + policyOnly, }) { // internal tasks const core = { @@ -185,6 +190,7 @@ function createScriptTasks({ return `./app/scripts/${label}.js`; }), ignoredFiles, + policyOnly, shouldLintFenceFiles, testing, }), @@ -241,6 +247,7 @@ function createScriptTasks({ runInChildProcess(subtask, { buildType, isLavaMoat, + policyOnly, shouldLintFenceFiles, }), ); @@ -258,6 +265,8 @@ function createScriptTasks({ entryFilepath: `./app/scripts/${label}.js`, ignoredFiles, label, + testing, + policyOnly, shouldLintFenceFiles, }); } @@ -272,6 +281,8 @@ function createScriptTasks({ entryFilepath: `./app/scripts/${label}.js`, ignoredFiles, label, + testing, + policyOnly, shouldLintFenceFiles, }); } @@ -286,6 +297,8 @@ function createScriptTasks({ entryFilepath: `./app/scripts/${label}.js`, ignoredFiles, label, + testing, + policyOnly, shouldLintFenceFiles, }); } @@ -303,6 +316,7 @@ function createScriptTasks({ entryFilepath: `./app/scripts/${inpage}.js`, label: inpage, ignoredFiles, + policyOnly, shouldLintFenceFiles, testing, }), @@ -314,6 +328,7 @@ function createScriptTasks({ entryFilepath: `./app/scripts/${contentscript}.js`, label: contentscript, ignoredFiles, + policyOnly, shouldLintFenceFiles, testing, }), @@ -327,6 +342,7 @@ function createFactoredBuild({ devMode, entryFiles, ignoredFiles, + policyOnly, shouldLintFenceFiles, testing, }) { @@ -346,6 +362,7 @@ function createFactoredBuild({ devMode, envVars, ignoredFiles, + policyOnly, minify, reloadOnChange, shouldLintFenceFiles, @@ -417,12 +434,17 @@ function createFactoredBuild({ // setup bundle destination browserPlatforms.forEach((platform) => { const dest = `./dist/${platform}/`; - pipeline.get('dest').push(gulp.dest(dest)); + const destination = policyOnly ? noopWriteStream : gulp.dest(dest); + pipeline.get('dest').push(destination); }); }); // wait for bundle completion for postprocessing events.on('bundleDone', () => { + // Skip HTML generation if nothing is to be written to disk + if (policyOnly) { + return; + } const commonSet = sizeGroupMap.get('common'); // create entry points for each file for (const [groupLabel, groupSet] of sizeGroupMap.entries()) { @@ -496,6 +518,7 @@ function createNormalBundle({ extraEntries = [], ignoredFiles, label, + policyOnly, modulesToExpose, shouldLintFenceFiles, testing, @@ -516,6 +539,7 @@ function createNormalBundle({ devMode, envVars, ignoredFiles, + policyOnly, minify, reloadOnChange, shouldLintFenceFiles, @@ -540,7 +564,8 @@ function createNormalBundle({ // setup bundle destination browserPlatforms.forEach((platform) => { const dest = `./dist/${platform}/`; - pipeline.get('dest').push(gulp.dest(dest)); + const destination = policyOnly ? noopWriteStream : gulp.dest(dest); + pipeline.get('dest').push(destination); }); }); @@ -570,6 +595,7 @@ function setupBundlerDefaults( devMode, envVars, ignoredFiles, + policyOnly, minify, reloadOnChange, shouldLintFenceFiles, @@ -613,12 +639,14 @@ function setupBundlerDefaults( setupReloadOnChange(buildConfiguration); } - if (minify) { - setupMinification(buildConfiguration); - } + if (!policyOnly) { + if (minify) { + setupMinification(buildConfiguration); + } - // Setup source maps - setupSourcemaps(buildConfiguration, { devMode }); + // Setup source maps + setupSourcemaps(buildConfiguration, { devMode }); + } } function setupReloadOnChange({ bundlerOpts, events }) { diff --git a/development/build/task.js b/development/build/task.js index 07cd6d0c3..316020259 100644 --- a/development/build/task.js +++ b/development/build/task.js @@ -50,7 +50,7 @@ function createTask(taskName, taskFn) { function runInChildProcess( task, - { buildType, isLavaMoat, shouldLintFenceFiles }, + { buildType, isLavaMoat, policyOnly, shouldLintFenceFiles }, ) { const taskName = typeof task === 'string' ? task : task.taskName; if (!taskName) { @@ -74,6 +74,7 @@ function runInChildProcess( '--lint-fence-files', shouldLintFenceFiles, '--skip-stats', + ...(policyOnly ? ['--policy-only'] : []), ], { env: process.env, @@ -90,6 +91,7 @@ function runInChildProcess( '--lint-fence-files', shouldLintFenceFiles, '--skip-stats', + ...(policyOnly ? ['--policy-only'] : []), ], { env: process.env, diff --git a/development/generate-lavamoat-policies.sh b/development/generate-lavamoat-policies.sh index d023cb942..a5a104a23 100755 --- a/development/generate-lavamoat-policies.sh +++ b/development/generate-lavamoat-policies.sh @@ -8,6 +8,6 @@ set -o pipefail # type. # ATTN: This may tax your device when running it locally. concurrently --kill-others-on-fail -n main,beta,flask \ - "WRITE_AUTO_POLICY=1 yarn dist" \ - "WRITE_AUTO_POLICY=1 yarn dist --build-type beta" \ - "WRITE_AUTO_POLICY=1 yarn dist --build-type flask" + "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only" \ + "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only --build-type beta" \ + "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only --build-type flask"