Switch from npm to yarn (#6843)

As a solution to the constant lockfile churn issues we've had with
`npm`, the project now uses `yarn` to manage dependencies.

The `package-lock.json` file has been replaced with `yarn.lock`, which
was created using `yarn import`. It should approximate the contents of
`package-lock.json` fairly well, though there may be some changes due to
deduplication. The codeowners file has been updated to reference this
new lockfile.

All documentation and npm scripts have been updated to reference `yarn`
rather than `npm`. Note that running scripts using `npm run` still works
fine, but it seemed better to switch those to `yarn` as well to avoid
confusion.

The `npm-audit` Bash script has been replaced with `yarn-audit`. The
output of `yarn audit` is a bit different than `npm audit` in that it
returns a bitmask to describe which severity issues were found. This
made it simpler to check the results directly from the Bash script, so
the associated `npm-audit-check.js` script was no longer required. The
output should be exactly the same, and the information is still sourced
from the same place (the npm registry).

The new `yarn-audit` script does have an external dependency: `jq`.
However, `jq` is already assumed to be present by another CI script, and
is present on all CI images we use. `jq` was not added to `package.json`
as a dependency because there is no official package on the npm
registry, just wrapper scripts. We don't need it anywhere exept on CI
anyway.

The section in `CONTRIBUTING` about how to develop inside the
`node_modules` folder was removed, as the advice was a bit dated, and
wasn't specific to this project anyway.

.circleci/config.yml

@@ -9,11 +9,11 @@ workflows:
             branches:
               only:
                 - /^Version-v(\d+)[.](\d+)[.](\d+)/
-      - prep-deps-npm
+      - prep-deps
       - test-deps
       - prep-build:
           requires:
-            - prep-deps-npm
+            - prep-deps
       - create_github_release:
           requires:
             - prep-build
@@ -23,33 +23,33 @@ workflows:
                 - develop
       # - prep-docs:
       #    requires:
-      #      - prep-deps-npm
+      #      - prep-deps
       - prep-scss:
           requires:
-            - prep-deps-npm
+            - prep-deps
       - test-lint:
           requires:
-            - prep-deps-npm
+            - prep-deps
       - test-e2e-chrome:
           requires:
-            - prep-deps-npm
+            - prep-deps
       - test-e2e-firefox:
           requires:
-            - prep-deps-npm
+            - prep-deps
       - test-unit:
           requires:
-            - prep-deps-npm
+            - prep-deps
       - test-mozilla-lint:
           requires:
-            - prep-deps-npm
+            - prep-deps
             - prep-build
       - test-integration-flat-chrome:
           requires:
-            - prep-deps-npm
+            - prep-deps
             - prep-scss
       - test-integration-flat-firefox:
           requires:
-            - prep-deps-npm
+            - prep-deps
             - prep-scss
       - all-tests-pass:
           requires:
@@ -62,7 +62,7 @@ workflows:
             - test-integration-flat-firefox
       - job-publish-prerelease:
           requires:
-            - prep-deps-npm
+            - prep-deps
             - prep-build
             - all-tests-pass
       - job-publish-release:
@@ -70,7 +70,7 @@ workflows:
             branches:
               only: master
           requires:
-            - prep-deps-npm
+            - prep-deps
             - prep-build
       #       - prep-docs
             - all-tests-pass
@@ -88,18 +88,15 @@ jobs:
             .circleci/scripts/release-bump-manifest-version
             .circleci/scripts/release-create-release-pr
 
-  prep-deps-npm:
+  prep-deps:
     docker:
       - image: circleci/node:10.16-browsers
     steps:
       - checkout
       - run:
-          name: Update npm
+          name: Install deps
-          command: sudo npm install -g npm@6.10.2
-      - run:
-          name: Install deps via npm
           command: |
-           npm ci
+           yarn --frozen-lockfile
       - persist_to_workspace:
           root: .
           paths:
@@ -114,7 +111,7 @@ jobs:
           at: .
       - run:
           name: build:dist
-          command: npm run dist
+          command: yarn dist
       - run:
           name: build:debug
           command: find dist/ -type f -exec md5sum {} \; | sort -k 2
@@ -133,7 +130,7 @@ jobs:
           at: .
       - run:
           name: build:dist
-          command: npm run doc
+          command: yarn doc
       - persist_to_workspace:
           root: .
           paths:
@@ -152,7 +149,7 @@ jobs:
           command: find ui/app/css -type f -exec md5sum {} \; | sort -k 2 > scss_checksum
       - run:
           name: Build for integration tests
-          command: npm run test:integration:build
+          command: yarn test:integration:build
       - persist_to_workspace:
           root: .
           paths:
@@ -167,7 +164,7 @@ jobs:
           at: .
       - run:
           name: Test
-          command: npm run lint
+          command: yarn lint
 
   test-deps:
     docker:
@@ -177,11 +174,8 @@ jobs:
       - attach_workspace:
           at: .
       - run:
-          name: Update npm
+          name: yarn audit
-          command: sudo npm install -g npm@6.10.2
+          command: .circleci/scripts/yarn-audit
-      - run:
-          name: npm audit
-          command: .circleci/scripts/npm-audit
 
   test-e2e-chrome:
     docker:
@@ -192,7 +186,7 @@ jobs:
           at: .
       - run:
           name: test:e2e:chrome
-          command: npm run build:test && npm run test:e2e:chrome
+          command: yarn build:test && yarn test:e2e:chrome
           no_output_timeout: 20m
       - store_artifacts:
           path: test-artifacts
@@ -210,7 +204,7 @@ jobs:
           at: .
       - run:
           name: test:e2e:firefox
-          command: npm run build:test && npm run test:e2e:chrome
+          command: yarn build:test && yarn test:e2e:chrome
           no_output_timeout: 20m
       - store_artifacts:
           path: test-artifacts
@@ -245,13 +239,13 @@ jobs:
           at: .
       - run:
           name: sentry sourcemaps upload
-          command: npm run sentry:publish
+          command: yarn sentry:publish
       # - run:
       #    name: github gh-pages docs publish
       #    command: >
       #      git config --global user.name "metamaskbot" &&
       #      git config --global user.email "admin@metamask.io" &&
-      #      npm run publish-docs
+      #      yarn publish-docs
 
   test-unit:
     docker:
@@ -262,7 +256,7 @@ jobs:
           at: .
       - run:
           name: test:coverage
-          command: npm run test:coverage
+          command: yarn test:coverage
   test-mozilla-lint:
     docker:
       - image: circleci/node:10.16-browsers
@@ -272,7 +266,7 @@ jobs:
           at: .
       - run:
           name: test:mozilla-lint
-          command: NODE_OPTIONS=--max_old_space_size=3072 npm run mozilla-lint
+          command: NODE_OPTIONS=--max_old_space_size=3072 yarn mozilla-lint
 
   test-integration-flat-firefox:
     docker:
@@ -286,7 +280,7 @@ jobs:
           command: ./.circleci/scripts/firefox-install
       - run:
           name: test:integration:flat
-          command: npm run test:flat
+          command: yarn test:flat
 
   test-integration-flat-chrome:
     environment:
@@ -299,7 +293,7 @@ jobs:
           at: .
       - run:
           name: test:integration:flat
-          command: npm run test:flat
+          command: yarn test:flat
 
   all-tests-pass:
     docker:

.circleci/scripts/npm-audit

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-if ! npm audit --production
-then
-    ! npm audit --production --json > audit.json
-    printf '%s\n' ''
-    node .circleci/scripts/npm-audit-check.js
-fi

.circleci/scripts/npm-audit-check.js

@@ -1,24 +0,0 @@
-const path = require('path')
-const audit = require(path.join(__dirname, '..', '..', 'audit.json'))
-const error = audit.error
-const advisories = Object.keys(audit.advisories || []).map((k) => audit.advisories[k])
-
-if (error) {
-  process.exit(1)
-}
-
-let count = 0
-for (const advisory of advisories) {
-  if (advisory.severity === 'low') {
-    continue
-  }
-
-  count += advisory.findings.length
-}
-
-if (count > 0) {
-  console.log(`Audit shows ${count} moderate or high severity advisories _in the production dependencies_`)
-  process.exit(1)
-} else {
-  console.log(`Audit shows _zero_ moderate or high severity advisories _in the production dependencies_`)
-}

.circleci/scripts/yarn-audit

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -u
+set -o pipefail
+
+yarn audit --level moderate --groups dependencies
+audit_status="$?"
+
+# Use a bitmask to ignore INFO and LOW severity audit results
+# See here: https://yarnpkg.com/lang/en/docs/cli/audit/
+audit_status="$(( audit_status & 11100 ))"
+
+if [[ "$audit_status" != 0 ]]
+then
+    count="$(yarn audit --level moderate --groups dependencies --json | tail -1 | jq '.data.vulnerabilities.moderate + .data.vulnerabilities.high + .data.vulnerabilities.critical')"
+    printf "Audit shows %s moderate or high severity advisories _in the production dependencies_\n" "$count"
+    exit 1
+else
+    printf "Audit shows _zero_ moderate or high severity advisories _in the production dependencies_\n"
+fi

.github/CODEOWNERS

@@ -1,7 +1,8 @@
 # Lines starting with '#' are comments.
 # Each line is a file pattern followed by one or more owners.
 
-package*.json                        @whymarrh
+package.json                         @whymarrh
+yarn.lock                            @whymarrh
 ui/                                  @danjm @whymarrh @Gudahtt
 app/scripts/controllers/transactions @frankiebee
 

.gitignore

@@ -1,6 +1,7 @@
 npm-debug.log
+yarn-error.log
 node_modules
-yarn.lock
+package-lock.json
 
 audit.json
 

.storybook/README.md

@@ -1,8 +1,8 @@
 # Storybook
 We're currently using [Storybook](https://storybook.js.org/) as part of our design system. To run Storybook and test some of our UI components, clone the repo and run the following:
 ```
-npm install
+yarn
-npm run storybook
+yarn storybook
 ```
 You should then see:
 > info Storybook started on => http://localhost:6006/

CONTRIBUTING.md

@@ -22,30 +22,3 @@ When you're done with your project / bugfix / feature and ready to submit a PR,
 - [ ] **Get reviewed by a core contributor**: Make sure you get a `:thumbsup`, `:+1`, or `LGTM` from a user with a `Member` badge before merging.
 
 And that's it! Thanks for helping out.
-
-### Developing inside a node_modules folder
-
-First make sure you are comfortable with [how require works](https://github.com/maxogden/art-of-node#how-require-works) in node.
-
-We recommend creating a folder somewhere manually called `node_modules`. For example in `~/code/node_modules`. Clone all of your git copies of modules that you want to work on into here, so for example:
-
-- `~/code/node_modules/dat`
-- `~/code/node_modules/hyperdrive`
-
-When you run `npm install` inside of `~/code/node_modules/dat`, dat will get its own copy of `hyperdrive` (one if its dependencies) inside `~/code/node_modules/dat/node_modules`. However, if you encounter a bug in hyperdrive that you need to fix, but you want to test your fix in dat, you want dat to use your git copy of hyperdrive at `~/code/node_modules/hyperdrive` and not the npm copy of hyperdrive at `~/code/node_modules/dat/node_modules/hyperdrive`.
-
-How do you get dat to use the git copy of hyperdrive? Just delete the npm copy!
-
-```
-rm -rf ~/code/node_modules/dat/node_modules/hyperdrive
-```
-
-Now when you run dat, and it tries to `require('hyperdrive')` it first looks in its own `node_modules` folder at `~/code/node_modules/dat/node_modules` but doesnt find hyperdrive. So it goes up to `~/code/node_modules` and finds `hyperdrive` there and uses that one, your git copy.
-
-If you want to switch back to an npm copy, just run `npm install` inside `~/code/node_modules/dat/` and npm will download any missing modules into `~/code/node_modules/dat/node_modules` but wont touch anything in `~/code/node_modules`.
-
-This might seem a bit complicated at first, but is simple once you get the hang of it. Here are some rules to help you get started:
-
-- Never make any meaningful edits to code inside an "npm-managed" node_modules folder (such as `~/code/node_modules/dat/node_modules`), because when you run `npm install` inside those folders it could inadvertently delete all of your edits when installing an updated copy of a module. This has happened to me many times, so I just always use my git copy and delete the npm copy (as described above) to make edits to a module.
-- You should never need to run any npm commands in terminal when at your "manually managed"" node_modules folder at `~/code/node_modules`. Never running npm commands at that folder also prevents npm from accidentally erasing your git copies of modules
-- The location of your "manually managed" node_modules folder should be somewhere isolated from your normal require path. E.g. if you put it at `~/node_modules`, then when you run `npm install dat` at `~/Desktop` npm might decide to erase your git copy of dat at `~/node_modules/dat` and replace it with a copy from npm, which could make you lose work. Putting your manually managed `node_modules` folder in a sub-folder like `~/code` gets it "out of the way" and prevents accidents like that from happening.

README.md

@@ -11,16 +11,14 @@ To learn how to contribute to the MetaMask project itself, visit our [Internal D
 
 ## Building locally
 
-- Install [Node.js](https://nodejs.org) version 10 and the latest available npm@6
+- Install [Node.js](https://nodejs.org) version 10
     - If you are using [nvm](https://github.com/creationix/nvm#installation) (recommended) running `nvm use` will automatically choose the right node version for you.
-    - If you install Node.js manually, ensure you're using npm@6
+- Install [Yarn](https://yarnpkg.com/en/docs/install)
-        - Install npm@6 using `npm install -g npm@6`
+- Install dependencies: `yarn`
-- Install dependencies: `npm ci`
+- Build the project to the `./dist/` folder with `yarn dist`.
-    - If you have issues with node-sass compilation, try `npm rebuild node-sass`
+- Optionally, to start a development build (e.g. with logging and file watching) run `yarn start` instead.
-- Build the project to the `./dist/` folder with `npm run dist`.
-- Optionally, to start a development build (e.g. with logging and file watching) run `npm start` instead.
     - To start the [React DevTools](https://github.com/facebook/react-devtools) and [Redux DevTools Extension](http://extension.remotedev.io)
-      alongside the app, use `npm run start:dev`.
+      alongside the app, use `yarn start:dev`.
       - React DevTools will open in a separate window; no browser extension is required
       - Redux DevTools will need to be installed as a browser extension. Open the Redux Remote Devtools to access Redux state logs. This can be done by either right clicking within the web browser to bring up the context menu, expanding the Redux DevTools panel and clicking Open Remote DevTools OR clicking the Redux DevTools extension icon and clicking Open Remote DevTools.
         - You will also need to check the "Use custom (local) server" checkbox in the Remote DevTools Settings, using the default server configuration (host `localhost`, port `8000`, secure connection checkbox unchecked)
@@ -31,15 +29,15 @@ Uncompressed builds can be found in `/dist`, compressed builds can be found in `
 
 You can read [our internal docs here](https://metamask.github.io/metamask-extension/).
 
-You can re-generate the docs locally by running `npm run doc`, and contributors can update the hosted docs by running `npm run publish-docs`.
+You can re-generate the docs locally by running `yarn doc`, and contributors can update the hosted docs by running `yarn publish-docs`.
 
 ### Running Tests
 
-Run tests with `npm test`.
+Run tests with `yarn test`.
 
-You can also test with a continuously watching process, via `npm run watch`.
+You can also test with a continuously watching process, via `yarn watch`.
 
-You can run the linter by itself with `npm run lint`.
+You can run the linter by itself with `yarn lint`.
 
 ## Architecture
 
@@ -48,14 +46,14 @@ You can run the linter by itself with `npm run lint`.
 ## Development
 
 ```bash
-npm install
+yarn
-npm start
+yarn start
 ```
 
 ## Build for Publishing
 
 ```bash
-npm run dist
+yarn dist
 ```
 
 #### Writing Browser Tests

development/rollback.sh

@@ -5,7 +5,7 @@ echo "Rolling back to version $1"
 
 # Checkout branch to increment version
 git checkout -b version-increment-$1
-npm run version:bump patch
+yarn version:bump patch
 
 # Store the new version name
 NEW_VERSION=$(cat app/manifest.json | jq -r .version)

docs/bumping_version.md

@@ -1,7 +1,7 @@
 # How to Bump MetaMask's Version Automatically
 
 ```
-npm run version:bump patch
+yarn version:bump patch
 ```
 
 MetaMask publishes using a loose [semver](https://semver.org/) interpretation. We divide the three segments of our version into three types of version bump:
@@ -20,7 +20,7 @@ Means a fix for a bug, or correcting something that should have been assumed to
 
 # Bumping the version
 
-`npm run version:bump $BUMP_TYPE` where `$BUMP_TYPE` is one of `major`, `minor`, or `patch`.
+`yarn version:bump $BUMP_TYPE` where `$BUMP_TYPE` is one of `major`, `minor`, or `patch`.
 
 This will increment the version in the `app/manifest.json` and `CHANGELOG.md` files according to our current protocol, where the manifest's version is updated, and any line items currently under the changelog's "master" section are now under the new dated version section.
 

docs/developing-on-deps.md

@@ -3,7 +3,7 @@
 To enjoy the live-reloading that `gulp dev` offers while working on the dependencies:
 
  1. Clone the dependency locally.
- 2. `npm install` in its folder.
+ 2. `npm install` or `yarn install` in its folder.
- 3. Run `npm link` in its folder.
+ 3. Run `yarn link` in its folder.
- 4. Run `npm link $DEP_NAME` in this project folder.
+ 4. Run `yarn link $DEP_NAME` in this project folder.
- 5. Next time you `npm start` it will watch the dependency for changes as well!
+ 5. Next time you `yarn start` it will watch the dependency for changes as well!

docs/publishing.md

@@ -41,7 +41,7 @@ With each pull request, the @MetaMaskBot will comment with a build of that new p
 3. Publish to [firefox addon marketplace](http://addons.mozilla.org/en-us/firefox/addon/ether-metamask).
 4. Publish to [Opera store](https://addons.opera.com/en/extensions/details/metamask/).
 5. Post on [Github releases](https://github.com/MetaMask/metamask-extension/releases) page.
-6. Run the `npm run announce` script, and post that announcement in our public places.
+6. Run the `yarn announce` script, and post that announcement in our public places.
 
 ## Hotfix Differences
 

package.json

@@ -9,26 +9,26 @@
     "publish-docs": "gh-pages -d docs/jsdocs",
     "start:test": "gulp dev:test",
     "build:test": "gulp build:test",
-    "test": "npm run test:unit && npm run lint",
+    "test": "yarn test:unit && yarn lint",
     "dapp": "static-server test/e2e/contract-test --port 8080",
-    "dapp-chain": "GANACHE_ARGS='-b 2' concurrently -k -n ganache,dapp -p '[{time}][{name}]' 'npm run ganache:start' 'sleep 5 && static-server test/e2e/contract-test --port 8080'",
+    "dapp-chain": "GANACHE_ARGS='-b 2' concurrently -k -n ganache,dapp -p '[{time}][{name}]' 'yarn ganache:start' 'sleep 5 && static-server test/e2e/contract-test --port 8080'",
-    "watch:test:unit": "nodemon --exec \"npm run test:unit\" ./test ./app ./ui",
+    "watch:test:unit": "nodemon --exec \"yarn test:unit\" ./test ./app ./ui",
     "test:unit": "cross-env METAMASK_ENV=test mocha --exit --require test/setup.js --recursive \"test/unit/**/*.js\" \"ui/app/**/*.test.js\"",
     "test:single": "cross-env METAMASK_ENV=test mocha --require test/helper.js",
-    "test:integration": "npm run test:integration:build && npm run test:flat",
+    "test:integration": "yarn test:integration:build && yarn test:flat",
     "test:integration:build": "gulp build:scss",
     "test:e2e:chrome": "SELENIUM_BROWSER=chrome test/e2e/run-all.sh",
     "test:web3:chrome": "SELENIUM_BROWSER=chrome test/e2e/run-web3.sh",
     "test:web3:firefox": "SELENIUM_BROWSER=firefox test/e2e/run-web3.sh",
     "test:e2e:firefox": "SELENIUM_BROWSER=firefox test/e2e/run-all.sh",
-    "test:coverage": "nyc --reporter=text --reporter=html npm run test:unit && npm run test:coveralls-upload",
+    "test:coverage": "nyc --reporter=text --reporter=html npm run test:unit && yarn test:coveralls-upload",
     "test:coveralls-upload": "if [ $COVERALLS_REPO_TOKEN ]; then nyc report --reporter=text-lcov | coveralls; fi",
-    "test:flat": "npm run test:flat:build && karma start test/flat.conf.js",
+    "test:flat": "yarn test:flat:build && karma start test/flat.conf.js",
-    "test:flat:build": "npm run test:flat:build:ui && npm run test:flat:build:tests && npm run test:flat:build:locales",
+    "test:flat:build": "yarn test:flat:build:ui && yarn test:flat:build:tests && yarn test:flat:build:locales",
     "test:flat:build:tests": "node test/integration/index.js",
     "test:flat:build:states": "node development/genStates.js",
     "test:flat:build:locales": "mkdirp dist/chrome && cp -R app/_locales dist/chrome/_locales",
-    "test:flat:build:ui": "npm run test:flat:build:states && browserify --transform babelify --transform brfs ./development/mock-dev.js -o ./development/bundle.js",
+    "test:flat:build:ui": "yarn test:flat:build:states && browserify --transform babelify --transform brfs ./development/mock-dev.js -o ./development/bundle.js",
     "ganache:start": "./development/run-ganache",
     "sentry:publish": "node ./development/sentry-publish.js",
     "lint": "eslint . --ext js,json",
@@ -37,7 +37,7 @@
     "watch": "cross-env METAMASK_ENV=test mocha --watch --require test/setup.js --reporter min --recursive \"test/unit/**/*.js\" \"ui/app/**/*.test.js\"",
     "devtools:react": "react-devtools",
     "devtools:redux": "remotedev --hostname=localhost --port=8000",
-    "start:dev": "concurrently -k -n build,react,redux npm:start npm:devtools:react npm:devtools:redux",
+    "start:dev": "concurrently -k -n build,react,redux yarn:start yarn:devtools:react yarn:devtools:redux",
     "announce": "node development/announcer.js",
     "version:bump": "node development/run-version-bump.js",
     "storybook": "start-storybook -p 6006 -c .storybook",
@@ -272,7 +272,7 @@
     "watchify": "^3.11.1"
   },
   "engines": {
-    "node": "10.16.0",
+    "node": "^10.16.0",
-    "npm": "^6.9.0"
+    "yarn": "^1.16.0"
   }
 }

test/e2e/run-all.sh

@@ -12,16 +12,16 @@ concurrently --kill-others \
   --names 'ganache,dapp,e2e' \
   --prefix '[{time}][{name}]' \
   --success first \
-  'npm run ganache:start' \
+  'yarn ganache:start' \
-  'npm run dapp' \
+  'yarn dapp' \
   'sleep 5 && mocha test/e2e/metamask-ui.spec'
 
 concurrently --kill-others \
   --names 'ganache,dapp,e2e' \
   --prefix '[{time}][{name}]' \
   --success first \
-  'npm run ganache:start' \
+  'yarn ganache:start' \
-  'npm run dapp' \
+  'yarn dapp' \
   'sleep 5 && mocha test/e2e/metamask-responsive-ui.spec'
 
 export GANACHE_ARGS="$GANACHE_ARGS --deterministic --account=0x53CB0AB5226EEBF4D872113D98332C1555DC304443BEE1CF759D15798D3C55A9,25000000000000000000"
@@ -29,5 +29,5 @@ concurrently --kill-others \
   --names 'ganache,e2e' \
   --prefix '[{time}][{name}]' \
   --success first \
-  'npm run ganache:start' \
+  'yarn ganache:start' \
   'sleep 5 && mocha test/e2e/from-import-ui.spec'