From e02b229df6661248f17dfac692f2ec4f12523dc0 Mon Sep 17 00:00:00 2001
From: Mark Stacey <markjstacey@gmail.com>
Date: Tue, 21 Jan 2020 17:12:40 -0400
Subject: [PATCH] Add margin of error metric (#7877)

A margin of error metric has been added, which is calculated from a 95%
confidence interval. This confidence interval is calculated using
Student's t-distribution, which is generally preferred for smaller
sample sizes (< ~30) of populations following a normal distribution.
---
 package.json          |  1 +
 test/e2e/benchmark.js |  5 +++++
 yarn.lock             | 25 +++++++++++++++++++++++++
 3 files changed, 31 insertions(+)

diff --git a/package.json b/package.json
index 0cbeef17b..eb7f465d2 100644
--- a/package.json
+++ b/package.json
@@ -302,6 +302,7 @@
     "stylelint-config-standard": "^18.2.0",
     "testem": "^2.16.0",
     "through2": "^2.0.3",
+    "ttest": "^2.1.1",
     "vinyl-buffer": "^1.0.1",
     "vinyl-source-stream": "^2.0.0",
     "watchify": "^3.11.1"
diff --git a/test/e2e/benchmark.js b/test/e2e/benchmark.js
index caaba39b3..d89cd5666 100644
--- a/test/e2e/benchmark.js
+++ b/test/e2e/benchmark.js
@@ -2,6 +2,7 @@
 
 const path = require('path')
 const { promises: fs, constants: fsConstants } = require('fs')
+const ttest = require('ttest')
 const { By, Key } = require('selenium-webdriver')
 const { withFixtures } = require('./helpers')
 const { PAGES } = require('./webdriver/driver')
@@ -41,6 +42,9 @@ const standardDeviationResult = calculateResult((array) => {
   const squareDiffs = array.map(value => Math.pow(value - average, 2))
   return Math.sqrt(calculateAverage(squareDiffs))
 })
+// 95% margin of error calculated using Student's t-distrbution
+const calculateMarginOfError = (array) => ttest(array).confidence()[1] - calculateAverage(array)
+const marginOfErrorResult = calculateResult((array) => calculateMarginOfError(array))
 
 async function profilePageLoad (pages, numSamples) {
   const results = {}
@@ -68,6 +72,7 @@ async function profilePageLoad (pages, numSamples) {
       max: maxResult(result),
       average: averageResult(result),
       standardDeviation: standardDeviationResult(result),
+      marginOfError: marginOfErrorResult(result),
     }
   }
   return results
diff --git a/yarn.lock b/yarn.lock
index 95a81b540..5679b8b42 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6606,6 +6606,11 @@ ccount@^1.0.0, ccount@^1.0.3:
   resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.0.4.tgz#9cf2de494ca84060a2a8d2854edd6dfb0445f386"
   integrity sha512-fpZ81yYfzentuieinmGnphk0pLkOTMm6MZdVqwd77ROvhko6iujLNGrHH5E7utq3ygWklwfmwuG+A7P+NpqT6w==
 
+cephes@^1.1.2:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/cephes/-/cephes-1.2.0.tgz#4d246b622bc488ab8c982a1f702f0c9f49705d96"
+  integrity sha512-twuUuJRrIrsELHz6foJtZlqrz6FC36zoHZJvvThsrM1UWPKxyoilw1Rka6Hk0AmPFKHKUoGwGfAtvNZNtNZu0g==
+
 chai-checkmark@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/chai-checkmark/-/chai-checkmark-1.0.1.tgz#9fbb3c9ad9101f097ef288328d30f4227d74fffb"
@@ -8970,6 +8975,13 @@ disposables@^1.0.1:
   resolved "https://registry.yarnpkg.com/disposables/-/disposables-1.0.2.tgz#36c6a674475f55a2d6913567a601444e487b4b6e"
   integrity sha1-NsamdEdfVaLWkTVnpgFETkh7S24=
 
+distributions@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/distributions/-/distributions-2.1.0.tgz#7005f01898937fbe6404abda20fa7f4aaaa058e0"
+  integrity sha512-4eov7WXRip5Tu0K7Y51W1oVt+DcvkoE0EzsvbLzgedC/n4PUpYXHqk6vo/OH1HteBHp5Vm6cmfnInf9NAXM4qQ==
+  dependencies:
+    cephes "^1.1.2"
+
 dlv@^1.1.2, dlv@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79"
@@ -26071,6 +26083,11 @@ sumchecker@^2.0.2:
   dependencies:
     debug "^2.2.0"
 
+summary@0.3.x:
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/summary/-/summary-0.3.2.tgz#3d7439b30dab3449dfd66840a822838d4a4a8561"
+  integrity sha1-PXQ5sw2rNEnf1mhAqCKDjUpKhWE=
+
 superagent-proxy@^1.0.3, superagent-proxy@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/superagent-proxy/-/superagent-proxy-2.0.0.tgz#9f57515cd660e2e9ce55c0e6bd70f92eb07c3ee0"
@@ -27056,6 +27073,14 @@ tsscmp@1.0.6:
   resolved "https://registry.yarnpkg.com/tsscmp/-/tsscmp-1.0.6.tgz#85b99583ac3589ec4bfef825b5000aa911d605eb"
   integrity sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==
 
+ttest@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/ttest/-/ttest-2.1.1.tgz#ee0c9118fff6580a5c8d6cfe69845219e075277f"
+  integrity sha512-YlRYswipptMVPLil4bsblnp2HXmcAR0McnfnuCM26EYrBZTCdUc/BVPuqZoaMb9h06smAPhe+s+wFMtYOULVGQ==
+  dependencies:
+    distributions "^2.1.0"
+    summary "0.3.x"
+
 tty-browserify@0.0.0:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"