Merge pull request #22 from bookingcom/multivariant

Add multi-variant calculations for even splits
bookingcom · Jul 22, 2019 · e97ab40 · e97ab40
2 parents c14ef1c + 82af3fe
commit e97ab40
Show file tree

Hide file tree

Showing 26 changed files with 17,090 additions and 6,866 deletions.
diff --git a/dist/powercalculator.css b/dist/powercalculator.css
diff --git a/dist/powercalculator.js b/dist/powercalculator.js
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -39,5 +39,8 @@
     "babel-core": "^6.26.0",
     "babel-jest": "^22.4.1",
     "regenerator-runtime": "^0.11.1"
+  },
+  "jest": {
+    "testURL": "http://localhost/"
   }
 }
diff --git a/rollup.config.js b/rollup.config.js
@@ -36,6 +36,7 @@ export default {
     })
   ],
   output: {
+    banner: banner,
     name: 'powercalculator',
     file: 'dist/powercalculator.js',
     format: 'umd',

diff --git a/src/components/impact-comp.vue b/src/components/impact-comp.vue
@@ -93,6 +93,7 @@
                     <span class="pc-input-details">
                         {{ testType == 'gTest' ? ' Incremental trials per day': ' Incremental change in the metric per day' }}
                     </span>
+                </label>
             </li>
         </ul>
     </div>

diff --git a/src/components/non-inferiority-comp.vue b/src/components/non-inferiority-comp.vue
@@ -10,20 +10,13 @@
         <ul class="pc-inputs">
             <li class="pc-input-item pc-input-left">
                 <label>
-                    <span class="pc-input-title">Acceptable Cost
-                        <small class="pc-input-sub-title">
-                            {{isRelative ?
-                                'relative difference of' :
-                                'absolute impact per day of'
-                            }}
-                        </small>
-                    </span>
+                    <span class="pc-input-title">Relative <small class="pc-input-sub-title">change</small></span>
 
                     <pc-block-field
-                        fieldProp="threshold"
-                        :suffix="isRelative ? '%' : ''"
+                        fieldProp="thresholdRelative"
+                        suffix="%"
 
-                        v-bind:fieldValue="threshold"
+                        v-bind:fieldValue="thresholdRelative"
                         v-bind:fieldFromBlock="fieldFromBlock"
                         v-bind:isBlockFocused="isBlockFocused"
                         v-bind:isReadOnly="isReadOnly"
@@ -35,22 +28,18 @@
 
             <li class="pc-input-item pc-input-right">
                 <label>
-                    <span class="pc-input-title">
-                        Type {{ isRelative ?
-                                '' :
-                                '(per day)'
-                            }}
-                        <small class="pc-input-sub-title">
-                        </small>
-                    </span>
+                    <span class="pc-input-title">Absolute <small class="pc-input-sub-title">impact per day</small></span>
 
-                    <div class="pc-non-inf-select-wrapper">
-                        <select v-model="selected" class="pc-non-inf-select">
-                            <option v-for="(option, index) in options" v-bind:key="index" v-bind:value="option.value">
-                                {{option.text}}
-                            </option>
-                        </select>
-                    </div>
+                    <pc-block-field
+                        fieldProp="thresholdAbsolute"
+                        suffix=""
+                        v-bind:fieldValue="thresholdAbsolute"
+                        v-bind:fieldFromBlock="fieldFromBlock"
+                        v-bind:isBlockFocused="isBlockFocused"
+                        v-bind:isReadOnly="isReadOnly"
+                        v-bind:enableEdit="true"
+
+                        v-on:update:focus="updateFocus"></pc-block-field>
                 </label>
             </li>
 
@@ -114,6 +103,12 @@ export default {
         threshold () {
             return this.$store.state.nonInferiority.threshold
         },
+        thresholdRelative () {
+            return this.$store.state.nonInferiority.thresholdRelative
+        },
+        thresholdAbsolute () {
+            return this.$store.state.nonInferiority.thresholdAbsolute
+        },
         isRelative () {
             return this.$store.state.nonInferiority.selected == 'relative'
         },

diff --git a/src/components/pc-block-field.vue b/src/components/pc-block-field.vue
@@ -61,6 +61,12 @@ let validateFunctions = {
                     return value > 0
                 },
                 defaultVal: 0
+            },
+            variants: {
+                fn (value) {
+                    return Number.isInteger(value) && value > 1
+                },
+                defaultVal: 1
             }
         },
         gTest: {
@@ -376,7 +382,8 @@ export default {
 
 .pc-non-inf-treshold-input,
 .pc-power-input,
-.pc-false-positive-input {
+.pc-false-positive-input,
+.pc-variants-input {
     display: inline-block;
     vertical-align: middle;
     padding: 4px 8px;
@@ -387,6 +394,10 @@ export default {
     font-size: inherit;
 }
 
+.pc-variants-input {
+    width: 6.5em;
+}
+
 .pc-top-fields-error {
     color: var(--red);
 }

diff --git a/src/components/sample-comp.vue b/src/components/sample-comp.vue
@@ -13,10 +13,10 @@
         </div>
 
 
-        <ul class="pc-inputs">
+        <ul class="pc-inputs" :class="{'pc-inputs-no-grid': onlyTotalVisitors}">
             <li class="pc-input-item pc-input-left">
                 <label>
-                    <span class="pc-input-title">Total # <small class="pc-input-sub-title">of visitors</small></span>
+                    <span class="pc-input-title">Total # <small class="pc-input-sub-title">of new visitors</small></span>
 
                     <pc-block-field
                         fieldProp="sample"
@@ -28,9 +28,9 @@
                         v-on:update:focus="updateFocus"></pc-block-field>
                 </label>
             </li>
-            <li class="pc-input-item pc-input-right pc-value-field--lockable" :class="getLockedStateClass('visitorsPerDay')">
+            <li class="pc-input-item pc-input-right pc-value-field--lockable" :class="[getLockedStateClass('visitorsPerDay'), {'pc-hidden': onlyTotalVisitors}]">
                 <label>
-                    <span class="pc-input-title">Daily # <small class="pc-input-sub-title">of visitors</small></span>
+                    <span class="pc-input-title">Daily # <small class="pc-input-sub-title">of new visitors</small></span>
 
                     <pc-block-field
                         fieldProp="visitorsPerDay"
@@ -74,7 +74,7 @@
                 </button>
 
             </li>
-            <li class="pc-input-item pc-input-right-swap pc-value-field--lockable" :class="getLockedStateClass('days')">
+            <li class="pc-input-item pc-input-right-swap pc-value-field--lockable" :class="[getLockedStateClass('days'), {'pc-hidden': onlyTotalVisitors}]">
                 <label>
                     <pc-block-field
                         fieldProp="runtime"
@@ -104,7 +104,6 @@ export default {
     extends: pcBlock,
     data () {
         return {
-            variants: 2,
             focusedBlock: ''
         }
     },
@@ -120,7 +119,10 @@ export default {
         },
         lockedField () {
             return this.$store.state.attributes.lockedField
-        }
+        },
+        onlyTotalVisitors () {
+            return this.$store.state.attributes.onlyTotalVisitors
+        },
     },
     methods: {
         updateFocus ({fieldProp, value}) {
@@ -209,4 +211,8 @@ export default {
     background: linear-gradient(0deg, var(--light-gray) 0%, var(--white) 100%);
 }
 
+.pc-inputs-no-grid {
+    display: block;
+}
+
 </style>
diff --git a/src/js/math.js b/src/js/math.js
@@ -1,56 +1,52 @@
 import jstat from 'jstat';
 
-// SOLVING FOR POWER
-function solveforpower_Gtest ({total_sample_size, base_rate, effect_size, alpha, alternative, mu}) {
-    var sample_size = total_sample_size/2;
+function get_alpha_sidaks_correction(alpha, variants) {
+    if (variants == 1) {
+        return alpha;
+    }
 
-    var mean_base = base_rate;
-    var mean_var = base_rate * (1+effect_size);
+    return 1 - (Math.pow(1-alpha, 1/variants));
+}
 
-    var mean_diff = mean_var - mean_base;
-    var delta = mean_diff - mu
+// SOLVING FOR POWER
+function solveforpower_Gtest(data) {
+    var { base_rate, effect_size } = data;
+    var mean_var = base_rate*(1+effect_size);
+    data.variance = base_rate*(1-base_rate) + mean_var*(1-mean_var);
 
-    var variance = mean_base * (1-mean_base) + mean_var * (1-mean_var);
-    var z = jstat.normal.inv(1-alpha/2, 0, 1);
-    var mean = delta*Math.sqrt(sample_size/variance);
+    return solve_for_power(data);
+}
 
-    var power;
-    if (alternative == 'lower') {
-        power = jstat.normal.cdf(jstat.normal.inv(alpha, 0, 1), mean, 1)
-    } else if (alternative == 'greater') {
-        power = 1-jstat.normal.cdf(jstat.normal.inv(1-alpha, 0, 1), mean, 1)
-    } else {
-        power = 1 - (jstat.normal.cdf(z, mean, 1) -
-            jstat.normal.cdf(-z, mean, 1))
-    }
+function solveforpower_Ttest(data) {
+    var { sd_rate } = data;
+    data.variance = 2*sd_rate**2;
 
-    return power
+    return solve_for_power(data);
 }
 
-function solveforpower_Ttest({total_sample_size, base_rate, sd_rate, effect_size, alpha, alternative, mu}) {
-    var sample_size = total_sample_size/2;
+function solve_for_power(data) {
+    var {total_sample_size, base_rate, variance, effect_size, alpha, variants, alternative, mu} = data;
+    var sample_size = total_sample_size / (1 + variants);
 
     var mean_base = base_rate;
     var mean_var = base_rate * (1+effect_size);
 
     var mean_diff = mean_var - mean_base;
-    var delta = mean_diff - mu
+    var delta = mean_diff - mu;
 
-    var variance = 2*sd_rate**2;
-    var z = jstat.normal.inv(1-alpha/2, 0, 1)
+    var z = jstat.normal.inv(1-alpha/2, 0, 1);
     var mean = delta*Math.sqrt(sample_size/variance);
 
     var power;
     if (alternative == 'lower') {
-        power = jstat.normal.cdf(jstat.normal.inv(alpha, 0, 1), mean, 1)
+        power = jstat.normal.cdf(jstat.normal.inv(alpha, 0, 1), mean, 1);
     } else if (alternative == 'greater') {
-	power = 1-jstat.normal.cdf(jstat.normal.inv(1-alpha, 0, 1), mean, 1)
+        power = 1-jstat.normal.cdf(jstat.normal.inv(1-alpha, 0, 1), mean, 1);
     } else {
-        power = 1 - (jstat.normal.cdf(z, mean, 1) -
-            jstat.normal.cdf(-z, mean, 1))
+        power = 1 - (jstat.normal.cdf(z, mean, 1) - jstat.normal.cdf(-z, mean, 1));
    }
 
-    return power
+    return power;
 }
 
 
@@ -109,64 +105,28 @@ function solve_quadratic_for_sample({mean_diff, Z, days, threshold, variance}) {
 }
 
 function solveforsample_Ttest(data){
-    var { base_rate, sd_rate, effect_size, alpha, beta, alternative, mu, opts } = data;
-    if (!is_valid_input(data)) {
-       return NaN;
-    }
-    var mean_base = base_rate;
-    var mean_var = base_rate * (1+effect_size);
-
-    var variance = 2*sd_rate**2;
-    var mean_diff = mean_var - mean_base;
-
-    var multiplier;
-    var sample_one_group;
-    if (opts && opts.type == 'absolutePerDay') {
-        if (opts.calculating == 'visitorsPerDay') {
-            var Z;
-            if (alternative == "greater") {
-                Z = jstat.normal.inv(beta, 0, 1) - jstat.normal.inv(1-alpha, 0, 1);
-            } else if (alternative == "lower") {
-                Z = jstat.normal.inv(1-beta, 0, 1) - jstat.normal.inv(alpha, 0, 1);
-            } else {
-                Z = jstat.normal.inv(1-beta, 0, 1) + jstat.normal.inv(1-alpha/2, 0, 1);
-            }
-            var sqrt_visitors_per_day = solve_quadratic_for_sample({mean_diff: mean_diff, Z: Z,
-                days: opts.days, threshold: opts.threshold, variance: variance});
-            sample_one_group = opts.days*sqrt_visitors_per_day**2;
-        } else {
-            multiplier = variance/(mean_diff*Math.sqrt(opts.visitors_per_day/2) - opts.threshold/(Math.sqrt(2*opts.visitors_per_day)))**2;
-            var days;
-            if (alternative == "greater" || alternative == "lower") {
-                days = multiplier * (jstat.normal.inv(beta, 0, 1) - jstat.normal.inv(1-alpha, 0, 1))**2
-            } else {
-                days = multiplier * (jstat.normal.inv(1-beta, 0, 1) + jstat.normal.inv(1-alpha/2, 0, 1))**2
-            }
-            sample_one_group = days*opts.visitors_per_day/2;
-        }
-    } else {
-        multiplier = variance/(mu - mean_diff)**2
+    var { sd_rate } = data;
+    data.variance = 2*sd_rate**2;
+    return sample_size_calculation(data);
+}
 
-        if (alternative == "greater" || alternative == "lower") {
-            sample_one_group = multiplier * (jstat.normal.inv(beta, 0, 1) - jstat.normal.inv(1-alpha, 0, 1))**2
-        } else {
-            sample_one_group = multiplier * (jstat.normal.inv(1-beta, 0, 1) + jstat.normal.inv(1-alpha/2, 0, 1))**2
-        }
-    }
+function solveforsample_Gtest(data){
+    var { base_rate, effect_size } = data;
+    var mean_var = base_rate*(1+effect_size);
+    data.variance = base_rate*(1-base_rate) + mean_var*(1-mean_var);
 
-    return 2*Math.ceil(sample_one_group);
+    return sample_size_calculation(data);
 }
 
-function solveforsample_Gtest(data){
-    var { base_rate, effect_size, alpha, beta, alternative, mu, opts } = data;
+function sample_size_calculation(data) {
+    var { base_rate, variance, effect_size, alpha, beta, variants, alternative, mu, opts } = data;
+
     if (!is_valid_input(data)) {
-       return NaN;
+        return NaN;
     }
+
     var mean_base = base_rate;
     var mean_var = base_rate*(1+effect_size);
-
-    var variance = mean_base*(1-mean_base) + mean_var*(1-mean_var);
-
     var mean_diff = mean_var - mean_base;
 
     var multiplier;
@@ -204,14 +164,14 @@ function solveforsample_Gtest(data){
         }
     }
 
-    return 2*Math.ceil(sample_one_group);
+    return (1+variants)*Math.ceil(sample_one_group);
 }
 
 
 
 // SOLVING FOR EFFECT SIZE
-function solveforeffectsize_Ttest({total_sample_size, base_rate, sd_rate, alpha, beta, alternative, mu}){
-    var sample_size = total_sample_size/2;
+function solveforeffectsize_Ttest({total_sample_size, base_rate, sd_rate, alpha, beta, variants, alternative, mu}){
+    var sample_size = total_sample_size / (1 + variants);
     var variance = 2*sd_rate**2;
 
     var z = jstat.normal.inv(1-beta, 0, 1);
@@ -245,8 +205,8 @@ function solve_quadratic(Z, sample_size, control_rate, mu) {
     return [sol_h, sol_l];
 }
 
-function solveforeffectsize_Gtest({total_sample_size, base_rate, alpha, beta, alternative, mu}){
-    var sample_size = total_sample_size / 2;
+function solveforeffectsize_Gtest({total_sample_size, base_rate, alpha, beta, variants, alternative, mu}){
+    var sample_size = total_sample_size / (1 + variants);
 
     var rel_effect_size;
     var Z;
@@ -340,4 +300,5 @@ export default {
     getMuFromRelativeDifference: get_mu_from_relative_difference,
     getMuFromAbsolutePerDay: get_mu_from_absolute_per_day,
     getAlternative: get_alternative,
+    getCorrectedAlpha: get_alpha_sidaks_correction,
 }