A test with a little power. I.e. add a small treatment effect and confirm the pvalues are pulled down to zero
@@ -211,3 +211,48 @@ def test_using_lots_of_AA_tests_ratio__always_1__withZeros():
assert 0.08 < np.percentile(all_ps,10) < 0.12
assert 0.48 < np.percentile(all_ps,50) < 0.52
assert 0.88 < np.percentile(all_ps,90) < 0.92
+
+def test_using_lots_of_AB_tests_with_FALSE_null():
+ n = TOTAL_SAMPLE_SIZE_FOR_AA_TESTS
+
+ # In this test we add a small difference in effect between
+ # treatment in order to confirm that the pvalues are
+ # pulled down from uniformity.
+
+ rng = np.random.RandomState()
+ rng.seed(1337)
+
+ # Arbitrary data here for the numerators and the denominators. Feel
+ # free to change it. This is just a lazy way to give every user
+ # a slightly different ratio.
+ revs = 1+np.arange(n)
+ sess = 2+np.arange(n)
+
+ worker = worker_table['fixed_horizon'](min_observations=0)
+
+ assignments = (np.arange(n) % 2 == 0) # 50/50
+
+ assert len(assignments) == len(revs)
+ assert len(assignments) == len(sess)
+
+ all_ps = []
+ for i in range(NUMBER_OF_SIMULATED_AA_TESTS_TO_RUN):
+ rng.shuffle(assignments) # randomly reassign everybody
+
+ # extract the 'controls':
+ x_num = revs[ assignments]
+ x_den = sess[ assignments]
+ # extract the 'treatments':
+ y_num = revs[~assignments] * 1.0005
+ y_den = sess[~assignments]
+ # call the 'fixed_horizon' worker:
+ res = worker(x_num,y_num,
+ x_den, y_den,
+ #np.mean(x_den), np.mean(y_den), # using means would replicated the old (wrong) behaviour
+ )
+ all_ps.append(res.p)
+
+ # The 'all_ps' should be approximately uniform between zero and one
+ assert 0.0001 < np.percentile(all_ps,10) < 0.001
+ assert 0.005 < np.percentile(all_ps,50) < 0.02
+ assert 0.20 < np.percentile(all_ps,90) < 0.30
