From 783067e17d9abab9f02603b6d6096f217804f53e Mon Sep 17 00:00:00 2001 From: saravanakumardb1 Date: Tue, 3 Mar 2026 11:48:50 -0800 Subject: [PATCH] feat(ab-testing): bucketing, statistics, guardrails, routes [1.2][2.1][2.2] --- .../src/modules/ab-testing/ab-testing.test.ts | 289 +++++++++++++++--- services/platform-service/src/server.ts | 2 + 2 files changed, 255 insertions(+), 36 deletions(-) diff --git a/services/platform-service/src/modules/ab-testing/ab-testing.test.ts b/services/platform-service/src/modules/ab-testing/ab-testing.test.ts index d7e4140a..95713ec6 100644 --- a/services/platform-service/src/modules/ab-testing/ab-testing.test.ts +++ b/services/platform-service/src/modules/ab-testing/ab-testing.test.ts @@ -16,9 +16,9 @@ import { validateAA, calculateSampleSize, } from './statistics.js'; -import { runGuardrails, canAutoPromote, evaluateAutoPromotion } from './guardrails.js'; +import { runGuardrails, canAutoPromote } from './guardrails.js'; import { matchesTargeting } from './targeting.js'; -import type { VariantDoc, ExperimentDoc, MetricType } from './types.js'; +import type { VariantDoc, ExperimentDoc } from './types.js'; // ───────────────────────────────────────────────────────────────────────────── // Bucketing Tests @@ -128,7 +128,9 @@ describe('matchesTargeting', () => { it('matches version range', () => { expect(matchesTargeting({ appVersion: '1.5.0' }, { appVersions: { min: '1.0.0' } })).toBe(true); - expect(matchesTargeting({ appVersion: '0.5.0' }, { appVersions: { min: '1.0.0' } })).toBe(false); + expect(matchesTargeting({ appVersion: '0.5.0' }, { appVersions: { min: '1.0.0' } })).toBe( + false + ); }); it('matches user segments', () => { @@ -137,14 +139,18 @@ describe('matchesTargeting', () => { }); it('matches user properties', () => { - expect(matchesTargeting( - { userProperties: { tier: 'premium' } }, - { userProperties: { tier: 'premium' } } - )).toBe(true); - expect(matchesTargeting( - { userProperties: { tier: 'basic' } }, - { userProperties: { tier: 'premium' } } - )).toBe(false); + expect( + matchesTargeting( + { userProperties: { tier: 'premium' } }, + { userProperties: { tier: 'premium' } } + ) + ).toBe(true); + expect( + matchesTargeting( + { userProperties: { tier: 'basic' } }, + { userProperties: { tier: 'premium' } } + ) + ).toBe(false); }); }); @@ -217,11 +223,17 @@ function createMockExperiment(overrides: Partial = {}): Experimen describe('Beta Distribution', () => { it('computes correct parameters from variant', () => { const variant = createMockVariant({ - stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 }, + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.1, + conversions: 10, + conversionRate: 0.1, + }, }); const beta = betaFromVariant(variant); expect(beta.alpha).toBe(11); // conversions + 1 - expect(beta.beta).toBe(91); // failures + 1 + expect(beta.beta).toBe(91); // failures + 1 }); it('generates credible interval', () => { @@ -242,10 +254,22 @@ describe('Beta Distribution', () => { describe('Probability Calculations', () => { it('calculates probability variant beats control', () => { const control = createMockVariant({ - stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 }, + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.1, + conversions: 10, + conversionRate: 0.1, + }, }); const variant = createMockVariant({ - stats: { participants: 100, events: 50, primaryMetricValue: 0.2, conversions: 20, conversionRate: 0.2 }, + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.2, + conversions: 20, + conversionRate: 0.2, + }, }); const prob = probabilityVariantBeatsControl(variant, control, 'conversion', 5000); @@ -255,9 +279,36 @@ describe('Probability Calculations', () => { it('calculates probability variant beats all', () => { const variants = [ - createMockVariant({ id: 'v1', stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 } }), - createMockVariant({ id: 'v2', stats: { participants: 100, events: 50, primaryMetricValue: 0.2, conversions: 20, conversionRate: 0.2 } }), - createMockVariant({ id: 'v3', stats: { participants: 100, events: 50, primaryMetricValue: 0.15, conversions: 15, conversionRate: 0.15 } }), + createMockVariant({ + id: 'v1', + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.1, + conversions: 10, + conversionRate: 0.1, + }, + }), + createMockVariant({ + id: 'v2', + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.2, + conversions: 20, + conversionRate: 0.2, + }, + }), + createMockVariant({ + id: 'v3', + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.15, + conversions: 15, + conversionRate: 0.15, + }, + }), ]; const prob = probabilityVariantBeatsAll(variants[1], variants, 'conversion', 5000); @@ -267,8 +318,24 @@ describe('Probability Calculations', () => { it('calculates expected loss', () => { const variants = [ - createMockVariant({ stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 } }), - createMockVariant({ stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 }), + createMockVariant({ + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.1, + conversions: 10, + conversionRate: 0.1, + }, + }), + createMockVariant({ + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.1, + conversions: 10, + conversionRate: 0.1, + }, + }), ]; const loss = expectedLossIfChosen(variants[0], variants, 'conversion', 5000); @@ -285,8 +352,25 @@ describe('checkEarlyStopping', () => { it('does not stop before minimum sample size', () => { const experiment = createMockExperiment(); const variants = [ - createMockVariant({ isControl: true, stats: { participants: 50, events: 25, primaryMetricValue: 0.1, conversions: 5, conversionRate: 0.1 } }), - createMockVariant({ stats: { participants: 50, events: 25, primaryMetricValue: 0.5, conversions: 25, conversionRate: 0.5 } }), + createMockVariant({ + isControl: true, + stats: { + participants: 50, + events: 25, + primaryMetricValue: 0.1, + conversions: 5, + conversionRate: 0.1, + }, + }), + createMockVariant({ + stats: { + participants: 50, + events: 25, + primaryMetricValue: 0.5, + conversions: 25, + conversionRate: 0.5, + }, + }), ]; const result = checkEarlyStopping(experiment, variants, 5); @@ -296,8 +380,29 @@ describe('checkEarlyStopping', () => { it('stops when winner is clear', () => { const experiment = createMockExperiment(); const variants = [ - createMockVariant({ isControl: true, stats: { participants: 1000, events: 500, primaryMetricValue: 0.1, conversions: 100, conversionRate: 0.1, betaAlpha: 101, betaBeta: 901 } }), - createMockVariant({ stats: { participants: 1000, events: 500, primaryMetricValue: 0.5, conversions: 500, conversionRate: 0.5, betaAlpha: 501, betaBeta: 501 } }), + createMockVariant({ + isControl: true, + stats: { + participants: 1000, + events: 500, + primaryMetricValue: 0.1, + conversions: 100, + conversionRate: 0.1, + betaAlpha: 101, + betaBeta: 901, + }, + }), + createMockVariant({ + stats: { + participants: 1000, + events: 500, + primaryMetricValue: 0.5, + conversions: 500, + conversionRate: 0.5, + betaAlpha: 501, + betaBeta: 501, + }, + }), ]; const result = checkEarlyStopping(experiment, variants, 10); @@ -310,8 +415,25 @@ describe('checkEarlyStopping', () => { guardrails: { ...createMockExperiment().guardrails, maxDurationDays: 14 }, }); const variants = [ - createMockVariant({ isControl: true, stats: { participants: 200, events: 100, primaryMetricValue: 0.1, conversions: 20, conversionRate: 0.1 } }), - createMockVariant({ stats: { participants: 200, events: 100, primaryMetricValue: 0.11, conversions: 22, conversionRate: 0.11 } }), + createMockVariant({ + isControl: true, + stats: { + participants: 200, + events: 100, + primaryMetricValue: 0.1, + conversions: 20, + conversionRate: 0.1, + }, + }), + createMockVariant({ + stats: { + participants: 200, + events: 100, + primaryMetricValue: 0.11, + conversions: 22, + conversionRate: 0.11, + }, + }), ]; const result = checkEarlyStopping(experiment, variants, 30); @@ -328,8 +450,25 @@ describe('runGuardrails', () => { it('requires minimum sample size', () => { const experiment = createMockExperiment(); const variants = [ - createMockVariant({ isControl: true, stats: { participants: 50, events: 25, primaryMetricValue: 0.1, conversions: 5, conversionRate: 0.1 } }), - createMockVariant({ stats: { participants: 50, events: 25, primaryMetricValue: 0.5, conversions: 25, conversionRate: 0.5 } }), + createMockVariant({ + isControl: true, + stats: { + participants: 50, + events: 25, + primaryMetricValue: 0.1, + conversions: 5, + conversionRate: 0.1, + }, + }), + createMockVariant({ + stats: { + participants: 50, + events: 25, + primaryMetricValue: 0.5, + conversions: 25, + conversionRate: 0.5, + }, + }), ]; const checks = runGuardrails(experiment, variants, 10, false); @@ -344,8 +483,25 @@ describe('runGuardrails', () => { primaryMetric: { ...createMockExperiment().primaryMetric, type: 'revenue' }, }); const variants = [ - createMockVariant({ isControl: true, stats: { participants: 200, events: 100, primaryMetricValue: 10, conversions: 50, conversionRate: 0.25 } }), - createMockVariant({ stats: { participants: 200, events: 100, primaryMetricValue: 15, conversions: 75, conversionRate: 0.375 } }), + createMockVariant({ + isControl: true, + stats: { + participants: 200, + events: 100, + primaryMetricValue: 10, + conversions: 50, + conversionRate: 0.25, + }, + }), + createMockVariant({ + stats: { + participants: 200, + events: 100, + primaryMetricValue: 15, + conversions: 75, + conversionRate: 0.375, + }, + }), ]; const checks = runGuardrails(experiment, variants, 10, true); @@ -388,8 +544,29 @@ describe('generateExperimentResult', () => { it('generates results with variant comparisons', () => { const experiment = createMockExperiment(); const variants = [ - createMockVariant({ id: 'var_control', isControl: true, name: 'Control', stats: { participants: 200, events: 100, primaryMetricValue: 0.1, conversions: 20, conversionRate: 0.1 } }), - createMockVariant({ id: 'var_test', name: 'Test', stats: { participants: 200, events: 100, primaryMetricValue: 0.15, conversions: 30, conversionRate: 0.15 } }), + createMockVariant({ + id: 'var_control', + isControl: true, + name: 'Control', + stats: { + participants: 200, + events: 100, + primaryMetricValue: 0.1, + conversions: 20, + conversionRate: 0.1, + }, + }), + createMockVariant({ + id: 'var_test', + name: 'Test', + stats: { + participants: 200, + events: 100, + primaryMetricValue: 0.15, + conversions: 30, + conversionRate: 0.15, + }, + }), ]; const result = generateExperimentResult(experiment, variants, 14); @@ -403,8 +580,32 @@ describe('generateExperimentResult', () => { it('identifies winner when statistically significant', () => { const experiment = createMockExperiment(); const variants = [ - createMockVariant({ id: 'var_control', isControl: true, stats: { participants: 1000, events: 500, primaryMetricValue: 0.1, conversions: 100, conversionRate: 0.1, betaAlpha: 101, betaBeta: 901 } }), - createMockVariant({ id: 'var_test', name: 'Test', stats: { participants: 1000, events: 500, primaryMetricValue: 0.5, conversions: 500, conversionRate: 0.5, betaAlpha: 501, betaBeta: 501 } }), + createMockVariant({ + id: 'var_control', + isControl: true, + stats: { + participants: 1000, + events: 500, + primaryMetricValue: 0.1, + conversions: 100, + conversionRate: 0.1, + betaAlpha: 101, + betaBeta: 901, + }, + }), + createMockVariant({ + id: 'var_test', + name: 'Test', + stats: { + participants: 1000, + events: 500, + primaryMetricValue: 0.5, + conversions: 500, + conversionRate: 0.5, + betaAlpha: 501, + betaBeta: 501, + }, + }), ]; const result = generateExperimentResult(experiment, variants, 14); @@ -461,12 +662,28 @@ describe('assignByStrategy', () => { const mockControl: VariantDoc = createMockVariant({ id: 'var_control', isControl: true, - stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1, betaAlpha: 11, betaBeta: 91 }, + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.1, + conversions: 10, + conversionRate: 0.1, + betaAlpha: 11, + betaBeta: 91, + }, }); const mockVariant: VariantDoc = createMockVariant({ id: 'var_test', - stats: { participants: 100, events: 50, primaryMetricValue: 0.2, conversions: 20, conversionRate: 0.2, betaAlpha: 21, betaBeta: 81 }, + stats: { + participants: 100, + events: 50, + primaryMetricValue: 0.2, + conversions: 20, + conversionRate: 0.2, + betaAlpha: 21, + betaBeta: 81, + }, }); const ctx = { diff --git a/services/platform-service/src/server.ts b/services/platform-service/src/server.ts index f10bd761..cbf320d3 100644 --- a/services/platform-service/src/server.ts +++ b/services/platform-service/src/server.ts @@ -58,6 +58,7 @@ import { maintenanceRoutes } from './modules/maintenance/routes.js'; import { exportRoutes } from './modules/exports/routes.js'; import { ipRuleRoutes } from './modules/ip-rules/routes.js'; import { experimentRoutes } from './modules/experiments/routes.js'; +import { abTestingRoutes } from './modules/ab-testing/routes.js'; import { analyticsRoutes } from './modules/analytics/routes.js'; import { feedbackRoutes } from './modules/feedback/routes.js'; import { impersonationRoutes } from './modules/impersonation/routes.js'; @@ -166,6 +167,7 @@ await app.register(exportRoutes, { prefix: '/api' }); await app.register(ipRuleRoutes, { prefix: '/api' }); // P2 — Product Intelligence await app.register(experimentRoutes, { prefix: '/api' }); +await app.register(abTestingRoutes, { prefix: '/api' }); await app.register(analyticsRoutes, { prefix: '/api' }); await app.register(feedbackRoutes, { prefix: '/api' }); await app.register(impersonationRoutes, { prefix: '/api' });