bytelyst/learning_ai_common_plat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ab-testing): bucketing, statistics, guardrails, routes [1.2][2.1][2.2]

Browse Source
This commit is contained in:
saravanakumardb1 2026-03-03 11:48:50 -08:00
parent 917ea03af9
commit 783067e17d
2 changed files with 255 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

289
services/platform-service/src/modules/ab-testing/ab-testing.test.ts
View File

@ -16,9 +16,9 @@ import {
validateAA,
calculateSampleSize,
} from './statistics.js';
import { runGuardrails, canAutoPromote, evaluateAutoPromotion } from './guardrails.js';
import { runGuardrails, canAutoPromote } from './guardrails.js';
import { matchesTargeting } from './targeting.js';
import type { VariantDoc, ExperimentDoc, MetricType } from './types.js';
import type { VariantDoc, ExperimentDoc } from './types.js';
// ─────────────────────────────────────────────────────────────────────────────
// Bucketing Tests
@ -128,7 +128,9 @@ describe('matchesTargeting', () => {
it('matches version range', () => {
expect(matchesTargeting({ appVersion: '1.5.0' }, { appVersions: { min: '1.0.0' } })).toBe(true);
expect(matchesTargeting({ appVersion: '0.5.0' }, { appVersions: { min: '1.0.0' } })).toBe(false);
expect(matchesTargeting({ appVersion: '0.5.0' }, { appVersions: { min: '1.0.0' } })).toBe(
false
);
});
it('matches user segments', () => {
@ -137,14 +139,18 @@ describe('matchesTargeting', () => {
});
it('matches user properties', () => {
expect(matchesTargeting(
{ userProperties: { tier: 'premium' } },
{ userProperties: { tier: 'premium' } }
)).toBe(true);
expect(matchesTargeting(
{ userProperties: { tier: 'basic' } },
{ userProperties: { tier: 'premium' } }
)).toBe(false);
expect(
matchesTargeting(
{ userProperties: { tier: 'premium' } },
{ userProperties: { tier: 'premium' } }
)
).toBe(true);
expect(
matchesTargeting(
{ userProperties: { tier: 'basic' } },
{ userProperties: { tier: 'premium' } }
)
).toBe(false);
});
});
@ -217,11 +223,17 @@ function createMockExperiment(overrides: Partial<ExperimentDoc> = {}): Experimen
describe('Beta Distribution', () => {
it('computes correct parameters from variant', () => {
const variant = createMockVariant({
stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 },
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.1,
conversions: 10,
conversionRate: 0.1,
},
});
const beta = betaFromVariant(variant);
expect(beta.alpha).toBe(11); // conversions + 1
expect(beta.beta).toBe(91); // failures + 1
expect(beta.beta).toBe(91); // failures + 1
});
it('generates credible interval', () => {
@ -242,10 +254,22 @@ describe('Beta Distribution', () => {
describe('Probability Calculations', () => {
it('calculates probability variant beats control', () => {
const control = createMockVariant({
stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 },
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.1,
conversions: 10,
conversionRate: 0.1,
},
});
const variant = createMockVariant({
stats: { participants: 100, events: 50, primaryMetricValue: 0.2, conversions: 20, conversionRate: 0.2 },
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.2,
conversions: 20,
conversionRate: 0.2,
},
});
const prob = probabilityVariantBeatsControl(variant, control, 'conversion', 5000);
@ -255,9 +279,36 @@ describe('Probability Calculations', () => {
it('calculates probability variant beats all', () => {
const variants = [
createMockVariant({ id: 'v1', stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 } }),
createMockVariant({ id: 'v2', stats: { participants: 100, events: 50, primaryMetricValue: 0.2, conversions: 20, conversionRate: 0.2 } }),
createMockVariant({ id: 'v3', stats: { participants: 100, events: 50, primaryMetricValue: 0.15, conversions: 15, conversionRate: 0.15 } }),
createMockVariant({
id: 'v1',
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.1,
conversions: 10,
conversionRate: 0.1,
},
}),
createMockVariant({
id: 'v2',
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.2,
conversions: 20,
conversionRate: 0.2,
},
}),
createMockVariant({
id: 'v3',
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.15,
conversions: 15,
conversionRate: 0.15,
},
}),
];
const prob = probabilityVariantBeatsAll(variants[1], variants, 'conversion', 5000);
@ -267,8 +318,24 @@ describe('Probability Calculations', () => {
it('calculates expected loss', () => {
const variants = [
createMockVariant({ stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 } }),
createMockVariant({ stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1 }),
createMockVariant({
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.1,
conversions: 10,
conversionRate: 0.1,
},
}),
createMockVariant({
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.1,
conversions: 10,
conversionRate: 0.1,
},
}),
];
const loss = expectedLossIfChosen(variants[0], variants, 'conversion', 5000);
@ -285,8 +352,25 @@ describe('checkEarlyStopping', () => {
it('does not stop before minimum sample size', () => {
const experiment = createMockExperiment();
const variants = [
createMockVariant({ isControl: true, stats: { participants: 50, events: 25, primaryMetricValue: 0.1, conversions: 5, conversionRate: 0.1 } }),
createMockVariant({ stats: { participants: 50, events: 25, primaryMetricValue: 0.5, conversions: 25, conversionRate: 0.5 } }),
createMockVariant({
isControl: true,
stats: {
participants: 50,
events: 25,
primaryMetricValue: 0.1,
conversions: 5,
conversionRate: 0.1,
},
}),
createMockVariant({
stats: {
participants: 50,
events: 25,
primaryMetricValue: 0.5,
conversions: 25,
conversionRate: 0.5,
},
}),
];
const result = checkEarlyStopping(experiment, variants, 5);
@ -296,8 +380,29 @@ describe('checkEarlyStopping', () => {
it('stops when winner is clear', () => {
const experiment = createMockExperiment();
const variants = [
createMockVariant({ isControl: true, stats: { participants: 1000, events: 500, primaryMetricValue: 0.1, conversions: 100, conversionRate: 0.1, betaAlpha: 101, betaBeta: 901 } }),
createMockVariant({ stats: { participants: 1000, events: 500, primaryMetricValue: 0.5, conversions: 500, conversionRate: 0.5, betaAlpha: 501, betaBeta: 501 } }),
createMockVariant({
isControl: true,
stats: {
participants: 1000,
events: 500,
primaryMetricValue: 0.1,
conversions: 100,
conversionRate: 0.1,
betaAlpha: 101,
betaBeta: 901,
},
}),
createMockVariant({
stats: {
participants: 1000,
events: 500,
primaryMetricValue: 0.5,
conversions: 500,
conversionRate: 0.5,
betaAlpha: 501,
betaBeta: 501,
},
}),
];
const result = checkEarlyStopping(experiment, variants, 10);
@ -310,8 +415,25 @@ describe('checkEarlyStopping', () => {
guardrails: { ...createMockExperiment().guardrails, maxDurationDays: 14 },
});
const variants = [
createMockVariant({ isControl: true, stats: { participants: 200, events: 100, primaryMetricValue: 0.1, conversions: 20, conversionRate: 0.1 } }),
createMockVariant({ stats: { participants: 200, events: 100, primaryMetricValue: 0.11, conversions: 22, conversionRate: 0.11 } }),
createMockVariant({
isControl: true,
stats: {
participants: 200,
events: 100,
primaryMetricValue: 0.1,
conversions: 20,
conversionRate: 0.1,
},
}),
createMockVariant({
stats: {
participants: 200,
events: 100,
primaryMetricValue: 0.11,
conversions: 22,
conversionRate: 0.11,
},
}),
];
const result = checkEarlyStopping(experiment, variants, 30);
@ -328,8 +450,25 @@ describe('runGuardrails', () => {
it('requires minimum sample size', () => {
const experiment = createMockExperiment();
const variants = [
createMockVariant({ isControl: true, stats: { participants: 50, events: 25, primaryMetricValue: 0.1, conversions: 5, conversionRate: 0.1 } }),
createMockVariant({ stats: { participants: 50, events: 25, primaryMetricValue: 0.5, conversions: 25, conversionRate: 0.5 } }),
createMockVariant({
isControl: true,
stats: {
participants: 50,
events: 25,
primaryMetricValue: 0.1,
conversions: 5,
conversionRate: 0.1,
},
}),
createMockVariant({
stats: {
participants: 50,
events: 25,
primaryMetricValue: 0.5,
conversions: 25,
conversionRate: 0.5,
},
}),
];
const checks = runGuardrails(experiment, variants, 10, false);
@ -344,8 +483,25 @@ describe('runGuardrails', () => {
primaryMetric: { ...createMockExperiment().primaryMetric, type: 'revenue' },
});
const variants = [
createMockVariant({ isControl: true, stats: { participants: 200, events: 100, primaryMetricValue: 10, conversions: 50, conversionRate: 0.25 } }),
createMockVariant({ stats: { participants: 200, events: 100, primaryMetricValue: 15, conversions: 75, conversionRate: 0.375 } }),
createMockVariant({
isControl: true,
stats: {
participants: 200,
events: 100,
primaryMetricValue: 10,
conversions: 50,
conversionRate: 0.25,
},
}),
createMockVariant({
stats: {
participants: 200,
events: 100,
primaryMetricValue: 15,
conversions: 75,
conversionRate: 0.375,
},
}),
];
const checks = runGuardrails(experiment, variants, 10, true);
@ -388,8 +544,29 @@ describe('generateExperimentResult', () => {
it('generates results with variant comparisons', () => {
const experiment = createMockExperiment();
const variants = [
createMockVariant({ id: 'var_control', isControl: true, name: 'Control', stats: { participants: 200, events: 100, primaryMetricValue: 0.1, conversions: 20, conversionRate: 0.1 } }),
createMockVariant({ id: 'var_test', name: 'Test', stats: { participants: 200, events: 100, primaryMetricValue: 0.15, conversions: 30, conversionRate: 0.15 } }),
createMockVariant({
id: 'var_control',
isControl: true,
name: 'Control',
stats: {
participants: 200,
events: 100,
primaryMetricValue: 0.1,
conversions: 20,
conversionRate: 0.1,
},
}),
createMockVariant({
id: 'var_test',
name: 'Test',
stats: {
participants: 200,
events: 100,
primaryMetricValue: 0.15,
conversions: 30,
conversionRate: 0.15,
},
}),
];
const result = generateExperimentResult(experiment, variants, 14);
@ -403,8 +580,32 @@ describe('generateExperimentResult', () => {
it('identifies winner when statistically significant', () => {
const experiment = createMockExperiment();
const variants = [
createMockVariant({ id: 'var_control', isControl: true, stats: { participants: 1000, events: 500, primaryMetricValue: 0.1, conversions: 100, conversionRate: 0.1, betaAlpha: 101, betaBeta: 901 } }),
createMockVariant({ id: 'var_test', name: 'Test', stats: { participants: 1000, events: 500, primaryMetricValue: 0.5, conversions: 500, conversionRate: 0.5, betaAlpha: 501, betaBeta: 501 } }),
createMockVariant({
id: 'var_control',
isControl: true,
stats: {
participants: 1000,
events: 500,
primaryMetricValue: 0.1,
conversions: 100,
conversionRate: 0.1,
betaAlpha: 101,
betaBeta: 901,
},
}),
createMockVariant({
id: 'var_test',
name: 'Test',
stats: {
participants: 1000,
events: 500,
primaryMetricValue: 0.5,
conversions: 500,
conversionRate: 0.5,
betaAlpha: 501,
betaBeta: 501,
},
}),
];
const result = generateExperimentResult(experiment, variants, 14);
@ -461,12 +662,28 @@ describe('assignByStrategy', () => {
const mockControl: VariantDoc = createMockVariant({
id: 'var_control',
isControl: true,
stats: { participants: 100, events: 50, primaryMetricValue: 0.1, conversions: 10, conversionRate: 0.1, betaAlpha: 11, betaBeta: 91 },
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.1,
conversions: 10,
conversionRate: 0.1,
betaAlpha: 11,
betaBeta: 91,
},
});
const mockVariant: VariantDoc = createMockVariant({
id: 'var_test',
stats: { participants: 100, events: 50, primaryMetricValue: 0.2, conversions: 20, conversionRate: 0.2, betaAlpha: 21, betaBeta: 81 },
stats: {
participants: 100,
events: 50,
primaryMetricValue: 0.2,
conversions: 20,
conversionRate: 0.2,
betaAlpha: 21,
betaBeta: 81,
},
});
const ctx = {

2
services/platform-service/src/server.ts
View File

@ -58,6 +58,7 @@ import { maintenanceRoutes } from './modules/maintenance/routes.js';
import { exportRoutes } from './modules/exports/routes.js';
import { ipRuleRoutes } from './modules/ip-rules/routes.js';
import { experimentRoutes } from './modules/experiments/routes.js';
import { abTestingRoutes } from './modules/ab-testing/routes.js';
import { analyticsRoutes } from './modules/analytics/routes.js';
import { feedbackRoutes } from './modules/feedback/routes.js';
import { impersonationRoutes } from './modules/impersonation/routes.js';
@ -166,6 +167,7 @@ await app.register(exportRoutes, { prefix: '/api' });
await app.register(ipRuleRoutes, { prefix: '/api' });
// P2 — Product Intelligence
await app.register(experimentRoutes, { prefix: '/api' });
await app.register(abTestingRoutes, { prefix: '/api' });
await app.register(analyticsRoutes, { prefix: '/api' });
await app.register(feedbackRoutes, { prefix: '/api' });
await app.register(impersonationRoutes, { prefix: '/api' });