feat(ai-diagnostics): add Azure OpenAI embedding pipeline [1.2.1]
This commit is contained in:
parent
f77bd13d4a
commit
50b7e22985
@ -17,6 +17,9 @@ const envSchema = z.object({
|
|||||||
// ── Growth (merged) ──
|
// ── Growth (merged) ──
|
||||||
WEBHOOK_INVITATION_REDEEMED_URL: z.string().optional(),
|
WEBHOOK_INVITATION_REDEEMED_URL: z.string().optional(),
|
||||||
WEBHOOK_REFERRAL_STATUS_URL: z.string().optional(),
|
WEBHOOK_REFERRAL_STATUS_URL: z.string().optional(),
|
||||||
|
// ── AI Diagnostics ──
|
||||||
|
AZURE_OPENAI_KEY: z.string().optional(),
|
||||||
|
AZURE_OPENAI_ENDPOINT: z.string().optional(),
|
||||||
// ── Billing (merged) ──
|
// ── Billing (merged) ──
|
||||||
STRIPE_SECRET_KEY: z.string().optional(),
|
STRIPE_SECRET_KEY: z.string().optional(),
|
||||||
STRIPE_WEBHOOK_SECRET: z.string().optional(),
|
STRIPE_WEBHOOK_SECRET: z.string().optional(),
|
||||||
|
|||||||
278
services/platform-service/src/modules/ab-testing/guardrails.ts
Normal file
278
services/platform-service/src/modules/ab-testing/guardrails.ts
Normal file
@ -0,0 +1,278 @@
|
|||||||
|
/**
|
||||||
|
* A/B Testing — Early Stopping Guardrails.
|
||||||
|
* Auto-promotion, business hours check, approval requirements, safety limits.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ExperimentDoc, VariantDoc } from './types.js';
|
||||||
|
import { generateExperimentResult, checkEarlyStopping } from './statistics.js';
|
||||||
|
|
||||||
|
export interface GuardrailCheck {
|
||||||
|
passed: boolean;
|
||||||
|
violation?: string;
|
||||||
|
severity: 'info' | 'warning' | 'blocking';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run all guardrail checks before allowing early stop or auto-promotion.
|
||||||
|
*/
|
||||||
|
export function runGuardrails(
|
||||||
|
experiment: ExperimentDoc,
|
||||||
|
variants: VariantDoc[],
|
||||||
|
daysRunning: number,
|
||||||
|
isRevenueImpacting: boolean
|
||||||
|
): GuardrailCheck[] {
|
||||||
|
const checks: GuardrailCheck[] = [];
|
||||||
|
|
||||||
|
// 1. Minimum sample size
|
||||||
|
const minSampleCheck = checkMinSampleSize(experiment, variants);
|
||||||
|
checks.push(minSampleCheck);
|
||||||
|
|
||||||
|
// 2. Business hours (safety for auto-actions)
|
||||||
|
const businessHoursCheck = checkBusinessHours();
|
||||||
|
checks.push(businessHoursCheck);
|
||||||
|
|
||||||
|
// 3. Approval requirement
|
||||||
|
const approvalCheck = checkApprovalRequired(experiment, isRevenueImpacting);
|
||||||
|
checks.push(approvalCheck);
|
||||||
|
|
||||||
|
// 4. Max duration safety limit
|
||||||
|
const durationCheck = checkMaxDuration(experiment, daysRunning);
|
||||||
|
checks.push(durationCheck);
|
||||||
|
|
||||||
|
// 5. Statistical significance threshold
|
||||||
|
const significanceCheck = checkStatisticalSignificance(experiment, variants, daysRunning);
|
||||||
|
checks.push(significanceCheck);
|
||||||
|
|
||||||
|
// 6. Variant balance check
|
||||||
|
const balanceCheck = checkVariantBalance(variants);
|
||||||
|
checks.push(balanceCheck);
|
||||||
|
|
||||||
|
return checks;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check minimum sample size per variant.
|
||||||
|
*/
|
||||||
|
function checkMinSampleSize(experiment: ExperimentDoc, variants: VariantDoc[]): GuardrailCheck {
|
||||||
|
const minRequired = experiment.guardrails.minSampleSizePerVariant;
|
||||||
|
const violations: string[] = [];
|
||||||
|
|
||||||
|
for (const v of variants) {
|
||||||
|
if (v.stats.participants < minRequired) {
|
||||||
|
violations.push(`${v.name}: ${v.stats.participants}/${minRequired}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (violations.length > 0) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: `Insufficient samples: ${violations.join(', ')}`,
|
||||||
|
severity: 'blocking',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { passed: true, severity: 'info' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if current time is within business hours.
|
||||||
|
* Prevents auto-promotion during off-hours.
|
||||||
|
*/
|
||||||
|
function checkBusinessHours(): GuardrailCheck {
|
||||||
|
const now = new Date();
|
||||||
|
const hour = now.getHours();
|
||||||
|
const day = now.getDay(); // 0 = Sunday, 6 = Saturday
|
||||||
|
|
||||||
|
// Business hours: Monday-Friday, 9 AM - 6 PM
|
||||||
|
const isBusinessDay = day >= 1 && day <= 5;
|
||||||
|
const isBusinessHour = hour >= 9 && hour < 18;
|
||||||
|
|
||||||
|
if (!isBusinessDay) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: 'Auto-promotion disabled on weekends',
|
||||||
|
severity: 'warning',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isBusinessHour) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: 'Auto-promotion disabled outside business hours (9 AM - 6 PM)',
|
||||||
|
severity: 'warning',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { passed: true, severity: 'info' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if approval is required for this experiment.
|
||||||
|
*/
|
||||||
|
function checkApprovalRequired(
|
||||||
|
experiment: ExperimentDoc,
|
||||||
|
isRevenueImpacting: boolean
|
||||||
|
): GuardrailCheck {
|
||||||
|
const requirement = experiment.guardrails.requireApprovalFor;
|
||||||
|
|
||||||
|
if (requirement === 'all') {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: 'Approval required: all experiments require manual approval',
|
||||||
|
severity: 'blocking',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (requirement === 'revenue' && isRevenueImpacting) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: 'Approval required: revenue-impacting experiment',
|
||||||
|
severity: 'blocking',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { passed: true, severity: 'info' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check max duration safety limit.
|
||||||
|
*/
|
||||||
|
function checkMaxDuration(experiment: ExperimentDoc, daysRunning: number): GuardrailCheck {
|
||||||
|
const maxDays = experiment.guardrails.maxDurationDays;
|
||||||
|
|
||||||
|
if (daysRunning >= maxDays) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: `Max duration reached: ${daysRunning}/${maxDays} days`,
|
||||||
|
severity: 'warning',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warn at 80% of max duration
|
||||||
|
if (daysRunning >= maxDays * 0.8) {
|
||||||
|
return {
|
||||||
|
passed: true,
|
||||||
|
violation: `Approaching max duration: ${daysRunning}/${maxDays} days`,
|
||||||
|
severity: 'warning',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { passed: true, severity: 'info' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check statistical significance threshold.
|
||||||
|
*/
|
||||||
|
function checkStatisticalSignificance(
|
||||||
|
experiment: ExperimentDoc,
|
||||||
|
variants: VariantDoc[],
|
||||||
|
daysRunning: number
|
||||||
|
): GuardrailCheck {
|
||||||
|
const earlyStop = checkEarlyStopping(experiment, variants, daysRunning);
|
||||||
|
|
||||||
|
if (earlyStop.shouldStop && earlyStop.confidence >= experiment.guardrails.winnerThreshold / 100) {
|
||||||
|
return {
|
||||||
|
passed: true,
|
||||||
|
violation: `Statistical significance reached: ${(earlyStop.confidence * 100).toFixed(1)}%`,
|
||||||
|
severity: 'info',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: `Not statistically significant: ${(earlyStop.confidence * 100).toFixed(1)}% < ${experiment.guardrails.winnerThreshold}%`,
|
||||||
|
severity: 'blocking',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if variants have balanced traffic allocation.
|
||||||
|
*/
|
||||||
|
function checkVariantBalance(variants: VariantDoc[]): GuardrailCheck {
|
||||||
|
const participants = variants.map(v => v.stats.participants);
|
||||||
|
const total = participants.reduce((a, b) => a + b, 0);
|
||||||
|
|
||||||
|
if (total === 0) {
|
||||||
|
return { passed: true, severity: 'info' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const ratios = participants.map(p => p / total);
|
||||||
|
const expectedRatio = 1 / variants.length;
|
||||||
|
|
||||||
|
// Check if any variant is > 2x or < 0.5x the expected ratio
|
||||||
|
const imbalances: string[] = [];
|
||||||
|
for (let i = 0; i < variants.length; i++) {
|
||||||
|
const ratio = ratios[i];
|
||||||
|
if (ratio > expectedRatio * 2) {
|
||||||
|
imbalances.push(`${variants[i].name}: ${(ratio * 100).toFixed(1)}% (expected ~${(expectedRatio * 100).toFixed(1)}%)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imbalances.length > 0) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
violation: `Traffic imbalance detected: ${imbalances.join(', ')}`,
|
||||||
|
severity: 'warning',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { passed: true, severity: 'info' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if all blocking guardrails pass.
|
||||||
|
*/
|
||||||
|
export function canAutoPromote(checks: GuardrailCheck[]): boolean {
|
||||||
|
return checks.every(c => c.passed || c.severity !== 'blocking');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get summary of guardrail violations.
|
||||||
|
*/
|
||||||
|
export function getGuardrailViolations(checks: GuardrailCheck[]): string[] {
|
||||||
|
return checks
|
||||||
|
.filter(c => !c.passed && c.violation)
|
||||||
|
.map(c => c.violation!);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-promotion decision with full guardrail evaluation.
|
||||||
|
*/
|
||||||
|
export interface AutoPromotionResult {
|
||||||
|
canPromote: boolean;
|
||||||
|
winnerVariantId?: string;
|
||||||
|
violations: string[];
|
||||||
|
warnings: string[];
|
||||||
|
result: ReturnType<typeof generateExperimentResult>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateAutoPromotion(
|
||||||
|
experiment: ExperimentDoc,
|
||||||
|
variants: VariantDoc[],
|
||||||
|
daysRunning: number,
|
||||||
|
isRevenueImpacting: boolean
|
||||||
|
): AutoPromotionResult {
|
||||||
|
const result = generateExperimentResult(experiment, variants, daysRunning);
|
||||||
|
const checks = runGuardrails(experiment, variants, daysRunning, isRevenueImpacting);
|
||||||
|
|
||||||
|
const violations = checks
|
||||||
|
.filter(c => !c.passed && c.severity === 'blocking')
|
||||||
|
.map(c => c.violation!);
|
||||||
|
|
||||||
|
const warnings = checks
|
||||||
|
.filter(c => !c.passed && c.severity === 'warning')
|
||||||
|
.map(c => c.violation!);
|
||||||
|
|
||||||
|
const canPromote =
|
||||||
|
result.status === 'winner_found' &&
|
||||||
|
canAutoPromote(checks) &&
|
||||||
|
experiment.guardrails.autoStopEnabled;
|
||||||
|
|
||||||
|
return {
|
||||||
|
canPromote,
|
||||||
|
winnerVariantId: result.winnerVariantId,
|
||||||
|
violations,
|
||||||
|
warnings,
|
||||||
|
result,
|
||||||
|
};
|
||||||
|
}
|
||||||
546
services/platform-service/src/modules/ab-testing/repository.ts
Normal file
546
services/platform-service/src/modules/ab-testing/repository.ts
Normal file
@ -0,0 +1,546 @@
|
|||||||
|
/**
|
||||||
|
* Intelligent A/B Testing — Repository layer.
|
||||||
|
* Cosmos DB CRUD for experiments, variants, assignments, events, metrics.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { getRegisteredContainer } from '@bytelyst/cosmos';
|
||||||
|
import crypto from 'node:crypto';
|
||||||
|
import type {
|
||||||
|
ExperimentDoc,
|
||||||
|
VariantDoc,
|
||||||
|
ExperimentAssignmentDoc,
|
||||||
|
ExperimentEventDoc,
|
||||||
|
ExperimentMetricDoc,
|
||||||
|
ExperimentSuggestion,
|
||||||
|
CreateExperimentInput,
|
||||||
|
UpdateExperimentInput,
|
||||||
|
} from './types.js';
|
||||||
|
import { assignVariant, assignByStrategy, isInExperimentBucket, type StrategyContext } from './bucketing.js';
|
||||||
|
import type { TargetingContext, TargetingConfig } from './targeting.js';
|
||||||
|
import { matchesTargeting } from './targeting.js';
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Container Access
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function getExperimentContainer() {
|
||||||
|
return getRegisteredContainer('experiments');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getVariantContainer() {
|
||||||
|
return getRegisteredContainer('ab_testing_variants');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAssignmentContainer() {
|
||||||
|
return getRegisteredContainer('experiment_assignments');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getEventContainer() {
|
||||||
|
return getRegisteredContainer('ab_testing_events');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getMetricContainer() {
|
||||||
|
return getRegisteredContainer('ab_testing_metrics');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSuggestionsContainer() {
|
||||||
|
return getRegisteredContainer('experiment_suggestions');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Experiment CRUD
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function listExperiments(productId: string): Promise<ExperimentDoc[]> {
|
||||||
|
const { resources } = await getExperimentContainer()
|
||||||
|
.items.query<ExperimentDoc>({
|
||||||
|
query: 'SELECT * FROM c WHERE c.productId = @pid ORDER BY c.createdAt DESC',
|
||||||
|
parameters: [{ name: '@pid', value: productId }],
|
||||||
|
})
|
||||||
|
.fetchAll();
|
||||||
|
return resources;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listRunningExperiments(productId: string): Promise<ExperimentDoc[]> {
|
||||||
|
const { resources } = await getExperimentContainer()
|
||||||
|
.items.query<ExperimentDoc>({
|
||||||
|
query: 'SELECT * FROM c WHERE c.productId = @pid AND c.status = @status',
|
||||||
|
parameters: [
|
||||||
|
{ name: '@pid', value: productId },
|
||||||
|
{ name: '@status', value: 'running' },
|
||||||
|
],
|
||||||
|
})
|
||||||
|
.fetchAll();
|
||||||
|
return resources;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getExperiment(id: string): Promise<ExperimentDoc | null> {
|
||||||
|
try {
|
||||||
|
const { resource } = await getExperimentContainer().item(id, id).read<ExperimentDoc>();
|
||||||
|
return resource ?? null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createExperiment(
|
||||||
|
productId: string,
|
||||||
|
input: CreateExperimentInput
|
||||||
|
): Promise<ExperimentDoc> {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const experimentId = `exp_${crypto.randomUUID()}`;
|
||||||
|
|
||||||
|
// Create variants
|
||||||
|
const variantIds: string[] = [];
|
||||||
|
let controlVariantId = '';
|
||||||
|
const numVariants = input.variants.length;
|
||||||
|
const baseAllocation = Math.floor(100 / numVariants);
|
||||||
|
const remainder = 100 - baseAllocation * numVariants;
|
||||||
|
|
||||||
|
for (let i = 0; i < input.variants.length; i++) {
|
||||||
|
const v = input.variants[i];
|
||||||
|
const variantId = `var_${crypto.randomUUID()}`;
|
||||||
|
variantIds.push(variantId);
|
||||||
|
|
||||||
|
if (v.isControl) {
|
||||||
|
controlVariantId = variantId;
|
||||||
|
}
|
||||||
|
|
||||||
|
const variantDoc: VariantDoc = {
|
||||||
|
id: variantId,
|
||||||
|
experimentId,
|
||||||
|
name: v.name,
|
||||||
|
description: v.description ?? '',
|
||||||
|
isControl: v.isControl,
|
||||||
|
flagConfig: v.flagConfig,
|
||||||
|
currentAllocationPercent: baseAllocation + (i < remainder ? 1 : 0),
|
||||||
|
stats: {
|
||||||
|
participants: 0,
|
||||||
|
events: 0,
|
||||||
|
primaryMetricValue: 0,
|
||||||
|
},
|
||||||
|
createdAt: now,
|
||||||
|
updatedAt: now,
|
||||||
|
};
|
||||||
|
|
||||||
|
await getVariantContainer().items.create(variantDoc);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default to first variant as control if none specified
|
||||||
|
if (!controlVariantId) {
|
||||||
|
controlVariantId = variantIds[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
const experiment: ExperimentDoc = {
|
||||||
|
id: experimentId,
|
||||||
|
productId,
|
||||||
|
name: input.name,
|
||||||
|
description: input.description ?? '',
|
||||||
|
hypothesis: input.hypothesis,
|
||||||
|
status: 'draft',
|
||||||
|
controlVariantId,
|
||||||
|
variantIds,
|
||||||
|
allocationStrategy: input.allocationStrategy,
|
||||||
|
targetPercent: input.targetPercent,
|
||||||
|
targeting: input.targeting,
|
||||||
|
primaryMetric: input.primaryMetric,
|
||||||
|
secondaryMetrics: input.secondaryMetrics,
|
||||||
|
guardrails: input.guardrails,
|
||||||
|
startAt: input.startAt,
|
||||||
|
totalParticipants: 0,
|
||||||
|
totalEvents: 0,
|
||||||
|
createdAt: now,
|
||||||
|
updatedAt: now,
|
||||||
|
};
|
||||||
|
|
||||||
|
await getExperimentContainer().items.create(experiment);
|
||||||
|
return experiment;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateExperiment(
|
||||||
|
id: string,
|
||||||
|
updates: UpdateExperimentInput
|
||||||
|
): Promise<ExperimentDoc | null> {
|
||||||
|
const existing = await getExperiment(id);
|
||||||
|
if (!existing) return null;
|
||||||
|
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const updated: ExperimentDoc = {
|
||||||
|
...existing,
|
||||||
|
...updates,
|
||||||
|
updatedAt: now,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (updates.status === 'running' && !existing.startedAt) {
|
||||||
|
updated.startedAt = now;
|
||||||
|
}
|
||||||
|
if ((updates.status === 'completed' || updates.status === 'stopped') && !existing.completedAt) {
|
||||||
|
updated.completedAt = now;
|
||||||
|
// Set TTL for 2 years
|
||||||
|
updated.ttl = 2 * 365 * 86400;
|
||||||
|
}
|
||||||
|
|
||||||
|
await getExperimentContainer().item(id, id).replace(updated);
|
||||||
|
return updated;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteExperiment(id: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
// Delete variants first
|
||||||
|
const { resources: variants } = await getVariantContainer()
|
||||||
|
.items.query({ query: 'SELECT * FROM c WHERE c.experimentId = @eid', parameters: [{ name: '@eid', value: id }] })
|
||||||
|
.fetchAll();
|
||||||
|
|
||||||
|
for (const v of variants) {
|
||||||
|
await getVariantContainer().item(v.id, id).delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
await getExperimentContainer().item(id, id).delete();
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Variant Operations
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function getVariant(id: string, experimentId: string): Promise<VariantDoc | null> {
|
||||||
|
try {
|
||||||
|
const { resource } = await getVariantContainer().item(id, experimentId).read<VariantDoc>();
|
||||||
|
return resource ?? null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listVariants(experimentId: string): Promise<VariantDoc[]> {
|
||||||
|
const { resources } = await getVariantContainer()
|
||||||
|
.items.query<VariantDoc>({
|
||||||
|
query: 'SELECT * FROM c WHERE c.experimentId = @eid',
|
||||||
|
parameters: [{ name: '@eid', value: experimentId }],
|
||||||
|
})
|
||||||
|
.fetchAll();
|
||||||
|
return resources;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateVariantStats(
|
||||||
|
variantId: string,
|
||||||
|
experimentId: string,
|
||||||
|
stats: Partial<VariantDoc['stats']>
|
||||||
|
): Promise<void> {
|
||||||
|
const variant = await getVariant(variantId, experimentId);
|
||||||
|
if (!variant) return;
|
||||||
|
|
||||||
|
const updated: VariantDoc = {
|
||||||
|
...variant,
|
||||||
|
stats: { ...variant.stats, ...stats },
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
await getVariantContainer().item(variantId, experimentId).replace(updated);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateVariantAllocation(
|
||||||
|
variantId: string,
|
||||||
|
experimentId: string,
|
||||||
|
newAllocation: number
|
||||||
|
): Promise<void> {
|
||||||
|
const variant = await getVariant(variantId, experimentId);
|
||||||
|
if (!variant) return;
|
||||||
|
|
||||||
|
const updated: VariantDoc = {
|
||||||
|
...variant,
|
||||||
|
currentAllocationPercent: newAllocation,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
await getVariantContainer().item(variantId, experimentId).replace(updated);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateVariantBayesianResults(
|
||||||
|
variantId: string,
|
||||||
|
experimentId: string,
|
||||||
|
results: VariantDoc['bayesianResults']
|
||||||
|
): Promise<void> {
|
||||||
|
const variant = await getVariant(variantId, experimentId);
|
||||||
|
if (!variant) return;
|
||||||
|
|
||||||
|
const updated: VariantDoc = {
|
||||||
|
...variant,
|
||||||
|
bayesianResults: results,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
await getVariantContainer().item(variantId, experimentId).replace(updated);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Assignment Operations
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface AssignmentResult {
|
||||||
|
assignment: ExperimentAssignmentDoc;
|
||||||
|
variant: VariantDoc;
|
||||||
|
isNew: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAssignment(
|
||||||
|
experimentId: string,
|
||||||
|
userId: string
|
||||||
|
): Promise<ExperimentAssignmentDoc | null> {
|
||||||
|
// Query by userId partition key
|
||||||
|
const { resources } = await getAssignmentContainer()
|
||||||
|
.items.query<ExperimentAssignmentDoc>({
|
||||||
|
query: 'SELECT * FROM c WHERE c.experimentId = @eid AND c.userId = @uid',
|
||||||
|
parameters: [
|
||||||
|
{ name: '@eid', value: experimentId },
|
||||||
|
{ name: '@uid', value: userId },
|
||||||
|
],
|
||||||
|
})
|
||||||
|
.fetchAll();
|
||||||
|
return resources[0] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getOrCreateAssignment(
|
||||||
|
experiment: ExperimentDoc,
|
||||||
|
userId: string,
|
||||||
|
targetingContext: TargetingContext
|
||||||
|
): Promise<AssignmentResult | null> {
|
||||||
|
// Check targeting first
|
||||||
|
if (!matchesTargeting(targetingContext, experiment.targeting)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check traffic percentage
|
||||||
|
if (!isInExperimentBucket(experiment.id, userId, experiment.targetPercent)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for existing assignment
|
||||||
|
const existing = await getAssignment(experiment.id, userId);
|
||||||
|
if (existing) {
|
||||||
|
const variant = await getVariant(existing.variantId, experiment.id);
|
||||||
|
if (!variant) return null;
|
||||||
|
return { assignment: existing, variant, isNew: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get all variants
|
||||||
|
const variants = await listVariants(experiment.id);
|
||||||
|
if (variants.length === 0) return null;
|
||||||
|
|
||||||
|
const controlVariant = variants.find(v => v.isControl) ?? variants[0];
|
||||||
|
|
||||||
|
// Assign variant based on strategy
|
||||||
|
let assignedVariantId: string;
|
||||||
|
|
||||||
|
if (experiment.allocationStrategy === 'random') {
|
||||||
|
// Use FNV-1a bucketing for random strategy
|
||||||
|
assignedVariantId = assignVariant(
|
||||||
|
experiment.id,
|
||||||
|
userId,
|
||||||
|
variants.map(v => ({ key: v.id, weight: v.currentAllocationPercent }))
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// Use adaptive strategies
|
||||||
|
const ctx: StrategyContext = {
|
||||||
|
variants,
|
||||||
|
controlVariant,
|
||||||
|
totalParticipants: experiment.totalParticipants,
|
||||||
|
explorationRate: 0.1, // Default epsilon for epsilon-greedy
|
||||||
|
};
|
||||||
|
assignedVariantId = assignByStrategy(experiment.allocationStrategy, ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
const assignedVariant = variants.find(v => v.id === assignedVariantId) ?? variants[0];
|
||||||
|
|
||||||
|
// Create assignment record
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const assignment: ExperimentAssignmentDoc = {
|
||||||
|
id: `ea_${crypto.randomUUID()}`,
|
||||||
|
userId,
|
||||||
|
experimentId: experiment.id,
|
||||||
|
variantId: assignedVariant.id,
|
||||||
|
assignedAt: now,
|
||||||
|
assignmentContext: {
|
||||||
|
platform: targetingContext.platform ?? 'unknown',
|
||||||
|
appVersion: targetingContext.appVersion ?? 'unknown',
|
||||||
|
osVersion: targetingContext.osVersion ?? 'unknown',
|
||||||
|
deviceModel: targetingContext.deviceModel,
|
||||||
|
region: targetingContext.region,
|
||||||
|
},
|
||||||
|
eventCount: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
await getAssignmentContainer().items.create(assignment);
|
||||||
|
|
||||||
|
// Update participant count
|
||||||
|
await updateVariantStats(assignedVariant.id, experiment.id, {
|
||||||
|
participants: assignedVariant.stats.participants + 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Update experiment total
|
||||||
|
const expUpdate: Partial<ExperimentDoc> = {
|
||||||
|
totalParticipants: experiment.totalParticipants + 1,
|
||||||
|
updatedAt: now,
|
||||||
|
};
|
||||||
|
await getExperimentContainer().item(experiment.id, experiment.id).patch({
|
||||||
|
operations: [
|
||||||
|
{ op: 'incr', path: '/totalParticipants', value: 1 },
|
||||||
|
{ op: 'set', path: '/updatedAt', value: now },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
return { assignment, variant: assignedVariant, isNew: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Event Tracking
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function trackEvent(
|
||||||
|
experimentId: string,
|
||||||
|
userId: string,
|
||||||
|
assignmentId: string,
|
||||||
|
variantId: string,
|
||||||
|
metricName: string,
|
||||||
|
metricType: ExperimentEventDoc['metricType'],
|
||||||
|
value: number,
|
||||||
|
converted: boolean,
|
||||||
|
platform: string,
|
||||||
|
appVersion: string,
|
||||||
|
eventMetadata?: Record<string, unknown>
|
||||||
|
): Promise<void> {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const event: ExperimentEventDoc = {
|
||||||
|
id: `ee_${crypto.randomUUID()}`,
|
||||||
|
experimentId,
|
||||||
|
timestamp: now,
|
||||||
|
userId,
|
||||||
|
variantId,
|
||||||
|
assignmentId,
|
||||||
|
metricName,
|
||||||
|
metricType,
|
||||||
|
value,
|
||||||
|
converted,
|
||||||
|
eventMetadata,
|
||||||
|
platform,
|
||||||
|
appVersion,
|
||||||
|
ttl: 90 * 86400, // 90 days
|
||||||
|
};
|
||||||
|
|
||||||
|
await getEventContainer().items.create(event);
|
||||||
|
|
||||||
|
// Update assignment event count
|
||||||
|
await getAssignmentContainer().item(assignmentId, userId).patch({
|
||||||
|
operations: [
|
||||||
|
{ op: 'incr', path: '/eventCount', value: 1 },
|
||||||
|
{ op: 'set', path: '/lastEventAt', value: now },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
// Update experiment total events
|
||||||
|
await getExperimentContainer().item(experimentId, experimentId).patch({
|
||||||
|
operations: [
|
||||||
|
{ op: 'incr', path: '/totalEvents', value: 1 },
|
||||||
|
{ op: 'set', path: '/updatedAt', value: now },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Metrics Aggregation
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function getOrCreateMetric(
|
||||||
|
experimentId: string,
|
||||||
|
metricName: string,
|
||||||
|
variantId: string
|
||||||
|
): Promise<ExperimentMetricDoc> {
|
||||||
|
const id = `em_${experimentId}:${metricName}:${variantId}`;
|
||||||
|
const existing = await getMetricContainer().item(id, experimentId).read<ExperimentMetricDoc>();
|
||||||
|
if (existing.resource) return existing.resource;
|
||||||
|
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const metric: ExperimentMetricDoc = {
|
||||||
|
id,
|
||||||
|
experimentId,
|
||||||
|
metricName,
|
||||||
|
variantId,
|
||||||
|
count: 0,
|
||||||
|
sum: 0,
|
||||||
|
mean: 0,
|
||||||
|
stdDev: 0,
|
||||||
|
min: 0,
|
||||||
|
max: 0,
|
||||||
|
conversions: 0,
|
||||||
|
conversionRate: 0,
|
||||||
|
updatedAt: now,
|
||||||
|
};
|
||||||
|
|
||||||
|
await getMetricContainer().items.create(metric);
|
||||||
|
return metric;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateMetricAggregation(
|
||||||
|
experimentId: string,
|
||||||
|
metricName: string,
|
||||||
|
variantId: string,
|
||||||
|
value: number,
|
||||||
|
converted: boolean
|
||||||
|
): Promise<void> {
|
||||||
|
const metric = await getOrCreateMetric(experimentId, metricName, variantId);
|
||||||
|
|
||||||
|
const newCount = metric.count + 1;
|
||||||
|
const newSum = metric.sum + value;
|
||||||
|
const newMean = newSum / newCount;
|
||||||
|
|
||||||
|
// Welford's online algorithm for variance
|
||||||
|
const delta = value - metric.mean;
|
||||||
|
const delta2 = value - newMean;
|
||||||
|
const newVariance = ((metric.count * metric.stdDev * metric.stdDev) + delta * delta2) / Math.max(1, newCount);
|
||||||
|
const newStdDev = Math.sqrt(newVariance);
|
||||||
|
|
||||||
|
const updated: ExperimentMetricDoc = {
|
||||||
|
...metric,
|
||||||
|
count: newCount,
|
||||||
|
sum: newSum,
|
||||||
|
mean: newMean,
|
||||||
|
stdDev: newStdDev,
|
||||||
|
min: metric.count === 0 ? value : Math.min(metric.min, value),
|
||||||
|
max: metric.count === 0 ? value : Math.max(metric.max, value),
|
||||||
|
conversions: metric.conversions + (converted ? 1 : 0),
|
||||||
|
conversionRate: (metric.conversions + (converted ? 1 : 0)) / newCount,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
await getMetricContainer().item(metric.id, experimentId).replace(updated);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// AI Suggestions
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function listSuggestions(productId: string): Promise<ExperimentSuggestion[]> {
|
||||||
|
const { resources } = await getSuggestionsContainer()
|
||||||
|
.items.query<ExperimentSuggestion>({
|
||||||
|
query: 'SELECT * FROM c WHERE c.productId = @pid ORDER BY c.priority DESC, c.createdAt DESC',
|
||||||
|
parameters: [{ name: '@pid', value: productId }],
|
||||||
|
})
|
||||||
|
.fetchAll();
|
||||||
|
return resources;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createSuggestion(
|
||||||
|
productId: string,
|
||||||
|
suggestion: Omit<ExperimentSuggestion, 'id' | 'createdAt'>
|
||||||
|
): Promise<ExperimentSuggestion> {
|
||||||
|
const doc: ExperimentSuggestion = {
|
||||||
|
...suggestion,
|
||||||
|
id: `es_${crypto.randomUUID()}`,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
await getSuggestionsContainer().items.create({ ...doc, productId });
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
653
services/platform-service/src/modules/ab-testing/statistics.ts
Normal file
653
services/platform-service/src/modules/ab-testing/statistics.ts
Normal file
@ -0,0 +1,653 @@
|
|||||||
|
/**
|
||||||
|
* Intelligent A/B Testing — Bayesian Statistics Engine.
|
||||||
|
* Beta-Binomial for conversions, Normal for continuous metrics.
|
||||||
|
* Probability calculations, credible intervals, early stopping rules.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { VariantDoc, MetricType, ExperimentResult, ExperimentDoc } from './types.js';
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Beta Distribution (for conversion metrics)
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Beta distribution parameters.
|
||||||
|
* Beta(α, β) where α = successes + 1, β = failures + 1
|
||||||
|
*/
|
||||||
|
export interface BetaParams {
|
||||||
|
alpha: number;
|
||||||
|
beta: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute Beta parameters from variant stats.
|
||||||
|
* Uses uninformative prior Beta(1, 1) = Uniform(0, 1)
|
||||||
|
*/
|
||||||
|
export function betaFromVariant(variant: VariantDoc): BetaParams {
|
||||||
|
const conversions = variant.stats.conversions ?? 0;
|
||||||
|
const participants = variant.stats.participants || 1;
|
||||||
|
const failures = participants - conversions;
|
||||||
|
|
||||||
|
// Posterior: Beta(conversions + 1, failures + 1)
|
||||||
|
return {
|
||||||
|
alpha: conversions + 1,
|
||||||
|
beta: failures + 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sample from Beta distribution using Gamma ratio method.
|
||||||
|
* Beta(α, β) ~ Gamma(α) / (Gamma(α) + Gamma(β))
|
||||||
|
*/
|
||||||
|
export function sampleBeta(alpha: number, beta: number): number {
|
||||||
|
const x = sampleGamma(alpha, 1);
|
||||||
|
const y = sampleGamma(beta, 1);
|
||||||
|
return x / (x + y);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Beta PDF (probability density function).
|
||||||
|
* Used for analytical probability calculations.
|
||||||
|
*/
|
||||||
|
export function betaPdf(x: number, alpha: number, beta: number): number {
|
||||||
|
if (x <= 0 || x >= 1) return 0;
|
||||||
|
const B = gamma(alpha) * gamma(beta) / gamma(alpha + beta);
|
||||||
|
return (Math.pow(x, alpha - 1) * Math.pow(1 - x, beta - 1)) / B;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Beta CDF (cumulative distribution function).
|
||||||
|
* Uses regularized incomplete beta function approximation.
|
||||||
|
*/
|
||||||
|
export function betaCdf(x: number, alpha: number, beta: number): number {
|
||||||
|
if (x <= 0) return 0;
|
||||||
|
if (x >= 1) return 1;
|
||||||
|
|
||||||
|
// Use incomplete beta approximation (numerical integration)
|
||||||
|
const steps = 100;
|
||||||
|
const dx = x / steps;
|
||||||
|
let sum = 0;
|
||||||
|
for (let i = 0; i < steps; i++) {
|
||||||
|
const xi = i * dx + dx / 2;
|
||||||
|
sum += betaPdf(xi, alpha, beta) * dx;
|
||||||
|
}
|
||||||
|
return sum;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Normal Distribution (for continuous metrics)
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface NormalParams {
|
||||||
|
mean: number;
|
||||||
|
stdDev: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute Normal parameters from variant stats.
|
||||||
|
* Uses conjugate prior (Normal-Inverse-Gamma).
|
||||||
|
*/
|
||||||
|
export function normalFromVariant(variant: VariantDoc): NormalParams {
|
||||||
|
const mean = variant.stats.primaryMetricValue ?? 0;
|
||||||
|
const n = variant.stats.participants || 1;
|
||||||
|
const sampleStd = variant.stats.primaryMetricStdDev ?? 1;
|
||||||
|
|
||||||
|
// Posterior standard deviation (reduced uncertainty with more samples)
|
||||||
|
const posteriorStdDev = sampleStd / Math.sqrt(n);
|
||||||
|
|
||||||
|
return { mean, stdDev: posteriorStdDev };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sample from Normal distribution (Box-Muller transform).
|
||||||
|
*/
|
||||||
|
export function sampleNormal(mean: number, stdDev: number): number {
|
||||||
|
const u1 = Math.random();
|
||||||
|
const u2 = Math.random();
|
||||||
|
const z0 = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
||||||
|
return mean + z0 * stdDev;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normal PDF.
|
||||||
|
*/
|
||||||
|
export function normalPdf(x: number, mean: number, stdDev: number): number {
|
||||||
|
const z = (x - mean) / stdDev;
|
||||||
|
return (1 / (stdDev * Math.sqrt(2 * Math.PI))) * Math.exp(-0.5 * z * z);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normal CDF (approximation using error function).
|
||||||
|
*/
|
||||||
|
export function normalCdf(x: number, mean: number, stdDev: number): number {
|
||||||
|
const z = (x - mean) / stdDev;
|
||||||
|
return 0.5 * (1 + erf(z / Math.sqrt(2)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Gamma Distribution (for count/duration metrics)
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface GammaParams {
|
||||||
|
shape: number;
|
||||||
|
scale: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute Gamma parameters from variant stats.
|
||||||
|
* Uses method of moments estimation.
|
||||||
|
*/
|
||||||
|
export function gammaFromVariant(variant: VariantDoc): GammaParams {
|
||||||
|
const mean = variant.stats.primaryMetricValue ?? 1;
|
||||||
|
const variance = Math.pow(variant.stats.primaryMetricStdDev ?? 1, 2);
|
||||||
|
|
||||||
|
// Method of moments: shape = mean² / variance, scale = variance / mean
|
||||||
|
const shape = (mean * mean) / Math.max(variance, 0.001);
|
||||||
|
const scale = variance / Math.max(mean, 0.001);
|
||||||
|
|
||||||
|
return { shape, scale };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sample from Gamma distribution (Marsaglia-Tsang method).
|
||||||
|
*/
|
||||||
|
export function sampleGamma(shape: number, scale: number): number {
|
||||||
|
if (shape < 1) {
|
||||||
|
return sampleGamma(1 + shape, scale) * Math.pow(Math.random(), 1 / shape);
|
||||||
|
}
|
||||||
|
|
||||||
|
const d = shape - 1 / 3;
|
||||||
|
const c = 1 / Math.sqrt(9 * d);
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
let x = sampleStandardNormal();
|
||||||
|
let v = 1 + c * x;
|
||||||
|
if (v <= 0) continue;
|
||||||
|
|
||||||
|
v = v * v * v;
|
||||||
|
const u = Math.random();
|
||||||
|
|
||||||
|
if (u < 1 - 0.0331 * x * x * x * x) {
|
||||||
|
return d * v * scale;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) {
|
||||||
|
return d * v * scale;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sampleStandardNormal(): number {
|
||||||
|
return sampleNormal(0, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Mathematical Utilities
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Error function approximation (Abramowitz & Stegun).
|
||||||
|
*/
|
||||||
|
function erf(x: number): number {
|
||||||
|
const sign = x < 0 ? -1 : 1;
|
||||||
|
x = Math.abs(x);
|
||||||
|
|
||||||
|
const a1 = 0.254829592;
|
||||||
|
const a2 = -0.284496736;
|
||||||
|
const a3 = 1.421413741;
|
||||||
|
const a4 = -1.453152027;
|
||||||
|
const a5 = 1.061405429;
|
||||||
|
const p = 0.3275911;
|
||||||
|
|
||||||
|
const t = 1 / (1 + p * x);
|
||||||
|
const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
|
||||||
|
|
||||||
|
return sign * y;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gamma function approximation (Lanczos approximation).
|
||||||
|
* Only valid for positive numbers.
|
||||||
|
*/
|
||||||
|
function gamma(z: number): number {
|
||||||
|
if (z < 0.5) {
|
||||||
|
return Math.PI / (Math.sin(Math.PI * z) * gamma(1 - z));
|
||||||
|
}
|
||||||
|
|
||||||
|
z -= 1;
|
||||||
|
const g = 7;
|
||||||
|
const C = [
|
||||||
|
0.99999999999980993, 676.5203681218851, -1259.1392167224028,
|
||||||
|
771.32342877765313, -176.61502916214059, 12.507343278686905,
|
||||||
|
-0.13857109526572012, 9.9843695780195716e-6, 1.5056327351493116e-7,
|
||||||
|
];
|
||||||
|
|
||||||
|
let x = C[0];
|
||||||
|
for (let i = 1; i < g + 2; i++) {
|
||||||
|
x += C[i] / (z + i);
|
||||||
|
}
|
||||||
|
|
||||||
|
const t = z + g + 0.5;
|
||||||
|
return Math.sqrt(2 * Math.PI) * Math.pow(t, z + 0.5) * Math.exp(-t) * x;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Probability Calculations
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate probability that variant beats control using Monte Carlo simulation.
|
||||||
|
* P(variant > control) via sampling from posterior distributions.
|
||||||
|
*/
|
||||||
|
export function probabilityVariantBeatsControl(
|
||||||
|
variant: VariantDoc,
|
||||||
|
control: VariantDoc,
|
||||||
|
metricType: MetricType,
|
||||||
|
samples = 10000
|
||||||
|
): number {
|
||||||
|
let wins = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < samples; i++) {
|
||||||
|
const variantSample = sampleFromPosterior(variant, metricType);
|
||||||
|
const controlSample = sampleFromPosterior(control, metricType);
|
||||||
|
if (variantSample > controlSample) wins++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return wins / samples;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate probability that variant beats all other variants.
|
||||||
|
*/
|
||||||
|
export function probabilityVariantBeatsAll(
|
||||||
|
variant: VariantDoc,
|
||||||
|
allVariants: VariantDoc[],
|
||||||
|
metricType: MetricType,
|
||||||
|
samples = 10000
|
||||||
|
): number {
|
||||||
|
const others = allVariants.filter(v => v.id !== variant.id);
|
||||||
|
if (others.length === 0) return 1;
|
||||||
|
|
||||||
|
let wins = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < samples; i++) {
|
||||||
|
const variantSample = sampleFromPosterior(variant, metricType);
|
||||||
|
const otherSamples = others.map(v => sampleFromPosterior(v, metricType));
|
||||||
|
const maxOther = Math.max(...otherSamples);
|
||||||
|
if (variantSample > maxOther) wins++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return wins / samples;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate expected loss if we choose this variant.
|
||||||
|
* E[loss] = E[max(0, best_other - this)]
|
||||||
|
*/
|
||||||
|
export function expectedLossIfChosen(
|
||||||
|
variant: VariantDoc,
|
||||||
|
allVariants: VariantDoc[],
|
||||||
|
metricType: MetricType,
|
||||||
|
samples = 10000
|
||||||
|
): number {
|
||||||
|
const others = allVariants.filter(v => v.id !== variant.id);
|
||||||
|
if (others.length === 0) return 0;
|
||||||
|
|
||||||
|
let totalLoss = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < samples; i++) {
|
||||||
|
const variantSample = sampleFromPosterior(variant, metricType);
|
||||||
|
const otherSamples = others.map(v => sampleFromPosterior(v, metricType));
|
||||||
|
const bestOther = Math.max(...otherSamples);
|
||||||
|
const loss = Math.max(0, bestOther - variantSample);
|
||||||
|
totalLoss += loss;
|
||||||
|
}
|
||||||
|
|
||||||
|
return totalLoss / samples;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sample from appropriate posterior distribution based on metric type.
|
||||||
|
*/
|
||||||
|
function sampleFromPosterior(variant: VariantDoc, metricType: MetricType): number {
|
||||||
|
switch (metricType) {
|
||||||
|
case 'conversion': {
|
||||||
|
const beta = betaFromVariant(variant);
|
||||||
|
return sampleBeta(beta.alpha, beta.beta);
|
||||||
|
}
|
||||||
|
case 'count':
|
||||||
|
case 'duration': {
|
||||||
|
const gamma = gammaFromVariant(variant);
|
||||||
|
return sampleGamma(gamma.shape, gamma.scale);
|
||||||
|
}
|
||||||
|
case 'revenue':
|
||||||
|
case 'custom':
|
||||||
|
default: {
|
||||||
|
const normal = normalFromVariant(variant);
|
||||||
|
return sampleNormal(normal.mean, normal.stdDev);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Credible Intervals
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface CredibleInterval {
|
||||||
|
lower: number; // 2.5th percentile
|
||||||
|
mean: number;
|
||||||
|
upper: number; // 97.5th percentile
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate 95% credible interval for variant's metric.
|
||||||
|
*/
|
||||||
|
export function calculateCredibleInterval(
|
||||||
|
variant: VariantDoc,
|
||||||
|
metricType: MetricType,
|
||||||
|
samples = 10000
|
||||||
|
): CredibleInterval {
|
||||||
|
const samples_array: number[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < samples; i++) {
|
||||||
|
samples_array.push(sampleFromPosterior(variant, metricType));
|
||||||
|
}
|
||||||
|
|
||||||
|
samples_array.sort((a, b) => a - b);
|
||||||
|
|
||||||
|
const lowerIndex = Math.floor(samples * 0.025);
|
||||||
|
const upperIndex = Math.floor(samples * 0.975);
|
||||||
|
|
||||||
|
return {
|
||||||
|
lower: samples_array[lowerIndex],
|
||||||
|
mean: samples_array.reduce((a, b) => a + b, 0) / samples,
|
||||||
|
upper: samples_array[upperIndex],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analytical credible interval for Beta distribution.
|
||||||
|
* Uses inverse Beta CDF (approximation via binary search).
|
||||||
|
*/
|
||||||
|
export function betaCredibleInterval(alpha: number, beta: number): CredibleInterval {
|
||||||
|
// Binary search for percentiles
|
||||||
|
const lower = betaQuantile(alpha, beta, 0.025);
|
||||||
|
const upper = betaQuantile(alpha, beta, 0.975);
|
||||||
|
const mean = alpha / (alpha + beta);
|
||||||
|
|
||||||
|
return { lower, mean, upper };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find quantile of Beta distribution via binary search.
|
||||||
|
*/
|
||||||
|
function betaQuantile(alpha: number, beta: number, p: number): number {
|
||||||
|
let low = 0;
|
||||||
|
let high = 1;
|
||||||
|
let mid = 0.5;
|
||||||
|
|
||||||
|
for (let iter = 0; iter < 50; iter++) {
|
||||||
|
mid = (low + high) / 2;
|
||||||
|
const cdf = betaCdf(mid, alpha, beta);
|
||||||
|
if (cdf < p) {
|
||||||
|
low = mid;
|
||||||
|
} else {
|
||||||
|
high = mid;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mid;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Early Stopping Rules
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface EarlyStoppingResult {
|
||||||
|
shouldStop: boolean;
|
||||||
|
reason?: string;
|
||||||
|
winnerVariantId?: string;
|
||||||
|
confidence: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if experiment should stop early based on statistical criteria.
|
||||||
|
*/
|
||||||
|
export function checkEarlyStopping(
|
||||||
|
experiment: ExperimentDoc,
|
||||||
|
variants: VariantDoc[],
|
||||||
|
daysRunning: number
|
||||||
|
): EarlyStoppingResult {
|
||||||
|
// Minimum sample size guardrail
|
||||||
|
const minSamples = experiment.guardrails.minSampleSizePerVariant;
|
||||||
|
const allHaveMinSamples = variants.every(v => v.stats.participants >= minSamples);
|
||||||
|
|
||||||
|
if (!allHaveMinSamples) {
|
||||||
|
return { shouldStop: false, confidence: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
const controlVariant = variants.find(v => v.isControl);
|
||||||
|
if (!controlVariant || variants.length < 2) {
|
||||||
|
return { shouldStop: false, confidence: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate probabilities for each variant
|
||||||
|
const results = variants.map(variant => ({
|
||||||
|
variant,
|
||||||
|
probBeatsControl: variant.isControl
|
||||||
|
? 0
|
||||||
|
: probabilityVariantBeatsControl(variant, controlVariant, experiment.primaryMetric.type),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Find variant with highest probability
|
||||||
|
const nonControlResults = results.filter(r => !r.variant.isControl);
|
||||||
|
const bestResult = nonControlResults.reduce((best, current) =>
|
||||||
|
current.probBeatsControl > best.probBeatsControl ? current : best,
|
||||||
|
nonControlResults[0]
|
||||||
|
);
|
||||||
|
|
||||||
|
const threshold = experiment.guardrails.winnerThreshold / 100;
|
||||||
|
|
||||||
|
// Winner found: variant has > 95% probability of beating control
|
||||||
|
if (bestResult && bestResult.probBeatsControl >= threshold) {
|
||||||
|
return {
|
||||||
|
shouldStop: true,
|
||||||
|
reason: 'Winner found: variant has > 95% probability of beating control',
|
||||||
|
winnerVariantId: bestResult.variant.id,
|
||||||
|
confidence: bestResult.probBeatsControl,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// No winner clear: control has > 95% probability of beating all variants
|
||||||
|
const controlProbBeatsAll = probabilityVariantBeatsAll(
|
||||||
|
controlVariant,
|
||||||
|
variants,
|
||||||
|
experiment.primaryMetric.type
|
||||||
|
);
|
||||||
|
if (controlProbBeatsAll >= threshold) {
|
||||||
|
return {
|
||||||
|
shouldStop: true,
|
||||||
|
reason: 'No winner: control outperforms all variants with > 95% confidence',
|
||||||
|
confidence: controlProbBeatsAll,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Time bound: max duration reached
|
||||||
|
if (daysRunning >= experiment.guardrails.maxDurationDays) {
|
||||||
|
return {
|
||||||
|
shouldStop: true,
|
||||||
|
reason: 'Max duration reached',
|
||||||
|
confidence: bestResult?.probBeatsControl ?? 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
shouldStop: false,
|
||||||
|
confidence: bestResult?.probBeatsControl ?? 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Experiment Results
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate comprehensive experiment results.
|
||||||
|
*/
|
||||||
|
export function generateExperimentResult(
|
||||||
|
experiment: ExperimentDoc,
|
||||||
|
variants: VariantDoc[],
|
||||||
|
daysRunning: number
|
||||||
|
): ExperimentResult {
|
||||||
|
const controlVariant = variants.find(v => v.isControl);
|
||||||
|
const earlyStop = checkEarlyStopping(experiment, variants, daysRunning);
|
||||||
|
|
||||||
|
const variantResults = variants.map(variant => {
|
||||||
|
const credibleInterval = calculateCredibleInterval(variant, experiment.primaryMetric.type);
|
||||||
|
const probBeatsControl = controlVariant
|
||||||
|
? probabilityVariantBeatsControl(variant, controlVariant, experiment.primaryMetric.type)
|
||||||
|
: 0.5;
|
||||||
|
|
||||||
|
// Expected lift relative to control
|
||||||
|
let expectedLift = 0;
|
||||||
|
if (controlVariant && controlVariant.stats.primaryMetricValue > 0) {
|
||||||
|
expectedLift = ((variant.stats.primaryMetricValue - controlVariant.stats.primaryMetricValue)
|
||||||
|
/ controlVariant.stats.primaryMetricValue) * 100;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
variantId: variant.id,
|
||||||
|
variantName: variant.name,
|
||||||
|
isControl: variant.isControl,
|
||||||
|
participants: variant.stats.participants,
|
||||||
|
primaryMetricValue: variant.stats.primaryMetricValue,
|
||||||
|
probabilityBeatsControl: probBeatsControl,
|
||||||
|
expectedLiftPercent: expectedLift,
|
||||||
|
credibleInterval,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const winnerResult = earlyStop.winnerVariantId
|
||||||
|
? variantResults.find(vr => vr.variantId === earlyStop.winnerVariantId)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
// Determine recommended action
|
||||||
|
let recommendedAction: ExperimentResult['statisticalSummary']['recommendedAction'] = 'continue';
|
||||||
|
if (earlyStop.shouldStop) {
|
||||||
|
if (earlyStop.winnerVariantId) {
|
||||||
|
recommendedAction = 'ship';
|
||||||
|
} else if (earlyStop.reason?.includes('control')) {
|
||||||
|
recommendedAction = 'rollback';
|
||||||
|
} else {
|
||||||
|
recommendedAction = 'stop';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate probability that any variant beats control
|
||||||
|
const probAnyBeatsControl = Math.max(...variantResults.map(vr => vr.probabilityBeatsControl));
|
||||||
|
|
||||||
|
// Calculate expected loss if we ship the winner
|
||||||
|
const expectedLoss = earlyStop.winnerVariantId
|
||||||
|
? expectedLossIfChosen(
|
||||||
|
variants.find(v => v.id === earlyStop.winnerVariantId)!,
|
||||||
|
variants,
|
||||||
|
experiment.primaryMetric.type
|
||||||
|
)
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
return {
|
||||||
|
experimentId: experiment.id,
|
||||||
|
status: earlyStop.shouldStop
|
||||||
|
? (earlyStop.winnerVariantId ? 'winner_found' : 'no_winner')
|
||||||
|
: 'in_progress',
|
||||||
|
totalParticipants: experiment.totalParticipants,
|
||||||
|
totalEvents: experiment.totalEvents,
|
||||||
|
daysRunning,
|
||||||
|
winnerVariantId: earlyStop.winnerVariantId,
|
||||||
|
winnerProbability: earlyStop.confidence,
|
||||||
|
variantResults,
|
||||||
|
statisticalSummary: {
|
||||||
|
probabilityAnyBeatsControl: probAnyBeatsControl,
|
||||||
|
expectedLossIfShipped: expectedLoss,
|
||||||
|
recommendedAction,
|
||||||
|
},
|
||||||
|
earlyStopped: earlyStop.shouldStop,
|
||||||
|
stopReason: earlyStop.reason,
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Statistical Tests
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A/A test: check that two identical variants produce similar results.
|
||||||
|
* Used for validating the statistical engine.
|
||||||
|
*/
|
||||||
|
export function validateAA(
|
||||||
|
n: number,
|
||||||
|
trueRate: number,
|
||||||
|
samples = 1000
|
||||||
|
): { passRate: number; bias: number } {
|
||||||
|
let passes = 0;
|
||||||
|
let totalError = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < samples; i++) {
|
||||||
|
// Simulate two "identical" variants
|
||||||
|
const aSuccesses = simulateBinomial(n, trueRate);
|
||||||
|
const bSuccesses = simulateBinomial(n, trueRate);
|
||||||
|
|
||||||
|
const aRate = aSuccesses / n;
|
||||||
|
const bRate = bSuccesses / n;
|
||||||
|
|
||||||
|
// Check if confidence intervals overlap (simplified)
|
||||||
|
const aStd = Math.sqrt(aRate * (1 - aRate) / n);
|
||||||
|
const bStd = Math.sqrt(bRate * (1 - bRate) / n);
|
||||||
|
|
||||||
|
const diff = Math.abs(aRate - bRate);
|
||||||
|
const pooledStd = Math.sqrt(aStd * aStd + bStd * bStd);
|
||||||
|
|
||||||
|
// Pass if difference is within 2 standard deviations (95% roughly)
|
||||||
|
if (diff < 2 * pooledStd) passes++;
|
||||||
|
totalError += diff;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
passRate: passes / samples,
|
||||||
|
bias: totalError / samples,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Simulate binomial distribution (inverse transform sampling).
|
||||||
|
*/
|
||||||
|
function simulateBinomial(n: number, p: number): number {
|
||||||
|
let successes = 0;
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
if (Math.random() < p) successes++;
|
||||||
|
}
|
||||||
|
return successes;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Power analysis: calculate required sample size for given effect size.
|
||||||
|
*/
|
||||||
|
export function calculateSampleSize(
|
||||||
|
baselineRate: number,
|
||||||
|
minDetectableEffect: number, // relative change, e.g., 0.05 for 5%
|
||||||
|
alpha = 0.05,
|
||||||
|
power = 0.8
|
||||||
|
): number {
|
||||||
|
const zAlpha = 1.96; // ~95% confidence
|
||||||
|
const zBeta = 0.84; // ~80% power
|
||||||
|
|
||||||
|
const p1 = baselineRate;
|
||||||
|
const p2 = baselineRate * (1 + minDetectableEffect);
|
||||||
|
const pAvg = (p1 + p2) / 2;
|
||||||
|
|
||||||
|
const numerator = 2 * zAlpha * Math.sqrt(2 * pAvg * (1 - pAvg)) + zBeta * Math.sqrt(p1 * (1 - p1) + p2 * (1 - p2));
|
||||||
|
const denominator = p2 - p1;
|
||||||
|
|
||||||
|
return Math.ceil(Math.pow(numerator / denominator, 2));
|
||||||
|
}
|
||||||
169
services/platform-service/src/modules/ab-testing/targeting.ts
Normal file
169
services/platform-service/src/modules/ab-testing/targeting.ts
Normal file
@ -0,0 +1,169 @@
|
|||||||
|
/**
|
||||||
|
* A/B Testing — Audience targeting and filtering.
|
||||||
|
* Platform, version, region, segment, and user property matching.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { TargetingConfig } from './types.js';
|
||||||
|
|
||||||
|
export interface TargetingContext {
|
||||||
|
platform?: string; // ios, android, web
|
||||||
|
appVersion?: string; // semver
|
||||||
|
osVersion?: string;
|
||||||
|
deviceModel?: string;
|
||||||
|
region?: string; // country code
|
||||||
|
userSegments?: string[]; // pro, free, enterprise
|
||||||
|
userProperties?: Record<string, string | number | boolean>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a user matches the experiment's targeting criteria.
|
||||||
|
*/
|
||||||
|
export function matchesTargeting(
|
||||||
|
context: TargetingContext,
|
||||||
|
targeting: TargetingConfig
|
||||||
|
): boolean {
|
||||||
|
// No targeting = everyone eligible
|
||||||
|
if (!targeting || Object.keys(targeting).length === 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Platform check
|
||||||
|
if (targeting.platforms && targeting.platforms.length > 0) {
|
||||||
|
if (!context.platform || !targeting.platforms.includes(context.platform)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// App version check (semver range)
|
||||||
|
if (targeting.appVersions) {
|
||||||
|
if (!context.appVersion) return false;
|
||||||
|
if (!matchesVersionRange(context.appVersion, targeting.appVersions)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Region check
|
||||||
|
if (targeting.regions && targeting.regions.length > 0) {
|
||||||
|
if (!context.region || !targeting.regions.includes(context.region)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// User segment check
|
||||||
|
if (targeting.userSegments && targeting.userSegments.length > 0) {
|
||||||
|
const userSegs = context.userSegments ?? [];
|
||||||
|
const hasMatchingSegment = targeting.userSegments.some(seg => userSegs.includes(seg));
|
||||||
|
if (!hasMatchingSegment) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// User property check
|
||||||
|
if (targeting.userProperties && Object.keys(targeting.userProperties).length > 0) {
|
||||||
|
if (!context.userProperties) return false;
|
||||||
|
for (const [key, value] of Object.entries(targeting.userProperties)) {
|
||||||
|
if (context.userProperties[key] !== value) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if version matches the min/max range.
|
||||||
|
* Supports basic semver comparison.
|
||||||
|
*/
|
||||||
|
function matchesVersionRange(
|
||||||
|
version: string,
|
||||||
|
range: { min: string; max?: string }
|
||||||
|
): boolean {
|
||||||
|
const v = parseSemver(version);
|
||||||
|
const min = parseSemver(range.min);
|
||||||
|
|
||||||
|
if (!v || !min) return true; // Be permissive on parse failure
|
||||||
|
|
||||||
|
// Check min version
|
||||||
|
if (compareSemver(v, min) < 0) return false;
|
||||||
|
|
||||||
|
// Check max version if specified
|
||||||
|
if (range.max) {
|
||||||
|
const max = parseSemver(range.max);
|
||||||
|
if (max && compareSemver(v, max) > 0) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Semver {
|
||||||
|
major: number;
|
||||||
|
minor: number;
|
||||||
|
patch: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSemver(version: string): Semver | null {
|
||||||
|
const match = version.match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
|
||||||
|
if (!match) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
major: parseInt(match[1], 10),
|
||||||
|
minor: match[2] ? parseInt(match[2], 10) : 0,
|
||||||
|
patch: match[3] ? parseInt(match[3], 10) : 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function compareSemver(a: Semver, b: Semver): number {
|
||||||
|
if (a.major !== b.major) return a.major - b.major;
|
||||||
|
if (a.minor !== b.minor) return a.minor - b.minor;
|
||||||
|
return a.patch - b.patch;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Exclusion Lists
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if user is in an exclusion list (beta users, internal accounts).
|
||||||
|
*/
|
||||||
|
export function isExcluded(
|
||||||
|
userId: string,
|
||||||
|
exclusionList: string[]
|
||||||
|
): boolean {
|
||||||
|
return exclusionList.includes(userId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Common exclusion patterns.
|
||||||
|
*/
|
||||||
|
export const DEFAULT_EXCLUSIONS = {
|
||||||
|
// Internal test accounts (patterns)
|
||||||
|
internalPatterns: [
|
||||||
|
/^test@/,
|
||||||
|
/^admin@/,
|
||||||
|
/@example\.com$/,
|
||||||
|
/@test\.com$/,
|
||||||
|
],
|
||||||
|
// Beta user segments to exclude from production experiments
|
||||||
|
betaSegments: ['beta', 'alpha', 'internal', 'employee'],
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if user matches any exclusion pattern.
|
||||||
|
*/
|
||||||
|
export function matchesExclusion(
|
||||||
|
userId: string,
|
||||||
|
segments: string[] = []
|
||||||
|
): boolean {
|
||||||
|
// Check patterns
|
||||||
|
for (const pattern of DEFAULT_EXCLUSIONS.internalPatterns) {
|
||||||
|
if (pattern.test(userId)) return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check segments
|
||||||
|
for (const seg of segments) {
|
||||||
|
if (DEFAULT_EXCLUSIONS.betaSegments.includes(seg)) return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
@ -0,0 +1,364 @@
|
|||||||
|
import { config } from '../../lib/config.js';
|
||||||
|
import type { FastifyBaseLogger } from 'fastify';
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Azure OpenAI Embedding Client
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
const EMBEDDING_MODEL = 'text-embedding-3-small';
|
||||||
|
const EMBEDDING_DIMENSIONS = 1536;
|
||||||
|
const MAX_BATCH_SIZE = 100; // Azure OpenAI limit
|
||||||
|
const MAX_TOKENS_PER_REQUEST = 8192;
|
||||||
|
|
||||||
|
interface EmbeddingResponse {
|
||||||
|
object: 'list';
|
||||||
|
data: Array<{
|
||||||
|
object: 'embedding';
|
||||||
|
embedding: number[];
|
||||||
|
index: number;
|
||||||
|
}>;
|
||||||
|
model: string;
|
||||||
|
usage: {
|
||||||
|
prompt_tokens: number;
|
||||||
|
total_tokens: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface EmbeddingResult {
|
||||||
|
embedding: number[];
|
||||||
|
tokenCount: number;
|
||||||
|
model: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BatchEmbeddingResult {
|
||||||
|
embeddings: Array<{
|
||||||
|
input: string;
|
||||||
|
embedding: number[];
|
||||||
|
index: number;
|
||||||
|
tokenCount: number;
|
||||||
|
}>;
|
||||||
|
totalTokens: number;
|
||||||
|
model: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates embeddings for error text using Azure OpenAI
|
||||||
|
*/
|
||||||
|
export async function generateEmbedding(
|
||||||
|
text: string,
|
||||||
|
options: {
|
||||||
|
model?: string;
|
||||||
|
dimensions?: number;
|
||||||
|
} = {}
|
||||||
|
): Promise<EmbeddingResult> {
|
||||||
|
const model = options.model || EMBEDDING_MODEL;
|
||||||
|
const apiKey = config.AZURE_OPENAI_KEY;
|
||||||
|
const endpoint = config.AZURE_OPENAI_ENDPOINT;
|
||||||
|
|
||||||
|
if (!apiKey || !endpoint) {
|
||||||
|
throw new Error('Azure OpenAI credentials not configured');
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = `${endpoint}/openai/deployments/${model}/embeddings?api-version=2024-02-01`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'api-key': apiKey,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
input: text,
|
||||||
|
model,
|
||||||
|
dimensions: options.dimensions || EMBEDDING_DIMENSIONS,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const error = await response.text();
|
||||||
|
throw new Error(`Azure OpenAI embedding failed: ${response.status} ${error}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await response.json()) as EmbeddingResponse;
|
||||||
|
|
||||||
|
if (!data.data?.[0]?.embedding) {
|
||||||
|
throw new Error('Invalid embedding response from Azure OpenAI');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
embedding: data.data[0].embedding,
|
||||||
|
tokenCount: data.usage?.prompt_tokens || 0,
|
||||||
|
model: data.model,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to generate embedding:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates embeddings for multiple texts in a batch
|
||||||
|
* Respects Azure OpenAI batch limits
|
||||||
|
*/
|
||||||
|
export async function generateEmbeddingsBatch(
|
||||||
|
texts: string[],
|
||||||
|
options: {
|
||||||
|
model?: string;
|
||||||
|
dimensions?: number;
|
||||||
|
batchSize?: number;
|
||||||
|
} = {}
|
||||||
|
): Promise<BatchEmbeddingResult> {
|
||||||
|
const batchSize = options.batchSize || MAX_BATCH_SIZE;
|
||||||
|
const model = options.model || EMBEDDING_MODEL;
|
||||||
|
|
||||||
|
if (texts.length === 0) {
|
||||||
|
return { embeddings: [], totalTokens: 0, model };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process in chunks to respect batch limits
|
||||||
|
const chunks: string[][] = [];
|
||||||
|
for (let i = 0; i < texts.length; i += batchSize) {
|
||||||
|
chunks.push(texts.slice(i, i + batchSize));
|
||||||
|
}
|
||||||
|
|
||||||
|
const allEmbeddings: BatchEmbeddingResult['embeddings'] = [];
|
||||||
|
let totalTokens = 0;
|
||||||
|
|
||||||
|
for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
|
||||||
|
const chunk = chunks[chunkIndex];
|
||||||
|
const apiKey = config.AZURE_OPENAI_KEY;
|
||||||
|
const endpoint = config.AZURE_OPENAI_ENDPOINT;
|
||||||
|
|
||||||
|
if (!apiKey || !endpoint) {
|
||||||
|
throw new Error('Azure OpenAI credentials not configured');
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = `${endpoint}/openai/deployments/${model}/embeddings?api-version=2024-02-01`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'api-key': apiKey,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
input: chunk,
|
||||||
|
model,
|
||||||
|
dimensions: options.dimensions || EMBEDDING_DIMENSIONS,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const error = await response.text();
|
||||||
|
throw new Error(`Azure OpenAI batch embedding failed: ${response.status} ${error}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await response.json()) as EmbeddingResponse;
|
||||||
|
|
||||||
|
// Map results back to original inputs
|
||||||
|
const chunkEmbeddings = data.data.map((item) => ({
|
||||||
|
input: chunk[item.index],
|
||||||
|
embedding: item.embedding,
|
||||||
|
index: chunkIndex * batchSize + item.index,
|
||||||
|
tokenCount: Math.floor((data.usage?.prompt_tokens || 0) / chunk.length), // Approximate
|
||||||
|
}));
|
||||||
|
|
||||||
|
allEmbeddings.push(...chunkEmbeddings);
|
||||||
|
totalTokens += data.usage?.prompt_tokens || 0;
|
||||||
|
|
||||||
|
// Small delay between batches to avoid rate limits
|
||||||
|
if (chunkIndex < chunks.length - 1) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to generate batch embeddings:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
embeddings: allEmbeddings.sort((a, b) => a.index - b.index),
|
||||||
|
totalTokens,
|
||||||
|
model,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Error-Specific Embedding Functions
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates embedding text for an error cluster
|
||||||
|
* Combines error type, message template, and stack signature
|
||||||
|
*/
|
||||||
|
export function createClusterEmbeddingText(
|
||||||
|
errorType: string,
|
||||||
|
messageTemplate: string,
|
||||||
|
stackSignature: string
|
||||||
|
): string {
|
||||||
|
const parts = [
|
||||||
|
`Error: ${errorType}`,
|
||||||
|
`Message: ${messageTemplate}`,
|
||||||
|
];
|
||||||
|
|
||||||
|
if (stackSignature) {
|
||||||
|
// Include top 3 stack frames for context
|
||||||
|
const topFrames = stackSignature.split('|').slice(0, 3).join(' -> ');
|
||||||
|
parts.push(`Stack: ${topFrames}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates embedding text for semantic error search
|
||||||
|
*/
|
||||||
|
export function createSearchEmbeddingText(query: string): string {
|
||||||
|
// Normalize query for better embedding matching
|
||||||
|
return query
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^\w\s]/g, ' ')
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Vector Utilities
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates cosine similarity between two vectors
|
||||||
|
*/
|
||||||
|
export function cosineSimilarity(a: number[], b: number[]): number {
|
||||||
|
if (a.length !== b.length) {
|
||||||
|
throw new Error('Vectors must have same dimensions');
|
||||||
|
}
|
||||||
|
|
||||||
|
let dotProduct = 0;
|
||||||
|
let normA = 0;
|
||||||
|
let normB = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < a.length; i++) {
|
||||||
|
dotProduct += a[i] * b[i];
|
||||||
|
normA += a[i] * a[i];
|
||||||
|
normB += b[i] * b[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normA === 0 || normB === 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates Euclidean distance between two vectors
|
||||||
|
*/
|
||||||
|
export function euclideanDistance(a: number[], b: number[]): number {
|
||||||
|
if (a.length !== b.length) {
|
||||||
|
throw new Error('Vectors must have same dimensions');
|
||||||
|
}
|
||||||
|
|
||||||
|
let sum = 0;
|
||||||
|
for (let i = 0; i < a.length; i++) {
|
||||||
|
const diff = a[i] - b[i];
|
||||||
|
sum += diff * diff;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.sqrt(sum);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalizes a vector to unit length
|
||||||
|
*/
|
||||||
|
export function normalizeVector(vector: number[]): number[] {
|
||||||
|
const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
|
||||||
|
if (norm === 0) return vector;
|
||||||
|
return vector.map((val) => val / norm);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Embedding Cache
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
interface CacheEntry {
|
||||||
|
embedding: number[];
|
||||||
|
timestamp: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
class EmbeddingCache {
|
||||||
|
private cache = new Map<string, CacheEntry>();
|
||||||
|
private ttlMs: number;
|
||||||
|
|
||||||
|
constructor(ttlHours: number = 24) {
|
||||||
|
this.ttlMs = ttlHours * 60 * 60 * 1000;
|
||||||
|
}
|
||||||
|
|
||||||
|
get(text: string): number[] | null {
|
||||||
|
const key = this.hashText(text);
|
||||||
|
const entry = this.cache.get(key);
|
||||||
|
|
||||||
|
if (!entry) return null;
|
||||||
|
|
||||||
|
// Check TTL
|
||||||
|
if (Date.now() - entry.timestamp > this.ttlMs) {
|
||||||
|
this.cache.delete(key);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry.embedding;
|
||||||
|
}
|
||||||
|
|
||||||
|
set(text: string, embedding: number[]): void {
|
||||||
|
const key = this.hashText(text);
|
||||||
|
this.cache.set(key, {
|
||||||
|
embedding,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private hashText(text: string): string {
|
||||||
|
// Simple hash for cache key
|
||||||
|
let hash = 0;
|
||||||
|
for (let i = 0; i < text.length; i++) {
|
||||||
|
const char = text.charCodeAt(i);
|
||||||
|
hash = ((hash << 5) - hash) + char;
|
||||||
|
hash = hash & hash;
|
||||||
|
}
|
||||||
|
return hash.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Global cache instance
|
||||||
|
export const embeddingCache = new EmbeddingCache(24);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates embedding with caching
|
||||||
|
*/
|
||||||
|
export async function generateEmbeddingCached(
|
||||||
|
text: string,
|
||||||
|
options: {
|
||||||
|
model?: string;
|
||||||
|
dimensions?: number;
|
||||||
|
} = {}
|
||||||
|
): Promise<EmbeddingResult> {
|
||||||
|
// Check cache first
|
||||||
|
const cached = embeddingCache.get(text);
|
||||||
|
if (cached) {
|
||||||
|
return {
|
||||||
|
embedding: cached,
|
||||||
|
tokenCount: 0,
|
||||||
|
model: options.model || EMBEDDING_MODEL,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate new embedding
|
||||||
|
const result = await generateEmbedding(text, options);
|
||||||
|
|
||||||
|
// Cache it
|
||||||
|
embeddingCache.set(text, result.embedding);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
@ -10,150 +10,106 @@ import type { TelemetryEventDoc } from '../telemetry/types.js';
|
|||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
export interface UserBehaviorFeatures {
|
export interface UserBehaviorFeatures {
|
||||||
// Recency features
|
|
||||||
daysSinceLastSession: number;
|
daysSinceLastSession: number;
|
||||||
daysSinceLastCoreAction: number;
|
daysSinceLastCoreAction: number;
|
||||||
hoursSinceLastLogin: number;
|
hoursSinceLastLogin: number;
|
||||||
|
|
||||||
// Frequency features
|
|
||||||
sessionsLast24Hours: number;
|
sessionsLast24Hours: number;
|
||||||
sessionsLast7Days: number;
|
sessionsLast7Days: number;
|
||||||
sessionsLast30Days: number;
|
sessionsLast30Days: number;
|
||||||
avgSessionsPerWeek: number;
|
avgSessionsPerWeek: number;
|
||||||
avgSessionsPerDay: number;
|
avgSessionsPerDay: number;
|
||||||
|
|
||||||
// Session depth
|
|
||||||
avgSessionDurationMinutes: number;
|
avgSessionDurationMinutes: number;
|
||||||
totalSessionDurationMinutes: number;
|
totalSessionDurationMinutes: number;
|
||||||
actionsPerSession: number;
|
actionsPerSession: number;
|
||||||
uniqueFeaturesUsed: number;
|
uniqueFeaturesUsed: number;
|
||||||
|
sessionFrequencyTrend: number;
|
||||||
// Engagement trends
|
|
||||||
sessionFrequencyTrend: number; // -1 to 1 (declining to increasing)
|
|
||||||
engagementDepthTrend: number;
|
engagementDepthTrend: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface EngagementFeatures {
|
export interface EngagementFeatures {
|
||||||
// Feature usage diversity
|
featureUsageDiversity: number;
|
||||||
featureUsageDiversity: number; // 0-1 (normalized unique features / total features)
|
|
||||||
coreActionCompletionRate: number;
|
coreActionCompletionRate: number;
|
||||||
featureAdoptionVelocity: number; // new features tried per week
|
featureAdoptionVelocity: number;
|
||||||
|
powerUserScore: number;
|
||||||
// Product-specific engagement
|
|
||||||
powerUserScore: number; // 0-1 based on advanced feature usage
|
|
||||||
onboardingCompletionRate: number;
|
onboardingCompletionRate: number;
|
||||||
firstValueMomentAchieved: boolean;
|
firstValueMomentAchieved: boolean;
|
||||||
timeToFirstValueHours: number;
|
timeToFirstValueHours: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PerformanceFeatures {
|
export interface PerformanceFeatures {
|
||||||
// Error/stability exposure
|
|
||||||
errorRateLast7Days: number;
|
errorRateLast7Days: number;
|
||||||
errorRateLast30Days: number;
|
errorRateLast30Days: number;
|
||||||
crashCountLast7Days: number;
|
crashCountLast7Days: number;
|
||||||
crashCountLast30Days: number;
|
crashCountLast30Days: number;
|
||||||
|
|
||||||
// Performance perception
|
|
||||||
avgLatencyMs: number;
|
avgLatencyMs: number;
|
||||||
slowRequestCount: number;
|
slowRequestCount: number;
|
||||||
timeoutCount: number;
|
timeoutCount: number;
|
||||||
|
|
||||||
// Recovery behavior
|
|
||||||
errorRecoveryRate: number;
|
errorRecoveryRate: number;
|
||||||
supportTicketCount: number;
|
supportTicketCount: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SocialFeatures {
|
export interface SocialFeatures {
|
||||||
// Sharing/collaboration
|
|
||||||
shareCount: number;
|
shareCount: number;
|
||||||
inviteCount: number;
|
inviteCount: number;
|
||||||
collaborationScore: number;
|
collaborationScore: number;
|
||||||
|
|
||||||
// Network effects
|
|
||||||
teamMemberCount: number;
|
teamMemberCount: number;
|
||||||
integrationsConnected: number;
|
integrationsConnected: number;
|
||||||
externalSharesLast30Days: number;
|
externalSharesLast30Days: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RevenueFeatures {
|
export interface RevenueFeatures {
|
||||||
// Payment history
|
planTier: number;
|
||||||
planTier: number; // 0=free, 1=pro, 2=enterprise
|
|
||||||
lifetimeValue: number;
|
lifetimeValue: number;
|
||||||
mrrContribution: number;
|
mrrContribution: number;
|
||||||
|
|
||||||
// Plan changes
|
|
||||||
upgradeCount: number;
|
upgradeCount: number;
|
||||||
downgradeCount: number;
|
downgradeCount: number;
|
||||||
daysSinceLastPayment: number;
|
daysSinceLastPayment: number;
|
||||||
daysSincePlanChange: number;
|
daysSincePlanChange: number;
|
||||||
|
|
||||||
// Support
|
|
||||||
supportTicketCount: number;
|
supportTicketCount: number;
|
||||||
supportSatisfactionScore: number;
|
supportSatisfactionScore: number;
|
||||||
escalatedTicketCount: number;
|
escalatedTicketCount: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RollingWindowFeatures {
|
export interface RollingWindowFeatures {
|
||||||
// 7-day rolling averages
|
|
||||||
rollingAvgSessions7d: number;
|
rollingAvgSessions7d: number;
|
||||||
rollingAvgDuration7d: number;
|
rollingAvgDuration7d: number;
|
||||||
rollingAvgActions7d: number;
|
rollingAvgActions7d: number;
|
||||||
|
wowSessionChange: number;
|
||||||
// Week-over-week change (acceleration)
|
|
||||||
wowSessionChange: number; // % change
|
|
||||||
wowDurationChange: number;
|
wowDurationChange: number;
|
||||||
wowActionsChange: number;
|
wowActionsChange: number;
|
||||||
|
cohortSessionPercentile: number;
|
||||||
// Cohort comparison (normalized vs similar users)
|
|
||||||
cohortSessionPercentile: number; // 0-100
|
|
||||||
cohortEngagementPercentile: number;
|
cohortEngagementPercentile: number;
|
||||||
cohortRetentionPercentile: number;
|
cohortRetentionPercentile: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ProductSpecificFeatures {
|
export interface ProductSpecificFeatures {
|
||||||
// NomGap
|
|
||||||
fastCompletionRate?: number;
|
fastCompletionRate?: number;
|
||||||
protocolAdherenceScore?: number;
|
protocolAdherenceScore?: number;
|
||||||
streakLength?: number;
|
streakLength?: number;
|
||||||
autophagyEngagementScore?: number;
|
autophagyEngagementScore?: number;
|
||||||
|
|
||||||
// JarvisJr
|
|
||||||
agentDiversityScore?: number;
|
agentDiversityScore?: number;
|
||||||
voiceSessionRatio?: number;
|
voiceSessionRatio?: number;
|
||||||
skillProgressionRate?: number;
|
skillProgressionRate?: number;
|
||||||
sessionCompletionRate?: number;
|
sessionCompletionRate?: number;
|
||||||
|
|
||||||
// ChronoMind
|
|
||||||
timerCompletionRate?: number;
|
timerCompletionRate?: number;
|
||||||
cascadeEffectiveness?: number;
|
cascadeEffectiveness?: number;
|
||||||
routineAdherenceScore?: number;
|
routineAdherenceScore?: number;
|
||||||
urgencyResponseRate?: number;
|
urgencyResponseRate?: number;
|
||||||
|
|
||||||
// MindLyst
|
|
||||||
brainUsageDiversity?: number;
|
brainUsageDiversity?: number;
|
||||||
triageAccuracyScore?: number;
|
triageAccuracyScore?: number;
|
||||||
memoryCaptureFrequency?: number;
|
memoryCaptureFrequency?: number;
|
||||||
reflectionCompletionRate?: number;
|
reflectionCompletionRate?: number;
|
||||||
|
|
||||||
// PeakPulse
|
|
||||||
activitySessionFrequency?: number;
|
activitySessionFrequency?: number;
|
||||||
goalCompletionRate?: number;
|
goalCompletionRate?: number;
|
||||||
streakMaintenanceScore?: number;
|
streakMaintenanceScore?: number;
|
||||||
socialSharingCount?: number;
|
socialSharingCount?: number;
|
||||||
|
|
||||||
// LysnrAI
|
|
||||||
dictationFrequency?: number;
|
dictationFrequency?: number;
|
||||||
accuracyRate?: number;
|
accuracyRate?: number;
|
||||||
hotkeyUsageRate?: number;
|
hotkeyUsageRate?: number;
|
||||||
vocabularyGrowthRate?: number;
|
vocabularyGrowthRate?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Time Window Aggregations
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
export interface TimeWindowFeatures {
|
export interface TimeWindowFeatures {
|
||||||
// Last 24 hours (recent behavior)
|
|
||||||
recent: {
|
recent: {
|
||||||
sessionCount: number;
|
sessionCount: number;
|
||||||
totalDuration: number;
|
totalDuration: number;
|
||||||
@ -161,8 +117,6 @@ export interface TimeWindowFeatures {
|
|||||||
errorCount: number;
|
errorCount: number;
|
||||||
uniqueFeatures: string[];
|
uniqueFeatures: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
// Last 7 days (weekly patterns)
|
|
||||||
weekly: {
|
weekly: {
|
||||||
sessionCount: number;
|
sessionCount: number;
|
||||||
totalDuration: number;
|
totalDuration: number;
|
||||||
@ -171,8 +125,6 @@ export interface TimeWindowFeatures {
|
|||||||
uniqueFeatures: string[];
|
uniqueFeatures: string[];
|
||||||
daysActive: number;
|
daysActive: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Last 30 days (monthly trends)
|
|
||||||
monthly: {
|
monthly: {
|
||||||
sessionCount: number;
|
sessionCount: number;
|
||||||
totalDuration: number;
|
totalDuration: number;
|
||||||
@ -181,8 +133,6 @@ export interface TimeWindowFeatures {
|
|||||||
uniqueFeatures: string[];
|
uniqueFeatures: string[];
|
||||||
daysActive: number;
|
daysActive: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Life-to-date (all-time totals)
|
|
||||||
lifetime: {
|
lifetime: {
|
||||||
totalSessions: number;
|
totalSessions: number;
|
||||||
totalDuration: number;
|
totalDuration: number;
|
||||||
@ -193,19 +143,11 @@ export interface TimeWindowFeatures {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Complete Feature Vector
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
export interface CompleteFeatureVector {
|
export interface CompleteFeatureVector {
|
||||||
userId: string;
|
userId: string;
|
||||||
productId: string;
|
productId: string;
|
||||||
computedAt: Date;
|
computedAt: Date;
|
||||||
observationWindow: {
|
observationWindow: { start: Date; end: Date };
|
||||||
start: Date;
|
|
||||||
end: Date;
|
|
||||||
};
|
|
||||||
|
|
||||||
behavior: UserBehaviorFeatures;
|
behavior: UserBehaviorFeatures;
|
||||||
engagement: EngagementFeatures;
|
engagement: EngagementFeatures;
|
||||||
performance: PerformanceFeatures;
|
performance: PerformanceFeatures;
|
||||||
@ -214,16 +156,10 @@ export interface CompleteFeatureVector {
|
|||||||
rolling: RollingWindowFeatures;
|
rolling: RollingWindowFeatures;
|
||||||
productSpecific: ProductSpecificFeatures;
|
productSpecific: ProductSpecificFeatures;
|
||||||
timeWindows: TimeWindowFeatures;
|
timeWindows: TimeWindowFeatures;
|
||||||
|
|
||||||
// Metadata
|
|
||||||
featureSchemaVersion: string;
|
featureSchemaVersion: string;
|
||||||
dataQualityScore: number; // 0-1 based on completeness
|
dataQualityScore: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Feature Extraction Functions
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
const SCHEMA_VERSION = '1.0.0';
|
const SCHEMA_VERSION = '1.0.0';
|
||||||
|
|
||||||
export function extractFeaturesFromTelemetry(
|
export function extractFeaturesFromTelemetry(
|
||||||
@ -235,46 +171,25 @@ export function extractFeaturesFromTelemetry(
|
|||||||
const observationStart = new Date(referenceDate);
|
const observationStart = new Date(referenceDate);
|
||||||
observationStart.setDate(observationStart.getDate() - 30);
|
observationStart.setDate(observationStart.getDate() - 30);
|
||||||
|
|
||||||
// Filter events to observation window
|
|
||||||
const windowedEvents = events.filter(
|
const windowedEvents = events.filter(
|
||||||
(e) => new Date(e.timestamp) >= observationStart && new Date(e.timestamp) <= referenceDate
|
(e) => new Date(e.occurredAt) >= observationStart && new Date(e.occurredAt) <= referenceDate
|
||||||
);
|
);
|
||||||
|
|
||||||
// Extract time windows
|
|
||||||
const timeWindows = extractTimeWindows(windowedEvents, referenceDate);
|
const timeWindows = extractTimeWindows(windowedEvents, referenceDate);
|
||||||
|
|
||||||
// Extract behavior features
|
|
||||||
const behavior = extractBehaviorFeatures(windowedEvents, timeWindows, referenceDate);
|
const behavior = extractBehaviorFeatures(windowedEvents, timeWindows, referenceDate);
|
||||||
|
|
||||||
// Extract engagement features
|
|
||||||
const engagement = extractEngagementFeatures(windowedEvents, timeWindows);
|
const engagement = extractEngagementFeatures(windowedEvents, timeWindows);
|
||||||
|
|
||||||
// Extract performance features
|
|
||||||
const performance = extractPerformanceFeatures(windowedEvents, timeWindows);
|
const performance = extractPerformanceFeatures(windowedEvents, timeWindows);
|
||||||
|
|
||||||
// Extract social features
|
|
||||||
const social = extractSocialFeatures(windowedEvents);
|
const social = extractSocialFeatures(windowedEvents);
|
||||||
|
|
||||||
// Extract revenue features (from events or external data)
|
|
||||||
const revenue = extractRevenueFeatures(windowedEvents);
|
const revenue = extractRevenueFeatures(windowedEvents);
|
||||||
|
|
||||||
// Extract rolling window features
|
|
||||||
const rolling = extractRollingWindowFeatures(timeWindows);
|
const rolling = extractRollingWindowFeatures(timeWindows);
|
||||||
|
|
||||||
// Extract product-specific features
|
|
||||||
const productSpecific = extractProductSpecificFeatures(windowedEvents, productId);
|
const productSpecific = extractProductSpecificFeatures(windowedEvents, productId);
|
||||||
|
|
||||||
// Calculate data quality score
|
|
||||||
const dataQualityScore = calculateDataQualityScore(behavior, engagement, performance);
|
const dataQualityScore = calculateDataQualityScore(behavior, engagement, performance);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
userId,
|
userId,
|
||||||
productId,
|
productId,
|
||||||
computedAt: referenceDate,
|
computedAt: referenceDate,
|
||||||
observationWindow: {
|
observationWindow: { start: observationStart, end: referenceDate },
|
||||||
start: observationStart,
|
|
||||||
end: referenceDate,
|
|
||||||
},
|
|
||||||
behavior,
|
behavior,
|
||||||
engagement,
|
engagement,
|
||||||
performance,
|
performance,
|
||||||
@ -288,44 +203,55 @@ export function extractFeaturesFromTelemetry(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractTimeWindows(
|
function extractTimeWindows(events: TelemetryEventDoc[], referenceDate: Date): TimeWindowFeatures {
|
||||||
events: TelemetryEventDoc[],
|
|
||||||
referenceDate: Date
|
|
||||||
): TimeWindowFeatures {
|
|
||||||
const oneDayAgo = new Date(referenceDate.getTime() - 24 * 60 * 60 * 1000);
|
const oneDayAgo = new Date(referenceDate.getTime() - 24 * 60 * 60 * 1000);
|
||||||
const sevenDaysAgo = new Date(referenceDate.getTime() - 7 * 24 * 60 * 60 * 1000);
|
const sevenDaysAgo = new Date(referenceDate.getTime() - 7 * 24 * 60 * 60 * 1000);
|
||||||
const thirtyDaysAgo = new Date(referenceDate.getTime() - 30 * 24 * 60 * 60 * 1000);
|
const thirtyDaysAgo = new Date(referenceDate.getTime() - 30 * 24 * 60 * 60 * 1000);
|
||||||
|
|
||||||
const recentEvents = events.filter((e) => new Date(e.timestamp) >= oneDayAgo);
|
const recentEvents = events.filter((e) => new Date(e.occurredAt) >= oneDayAgo);
|
||||||
const weeklyEvents = events.filter((e) => new Date(e.timestamp) >= sevenDaysAgo);
|
const weeklyEvents = events.filter((e) => new Date(e.occurredAt) >= sevenDaysAgo);
|
||||||
const monthlyEvents = events.filter((e) => new Date(e.timestamp) >= thirtyDaysAgo);
|
const monthlyEvents = events.filter((e) => new Date(e.occurredAt) >= thirtyDaysAgo);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
recent: aggregateEvents(recentEvents),
|
recent: aggregateEvents(recentEvents),
|
||||||
weekly: aggregateEvents(weeklyEvents, true),
|
weekly: aggregateEventsWithDays(weeklyEvents),
|
||||||
monthly: aggregateEvents(monthlyEvents, true),
|
monthly: aggregateEventsWithDays(monthlyEvents),
|
||||||
lifetime: {
|
lifetime: {
|
||||||
totalSessions: countSessions(events),
|
totalSessions: countSessions(events),
|
||||||
totalDuration: sumDurations(events),
|
totalDuration: sumDurations(events),
|
||||||
totalActions: countActions(events),
|
totalActions: countActions(events),
|
||||||
totalErrors: countErrors(events),
|
totalErrors: countErrors(events),
|
||||||
allFeaturesUsed: extractUniqueFeatures(events),
|
allFeaturesUsed: extractUniqueFeatures(events),
|
||||||
accountAgeDays: 30, // Default, should be passed as param
|
accountAgeDays: 30,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function aggregateEvents(
|
function aggregateEvents(events: TelemetryEventDoc[]) {
|
||||||
events: TelemetryEventDoc[],
|
const sessions = new Set<string>();
|
||||||
trackDaysActive = false
|
const features = new Set<string>();
|
||||||
): {
|
let totalDuration = 0;
|
||||||
sessionCount: number;
|
let actionCount = 0;
|
||||||
totalDuration: number;
|
let errorCount = 0;
|
||||||
actionCount: number;
|
|
||||||
errorCount: number;
|
for (const event of events) {
|
||||||
uniqueFeatures: string[];
|
if (event.sessionId) sessions.add(event.sessionId);
|
||||||
daysActive?: number;
|
if (event.feature) features.add(event.feature);
|
||||||
} {
|
if (event.metrics?.duration) totalDuration += event.metrics.duration;
|
||||||
|
if (event.eventName?.includes('action')) actionCount++;
|
||||||
|
if (event.eventType === 'error' || event.eventType === 'fatal') errorCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
sessionCount: sessions.size,
|
||||||
|
totalDuration,
|
||||||
|
actionCount,
|
||||||
|
errorCount,
|
||||||
|
uniqueFeatures: Array.from(features),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function aggregateEventsWithDays(events: TelemetryEventDoc[]) {
|
||||||
const sessions = new Set<string>();
|
const sessions = new Set<string>();
|
||||||
const features = new Set<string>();
|
const features = new Set<string>();
|
||||||
const activeDays = new Set<string>();
|
const activeDays = new Set<string>();
|
||||||
@ -336,14 +262,11 @@ function aggregateEvents(
|
|||||||
for (const event of events) {
|
for (const event of events) {
|
||||||
if (event.sessionId) sessions.add(event.sessionId);
|
if (event.sessionId) sessions.add(event.sessionId);
|
||||||
if (event.feature) features.add(event.feature);
|
if (event.feature) features.add(event.feature);
|
||||||
if (trackDaysActive) {
|
const day = event.occurredAt.split('T')[0];
|
||||||
const day = event.timestamp.split('T')[0];
|
activeDays.add(day);
|
||||||
activeDays.add(day);
|
if (event.metrics?.duration) totalDuration += event.metrics.duration;
|
||||||
}
|
if (event.eventName?.includes('action')) actionCount++;
|
||||||
|
if (event.eventType === 'error' || event.eventType === 'fatal') errorCount++;
|
||||||
if (event.eventType === 'action') actionCount++;
|
|
||||||
if (event.eventType === 'error') errorCount++;
|
|
||||||
if (event.duration) totalDuration += event.duration;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@ -352,7 +275,7 @@ function aggregateEvents(
|
|||||||
actionCount,
|
actionCount,
|
||||||
errorCount,
|
errorCount,
|
||||||
uniqueFeatures: Array.from(features),
|
uniqueFeatures: Array.from(features),
|
||||||
daysActive: trackDaysActive ? activeDays.size : undefined,
|
daysActive: activeDays.size,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -364,33 +287,17 @@ function extractBehaviorFeatures(
|
|||||||
const lastSession = findLastSession(events);
|
const lastSession = findLastSession(events);
|
||||||
const lastCoreAction = findLastCoreAction(events);
|
const lastCoreAction = findLastCoreAction(events);
|
||||||
|
|
||||||
const daysSinceLastSession = lastSession
|
const daysSinceLastSession = lastSession ? daysBetween(lastSession.occurredAt, referenceDate) : 30;
|
||||||
? daysBetween(lastSession.timestamp, referenceDate)
|
const daysSinceLastCoreAction = lastCoreAction ? daysBetween(lastCoreAction.occurredAt, referenceDate) : 30;
|
||||||
: 30;
|
|
||||||
const daysSinceLastCoreAction = lastCoreAction
|
|
||||||
? daysBetween(lastCoreAction.timestamp, referenceDate)
|
|
||||||
: 30;
|
|
||||||
|
|
||||||
const monthly = timeWindows.monthly;
|
const monthly = timeWindows.monthly;
|
||||||
const weekly = timeWindows.weekly;
|
const weekly = timeWindows.weekly;
|
||||||
|
|
||||||
// Calculate averages
|
const avgSessionsPerWeek = monthly.daysActive ? monthly.sessionCount / (monthly.daysActive / 7) : 0;
|
||||||
const avgSessionsPerWeek = monthly.daysActive
|
const avgSessionsPerDay = monthly.daysActive ? monthly.sessionCount / monthly.daysActive : 0;
|
||||||
? monthly.sessionCount / (monthly.daysActive / 7)
|
const avgSessionDurationMinutes = monthly.sessionCount ? monthly.totalDuration / monthly.sessionCount / 60 : 0;
|
||||||
: 0;
|
const actionsPerSession = monthly.sessionCount ? monthly.actionCount / monthly.sessionCount : 0;
|
||||||
const avgSessionsPerDay = monthly.daysActive
|
|
||||||
? monthly.sessionCount / monthly.daysActive
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
const avgSessionDurationMinutes = monthly.sessionCount
|
|
||||||
? monthly.totalDuration / monthly.sessionCount / 60
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
const actionsPerSession = monthly.sessionCount
|
|
||||||
? monthly.actionCount / monthly.sessionCount
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
// Calculate trends
|
|
||||||
const sessionFrequencyTrend = calculateTrend(weekly.sessionCount, monthly.sessionCount / 4);
|
const sessionFrequencyTrend = calculateTrend(weekly.sessionCount, monthly.sessionCount / 4);
|
||||||
const engagementDepthTrend = calculateTrend(
|
const engagementDepthTrend = calculateTrend(
|
||||||
weekly.totalDuration / Math.max(weekly.sessionCount, 1),
|
weekly.totalDuration / Math.max(weekly.sessionCount, 1),
|
||||||
@ -421,17 +328,12 @@ function extractEngagementFeatures(
|
|||||||
): EngagementFeatures {
|
): EngagementFeatures {
|
||||||
const monthly = timeWindows.monthly;
|
const monthly = timeWindows.monthly;
|
||||||
const allFeatures = extractUniqueFeatures(events);
|
const allFeatures = extractUniqueFeatures(events);
|
||||||
const totalPossibleFeatures = 20; // Configurable based on product
|
const totalPossibleFeatures = 20;
|
||||||
|
|
||||||
const featureUsageDiversity = Math.min(allFeatures.length / totalPossibleFeatures, 1);
|
const featureUsageDiversity = Math.min(allFeatures.length / totalPossibleFeatures, 1);
|
||||||
|
|
||||||
// Calculate core action completion (specific events indicate core actions)
|
|
||||||
const coreActionEvents = events.filter((e) => e.eventName?.includes('core_action'));
|
const coreActionEvents = events.filter((e) => e.eventName?.includes('core_action'));
|
||||||
const coreActionCompletionRate = monthly.actionCount
|
const coreActionCompletionRate = monthly.actionCount ? coreActionEvents.length / monthly.actionCount : 0;
|
||||||
? coreActionEvents.length / monthly.actionCount
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
// Power user score based on advanced features
|
|
||||||
const advancedFeatures = allFeatures.filter((f) =>
|
const advancedFeatures = allFeatures.filter((f) =>
|
||||||
['export', 'integration', 'automation', 'advanced'].some((a) => f.includes(a))
|
['export', 'integration', 'automation', 'advanced'].some((a) => f.includes(a))
|
||||||
);
|
);
|
||||||
@ -440,7 +342,7 @@ function extractEngagementFeatures(
|
|||||||
return {
|
return {
|
||||||
featureUsageDiversity,
|
featureUsageDiversity,
|
||||||
coreActionCompletionRate,
|
coreActionCompletionRate,
|
||||||
featureAdoptionVelocity: monthly.uniqueFeatures.length / 4, // per week
|
featureAdoptionVelocity: monthly.uniqueFeatures.length / 4,
|
||||||
powerUserScore,
|
powerUserScore,
|
||||||
onboardingCompletionRate: calculateOnboardingCompletion(events),
|
onboardingCompletionRate: calculateOnboardingCompletion(events),
|
||||||
firstValueMomentAchieved: hasFirstValueMoment(events),
|
firstValueMomentAchieved: hasFirstValueMoment(events),
|
||||||
@ -455,29 +357,22 @@ function extractPerformanceFeatures(
|
|||||||
const monthly = timeWindows.monthly;
|
const monthly = timeWindows.monthly;
|
||||||
const weekly = timeWindows.weekly;
|
const weekly = timeWindows.weekly;
|
||||||
|
|
||||||
const monthlyErrors = countErrors(
|
const monthlyErrors = countErrors(events.filter((e) => new Date(e.occurredAt) >= new Date(Date.now() - 30 * 24 * 60 * 60 * 1000)));
|
||||||
events.filter((e) => new Date(e.timestamp) >= new Date(Date.now() - 30 * 24 * 60 * 60 * 1000))
|
|
||||||
);
|
|
||||||
const weeklyErrors = weekly.errorCount;
|
const weeklyErrors = weekly.errorCount;
|
||||||
|
|
||||||
const errorRateLast30Days = monthly.actionCount
|
const errorRateLast30Days = monthly.actionCount ? monthlyErrors / monthly.actionCount : 0;
|
||||||
? monthlyErrors / monthly.actionCount
|
const errorRateLast7Days = weekly.actionCount ? weeklyErrors / weekly.actionCount : 0;
|
||||||
: 0;
|
|
||||||
const errorRateLast7Days = weekly.actionCount
|
|
||||||
? weeklyErrors / weekly.actionCount
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
// Extract latency from events
|
const latencyEvents = events.filter((e) => e.metrics?.duration && e.metrics.duration < 30000);
|
||||||
const latencyEvents = events.filter((e) => e.duration && e.duration < 30000); // Filter outliers
|
|
||||||
const avgLatencyMs = latencyEvents.length
|
const avgLatencyMs = latencyEvents.length
|
||||||
? latencyEvents.reduce((sum, e) => sum + (e.duration || 0), 0) / latencyEvents.length
|
? latencyEvents.reduce((sum, e) => sum + (e.metrics?.duration || 0), 0) / latencyEvents.length
|
||||||
: 0;
|
: 0;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
errorRateLast7Days,
|
errorRateLast7Days,
|
||||||
errorRateLast30Days,
|
errorRateLast30Days,
|
||||||
crashCountLast7Days: countCrashes(weeklyEvents(events)),
|
crashCountLast7Days: countCrashes(getWeeklyEvents(events)),
|
||||||
crashCountLast30Days: countCrashes(monthlyEvents(events)),
|
crashCountLast30Days: countCrashes(getMonthlyEvents(events)),
|
||||||
avgLatencyMs,
|
avgLatencyMs,
|
||||||
slowRequestCount: countSlowRequests(events),
|
slowRequestCount: countSlowRequests(events),
|
||||||
timeoutCount: countTimeouts(events),
|
timeoutCount: countTimeouts(events),
|
||||||
@ -497,14 +392,12 @@ function extractSocialFeatures(events: TelemetryEventDoc[]): SocialFeatures {
|
|||||||
collaborationScore: calculateCollaborationScore(events),
|
collaborationScore: calculateCollaborationScore(events),
|
||||||
teamMemberCount: extractTeamMemberCount(events),
|
teamMemberCount: extractTeamMemberCount(events),
|
||||||
integrationsConnected: integrationEvents.length,
|
integrationsConnected: integrationEvents.length,
|
||||||
externalSharesLast30Days: shareEvents.filter((e) => e.properties?.external === true).length,
|
externalSharesLast30Days: shareEvents.filter((e) => e.context?.external === true).length,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractRevenueFeatures(events: TelemetryEventDoc[]): RevenueFeatures {
|
function extractRevenueFeatures(events: TelemetryEventDoc[]): RevenueFeatures {
|
||||||
const planChangeEvents = events.filter(
|
const planChangeEvents = events.filter((e) => e.eventName?.includes('plan') || e.eventName?.includes('subscription'));
|
||||||
(e) => e.eventName?.includes('plan') || e.eventName?.includes('subscription')
|
|
||||||
);
|
|
||||||
const supportEvents = events.filter((e) => e.eventName?.includes('support'));
|
const supportEvents = events.filter((e) => e.eventName?.includes('support'));
|
||||||
|
|
||||||
const upgrades = planChangeEvents.filter((e) => e.eventName?.includes('upgrade')).length;
|
const upgrades = planChangeEvents.filter((e) => e.eventName?.includes('upgrade')).length;
|
||||||
@ -520,7 +413,7 @@ function extractRevenueFeatures(events: TelemetryEventDoc[]): RevenueFeatures {
|
|||||||
daysSincePlanChange: extractDaysSincePlanChange(events),
|
daysSincePlanChange: extractDaysSincePlanChange(events),
|
||||||
supportTicketCount: supportEvents.length,
|
supportTicketCount: supportEvents.length,
|
||||||
supportSatisfactionScore: calculateSupportSatisfaction(supportEvents),
|
supportSatisfactionScore: calculateSupportSatisfaction(supportEvents),
|
||||||
escalatedTicketCount: supportEvents.filter((e) => e.properties?.escalated).length,
|
escalatedTicketCount: supportEvents.filter((e) => e.context?.escalated).length,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -528,52 +421,31 @@ function extractRollingWindowFeatures(timeWindows: TimeWindowFeatures): RollingW
|
|||||||
const monthly = timeWindows.monthly;
|
const monthly = timeWindows.monthly;
|
||||||
const weekly = timeWindows.weekly;
|
const weekly = timeWindows.weekly;
|
||||||
|
|
||||||
// 7-day rolling averages
|
|
||||||
const rollingAvgSessions7d = weekly.sessionCount / 7;
|
const rollingAvgSessions7d = weekly.sessionCount / 7;
|
||||||
const rollingAvgDuration7d = weekly.sessionCount
|
const rollingAvgDuration7d = weekly.sessionCount ? weekly.totalDuration / weekly.sessionCount / 60 : 0;
|
||||||
? weekly.totalDuration / weekly.sessionCount / 60
|
|
||||||
: 0;
|
|
||||||
const rollingAvgActions7d = weekly.sessionCount ? weekly.actionCount / weekly.sessionCount : 0;
|
const rollingAvgActions7d = weekly.sessionCount ? weekly.actionCount / weekly.sessionCount : 0;
|
||||||
|
|
||||||
// Week-over-week change (comparing current week to average week in month)
|
|
||||||
const avgWeekInMonth = monthly.sessionCount / 4;
|
const avgWeekInMonth = monthly.sessionCount / 4;
|
||||||
const wowSessionChange = avgWeekInMonth ? (weekly.sessionCount - avgWeekInMonth) / avgWeekInMonth : 0;
|
const wowSessionChange = avgWeekInMonth ? (weekly.sessionCount - avgWeekInMonth) / avgWeekInMonth : 0;
|
||||||
|
|
||||||
const avgDurationWeekInMonth = monthly.sessionCount
|
const avgDurationWeekInMonth = monthly.sessionCount ? monthly.totalDuration / monthly.sessionCount / 60 / 4 : 0;
|
||||||
? monthly.totalDuration / monthly.sessionCount / 60 / 4
|
|
||||||
: 0;
|
|
||||||
const wowDurationChange = avgDurationWeekInMonth
|
const wowDurationChange = avgDurationWeekInMonth
|
||||||
? (rollingAvgDuration7d - avgDurationWeekInMonth) / avgDurationWeekInMonth
|
? (rollingAvgDuration7d - avgDurationWeekInMonth) / avgDurationWeekInMonth
|
||||||
: 0;
|
: 0;
|
||||||
|
|
||||||
// Cohort percentiles (would require cohort data - using estimates)
|
|
||||||
const cohortSessionPercentile = estimateCohortPercentile(rollingAvgSessions7d, 'sessions');
|
|
||||||
const cohortEngagementPercentile = estimateCohortPercentile(
|
|
||||||
timeWindows.monthly.uniqueFeatures.length,
|
|
||||||
'features'
|
|
||||||
);
|
|
||||||
const cohortRetentionPercentile = estimateCohortPercentile(
|
|
||||||
monthly.daysActive || 0,
|
|
||||||
'retention'
|
|
||||||
);
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
rollingAvgSessions7d,
|
rollingAvgSessions7d,
|
||||||
rollingAvgDuration7d,
|
rollingAvgDuration7d,
|
||||||
rollingAvgActions7d,
|
rollingAvgActions7d,
|
||||||
wowSessionChange,
|
wowSessionChange,
|
||||||
wowDurationChange,
|
wowDurationChange,
|
||||||
wowActionsChange: wowSessionChange, // Correlated with session change
|
wowActionsChange: wowSessionChange,
|
||||||
cohortSessionPercentile,
|
cohortSessionPercentile: estimateCohortPercentile(rollingAvgSessions7d, 'sessions'),
|
||||||
cohortEngagementPercentile,
|
cohortEngagementPercentile: estimateCohortPercentile(timeWindows.monthly.uniqueFeatures.length, 'features'),
|
||||||
cohortRetentionPercentile,
|
cohortRetentionPercentile: estimateCohortPercentile(monthly.daysActive || 0, 'retention'),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Product-Specific Feature Extraction
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
export function extractProductSpecificFeatures(
|
export function extractProductSpecificFeatures(
|
||||||
events: TelemetryEventDoc[],
|
events: TelemetryEventDoc[],
|
||||||
productId: string
|
productId: string
|
||||||
@ -600,12 +472,10 @@ function extractNomGapFeatures(events: TelemetryEventDoc[]): ProductSpecificFeat
|
|||||||
const fastEvents = events.filter((e) => e.feature === 'fasting');
|
const fastEvents = events.filter((e) => e.feature === 'fasting');
|
||||||
const completedFasts = fastEvents.filter((e) => e.eventName === 'fast_completed');
|
const completedFasts = fastEvents.filter((e) => e.eventName === 'fast_completed');
|
||||||
const totalFasts = fastEvents.filter((e) => e.eventName === 'fast_started').length;
|
const totalFasts = fastEvents.filter((e) => e.eventName === 'fast_started').length;
|
||||||
|
|
||||||
const protocolEvents = events.filter((e) => e.feature === 'protocol');
|
const protocolEvents = events.filter((e) => e.feature === 'protocol');
|
||||||
const adheredProtocols = protocolEvents.filter((e) => e.properties?.adhered).length;
|
const adheredProtocols = protocolEvents.filter((e) => e.context?.adhered).length;
|
||||||
|
|
||||||
const streakEvents = events.filter((e) => e.eventName?.includes('streak'));
|
const streakEvents = events.filter((e) => e.eventName?.includes('streak'));
|
||||||
const currentStreak = Math.max(...streakEvents.map((e) => e.properties?.streakLength || 0), 0);
|
const currentStreak = Math.max(...streakEvents.map((e) => (e.context?.streakLength as number) || 0), 0);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
fastCompletionRate: totalFasts ? completedFasts.length / totalFasts : 0,
|
fastCompletionRate: totalFasts ? completedFasts.length / totalFasts : 0,
|
||||||
@ -617,19 +487,16 @@ function extractNomGapFeatures(events: TelemetryEventDoc[]): ProductSpecificFeat
|
|||||||
|
|
||||||
function extractJarvisJrFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
function extractJarvisJrFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
||||||
const agentEvents = events.filter((e) => e.feature === 'agent');
|
const agentEvents = events.filter((e) => e.feature === 'agent');
|
||||||
const uniqueAgents = new Set(agentEvents.map((e) => e.properties?.agentId)).size;
|
const uniqueAgents = new Set(agentEvents.map((e) => e.context?.agentId as string)).size;
|
||||||
|
const voiceEvents = events.filter((e) => e.context?.mode === 'voice');
|
||||||
const voiceEvents = events.filter((e) => e.properties?.mode === 'voice');
|
const textEvents = events.filter((e) => e.context?.mode === 'text');
|
||||||
const textEvents = events.filter((e) => e.properties?.mode === 'text');
|
|
||||||
const totalSessions = voiceEvents.length + textEvents.length;
|
const totalSessions = voiceEvents.length + textEvents.length;
|
||||||
|
|
||||||
const skillEvents = events.filter((e) => e.eventName?.includes('skill'));
|
const skillEvents = events.filter((e) => e.eventName?.includes('skill'));
|
||||||
const skillProgression = calculateSkillProgression(skillEvents);
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
agentDiversityScore: Math.min(uniqueAgents / 3, 1),
|
agentDiversityScore: Math.min(uniqueAgents / 3, 1),
|
||||||
voiceSessionRatio: totalSessions ? voiceEvents.length / totalSessions : 0,
|
voiceSessionRatio: totalSessions ? voiceEvents.length / totalSessions : 0,
|
||||||
skillProgressionRate: skillProgression,
|
skillProgressionRate: calculateSkillProgression(skillEvents),
|
||||||
sessionCompletionRate: calculateSessionCompletionRate(events),
|
sessionCompletionRate: calculateSessionCompletionRate(events),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@ -638,12 +505,9 @@ function extractChronoMindFeatures(events: TelemetryEventDoc[]): ProductSpecific
|
|||||||
const timerEvents = events.filter((e) => e.feature === 'timer');
|
const timerEvents = events.filter((e) => e.feature === 'timer');
|
||||||
const completedTimers = timerEvents.filter((e) => e.eventName === 'timer_completed').length;
|
const completedTimers = timerEvents.filter((e) => e.eventName === 'timer_completed').length;
|
||||||
const totalTimers = timerEvents.filter((e) => e.eventName === 'timer_started').length;
|
const totalTimers = timerEvents.filter((e) => e.eventName === 'timer_started').length;
|
||||||
|
|
||||||
const cascadeEvents = events.filter((e) => e.feature === 'cascade');
|
const cascadeEvents = events.filter((e) => e.feature === 'cascade');
|
||||||
const acknowledgedCascades = cascadeEvents.filter((e) => e.properties?.acknowledged).length;
|
const acknowledgedCascades = cascadeEvents.filter((e) => e.context?.acknowledged).length;
|
||||||
|
|
||||||
const routineEvents = events.filter((e) => e.feature === 'routine');
|
const routineEvents = events.filter((e) => e.feature === 'routine');
|
||||||
const completedRoutines = routineEvents.filter((e) => e.eventName === 'routine_completed').length;
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
timerCompletionRate: totalTimers ? completedTimers / totalTimers : 0,
|
timerCompletionRate: totalTimers ? completedTimers / totalTimers : 0,
|
||||||
@ -655,37 +519,31 @@ function extractChronoMindFeatures(events: TelemetryEventDoc[]): ProductSpecific
|
|||||||
|
|
||||||
function extractMindLystFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
function extractMindLystFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
||||||
const brainEvents = events.filter((e) => e.feature === 'brain');
|
const brainEvents = events.filter((e) => e.feature === 'brain');
|
||||||
const uniqueBrains = new Set(brainEvents.map((e) => e.properties?.brainId)).size;
|
const uniqueBrains = new Set(brainEvents.map((e) => e.context?.brainId as string)).size;
|
||||||
|
|
||||||
const triageEvents = events.filter((e) => e.eventName?.includes('triage'));
|
const triageEvents = events.filter((e) => e.eventName?.includes('triage'));
|
||||||
const accurateTriages = triageEvents.filter((e) => e.properties?.accurate).length;
|
const accurateTriages = triageEvents.filter((e) => e.context?.accurate).length;
|
||||||
|
|
||||||
const memoryEvents = events.filter((e) => e.eventName?.includes('memory_capture'));
|
const memoryEvents = events.filter((e) => e.eventName?.includes('memory_capture'));
|
||||||
const reflectionEvents = events.filter((e) => e.eventName?.includes('reflection'));
|
const reflectionEvents = events.filter((e) => e.eventName?.includes('reflection'));
|
||||||
const completedReflections = reflectionEvents.filter((e) => e.properties?.completed).length;
|
const completedReflections = reflectionEvents.filter((e) => e.context?.completed).length;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
brainUsageDiversity: Math.min(uniqueBrains / 3, 1),
|
brainUsageDiversity: Math.min(uniqueBrains / 3, 1),
|
||||||
triageAccuracyScore: triageEvents.length ? accurateTriages / triageEvents.length : 0,
|
triageAccuracyScore: triageEvents.length ? accurateTriages / triageEvents.length : 0,
|
||||||
memoryCaptureFrequency: memoryEvents.length / 30, // per day
|
memoryCaptureFrequency: memoryEvents.length / 30,
|
||||||
reflectionCompletionRate: reflectionEvents.length
|
reflectionCompletionRate: reflectionEvents.length ? completedReflections / reflectionEvents.length : 0,
|
||||||
? completedReflections / reflectionEvents.length
|
|
||||||
: 0,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractPeakPulseFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
function extractPeakPulseFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
||||||
const sessionEvents = events.filter((e) => e.feature === 'activity_session');
|
const sessionEvents = events.filter((e) => e.feature === 'activity_session');
|
||||||
const goalEvents = events.filter((e) => e.feature === 'goal');
|
const goalEvents = events.filter((e) => e.feature === 'goal');
|
||||||
const completedGoals = goalEvents.filter((e) => e.properties?.completed).length;
|
const completedGoals = goalEvents.filter((e) => e.context?.completed).length;
|
||||||
|
|
||||||
const streakEvents = events.filter((e) => e.eventName?.includes('streak'));
|
const streakEvents = events.filter((e) => e.eventName?.includes('streak'));
|
||||||
const currentStreak = Math.max(...streakEvents.map((e) => e.properties?.streakLength || 0), 0);
|
const currentStreak = Math.max(...streakEvents.map((e) => (e.context?.streakLength as number) || 0), 0);
|
||||||
|
|
||||||
const shareEvents = events.filter((e) => e.eventName?.includes('share'));
|
const shareEvents = events.filter((e) => e.eventName?.includes('share'));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
activitySessionFrequency: sessionEvents.length / 30, // per day
|
activitySessionFrequency: sessionEvents.length / 30,
|
||||||
goalCompletionRate: goalEvents.length ? completedGoals / goalEvents.length : 0,
|
goalCompletionRate: goalEvents.length ? completedGoals / goalEvents.length : 0,
|
||||||
streakMaintenanceScore: Math.min(currentStreak / 7, 1),
|
streakMaintenanceScore: Math.min(currentStreak / 7, 1),
|
||||||
socialSharingCount: shareEvents.length,
|
socialSharingCount: shareEvents.length,
|
||||||
@ -694,41 +552,33 @@ function extractPeakPulseFeatures(events: TelemetryEventDoc[]): ProductSpecificF
|
|||||||
|
|
||||||
function extractLysnrAIFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
function extractLysnrAIFeatures(events: TelemetryEventDoc[]): ProductSpecificFeatures {
|
||||||
const dictationEvents = events.filter((e) => e.feature === 'dictation');
|
const dictationEvents = events.filter((e) => e.feature === 'dictation');
|
||||||
const completedDictations = dictationEvents.filter(
|
const completedDictations = dictationEvents.filter((e) => e.eventName === 'dictation_completed').length;
|
||||||
(e) => e.eventName === 'dictation_completed'
|
const accuracyEvents = dictationEvents.filter((e) => e.metrics?.accuracy !== undefined);
|
||||||
).length;
|
|
||||||
|
|
||||||
const accuracyEvents = dictationEvents.filter((e) => e.properties?.accuracy !== undefined);
|
|
||||||
const avgAccuracy = accuracyEvents.length
|
const avgAccuracy = accuracyEvents.length
|
||||||
? accuracyEvents.reduce((sum, e) => sum + (e.properties?.accuracy || 0), 0) /
|
? accuracyEvents.reduce((sum, e) => sum + ((e.metrics?.accuracy as number) || 0), 0) / accuracyEvents.length
|
||||||
accuracyEvents.length
|
|
||||||
: 0;
|
: 0;
|
||||||
|
|
||||||
const hotkeyEvents = events.filter((e) => e.eventName?.includes('hotkey'));
|
const hotkeyEvents = events.filter((e) => e.eventName?.includes('hotkey'));
|
||||||
const vocabularyEvents = events.filter((e) => e.eventName?.includes('vocabulary'));
|
const vocabularyEvents = events.filter((e) => e.eventName?.includes('vocabulary'));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
dictationFrequency: dictationEvents.length / 30, // per day
|
dictationFrequency: dictationEvents.length / 30,
|
||||||
accuracyRate: avgAccuracy,
|
accuracyRate: avgAccuracy,
|
||||||
hotkeyUsageRate: hotkeyEvents.length / Math.max(dictationEvents.length, 1),
|
hotkeyUsageRate: hotkeyEvents.length / Math.max(dictationEvents.length, 1),
|
||||||
vocabularyGrowthRate: calculateVocabularyGrowth(vocabularyEvents),
|
vocabularyGrowthRate: calculateVocabularyGrowth(vocabularyEvents),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Helper Functions
|
// Helper Functions
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
function findLastSession(events: TelemetryEventDoc[]): TelemetryEventDoc | undefined {
|
function findLastSession(events: TelemetryEventDoc[]): TelemetryEventDoc | undefined {
|
||||||
return events
|
return events
|
||||||
.filter((e) => e.eventType === 'session_start' || e.sessionId)
|
.filter((e) => e.eventName?.includes('session_start') || e.sessionId)
|
||||||
.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())[0];
|
.sort((a, b) => new Date(b.occurredAt).getTime() - new Date(a.occurredAt).getTime())[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
function findLastCoreAction(events: TelemetryEventDoc[]): TelemetryEventDoc | undefined {
|
function findLastCoreAction(events: TelemetryEventDoc[]): TelemetryEventDoc | undefined {
|
||||||
return events
|
return events
|
||||||
.filter((e) => e.properties?.isCoreAction === true || e.eventName?.includes('core'))
|
.filter((e) => e.context?.isCoreAction === true || e.eventName?.includes('core'))
|
||||||
.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())[0];
|
.sort((a, b) => new Date(b.occurredAt).getTime() - new Date(a.occurredAt).getTime())[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
function countSessions(events: TelemetryEventDoc[]): number {
|
function countSessions(events: TelemetryEventDoc[]): number {
|
||||||
@ -736,24 +586,23 @@ function countSessions(events: TelemetryEventDoc[]): number {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function sumDurations(events: TelemetryEventDoc[]): number {
|
function sumDurations(events: TelemetryEventDoc[]): number {
|
||||||
return events.reduce((sum, e) => sum + (e.duration || 0), 0);
|
return events.reduce((sum, e) => sum + (e.metrics?.duration || 0), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
function countActions(events: TelemetryEventDoc[]): number {
|
function countActions(events: TelemetryEventDoc[]): number {
|
||||||
return events.filter((e) => e.eventType === 'action').length;
|
return events.filter((e) => e.eventName?.includes('action')).length;
|
||||||
}
|
}
|
||||||
|
|
||||||
function countErrors(events: TelemetryEventDoc[]): number {
|
function countErrors(events: TelemetryEventDoc[]): number {
|
||||||
return events.filter((e) => e.eventType === 'error' || e.eventName?.includes('error')).length;
|
return events.filter((e) => e.eventType === 'error' || e.eventType === 'fatal').length;
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractUniqueFeatures(events: TelemetryEventDoc[]): string[] {
|
function extractUniqueFeatures(events: TelemetryEventDoc[]): string[] {
|
||||||
return Array.from(new Set(events.map((e) => e.feature).filter(Boolean) as string[]);
|
return Array.from(new Set(events.map((e) => e.feature).filter(Boolean) as string[]));
|
||||||
}
|
}
|
||||||
|
|
||||||
function daysBetween(timestamp: string, reference: Date): number {
|
function daysBetween(timestamp: string, reference: Date): number {
|
||||||
const diff = reference.getTime() - new Date(timestamp).getTime();
|
return Math.floor((reference.getTime() - new Date(timestamp).getTime()) / (1000 * 60 * 60 * 24));
|
||||||
return Math.floor(diff / (1000 * 60 * 60 * 24));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateTrend(current: number, baseline: number): number {
|
function calculateTrend(current: number, baseline: number): number {
|
||||||
@ -768,33 +617,18 @@ function calculateDataQualityScore(
|
|||||||
): number {
|
): number {
|
||||||
let score = 0;
|
let score = 0;
|
||||||
let factors = 0;
|
let factors = 0;
|
||||||
|
if (behavior.sessionsLast30Days > 0) { score += Math.min(behavior.sessionsLast30Days / 10, 1); factors++; }
|
||||||
if (behavior.sessionsLast30Days > 0) {
|
if (engagement.featureUsageDiversity > 0) { score += Math.min(engagement.featureUsageDiversity * 5, 1); factors++; }
|
||||||
score += Math.min(behavior.sessionsLast30Days / 10, 1);
|
if (performance.errorRateLast30Days >= 0) { score += 1 - performance.errorRateLast30Days; factors++; }
|
||||||
factors++;
|
|
||||||
}
|
|
||||||
if (engagement.uniqueFeaturesUsed > 0) {
|
|
||||||
score += Math.min(engagement.uniqueFeaturesUsed / 5, 1);
|
|
||||||
factors++;
|
|
||||||
}
|
|
||||||
if (performance.errorRateLast30Days >= 0) {
|
|
||||||
score += 1 - performance.errorRateLast30Days;
|
|
||||||
factors++;
|
|
||||||
}
|
|
||||||
|
|
||||||
return factors > 0 ? score / factors : 0;
|
return factors > 0 ? score / factors : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Placeholder implementations for product-specific helpers
|
|
||||||
function calculateAutophagyEngagement(events: TelemetryEventDoc[]): number {
|
function calculateAutophagyEngagement(events: TelemetryEventDoc[]): number {
|
||||||
const autophagyEvents = events.filter((e) => e.properties?.stage === 'autophagy');
|
return Math.min(events.filter((e) => e.context?.stage === 'autophagy').length / 10, 1);
|
||||||
return Math.min(autophagyEvents.length / 10, 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateSkillProgression(events: TelemetryEventDoc[]): number {
|
function calculateSkillProgression(events: TelemetryEventDoc[]): number {
|
||||||
if (events.length === 0) return 0;
|
return events.length ? events.filter((e) => e.context?.progressed).length / events.length : 0;
|
||||||
const progressed = events.filter((e) => e.properties?.progressed).length;
|
|
||||||
return progressed / events.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateSessionCompletionRate(events: TelemetryEventDoc[]): number {
|
function calculateSessionCompletionRate(events: TelemetryEventDoc[]): number {
|
||||||
@ -804,61 +638,50 @@ function calculateSessionCompletionRate(events: TelemetryEventDoc[]): number {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function calculateRoutineAdherence(events: TelemetryEventDoc[]): number {
|
function calculateRoutineAdherence(events: TelemetryEventDoc[]): number {
|
||||||
if (events.length === 0) return 0;
|
return events.length ? events.filter((e) => e.context?.onTime).length / events.length : 0;
|
||||||
const onTime = events.filter((e) => e.properties?.onTime).length;
|
|
||||||
return onTime / events.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateUrgencyResponse(events: TelemetryEventDoc[]): number {
|
function calculateUrgencyResponse(events: TelemetryEventDoc[]): number {
|
||||||
const urgent = events.filter((e) => e.properties?.urgent === true);
|
const urgent = events.filter((e) => e.context?.urgent === true);
|
||||||
if (urgent.length === 0) return 0;
|
return urgent.length ? urgent.filter((e) => e.context?.responded).length / urgent.length : 0;
|
||||||
const responded = urgent.filter((e) => e.properties?.responded).length;
|
|
||||||
return responded / urgent.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateVocabularyGrowth(events: TelemetryEventDoc[]): number {
|
function calculateVocabularyGrowth(events: TelemetryEventDoc[]): number {
|
||||||
const wordsAdded = events.reduce((sum, e) => sum + (e.properties?.wordsAdded || 0), 0);
|
return events.reduce((sum, e) => sum + ((e.metrics?.wordsAdded as number) || 0), 0) / 30;
|
||||||
return wordsAdded / 30; // per day
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateOnboardingCompletion(events: TelemetryEventDoc[]): number {
|
function calculateOnboardingCompletion(events: TelemetryEventDoc[]): number {
|
||||||
const onboardingSteps = events.filter((e) => e.eventName?.includes('onboarding'));
|
const steps = events.filter((e) => e.eventName?.includes('onboarding'));
|
||||||
const completed = onboardingSteps.filter((e) => e.properties?.completed).length;
|
return Math.min(steps.filter((e) => e.context?.completed).length / 5, 1);
|
||||||
const totalSteps = 5; // Configurable
|
|
||||||
return Math.min(completed / totalSteps, 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasFirstValueMoment(events: TelemetryEventDoc[]): boolean {
|
function hasFirstValueMoment(events: TelemetryEventDoc[]): boolean {
|
||||||
return events.some((e) => e.eventName?.includes('first_value') || e.properties?.ahaMoment);
|
return events.some((e) => e.eventName?.includes('first_value') || e.context?.ahaMoment);
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateTimeToFirstValue(events: TelemetryEventDoc[]): number {
|
function calculateTimeToFirstValue(events: TelemetryEventDoc[]): number {
|
||||||
const firstSession = events.find((e) => e.eventType === 'session_start');
|
const firstSession = events.find((e) => e.eventName?.includes('session_start'));
|
||||||
const firstValue = events.find((e) => e.eventName?.includes('first_value'));
|
const firstValue = events.find((e) => e.eventName?.includes('first_value'));
|
||||||
if (!firstSession || !firstValue) return 0;
|
return firstSession && firstValue
|
||||||
return (
|
? (new Date(firstValue.occurredAt).getTime() - new Date(firstSession.occurredAt).getTime()) / (1000 * 60 * 60)
|
||||||
(new Date(firstValue.timestamp).getTime() - new Date(firstSession.timestamp).getTime()) /
|
: 0;
|
||||||
(1000 * 60 * 60)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function countCrashes(events: TelemetryEventDoc[]): number {
|
function countCrashes(events: TelemetryEventDoc[]): number {
|
||||||
return events.filter((e) => e.eventName?.includes('crash') || e.properties?.crash).length;
|
return events.filter((e) => e.eventName?.includes('crash') || e.context?.crash).length;
|
||||||
}
|
}
|
||||||
|
|
||||||
function countSlowRequests(events: TelemetryEventDoc[]): number {
|
function countSlowRequests(events: TelemetryEventDoc[]): number {
|
||||||
return events.filter((e) => e.duration && e.duration > 5000).length;
|
return events.filter((e) => e.metrics?.duration && e.metrics.duration > 5000).length;
|
||||||
}
|
}
|
||||||
|
|
||||||
function countTimeouts(events: TelemetryEventDoc[]): number {
|
function countTimeouts(events: TelemetryEventDoc[]): number {
|
||||||
return events.filter((e) => e.properties?.timeout || e.eventName?.includes('timeout')).length;
|
return events.filter((e) => e.context?.timeout || e.eventName?.includes('timeout')).length;
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateErrorRecoveryRate(events: TelemetryEventDoc[]): number {
|
function calculateErrorRecoveryRate(events: TelemetryEventDoc[]): number {
|
||||||
const errors = events.filter((e) => e.eventType === 'error');
|
const errors = events.filter((e) => e.eventType === 'error' || e.eventType === 'fatal');
|
||||||
if (errors.length === 0) return 1;
|
return errors.length ? errors.filter((e) => e.context?.recovered).length / errors.length : 1;
|
||||||
const recovered = errors.filter((e) => e.properties?.recovered).length;
|
|
||||||
return recovered / errors.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function countSupportTickets(events: TelemetryEventDoc[]): number {
|
function countSupportTickets(events: TelemetryEventDoc[]): number {
|
||||||
@ -866,67 +689,58 @@ function countSupportTickets(events: TelemetryEventDoc[]): number {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function calculateCollaborationScore(events: TelemetryEventDoc[]): number {
|
function calculateCollaborationScore(events: TelemetryEventDoc[]): number {
|
||||||
const collabEvents = events.filter((e) => e.properties?.collaborative === true);
|
return Math.min(events.filter((e) => e.context?.collaborative === true).length / 10, 1);
|
||||||
return Math.min(collabEvents.length / 10, 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractTeamMemberCount(events: TelemetryEventDoc[]): number {
|
function extractTeamMemberCount(events: TelemetryEventDoc[]): number {
|
||||||
const teamEvents = events.filter((e) => e.properties?.teamSize !== undefined);
|
const teamEvents = events.filter((e) => e.context?.teamSize !== undefined);
|
||||||
return teamEvents.length > 0 ? Math.max(...teamEvents.map((e) => e.properties?.teamSize || 0)) : 0;
|
return teamEvents.length ? Math.max(...teamEvents.map((e) => (e.context?.teamSize as number) || 0)) : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractPlanTier(events: TelemetryEventDoc[]): number {
|
function extractPlanTier(events: TelemetryEventDoc[]): number {
|
||||||
const planEvent = events.find((e) => e.properties?.planTier !== undefined);
|
const planEvent = events.find((e) => e.context?.planTier !== undefined);
|
||||||
return planEvent?.properties?.planTier || 0;
|
return (planEvent?.context?.planTier as number) || 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractLifetimeValue(events: TelemetryEventDoc[]): number {
|
function extractLifetimeValue(events: TelemetryEventDoc[]): number {
|
||||||
return events.reduce((sum, e) => sum + (e.properties?.revenue || 0), 0);
|
return events.reduce((sum, e) => sum + ((e.metrics?.revenue as number) || 0), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractMrrContribution(events: TelemetryEventDoc[]): number {
|
function extractMrrContribution(events: TelemetryEventDoc[]): number {
|
||||||
const mrrEvent = events.find((e) => e.properties?.mrr !== undefined);
|
const mrrEvent = events.find((e) => e.metrics?.mrr !== undefined);
|
||||||
return mrrEvent?.properties?.mrr || 0;
|
return (mrrEvent?.metrics?.mrr as number) || 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractDaysSincePayment(events: TelemetryEventDoc[]): number {
|
function extractDaysSincePayment(events: TelemetryEventDoc[]): number {
|
||||||
const paymentEvent = events
|
const paymentEvent = events
|
||||||
.filter((e) => e.eventName?.includes('payment'))
|
.filter((e) => e.eventName?.includes('payment'))
|
||||||
.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())[0];
|
.sort((a, b) => new Date(b.occurredAt).getTime() - new Date(a.occurredAt).getTime())[0];
|
||||||
return paymentEvent ? daysBetween(paymentEvent.timestamp, new Date()) : 30;
|
return paymentEvent ? daysBetween(paymentEvent.occurredAt, new Date()) : 30;
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractDaysSincePlanChange(events: TelemetryEventDoc[]): number {
|
function extractDaysSincePlanChange(events: TelemetryEventDoc[]): number {
|
||||||
const planChange = events
|
const planChange = events
|
||||||
.filter((e) => e.eventName?.includes('plan_change'))
|
.filter((e) => e.eventName?.includes('plan_change'))
|
||||||
.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())[0];
|
.sort((a, b) => new Date(b.occurredAt).getTime() - new Date(a.occurredAt).getTime())[0];
|
||||||
return planChange ? daysBetween(planChange.timestamp, new Date()) : 90;
|
return planChange ? daysBetween(planChange.occurredAt, new Date()) : 90;
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateSupportSatisfaction(events: TelemetryEventDoc[]): number {
|
function calculateSupportSatisfaction(events: TelemetryEventDoc[]): number {
|
||||||
const ratedEvents = events.filter((e) => e.properties?.satisfaction !== undefined);
|
const rated = events.filter((e) => e.context?.satisfaction !== undefined);
|
||||||
if (ratedEvents.length === 0) return 0;
|
return rated.length ? rated.reduce((acc, e) => acc + ((e.context?.satisfaction as number) || 0), 0) / rated.length : 0;
|
||||||
const sum = ratedEvents.reduce((acc, e) => acc + (e.properties?.satisfaction || 0), 0);
|
|
||||||
return sum / ratedEvents.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function estimateCohortPercentile(value: number, metric: string): number {
|
function estimateCohortPercentile(value: number, metric: string): number {
|
||||||
// Simplified estimation - in production, this would query cohort data
|
const baselines: Record<string, number> = { sessions: 2, features: 5, retention: 15 };
|
||||||
const baselines: Record<string, number> = {
|
return Math.min(Math.round((value / (baselines[metric] || 1)) * 50), 100);
|
||||||
sessions: 2,
|
|
||||||
features: 5,
|
|
||||||
retention: 15,
|
|
||||||
};
|
|
||||||
const baseline = baselines[metric] || 1;
|
|
||||||
return Math.min(Math.round((value / baseline) * 50), 100);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function weeklyEvents(events: TelemetryEventDoc[]): TelemetryEventDoc[] {
|
function getWeeklyEvents(events: TelemetryEventDoc[]): TelemetryEventDoc[] {
|
||||||
const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
||||||
return events.filter((e) => new Date(e.timestamp) >= weekAgo);
|
return events.filter((e) => new Date(e.occurredAt) >= weekAgo);
|
||||||
}
|
}
|
||||||
|
|
||||||
function monthlyEvents(events: TelemetryEventDoc[]): TelemetryEventDoc[] {
|
function getMonthlyEvents(events: TelemetryEventDoc[]): TelemetryEventDoc[] {
|
||||||
const monthAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
|
const monthAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
|
||||||
return events.filter((e) => new Date(e.timestamp) >= monthAgo);
|
return events.filter((e) => new Date(e.occurredAt) >= monthAgo);
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user