diff --git a/services/mcp-server/src/modules/platform/experiments-tools.ts b/services/mcp-server/src/modules/platform/experiments-tools.ts index 73daaa2c..d4757b23 100644 --- a/services/mcp-server/src/modules/platform/experiments-tools.ts +++ b/services/mcp-server/src/modules/platform/experiments-tools.ts @@ -2,6 +2,8 @@ import { z } from 'zod'; import { registerTool } from '../tools/registry.js'; import { platformFetch } from '../../lib/platform-client.js'; +const metricTypeSchema = z.enum(['conversion', 'count', 'duration', 'revenue', 'custom']); + registerTool({ name: 'experiments.create', description: @@ -29,7 +31,7 @@ registerTool({ trafficPercent: z .number() .int() - .min(0) + .min(1) .max(100) .default(10) .describe('Percentage of eligible users to enroll'), @@ -50,25 +52,53 @@ registerTool({ } = args; const experiment = { - productId, - key, name, - description, - variants, - targetSegments, - trafficPercent, - hypothesis, - primaryMetric, - status: 'draft' as const, + description: description ?? '', + hypothesis: hypothesis ?? `Test whether ${name} improves ${primaryMetric} for ${productId}`, + variants: variants.map( + (variant: { key: string; weight: number; description: string }, index: number) => ({ + key: variant.key, + name: variant.key, + description: variant.description, + isControl: index === 0 || variant.key === 'control', + flagConfig: { + key, + variantKey: variant.key, + }, + }) + ), + allocationStrategy: 'random' as const, + targetPercent: trafficPercent, + targeting: targetSegments?.length ? { userSegments: targetSegments } : {}, + primaryMetric: { + name: primaryMetric, + type: 'conversion' as const, + eventName: primaryMetric, + aggregation: 'count' as const, + direction: 'increase' as const, + minimumDetectableEffect: 5, + }, + secondaryMetrics: [], + guardrails: { + minSampleSizePerVariant: 100, + maxDurationDays: 30, + autoStopEnabled: true, + winnerThreshold: 95, + requireApprovalFor: 'none' as const, + }, }; - const response = await platformFetch<{ id: string }>( - '/api/experiments', + const response = await platformFetch<{ id: string; status: string }>( + '/api/ab-testing/experiments', { method: 'POST', body: JSON.stringify(experiment), }, - { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } + { + token: req.headers.authorization?.replace('Bearer ', '') || '', + requestId: req.id, + productId, + } ); return { @@ -76,7 +106,7 @@ registerTool({ productId, key, name, - status: 'draft', + status: response.status, summary: `Created experiment "${name}" with ${variants.length} variants for ${productId}`, }; }, @@ -93,7 +123,7 @@ registerTool({ const { experimentId } = args; const response = await platformFetch<{ id: string; name: string; status: string }>( - `/api/experiments/${experimentId}/start`, + `/api/ab-testing/experiments/${experimentId}/start`, { method: 'POST' }, { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } ); @@ -134,31 +164,36 @@ registerTool({ if (status) params.set('status', status); params.set('limit', limit.toString()); - const response = await platformFetch<{ - experiments: Array<{ + const response = await platformFetch< + Array<{ id: string; - key: string; name: string; - productId: string; + description: string; status: string; - variants: Array<{ key: string; weight: number; description: string }>; - trafficPercent: number; - startedAt: string | null; - endedAt: string | null; + targetPercent: number; createdAt: string; - }>; - total: number; - }>( - `/api/experiments?${params}`, + startedAt?: string; + completedAt?: string; + }> + >( + '/api/ab-testing/experiments', { method: 'GET' }, - { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } + { + token: req.headers.authorization?.replace('Bearer ', '') || '', + requestId: req.id, + productId, + } ); + const experiments = response + .filter(experiment => (status ? experiment.status === status : true)) + .slice(0, limit); + return { - experiments: response.experiments, - total: response.total, + experiments, + total: experiments.length, filters: { productId, status, limit }, - summary: `Found ${response.experiments.length} experiments matching criteria`, + summary: `Found ${experiments.length} experiments matching criteria`, }; }, }); @@ -181,39 +216,38 @@ registerTool({ const { experimentId, confidenceLevel } = args; const response = await platformFetch<{ - experiment: { - id: string; - name: string; - status: string; - primaryMetric: string; - variants: Array<{ - key: string; - description: string; - sampleSize: number; - conversionRate?: number; - meanValue?: number; - statisticalSignificance?: number; - isWinner?: boolean; - }>; - }; - insights: { - totalSampleSize: number; - duration: number; // days - statisticalPower: number; - recommendation: 'continue' | 'stop' | 'inconclusive'; - confidence: number; + experimentId: string; + status: string; + totalParticipants: number; + totalEvents: number; + daysRunning: number; + winnerVariantId?: string; + winnerProbability?: number; + variantResults: Array<{ + variantId: string; + variantName: string; + isControl: boolean; + participants: number; + primaryMetricValue: number; + probabilityBeatsControl: number; + expectedLiftPercent: number; + credibleInterval: { lower: number; mean: number; upper: number }; + }>; + statisticalSummary: { + probabilityAnyBeatsControl: number; + expectedLossIfShipped: number; + recommendedAction: 'ship' | 'rollback' | 'continue' | 'stop'; }; }>( - `/api/experiments/${experimentId}/results?confidence=${confidenceLevel}`, + `/api/ab-testing/experiments/${experimentId}/results`, { method: 'GET' }, { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } ); return { - experiment: response.experiment, - insights: response.insights, + results: response, confidenceLevel, - summary: `Results for "${response.experiment.name}": ${response.insights.recommendation} with ${response.insights.confidence}% confidence`, + summary: `Results for ${response.experimentId}: ${response.statisticalSummary.recommendedAction} after ${response.totalParticipants} participants`, }; }, }); @@ -225,36 +259,41 @@ registerTool({ requiredRole: 'viewer', inputSchema: z.object({ experimentKey: z.string().min(1).describe('Experiment key (slug)'), - userId: z.string().min(1).describe('User ID to assign variant'), context: z.record(z.unknown()).optional().describe('Optional user context for targeting'), }), async execute(args, req) { - const { experimentKey, userId, context } = args; + const { experimentKey, context } = args; const response = await platformFetch<{ - variant: { - key: string; - description: string; - }; - enrolled: boolean; - reason?: string; // Why not enrolled if enrolled=false + assigned: boolean; + reason?: string; + experimentId?: string; + variantId?: string; + variantName?: string; + isControl?: boolean; + flagConfig?: Record; + isNew?: boolean; }>( - `/api/experiments/${experimentKey}/assign`, + '/api/ab-testing/assign', { method: 'POST', - body: JSON.stringify({ userId, context }), + body: JSON.stringify({ experimentKey, context }), }, { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } ); return { experimentKey, - userId, - variant: response.variant, - enrolled: response.enrolled, + assigned: response.assigned, reason: response.reason, - summary: response.enrolled - ? `User assigned to variant "${response.variant.key}"` + experimentId: response.experimentId, + variantId: response.variantId, + variantName: response.variantName, + isControl: response.isControl, + flagConfig: response.flagConfig, + isNew: response.isNew, + summary: response.assigned + ? `User assigned to variant "${response.variantName}"` : `User not enrolled: ${response.reason}`, }; }, @@ -266,48 +305,50 @@ registerTool({ 'Track an event for a user in an experiment (conversion, retention, etc.). Requires viewer role.', requiredRole: 'viewer', inputSchema: z.object({ - experimentKey: z.string().min(1).describe('Experiment key'), + experimentId: z.string().min(1).describe('Experiment ID'), userId: z.string().min(1).describe('User ID'), - eventType: z.string().min(1).describe('Event type (conversion, retention, custom)'), - eventName: z.string().min(1).describe('Specific event name'), - value: z.number().optional().describe('Numeric value for the event'), + metricType: metricTypeSchema.describe('Metric type'), + metricName: z.string().min(1).describe('Metric/event name'), + value: z.number().describe('Numeric value for the event'), + platform: z.string().optional().describe('Platform context (web, ios, android)'), + appVersion: z.string().optional().describe('App version context'), metadata: z.record(z.unknown()).optional().describe('Additional event metadata'), }), async execute(args, req) { - const { experimentKey, userId, eventType, eventName, value, metadata } = args; + const { experimentId, userId, metricType, metricName, value, platform, appVersion, metadata } = + args; const response = await platformFetch<{ tracked: boolean; - variant?: { - key: string; - description: string; - }; }>( - `/api/experiments/${experimentKey}/track`, + '/api/ab-testing/events', { method: 'POST', body: JSON.stringify({ + experimentId, userId, - eventType, - eventName, + metricType, + metricName, value, - metadata, + converted: true, + eventMetadata: metadata, + platform: platform ?? 'unknown', + appVersion: appVersion ?? 'unknown', }), }, { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } ); return { - experimentKey, + experimentId, userId, - eventType, - eventName, + metricType, + metricName, tracked: response.tracked, - variant: response.variant, value, summary: response.tracked - ? `Tracked ${eventType} event for user in variant "${response.variant?.key}"` - : 'Event not tracked (user not enrolled in experiment)', + ? `Tracked ${metricType} metric "${metricName}" for experiment ${experimentId}` + : `Event not tracked for experiment ${experimentId}`, }; }, }); diff --git a/services/mcp-server/src/modules/platform/secrets-tools.ts b/services/mcp-server/src/modules/platform/secrets-tools.ts index c1d253e0..fcd78db8 100644 --- a/services/mcp-server/src/modules/platform/secrets-tools.ts +++ b/services/mcp-server/src/modules/platform/secrets-tools.ts @@ -1,23 +1,51 @@ import { z } from 'zod'; +import { resolveSecrets, type SecretMapping } from '@bytelyst/config'; import { registerTool } from '../tools/registry.js'; -// Local interfaces since import is failing -interface SecretMapping { - kvName: string; - envVar: string; -} +const COMMON_MAPPINGS: SecretMapping[] = [ + { kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' }, + { kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' }, + { kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' }, + { kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, + { kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, +]; -// Mock resolveSecrets function for now - will need proper implementation -async function resolveSecrets( - secrets: SecretMapping[], - _opts?: { vaultUrl?: string } -): Promise { - // This is a placeholder - in real implementation would fetch from Key Vault - for (const secret of secrets) { - if (!process.env[secret.envVar]) { - process.env[secret.envVar] = `mock-value-for-${secret.kvName}`; - } +const PRODUCT_MAPPINGS: Record = { + lysnrai: [ + { kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, + { kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, + { kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' }, + { kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' }, + { kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' }, + ], + chronomind: [ + { kvName: 'chronomind-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, + { kvName: 'chronomind-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, + ], + jarvisjr: [ + { kvName: 'jarvis-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, + { kvName: 'jarvis-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, + ], + nomgap: [ + { kvName: 'nomgap-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, + { kvName: 'nomgap-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, + ], + peakpulse: [ + { kvName: 'peakpulse-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, + { kvName: 'peakpulse-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, + ], + mindlyst: [ + { kvName: 'mindlyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, + { kvName: 'mindlyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, + ], +}; + +function getMappings(productId?: string): SecretMapping[] { + if (productId && PRODUCT_MAPPINGS[productId]) { + return [...COMMON_MAPPINGS, ...PRODUCT_MAPPINGS[productId]]; } + + return [...COMMON_MAPPINGS]; } registerTool({ @@ -33,51 +61,7 @@ registerTool({ }), async execute(args, _req) { const { productId } = args; - - // Common platform mappings - const commonMappings: SecretMapping[] = [ - { kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' }, - { kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' }, - { kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' }, - { kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - ]; - - // Product-specific mappings - const productMappings: Record = { - lysnrai: [ - { kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - { kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' }, - { kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' }, - { kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' }, - ], - chronomind: [ - { kvName: 'chronomind-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'chronomind-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - ], - jarvisjr: [ - { kvName: 'jarvis-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'jarvis-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - ], - nomgap: [ - { kvName: 'nomgap-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'nomgap-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - ], - peakpulse: [ - { kvName: 'peakpulse-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'peakpulse-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - ], - mindlyst: [ - { kvName: 'mindlyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'mindlyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - ], - }; - - let mappings = [...commonMappings]; - if (productId && productMappings[productId]) { - mappings = [...mappings, ...productMappings[productId]]; - } + const mappings = getMappings(productId); // Show current status const status = mappings.map(mapping => ({ @@ -196,29 +180,7 @@ registerTool({ req.log.info({ runId, productId, dryRun }, 'Starting secret mapping validation'); try { - // Get mappings (reuse logic from listMappings) - const commonMappings: SecretMapping[] = [ - { kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' }, - { kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' }, - { kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' }, - { kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - ]; - - const productMappings: Record = { - lysnrai: [ - { kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' }, - { kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' }, - { kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' }, - { kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' }, - { kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' }, - ], - }; - - let mappings = [...commonMappings]; - if (productId && productMappings[productId]) { - mappings = [...mappings, ...productMappings[productId]]; - } + const mappings = getMappings(productId); const results = []; let successCount = 0;