bytelyst/learning_ai_common_plat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(mcp-server): align secret and experiment tools with real services

Browse Source
This commit is contained in:
saravanakumardb1 2026-03-05 22:36:30 -08:00
parent 53f34851df
commit b199ea7976
2 changed files with 176 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

221
services/mcp-server/src/modules/platform/experiments-tools.ts
View File

@ -2,6 +2,8 @@ import { z } from 'zod';
import { registerTool } from '../tools/registry.js'; import { registerTool } from '../tools/registry.js';
import { platformFetch } from '../../lib/platform-client.js'; import { platformFetch } from '../../lib/platform-client.js';
const metricTypeSchema = z.enum(['conversion', 'count', 'duration', 'revenue', 'custom']);
registerTool({ registerTool({
name: 'experiments.create', name: 'experiments.create',
description: description:
@ -29,7 +31,7 @@ registerTool({
trafficPercent: z trafficPercent: z
.number() .number()
.int() .int()
.min(0) .min(1)
.max(100) .max(100)
.default(10) .default(10)
.describe('Percentage of eligible users to enroll'), .describe('Percentage of eligible users to enroll'),
@ -50,25 +52,53 @@ registerTool({
} = args; } = args;
const experiment = { const experiment = {
productId,
key,
name, name,
description, description: description ?? '',
variants, hypothesis: hypothesis ?? `Test whether ${name} improves ${primaryMetric} for ${productId}`,
targetSegments, variants: variants.map(
trafficPercent, (variant: { key: string; weight: number; description: string }, index: number) => ({
hypothesis, key: variant.key,
primaryMetric, name: variant.key,
status: 'draft' as const, description: variant.description,
isControl: index === 0 || variant.key === 'control',
flagConfig: {
key,
variantKey: variant.key,
},
})
),
allocationStrategy: 'random' as const,
targetPercent: trafficPercent,
targeting: targetSegments?.length ? { userSegments: targetSegments } : {},
primaryMetric: {
name: primaryMetric,
type: 'conversion' as const,
eventName: primaryMetric,
aggregation: 'count' as const,
direction: 'increase' as const,
minimumDetectableEffect: 5,
},
secondaryMetrics: [],
guardrails: {
minSampleSizePerVariant: 100,
maxDurationDays: 30,
autoStopEnabled: true,
winnerThreshold: 95,
requireApprovalFor: 'none' as const,
},
}; };
const response = await platformFetch<{ id: string }>( const response = await platformFetch<{ id: string; status: string }>(
'/api/experiments', '/api/ab-testing/experiments',
{ {
method: 'POST', method: 'POST',
body: JSON.stringify(experiment), body: JSON.stringify(experiment),
}, },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } {
token: req.headers.authorization?.replace('Bearer ', '') || '',
requestId: req.id,
productId,
}
); );
return { return {
@ -76,7 +106,7 @@ registerTool({
productId, productId,
key, key,
name, name,
status: 'draft', status: response.status,
summary: `Created experiment "${name}" with ${variants.length} variants for ${productId}`, summary: `Created experiment "${name}" with ${variants.length} variants for ${productId}`,
}; };
}, },
@ -93,7 +123,7 @@ registerTool({
const { experimentId } = args; const { experimentId } = args;
const response = await platformFetch<{ id: string; name: string; status: string }>( const response = await platformFetch<{ id: string; name: string; status: string }>(
`/api/experiments/${experimentId}/start`, `/api/ab-testing/experiments/${experimentId}/start`,
{ method: 'POST' }, { method: 'POST' },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
); );
@ -134,31 +164,36 @@ registerTool({
if (status) params.set('status', status); if (status) params.set('status', status);
params.set('limit', limit.toString()); params.set('limit', limit.toString());
const response = await platformFetch<{ const response = await platformFetch<
experiments: Array<{ Array<{
id: string; id: string;
key: string;
name: string; name: string;
productId: string; description: string;
status: string; status: string;
variants: Array<{ key: string; weight: number; description: string }>; targetPercent: number;
trafficPercent: number;
startedAt: string | null;
endedAt: string | null;
createdAt: string; createdAt: string;
}>; startedAt?: string;
total: number; completedAt?: string;
}>( }>
`/api/experiments?${params}`, >(
'/api/ab-testing/experiments',
{ method: 'GET' }, { method: 'GET' },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } {
token: req.headers.authorization?.replace('Bearer ', '') || '',
requestId: req.id,
productId,
}
); );
const experiments = response
.filter(experiment => (status ? experiment.status === status : true))
.slice(0, limit);
return { return {
experiments: response.experiments, experiments,
total: response.total, total: experiments.length,
filters: { productId, status, limit }, filters: { productId, status, limit },
summary: `Found ${response.experiments.length} experiments matching criteria`, summary: `Found ${experiments.length} experiments matching criteria`,
}; };
}, },
}); });
@ -181,39 +216,38 @@ registerTool({
const { experimentId, confidenceLevel } = args; const { experimentId, confidenceLevel } = args;
const response = await platformFetch<{ const response = await platformFetch<{
experiment: { experimentId: string;
id: string; status: string;
name: string; totalParticipants: number;
status: string; totalEvents: number;
primaryMetric: string; daysRunning: number;
variants: Array<{ winnerVariantId?: string;
key: string; winnerProbability?: number;
description: string; variantResults: Array<{
sampleSize: number; variantId: string;
conversionRate?: number; variantName: string;
meanValue?: number; isControl: boolean;
statisticalSignificance?: number; participants: number;
isWinner?: boolean; primaryMetricValue: number;
}>; probabilityBeatsControl: number;
}; expectedLiftPercent: number;
insights: { credibleInterval: { lower: number; mean: number; upper: number };
totalSampleSize: number; }>;
duration: number; // days statisticalSummary: {
statisticalPower: number; probabilityAnyBeatsControl: number;
recommendation: 'continue' | 'stop' | 'inconclusive'; expectedLossIfShipped: number;
confidence: number; recommendedAction: 'ship' | 'rollback' | 'continue' | 'stop';
}; };
}>( }>(
`/api/experiments/${experimentId}/results?confidence=${confidenceLevel}`, `/api/ab-testing/experiments/${experimentId}/results`,
{ method: 'GET' }, { method: 'GET' },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
); );
return { return {
experiment: response.experiment, results: response,
insights: response.insights,
confidenceLevel, confidenceLevel,
summary: `Results for "${response.experiment.name}": ${response.insights.recommendation} with ${response.insights.confidence}% confidence`, summary: `Results for ${response.experimentId}: ${response.statisticalSummary.recommendedAction} after ${response.totalParticipants} participants`,
}; };
}, },
}); });
@ -225,36 +259,41 @@ registerTool({
requiredRole: 'viewer', requiredRole: 'viewer',
inputSchema: z.object({ inputSchema: z.object({
experimentKey: z.string().min(1).describe('Experiment key (slug)'), experimentKey: z.string().min(1).describe('Experiment key (slug)'),
userId: z.string().min(1).describe('User ID to assign variant'),
context: z.record(z.unknown()).optional().describe('Optional user context for targeting'), context: z.record(z.unknown()).optional().describe('Optional user context for targeting'),
}), }),
async execute(args, req) { async execute(args, req) {
const { experimentKey, userId, context } = args; const { experimentKey, context } = args;
const response = await platformFetch<{ const response = await platformFetch<{
variant: { assigned: boolean;
key: string; reason?: string;
description: string; experimentId?: string;
}; variantId?: string;
enrolled: boolean; variantName?: string;
reason?: string; // Why not enrolled if enrolled=false isControl?: boolean;
flagConfig?: Record<string, unknown>;
isNew?: boolean;
}>( }>(
`/api/experiments/${experimentKey}/assign`, '/api/ab-testing/assign',
{ {
method: 'POST', method: 'POST',
body: JSON.stringify({ userId, context }), body: JSON.stringify({ experimentKey, context }),
}, },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
); );
return { return {
experimentKey, experimentKey,
userId, assigned: response.assigned,
variant: response.variant,
enrolled: response.enrolled,
reason: response.reason, reason: response.reason,
summary: response.enrolled experimentId: response.experimentId,
? `User assigned to variant "${response.variant.key}"` variantId: response.variantId,
variantName: response.variantName,
isControl: response.isControl,
flagConfig: response.flagConfig,
isNew: response.isNew,
summary: response.assigned
? `User assigned to variant "${response.variantName}"`
: `User not enrolled: ${response.reason}`, : `User not enrolled: ${response.reason}`,
}; };
}, },
@ -266,48 +305,50 @@ registerTool({
'Track an event for a user in an experiment (conversion, retention, etc.). Requires viewer role.', 'Track an event for a user in an experiment (conversion, retention, etc.). Requires viewer role.',
requiredRole: 'viewer', requiredRole: 'viewer',
inputSchema: z.object({ inputSchema: z.object({
experimentKey: z.string().min(1).describe('Experiment key'), experimentId: z.string().min(1).describe('Experiment ID'),
userId: z.string().min(1).describe('User ID'), userId: z.string().min(1).describe('User ID'),
eventType: z.string().min(1).describe('Event type (conversion, retention, custom)'), metricType: metricTypeSchema.describe('Metric type'),
eventName: z.string().min(1).describe('Specific event name'), metricName: z.string().min(1).describe('Metric/event name'),
value: z.number().optional().describe('Numeric value for the event'), value: z.number().describe('Numeric value for the event'),
platform: z.string().optional().describe('Platform context (web, ios, android)'),
appVersion: z.string().optional().describe('App version context'),
metadata: z.record(z.unknown()).optional().describe('Additional event metadata'), metadata: z.record(z.unknown()).optional().describe('Additional event metadata'),
}), }),
async execute(args, req) { async execute(args, req) {
const { experimentKey, userId, eventType, eventName, value, metadata } = args; const { experimentId, userId, metricType, metricName, value, platform, appVersion, metadata } =
args;
const response = await platformFetch<{ const response = await platformFetch<{
tracked: boolean; tracked: boolean;
variant?: {
key: string;
description: string;
};
}>( }>(
`/api/experiments/${experimentKey}/track`, '/api/ab-testing/events',
{ {
method: 'POST', method: 'POST',
body: JSON.stringify({ body: JSON.stringify({
experimentId,
userId, userId,
eventType, metricType,
eventName, metricName,
value, value,
metadata, converted: true,
eventMetadata: metadata,
platform: platform ?? 'unknown',
appVersion: appVersion ?? 'unknown',
}), }),
}, },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id } { token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
); );
return { return {
experimentKey, experimentId,
userId, userId,
eventType, metricType,
eventName, metricName,
tracked: response.tracked, tracked: response.tracked,
variant: response.variant,
value, value,
summary: response.tracked summary: response.tracked
? `Tracked ${eventType} event for user in variant "${response.variant?.key}"` ? `Tracked ${metricType} metric "${metricName}" for experiment ${experimentId}`
: 'Event not tracked (user not enrolled in experiment)', : `Event not tracked for experiment ${experimentId}`,
}; };
}, },
}); });

128
services/mcp-server/src/modules/platform/secrets-tools.ts
View File

@ -1,23 +1,51 @@
import { z } from 'zod'; import { z } from 'zod';
import { resolveSecrets, type SecretMapping } from '@bytelyst/config';
import { registerTool } from '../tools/registry.js'; import { registerTool } from '../tools/registry.js';
// Local interfaces since import is failing const COMMON_MAPPINGS: SecretMapping[] = [
interface SecretMapping { { kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' },
kvName: string; { kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' },
envVar: string; { kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' },
} { kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
];
// Mock resolveSecrets function for now - will need proper implementation const PRODUCT_MAPPINGS: Record<string, SecretMapping[]> = {
async function resolveSecrets( lysnrai: [
secrets: SecretMapping[], { kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
_opts?: { vaultUrl?: string } { kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
): Promise<void> { { kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' },
// This is a placeholder - in real implementation would fetch from Key Vault { kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' },
for (const secret of secrets) { { kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' },
if (!process.env[secret.envVar]) { ],
process.env[secret.envVar] = `mock-value-for-${secret.kvName}`; chronomind: [
} { kvName: 'chronomind-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'chronomind-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
jarvisjr: [
{ kvName: 'jarvis-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'jarvis-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
nomgap: [
{ kvName: 'nomgap-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'nomgap-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
peakpulse: [
{ kvName: 'peakpulse-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'peakpulse-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
mindlyst: [
{ kvName: 'mindlyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'mindlyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
};
function getMappings(productId?: string): SecretMapping[] {
if (productId && PRODUCT_MAPPINGS[productId]) {
return [...COMMON_MAPPINGS, ...PRODUCT_MAPPINGS[productId]];
} }
return [...COMMON_MAPPINGS];
} }
registerTool({ registerTool({
@ -33,51 +61,7 @@ registerTool({
}), }),
async execute(args, _req) { async execute(args, _req) {
const { productId } = args; const { productId } = args;
const mappings = getMappings(productId);
// Common platform mappings
const commonMappings: SecretMapping[] = [
{ kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' },
{ kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' },
{ kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' },
{ kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
];
// Product-specific mappings
const productMappings: Record<string, SecretMapping[]> = {
lysnrai: [
{ kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
{ kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' },
{ kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' },
{ kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' },
],
chronomind: [
{ kvName: 'chronomind-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'chronomind-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
jarvisjr: [
{ kvName: 'jarvis-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'jarvis-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
nomgap: [
{ kvName: 'nomgap-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'nomgap-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
peakpulse: [
{ kvName: 'peakpulse-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'peakpulse-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
mindlyst: [
{ kvName: 'mindlyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'mindlyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
};
let mappings = [...commonMappings];
if (productId && productMappings[productId]) {
mappings = [...mappings, ...productMappings[productId]];
}
// Show current status // Show current status
const status = mappings.map(mapping => ({ const status = mappings.map(mapping => ({
@ -196,29 +180,7 @@ registerTool({
req.log.info({ runId, productId, dryRun }, 'Starting secret mapping validation'); req.log.info({ runId, productId, dryRun }, 'Starting secret mapping validation');
try { try {
// Get mappings (reuse logic from listMappings) const mappings = getMappings(productId);
const commonMappings: SecretMapping[] = [
{ kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' },
{ kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' },
{ kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' },
{ kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
];
const productMappings: Record<string, SecretMapping[]> = {
lysnrai: [
{ kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
{ kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' },
{ kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' },
{ kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' },
],
};
let mappings = [...commonMappings];
if (productId && productMappings[productId]) {
mappings = [...mappings, ...productMappings[productId]];
}
const results = []; const results = [];
let successCount = 0; let successCount = 0;