bytelyst/learning_ai_common_plat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(mcp-server): align secret and experiment tools with real services

Browse Source
This commit is contained in:
saravanakumardb1 2026-03-05 22:36:30 -08:00
parent 53f34851df
commit b199ea7976
2 changed files with 176 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

221
services/mcp-server/src/modules/platform/experiments-tools.ts
View File

@ -2,6 +2,8 @@ import { z } from 'zod';
import { registerTool } from '../tools/registry.js';
import { platformFetch } from '../../lib/platform-client.js';
const metricTypeSchema = z.enum(['conversion', 'count', 'duration', 'revenue', 'custom']);
registerTool({
name: 'experiments.create',
description:
@ -29,7 +31,7 @@ registerTool({
trafficPercent: z
.number()
.int()
.min(0)
.min(1)
.max(100)
.default(10)
.describe('Percentage of eligible users to enroll'),
@ -50,25 +52,53 @@ registerTool({
} = args;
const experiment = {
productId,
key,
name,
description,
variants,
targetSegments,
trafficPercent,
hypothesis,
primaryMetric,
status: 'draft' as const,
description: description ?? '',
hypothesis: hypothesis ?? `Test whether ${name} improves ${primaryMetric} for ${productId}`,
variants: variants.map(
(variant: { key: string; weight: number; description: string }, index: number) => ({
key: variant.key,
name: variant.key,
description: variant.description,
isControl: index === 0 || variant.key === 'control',
flagConfig: {
key,
variantKey: variant.key,
},
})
),
allocationStrategy: 'random' as const,
targetPercent: trafficPercent,
targeting: targetSegments?.length ? { userSegments: targetSegments } : {},
primaryMetric: {
name: primaryMetric,
type: 'conversion' as const,
eventName: primaryMetric,
aggregation: 'count' as const,
direction: 'increase' as const,
minimumDetectableEffect: 5,
},
secondaryMetrics: [],
guardrails: {
minSampleSizePerVariant: 100,
maxDurationDays: 30,
autoStopEnabled: true,
winnerThreshold: 95,
requireApprovalFor: 'none' as const,
},
};
const response = await platformFetch<{ id: string }>(
'/api/experiments',
const response = await platformFetch<{ id: string; status: string }>(
'/api/ab-testing/experiments',
{
method: 'POST',
body: JSON.stringify(experiment),
},
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
{
token: req.headers.authorization?.replace('Bearer ', '') || '',
requestId: req.id,
productId,
}
);
return {
@ -76,7 +106,7 @@ registerTool({
productId,
key,
name,
status: 'draft',
status: response.status,
summary: `Created experiment "${name}" with ${variants.length} variants for ${productId}`,
};
},
@ -93,7 +123,7 @@ registerTool({
const { experimentId } = args;
const response = await platformFetch<{ id: string; name: string; status: string }>(
`/api/experiments/${experimentId}/start`,
`/api/ab-testing/experiments/${experimentId}/start`,
{ method: 'POST' },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
);
@ -134,31 +164,36 @@ registerTool({
if (status) params.set('status', status);
params.set('limit', limit.toString());
const response = await platformFetch<{
experiments: Array<{
const response = await platformFetch<
Array<{
id: string;
key: string;
name: string;
productId: string;
description: string;
status: string;
variants: Array<{ key: string; weight: number; description: string }>;
trafficPercent: number;
startedAt: string | null;
endedAt: string | null;
targetPercent: number;
createdAt: string;
}>;
total: number;
}>(
`/api/experiments?${params}`,
startedAt?: string;
completedAt?: string;
}>
>(
'/api/ab-testing/experiments',
{ method: 'GET' },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
{
token: req.headers.authorization?.replace('Bearer ', '') || '',
requestId: req.id,
productId,
}
);
const experiments = response
.filter(experiment => (status ? experiment.status === status : true))
.slice(0, limit);
return {
experiments: response.experiments,
total: response.total,
experiments,
total: experiments.length,
filters: { productId, status, limit },
summary: `Found ${response.experiments.length} experiments matching criteria`,
summary: `Found ${experiments.length} experiments matching criteria`,
};
},
});
@ -181,39 +216,38 @@ registerTool({
const { experimentId, confidenceLevel } = args;
const response = await platformFetch<{
experiment: {
id: string;
name: string;
status: string;
primaryMetric: string;
variants: Array<{
key: string;
description: string;
sampleSize: number;
conversionRate?: number;
meanValue?: number;
statisticalSignificance?: number;
isWinner?: boolean;
}>;
};
insights: {
totalSampleSize: number;
duration: number; // days
statisticalPower: number;
recommendation: 'continue' | 'stop' | 'inconclusive';
confidence: number;
experimentId: string;
status: string;
totalParticipants: number;
totalEvents: number;
daysRunning: number;
winnerVariantId?: string;
winnerProbability?: number;
variantResults: Array<{
variantId: string;
variantName: string;
isControl: boolean;
participants: number;
primaryMetricValue: number;
probabilityBeatsControl: number;
expectedLiftPercent: number;
credibleInterval: { lower: number; mean: number; upper: number };
}>;
statisticalSummary: {
probabilityAnyBeatsControl: number;
expectedLossIfShipped: number;
recommendedAction: 'ship' | 'rollback' | 'continue' | 'stop';
};
}>(
`/api/experiments/${experimentId}/results?confidence=${confidenceLevel}`,
`/api/ab-testing/experiments/${experimentId}/results`,
{ method: 'GET' },
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
);
return {
experiment: response.experiment,
insights: response.insights,
results: response,
confidenceLevel,
summary: `Results for "${response.experiment.name}": ${response.insights.recommendation} with ${response.insights.confidence}% confidence`,
summary: `Results for ${response.experimentId}: ${response.statisticalSummary.recommendedAction} after ${response.totalParticipants} participants`,
};
},
});
@ -225,36 +259,41 @@ registerTool({
requiredRole: 'viewer',
inputSchema: z.object({
experimentKey: z.string().min(1).describe('Experiment key (slug)'),
userId: z.string().min(1).describe('User ID to assign variant'),
context: z.record(z.unknown()).optional().describe('Optional user context for targeting'),
}),
async execute(args, req) {
const { experimentKey, userId, context } = args;
const { experimentKey, context } = args;
const response = await platformFetch<{
variant: {
key: string;
description: string;
};
enrolled: boolean;
reason?: string; // Why not enrolled if enrolled=false
assigned: boolean;
reason?: string;
experimentId?: string;
variantId?: string;
variantName?: string;
isControl?: boolean;
flagConfig?: Record<string, unknown>;
isNew?: boolean;
}>(
`/api/experiments/${experimentKey}/assign`,
'/api/ab-testing/assign',
{
method: 'POST',
body: JSON.stringify({ userId, context }),
body: JSON.stringify({ experimentKey, context }),
},
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
);
return {
experimentKey,
userId,
variant: response.variant,
enrolled: response.enrolled,
assigned: response.assigned,
reason: response.reason,
summary: response.enrolled
? `User assigned to variant "${response.variant.key}"`
experimentId: response.experimentId,
variantId: response.variantId,
variantName: response.variantName,
isControl: response.isControl,
flagConfig: response.flagConfig,
isNew: response.isNew,
summary: response.assigned
? `User assigned to variant "${response.variantName}"`
: `User not enrolled: ${response.reason}`,
};
},
@ -266,48 +305,50 @@ registerTool({
'Track an event for a user in an experiment (conversion, retention, etc.). Requires viewer role.',
requiredRole: 'viewer',
inputSchema: z.object({
experimentKey: z.string().min(1).describe('Experiment key'),
experimentId: z.string().min(1).describe('Experiment ID'),
userId: z.string().min(1).describe('User ID'),
eventType: z.string().min(1).describe('Event type (conversion, retention, custom)'),
eventName: z.string().min(1).describe('Specific event name'),
value: z.number().optional().describe('Numeric value for the event'),
metricType: metricTypeSchema.describe('Metric type'),
metricName: z.string().min(1).describe('Metric/event name'),
value: z.number().describe('Numeric value for the event'),
platform: z.string().optional().describe('Platform context (web, ios, android)'),
appVersion: z.string().optional().describe('App version context'),
metadata: z.record(z.unknown()).optional().describe('Additional event metadata'),
}),
async execute(args, req) {
const { experimentKey, userId, eventType, eventName, value, metadata } = args;
const { experimentId, userId, metricType, metricName, value, platform, appVersion, metadata } =
args;
const response = await platformFetch<{
tracked: boolean;
variant?: {
key: string;
description: string;
};
}>(
`/api/experiments/${experimentKey}/track`,
'/api/ab-testing/events',
{
method: 'POST',
body: JSON.stringify({
experimentId,
userId,
eventType,
eventName,
metricType,
metricName,
value,
metadata,
converted: true,
eventMetadata: metadata,
platform: platform ?? 'unknown',
appVersion: appVersion ?? 'unknown',
}),
},
{ token: req.headers.authorization?.replace('Bearer ', '') || '', requestId: req.id }
);
return {
experimentKey,
experimentId,
userId,
eventType,
eventName,
metricType,
metricName,
tracked: response.tracked,
variant: response.variant,
value,
summary: response.tracked
? `Tracked ${eventType} event for user in variant "${response.variant?.key}"`
: 'Event not tracked (user not enrolled in experiment)',
? `Tracked ${metricType} metric "${metricName}" for experiment ${experimentId}`
: `Event not tracked for experiment ${experimentId}`,
};
},
});

128
services/mcp-server/src/modules/platform/secrets-tools.ts
View File

@ -1,23 +1,51 @@
import { z } from 'zod';
import { resolveSecrets, type SecretMapping } from '@bytelyst/config';
import { registerTool } from '../tools/registry.js';
// Local interfaces since import is failing
interface SecretMapping {
kvName: string;
envVar: string;
}
const COMMON_MAPPINGS: SecretMapping[] = [
{ kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' },
{ kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' },
{ kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' },
{ kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
];
// Mock resolveSecrets function for now - will need proper implementation
async function resolveSecrets(
secrets: SecretMapping[],
_opts?: { vaultUrl?: string }
): Promise<void> {
// This is a placeholder - in real implementation would fetch from Key Vault
for (const secret of secrets) {
if (!process.env[secret.envVar]) {
process.env[secret.envVar] = `mock-value-for-${secret.kvName}`;
}
const PRODUCT_MAPPINGS: Record<string, SecretMapping[]> = {
lysnrai: [
{ kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
{ kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' },
{ kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' },
{ kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' },
],
chronomind: [
{ kvName: 'chronomind-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'chronomind-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
jarvisjr: [
{ kvName: 'jarvis-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'jarvis-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
nomgap: [
{ kvName: 'nomgap-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'nomgap-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
peakpulse: [
{ kvName: 'peakpulse-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'peakpulse-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
mindlyst: [
{ kvName: 'mindlyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'mindlyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
};
function getMappings(productId?: string): SecretMapping[] {
if (productId && PRODUCT_MAPPINGS[productId]) {
return [...COMMON_MAPPINGS, ...PRODUCT_MAPPINGS[productId]];
}
return [...COMMON_MAPPINGS];
}
registerTool({
@ -33,51 +61,7 @@ registerTool({
}),
async execute(args, _req) {
const { productId } = args;
// Common platform mappings
const commonMappings: SecretMapping[] = [
{ kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' },
{ kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' },
{ kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' },
{ kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
];
// Product-specific mappings
const productMappings: Record<string, SecretMapping[]> = {
lysnrai: [
{ kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
{ kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' },
{ kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' },
{ kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' },
],
chronomind: [
{ kvName: 'chronomind-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'chronomind-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
jarvisjr: [
{ kvName: 'jarvis-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'jarvis-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
nomgap: [
{ kvName: 'nomgap-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'nomgap-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
peakpulse: [
{ kvName: 'peakpulse-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'peakpulse-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
mindlyst: [
{ kvName: 'mindlyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'mindlyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
],
};
let mappings = [...commonMappings];
if (productId && productMappings[productId]) {
mappings = [...mappings, ...productMappings[productId]];
}
const mappings = getMappings(productId);
// Show current status
const status = mappings.map(mapping => ({
@ -196,29 +180,7 @@ registerTool({
req.log.info({ runId, productId, dryRun }, 'Starting secret mapping validation');
try {
// Get mappings (reuse logic from listMappings)
const commonMappings: SecretMapping[] = [
{ kvName: 'bytelyst-cosmos-key', envVar: 'COSMOS_KEY' },
{ kvName: 'bytelyst-cosmos-endpoint', envVar: 'COSMOS_ENDPOINT' },
{ kvName: 'bytelyst-jwt-secret', envVar: 'JWT_SECRET' },
{ kvName: 'bytelyst-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'bytelyst-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
];
const productMappings: Record<string, SecretMapping[]> = {
lysnrai: [
{ kvName: 'lysnr-stripe-secret-key', envVar: 'STRIPE_SECRET_KEY' },
{ kvName: 'lysnr-stripe-webhook-secret', envVar: 'STRIPE_WEBHOOK_SECRET' },
{ kvName: 'lysnr-billing-internal-key', envVar: 'BILLING_INTERNAL_KEY' },
{ kvName: 'lysnr-blob-connection-string', envVar: 'AZURE_BLOB_CONNECTION_STRING' },
{ kvName: 'lysnr-blob-account-key', envVar: 'AZURE_BLOB_ACCOUNT_KEY' },
],
};
let mappings = [...commonMappings];
if (productId && productMappings[productId]) {
mappings = [...mappings, ...productMappings[productId]];
}
const mappings = getMappings(productId);
const results = [];
let successCount = 0;