feat(ai-diagnostics): add LLM analyzer with prompts and insight generation [2.1-2.2]
This commit is contained in:
saravanakumardb1
parent
bfa3d088a4
commit
97b3ffb21d
@ -7,7 +7,7 @@ description: Refresh the Windsurf chat history archive (re-scan all repos, updat
|
|||||||
Refreshes the centralized Windsurf chat history archive at `__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/`.
|
Refreshes the centralized Windsurf chat history archive at `__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/`.
|
||||||
Auto-discovers new repos, updates symlinks, and re-copies docs + workflows.
|
Auto-discovers new repos, updates symlinks, and re-copies docs + workflows.
|
||||||
|
|
||||||
## Covered Repos (All 7 workspaces)
|
## Covered Repos (All 8 workspaces)
|
||||||
|
|
||||||
| Repo | Product | Workflows | Docs |
|
| Repo | Product | Workflows | Docs |
|
||||||
|------|---------|-----------|------|
|
|------|---------|-----------|------|
|
||||||
@ -18,6 +18,7 @@ Auto-discovers new repos, updates symlinks, and re-copies docs + workflows.
|
|||||||
| `learning_ai_fastgap` | NomGap | ✅ | — |
|
| `learning_ai_fastgap` | NomGap | ✅ | — |
|
||||||
| `learning_ai_jarvis_jr` | JarvisJr | ✅ | — |
|
| `learning_ai_jarvis_jr` | JarvisJr | ✅ | — |
|
||||||
| `learning_ai_common_plat` | Common Platform | ✅ | — |
|
| `learning_ai_common_plat` | Common Platform | ✅ | — |
|
||||||
|
| `learning_agent_monitoring_fx` | Agent Monitoring | ✅ | — |
|
||||||
|
|
||||||
## Steps
|
## Steps
|
||||||
|
|
||||||
|
|||||||
@ -1,9 +1,9 @@
|
|||||||
Last refresh: 2026-03-03T19:50:04Z (2026-03-03 11:50:04 PST)
|
Last refresh: 2026-03-03T19:50:43Z (2026-03-03 11:50:43 PST)
|
||||||
Cascade conversations: 50 (297M)
|
Cascade conversations: 50 (297M)
|
||||||
Memories: 65
|
Memories: 65
|
||||||
Implicit context: 20
|
Implicit context: 20
|
||||||
Code tracker dirs: 188
|
Code tracker dirs: 190
|
||||||
File edit history: 2343 entries
|
File edit history: 2343 entries
|
||||||
Workspace storage: 28 workspaces
|
Workspace storage: 28 workspaces
|
||||||
Repo docs: 7 files across 2 repos
|
Repo docs: 7 files across 2 repos
|
||||||
Repo workflows: 35 files across 6 repos
|
Repo workflows: 37 files across 8 repos
|
||||||
|
|||||||
@ -0,0 +1,5 @@
|
|||||||
|
# NomGap Workflows
|
||||||
|
|
||||||
|
Workflows for NomGap fasting app.
|
||||||
|
|
||||||
|
See `/refresh-chat-history` for archive refresh.
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
# JarvisJr Workflows
|
||||||
|
|
||||||
|
Workflows for JarvisJr voice coaching app.
|
||||||
|
|
||||||
|
See `/refresh-chat-history` for archive refresh.
|
||||||
@ -0,0 +1,479 @@
|
|||||||
|
/**
|
||||||
|
* Experiment Detail Page — Live Dashboard with Bayesian Statistics
|
||||||
|
* Shows real-time metrics, variant comparisons, and statistical results.
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState, useEffect } from 'react';
|
||||||
|
import { useParams } from 'next/navigation';
|
||||||
|
import Link from 'next/link';
|
||||||
|
import {
|
||||||
|
ArrowLeft,
|
||||||
|
Play,
|
||||||
|
Pause,
|
||||||
|
Square,
|
||||||
|
Trophy,
|
||||||
|
AlertTriangle,
|
||||||
|
Users,
|
||||||
|
BarChart3,
|
||||||
|
TrendingUp,
|
||||||
|
Clock,
|
||||||
|
Sparkles,
|
||||||
|
Download,
|
||||||
|
} from 'lucide-react';
|
||||||
|
import { Button } from '@/components/ui/button';
|
||||||
|
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||||
|
import { Badge } from '@/components/ui/badge';
|
||||||
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||||
|
import { Progress } from '@/components/ui/progress';
|
||||||
|
import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
|
||||||
|
import type { ExperimentDoc, VariantDoc, ExperimentResult } from '@/lib/experiments-types';
|
||||||
|
|
||||||
|
interface ExperimentData {
|
||||||
|
experiment: ExperimentDoc;
|
||||||
|
variants: VariantDoc[];
|
||||||
|
results?: ExperimentResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function ExperimentDetailPage() {
|
||||||
|
const params = useParams();
|
||||||
|
const experimentId = params.id as string;
|
||||||
|
const [data, setData] = useState<ExperimentData | null>(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
fetchExperimentData();
|
||||||
|
const interval = setInterval(fetchExperimentData, 30000); // Auto-refresh every 30s
|
||||||
|
return () => clearInterval(interval);
|
||||||
|
}, [experimentId]);
|
||||||
|
|
||||||
|
async function fetchExperimentData() {
|
||||||
|
try {
|
||||||
|
const [expResponse, resultsResponse] = await Promise.all([
|
||||||
|
fetch(`/api/experiments/${experimentId}`),
|
||||||
|
fetch(`/api/experiments/${experimentId}/results`),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!expResponse.ok) throw new Error('Failed to fetch experiment');
|
||||||
|
|
||||||
|
const experiment = await expResponse.json();
|
||||||
|
const results = resultsResponse.ok ? await resultsResponse.json() : null;
|
||||||
|
|
||||||
|
setData({ experiment, variants: experiment.variants, results });
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function updateStatus(status: string) {
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/experiments/${experimentId}/${status}`, {
|
||||||
|
method: 'POST',
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error('Failed to update status');
|
||||||
|
fetchExperimentData();
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : 'Failed to update');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (loading) {
|
||||||
|
return (
|
||||||
|
<div className="container mx-auto py-8">
|
||||||
|
<div className="flex items-center justify-center h-64">
|
||||||
|
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!data) {
|
||||||
|
return (
|
||||||
|
<div className="container mx-auto py-8">
|
||||||
|
<Alert variant="destructive">
|
||||||
|
<AlertTriangle className="h-4 w-4" />
|
||||||
|
<AlertTitle>Error</AlertTitle>
|
||||||
|
<AlertDescription>{error || 'Experiment not found'}</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { experiment, variants, results } = data;
|
||||||
|
const daysRunning = experiment.startedAt
|
||||||
|
? Math.floor((Date.now() - new Date(experiment.startedAt).getTime()) / (1000 * 60 * 60 * 24))
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="container mx-auto py-8">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center justify-between mb-6">
|
||||||
|
<div className="flex items-center gap-4">
|
||||||
|
<Button variant="ghost" size="sm" asChild>
|
||||||
|
<Link href="/experiments">
|
||||||
|
<ArrowLeft className="h-4 w-4 mr-2" />
|
||||||
|
Back
|
||||||
|
</Link>
|
||||||
|
</Button>
|
||||||
|
<div>
|
||||||
|
<h1 className="text-2xl font-bold">{experiment.name}</h1>
|
||||||
|
<p className="text-muted-foreground text-sm">{experiment.hypothesis}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
{experiment.status === 'draft' && (
|
||||||
|
<Button onClick={() => updateStatus('start')}>
|
||||||
|
<Play className="h-4 w-4 mr-2" />
|
||||||
|
Start Experiment
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
{experiment.status === 'running' && (
|
||||||
|
<>
|
||||||
|
<Button variant="outline" onClick={() => updateStatus('pause')}>
|
||||||
|
<Pause className="h-4 w-4 mr-2" />
|
||||||
|
Pause
|
||||||
|
</Button>
|
||||||
|
<Button variant="destructive" onClick={() => updateStatus('stop')}>
|
||||||
|
<Square className="h-4 w-4 mr-2" />
|
||||||
|
Stop
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{experiment.status === 'paused' && (
|
||||||
|
<Button onClick={() => updateStatus('start')}>
|
||||||
|
<Play className="h-4 w-4 mr-2" />
|
||||||
|
Resume
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Status Banner */}
|
||||||
|
{results?.earlyStopped && (
|
||||||
|
<Alert className="mb-6 bg-yellow-50 border-yellow-200">
|
||||||
|
<AlertTriangle className="h-4 w-4 text-yellow-600" />
|
||||||
|
<AlertTitle>Early Stopping Triggered</AlertTitle>
|
||||||
|
<AlertDescription>{results.stopReason}</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{results?.winnerVariantId && (
|
||||||
|
<Alert className="mb-6 bg-green-50 border-green-200">
|
||||||
|
<Trophy className="h-4 w-4 text-green-600" />
|
||||||
|
<AlertTitle>Winner Found!</AlertTitle>
|
||||||
|
<AlertDescription>
|
||||||
|
Variant has {((results.winnerProbability || 0) * 100).toFixed(1)}% probability of being best.
|
||||||
|
Recommended action: {results.statisticalSummary.recommendedAction}.
|
||||||
|
</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Stats Overview */}
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-4 gap-4 mb-8">
|
||||||
|
<StatCard
|
||||||
|
title="Status"
|
||||||
|
value={experiment.status}
|
||||||
|
badge={getStatusBadge(experiment.status)}
|
||||||
|
/>
|
||||||
|
<StatCard
|
||||||
|
title="Participants"
|
||||||
|
value={experiment.totalParticipants?.toLocaleString() || '0'}
|
||||||
|
icon={<Users className="h-4 w-4" />}
|
||||||
|
/>
|
||||||
|
<StatCard
|
||||||
|
title="Days Running"
|
||||||
|
value={daysRunning.toString()}
|
||||||
|
icon={<Clock className="h-4 w-4" />}
|
||||||
|
/>
|
||||||
|
<StatCard
|
||||||
|
title="Total Events"
|
||||||
|
value={experiment.totalEvents?.toLocaleString() || '0'}
|
||||||
|
icon={<BarChart3 className="h-4 w-4" />}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Main Content */}
|
||||||
|
<Tabs defaultValue="variants" className="space-y-6">
|
||||||
|
<TabsList>
|
||||||
|
<TabsTrigger value="variants">Variant Performance</TabsTrigger>
|
||||||
|
<TabsTrigger value="statistics">Statistical Results</TabsTrigger>
|
||||||
|
<TabsTrigger value="ai-insights">AI Insights</TabsTrigger>
|
||||||
|
<TabsTrigger value="settings">Settings</TabsTrigger>
|
||||||
|
</TabsList>
|
||||||
|
|
||||||
|
<TabsContent value="variants" className="space-y-4">
|
||||||
|
{variants.map(variant => (
|
||||||
|
<VariantCard
|
||||||
|
key={variant.id}
|
||||||
|
variant={variant}
|
||||||
|
isControl={variant.isControl}
|
||||||
|
experiment={experiment}
|
||||||
|
result={results?.variantResults.find(vr => vr.variantId === variant.id)}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</TabsContent>
|
||||||
|
|
||||||
|
<TabsContent value="statistics">
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>Statistical Summary</CardTitle>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-6">
|
||||||
|
{results ? (
|
||||||
|
<>
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||||
|
<div className="p-4 bg-muted rounded-lg">
|
||||||
|
<div className="text-sm text-muted-foreground mb-1">
|
||||||
|
Probability Any Beats Control
|
||||||
|
</div>
|
||||||
|
<div className="text-2xl font-bold">
|
||||||
|
{(results.statisticalSummary.probabilityAnyBeatsControl * 100).toFixed(1)}%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="p-4 bg-muted rounded-lg">
|
||||||
|
<div className="text-sm text-muted-foreground mb-1">
|
||||||
|
Expected Loss If Shipped
|
||||||
|
</div>
|
||||||
|
<div className="text-2xl font-bold">
|
||||||
|
{(results.statisticalSummary.expectedLossIfShipped * 100).toFixed(2)}%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="p-4 bg-muted rounded-lg">
|
||||||
|
<div className="text-sm text-muted-foreground mb-1">
|
||||||
|
Recommended Action
|
||||||
|
</div>
|
||||||
|
<div className="text-2xl font-bold capitalize">
|
||||||
|
{results.statisticalSummary.recommendedAction}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<h4 className="font-semibold mb-4">Variant Comparison</h4>
|
||||||
|
<div className="space-y-3">
|
||||||
|
{results.variantResults.map(vr => (
|
||||||
|
<div key={vr.variantId} className="flex items-center gap-4 p-3 border rounded-lg">
|
||||||
|
<div className="w-32 font-medium">{vr.variantName}</div>
|
||||||
|
<div className="flex-1">
|
||||||
|
<div className="flex items-center gap-2 mb-1">
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
P(beats control): {(vr.probabilityBeatsControl * 100).toFixed(1)}%
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<Progress value={vr.probabilityBeatsControl * 100} className="h-2" />
|
||||||
|
</div>
|
||||||
|
<div className="w-24 text-right">
|
||||||
|
<div className={`text-sm font-medium ${vr.expectedLiftPercent > 0 ? 'text-green-600' : 'text-red-600'}`}>
|
||||||
|
{vr.expectedLiftPercent > 0 ? '+' : ''}{vr.expectedLiftPercent.toFixed(1)}%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<p className="text-muted-foreground">No statistical results available yet.</p>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
</TabsContent>
|
||||||
|
|
||||||
|
<TabsContent value="ai-insights">
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle className="flex items-center gap-2">
|
||||||
|
<Sparkles className="h-5 w-5" />
|
||||||
|
AI-Generated Insights
|
||||||
|
</CardTitle>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-4">
|
||||||
|
<div className="p-4 bg-blue-50 border border-blue-200 rounded-lg">
|
||||||
|
<h4 className="font-semibold mb-2">Summary</h4>
|
||||||
|
<p className="text-sm">
|
||||||
|
{results?.winnerVariantId
|
||||||
|
? `The ${results.variantResults.find(v => v.variantId === results.winnerVariantId)?.variantName || 'winning variant'} significantly outperformed control with ${((results.winnerProbability || 0) * 100).toFixed(1)}% probability of superiority.`
|
||||||
|
: 'Results are still inconclusive. Consider running the experiment longer or testing more distinct variations.'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<h4 className="font-semibold mb-2">Recommended Follow-up Experiments</h4>
|
||||||
|
<ul className="space-y-2">
|
||||||
|
<li className="flex items-center gap-2 text-sm">
|
||||||
|
<TrendingUp className="h-4 w-4 text-muted-foreground" />
|
||||||
|
Test the winning variant against additional success metrics
|
||||||
|
</li>
|
||||||
|
<li className="flex items-center gap-2 text-sm">
|
||||||
|
<Users className="h-4 w-4 text-muted-foreground" />
|
||||||
|
Run a validation experiment with a new user cohort
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
</TabsContent>
|
||||||
|
|
||||||
|
<TabsContent value="settings">
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>Experiment Configuration</CardTitle>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-4">
|
||||||
|
<div className="grid grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<label className="text-sm font-medium">Allocation Strategy</label>
|
||||||
|
<p className="text-muted-foreground capitalize">{experiment.allocationStrategy}</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label className="text-sm font-medium">Target Traffic</label>
|
||||||
|
<p className="text-muted-foreground">{experiment.targetPercent}%</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label className="text-sm font-medium">Primary Metric</label>
|
||||||
|
<p className="text-muted-foreground">{experiment.primaryMetric?.name}</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label className="text-sm font-medium">Min Sample Size</label>
|
||||||
|
<p className="text-muted-foreground">{experiment.guardrails?.minSampleSizePerVariant} per variant</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label className="text-sm font-medium">Max Duration</label>
|
||||||
|
<p className="text-muted-foreground">{experiment.guardrails?.maxDurationDays} days</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label className="text-sm font-medium">Auto Stop</label>
|
||||||
|
<p className="text-muted-foreground">{experiment.guardrails?.autoStopEnabled ? 'Enabled' : 'Disabled'}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
</TabsContent>
|
||||||
|
</Tabs>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function StatCard({
|
||||||
|
title,
|
||||||
|
value,
|
||||||
|
badge,
|
||||||
|
icon,
|
||||||
|
}: {
|
||||||
|
title: string;
|
||||||
|
value: string;
|
||||||
|
badge?: React.ReactNode;
|
||||||
|
icon?: React.ReactNode;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<Card>
|
||||||
|
<CardContent className="p-4">
|
||||||
|
<div className="flex items-center justify-between mb-2">
|
||||||
|
<span className="text-sm text-muted-foreground">{title}</span>
|
||||||
|
{icon}
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="text-2xl font-bold capitalize">{value}</span>
|
||||||
|
{badge}
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function VariantCard({
|
||||||
|
variant,
|
||||||
|
isControl,
|
||||||
|
experiment,
|
||||||
|
result,
|
||||||
|
}: {
|
||||||
|
variant: VariantDoc;
|
||||||
|
isControl: boolean;
|
||||||
|
experiment: ExperimentDoc;
|
||||||
|
result?: { probabilityBeatsControl: number; expectedLiftPercent: number; credibleInterval: { lower: number; mean: number; upper: number } };
|
||||||
|
}) {
|
||||||
|
const conversionRate = variant.stats?.conversionRate || 0;
|
||||||
|
const participants = variant.stats?.participants || 0;
|
||||||
|
const events = variant.stats?.events || 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Card className={isControl ? 'border-blue-200' : ''}>
|
||||||
|
<CardContent className="p-6">
|
||||||
|
<div className="flex items-start justify-between">
|
||||||
|
<div>
|
||||||
|
<div className="flex items-center gap-2 mb-1">
|
||||||
|
<h3 className="text-lg font-semibold">{variant.name}</h3>
|
||||||
|
{isControl && (
|
||||||
|
<Badge variant="outline" className="border-blue-300 text-blue-700">
|
||||||
|
Control
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
{variant.bayesianResults?.probabilityBeatsControl && variant.bayesianResults.probabilityBeatsControl > 0.95 && (
|
||||||
|
<Badge className="bg-green-500 text-white">
|
||||||
|
<Trophy className="h-3 w-3 mr-1" />
|
||||||
|
Winner
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<p className="text-sm text-muted-foreground">{variant.description}</p>
|
||||||
|
</div>
|
||||||
|
<div className="text-right">
|
||||||
|
<div className="text-2xl font-bold">{(conversionRate * 100).toFixed(1)}%</div>
|
||||||
|
<div className="text-sm text-muted-foreground">conversion rate</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-3 gap-4 mt-4 pt-4 border-t">
|
||||||
|
<div>
|
||||||
|
<div className="text-2xl font-semibold">{participants.toLocaleString()}</div>
|
||||||
|
<div className="text-sm text-muted-foreground">participants</div>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div className="text-2xl font-semibold">{events.toLocaleString()}</div>
|
||||||
|
<div className="text-sm text-muted-foreground">events</div>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div className="text-2xl font-semibold">
|
||||||
|
{result ? `${(result.probabilityBeatsControl * 100).toFixed(0)}%` : '-'}
|
||||||
|
</div>
|
||||||
|
<div className="text-sm text-muted-foreground">P(beats control)</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{result && result.credibleInterval && (
|
||||||
|
<div className="mt-4 pt-4 border-t">
|
||||||
|
<div className="text-sm text-muted-foreground mb-2">95% Credible Interval</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="text-sm">{(result.credibleInterval.lower * 100).toFixed(1)}%</span>
|
||||||
|
<div className="flex-1 h-2 bg-muted rounded-full relative">
|
||||||
|
<div
|
||||||
|
className="absolute h-full bg-primary rounded-full"
|
||||||
|
style={{
|
||||||
|
left: `${(result.credibleInterval.lower / result.credibleInterval.upper) * 50}%`,
|
||||||
|
width: `${((result.credibleInterval.upper - result.credibleInterval.lower) / result.credibleInterval.upper) * 50}%`,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<span className="text-sm">{(result.credibleInterval.upper * 100).toFixed(1)}%</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getStatusBadge(status: string) {
|
||||||
|
const colors: Record<string, string> = {
|
||||||
|
draft: 'bg-gray-500',
|
||||||
|
running: 'bg-green-500',
|
||||||
|
paused: 'bg-yellow-500',
|
||||||
|
stopped: 'bg-red-500',
|
||||||
|
completed: 'bg-blue-500',
|
||||||
|
};
|
||||||
|
return <Badge className={`${colors[status] || colors.draft} text-white capitalize`}>{status}</Badge>;
|
||||||
|
}
|
||||||
@ -0,0 +1,735 @@
|
|||||||
|
/**
|
||||||
|
* New Experiment Wizard — Admin Dashboard
|
||||||
|
* Multi-step wizard for creating experiments with AI suggestions.
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { useRouter } from 'next/navigation';
|
||||||
|
import Link from 'next/link';
|
||||||
|
import {
|
||||||
|
ArrowLeft,
|
||||||
|
ArrowRight,
|
||||||
|
Sparkles,
|
||||||
|
FlaskConical,
|
||||||
|
Target,
|
||||||
|
Sliders,
|
||||||
|
Check,
|
||||||
|
Lightbulb,
|
||||||
|
} from 'lucide-react';
|
||||||
|
import { Button } from '@/components/ui/button';
|
||||||
|
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card';
|
||||||
|
import { Input } from '@/components/ui/input';
|
||||||
|
import { Label } from '@/components/ui/label';
|
||||||
|
import { Textarea } from '@/components/ui/textarea';
|
||||||
|
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
|
||||||
|
import { Badge } from '@/components/ui/badge';
|
||||||
|
import { Alert, AlertDescription } from '@/components/ui/alert';
|
||||||
|
import { Progress } from '@/components/ui/progress';
|
||||||
|
import type { CreateExperimentInput, ExperimentSuggestion } from '@/lib/experiments-types';
|
||||||
|
|
||||||
|
const steps = [
|
||||||
|
{ id: 'hypothesis', title: 'Hypothesis', icon: Lightbulb },
|
||||||
|
{ id: 'variants', title: 'Variants', icon: FlaskConical },
|
||||||
|
{ id: 'metrics', title: 'Metrics', icon: Target },
|
||||||
|
{ id: 'targeting', title: 'Targeting', icon: Sliders },
|
||||||
|
{ id: 'review', title: 'Review', icon: Check },
|
||||||
|
];
|
||||||
|
|
||||||
|
export default function NewExperimentPage() {
|
||||||
|
const router = useRouter();
|
||||||
|
const [currentStep, setCurrentStep] = useState(0);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [aiSuggestions, setAiSuggestions] = useState<ExperimentSuggestion[]>([]);
|
||||||
|
|
||||||
|
const [formData, setFormData] = useState<Partial<CreateExperimentInput>>({
|
||||||
|
name: '',
|
||||||
|
description: '',
|
||||||
|
hypothesis: '',
|
||||||
|
variants: [
|
||||||
|
{ key: 'control', name: 'Control', description: 'Current implementation', isControl: true, flagConfig: {} },
|
||||||
|
{ key: 'variant_a', name: 'Variant A', description: '', isControl: false, flagConfig: {} },
|
||||||
|
],
|
||||||
|
allocationStrategy: 'random',
|
||||||
|
targetPercent: 100,
|
||||||
|
targeting: {},
|
||||||
|
primaryMetric: {
|
||||||
|
name: 'conversion',
|
||||||
|
type: 'conversion',
|
||||||
|
eventName: 'purchase',
|
||||||
|
aggregation: 'count',
|
||||||
|
direction: 'increase',
|
||||||
|
minimumDetectableEffect: 5,
|
||||||
|
},
|
||||||
|
guardrails: {
|
||||||
|
minSampleSizePerVariant: 100,
|
||||||
|
maxDurationDays: 30,
|
||||||
|
autoStopEnabled: true,
|
||||||
|
winnerThreshold: 95,
|
||||||
|
requireApprovalFor: 'none',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
async function loadAiSuggestions() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/ab-testing/suggestions');
|
||||||
|
if (response.ok) {
|
||||||
|
const data = await response.json();
|
||||||
|
setAiSuggestions(data.slice(0, 3));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Silently fail - AI suggestions are optional
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleSubmit() {
|
||||||
|
setLoading(true);
|
||||||
|
setError(null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/ab-testing/experiments', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(formData),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorData = await response.json();
|
||||||
|
throw new Error(errorData.message || 'Failed to create experiment');
|
||||||
|
}
|
||||||
|
|
||||||
|
const experiment = await response.json();
|
||||||
|
router.push(`/experiments/${experiment.id}`);
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : 'Failed to create experiment');
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyAiSuggestion(suggestion: ExperimentSuggestion) {
|
||||||
|
setFormData(prev => ({
|
||||||
|
...prev,
|
||||||
|
name: prev.name || `Experiment: ${suggestion.hypothesis.primary.slice(0, 50)}...`,
|
||||||
|
hypothesis: suggestion.hypothesis.primary,
|
||||||
|
description: suggestion.hypothesis.alternatives[0] || '',
|
||||||
|
primaryMetric: suggestion.suggestedMetrics[0] || prev.primaryMetric,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
const progress = ((currentStep + 1) / steps.length) * 100;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="container mx-auto py-8 max-w-4xl">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center gap-4 mb-8">
|
||||||
|
<Button variant="ghost" size="sm" asChild>
|
||||||
|
<Link href="/experiments">
|
||||||
|
<ArrowLeft className="h-4 w-4 mr-2" />
|
||||||
|
Back
|
||||||
|
</Link>
|
||||||
|
</Button>
|
||||||
|
<div>
|
||||||
|
<h1 className="text-2xl font-bold">Create New Experiment</h1>
|
||||||
|
<p className="text-muted-foreground text-sm">
|
||||||
|
Step {currentStep + 1} of {steps.length}: {steps[currentStep].title}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Progress */}
|
||||||
|
<Progress value={progress} className="mb-8" />
|
||||||
|
|
||||||
|
{/* Step Indicators */}
|
||||||
|
<div className="flex justify-between mb-8">
|
||||||
|
{steps.map((step, index) => {
|
||||||
|
const Icon = step.icon;
|
||||||
|
const isActive = index === currentStep;
|
||||||
|
const isCompleted = index < currentStep;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={step.id}
|
||||||
|
className={`flex flex-col items-center ${
|
||||||
|
isActive ? 'text-primary' : isCompleted ? 'text-green-600' : 'text-muted-foreground'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className={`w-10 h-10 rounded-full flex items-center justify-center mb-2 ${
|
||||||
|
isActive
|
||||||
|
? 'bg-primary text-white'
|
||||||
|
: isCompleted
|
||||||
|
? 'bg-green-100 text-green-600'
|
||||||
|
: 'bg-muted'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<Icon className="h-5 w-5" />
|
||||||
|
</div>
|
||||||
|
<span className="text-xs font-medium">{step.title}</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Error */}
|
||||||
|
{error && (
|
||||||
|
<Alert variant="destructive" className="mb-6">
|
||||||
|
<AlertDescription>{error}</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Step Content */}
|
||||||
|
<Card className="mb-6">
|
||||||
|
<CardContent className="p-6">
|
||||||
|
{currentStep === 0 && (
|
||||||
|
<HypothesisStep
|
||||||
|
formData={formData}
|
||||||
|
setFormData={setFormData}
|
||||||
|
aiSuggestions={aiSuggestions}
|
||||||
|
onLoadSuggestions={loadAiSuggestions}
|
||||||
|
onApplySuggestion={applyAiSuggestion}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
{currentStep === 1 && (
|
||||||
|
<VariantsStep formData={formData} setFormData={setFormData} />
|
||||||
|
)}
|
||||||
|
{currentStep === 2 && (
|
||||||
|
<MetricsStep formData={formData} setFormData={setFormData} />
|
||||||
|
)}
|
||||||
|
{currentStep === 3 && (
|
||||||
|
<TargetingStep formData={formData} setFormData={setFormData} />
|
||||||
|
)}
|
||||||
|
{currentStep === 4 && (
|
||||||
|
<ReviewStep formData={formData} />
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
{/* Navigation */}
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
onClick={() => setCurrentStep(Math.max(0, currentStep - 1))}
|
||||||
|
disabled={currentStep === 0}
|
||||||
|
>
|
||||||
|
<ArrowLeft className="h-4 w-4 mr-2" />
|
||||||
|
Previous
|
||||||
|
</Button>
|
||||||
|
|
||||||
|
{currentStep < steps.length - 1 ? (
|
||||||
|
<Button onClick={() => setCurrentStep(currentStep + 1)}>
|
||||||
|
Next
|
||||||
|
<ArrowRight className="h-4 w-4 ml-2" />
|
||||||
|
</Button>
|
||||||
|
) : (
|
||||||
|
<Button onClick={handleSubmit} disabled={loading}>
|
||||||
|
{loading ? (
|
||||||
|
<>
|
||||||
|
<div className="animate-spin mr-2 h-4 w-4 border-2 border-current border-t-transparent rounded-full" />
|
||||||
|
Creating...
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Check className="h-4 w-4 mr-2" />
|
||||||
|
Create Experiment
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Step Components
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function HypothesisStep({
|
||||||
|
formData,
|
||||||
|
setFormData,
|
||||||
|
aiSuggestions,
|
||||||
|
onLoadSuggestions,
|
||||||
|
onApplySuggestion,
|
||||||
|
}: {
|
||||||
|
formData: Partial<CreateExperimentInput>;
|
||||||
|
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||||
|
aiSuggestions: ExperimentSuggestion[];
|
||||||
|
onLoadSuggestions: () => void;
|
||||||
|
onApplySuggestion: (s: ExperimentSuggestion) => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div className="space-y-6">
|
||||||
|
<div>
|
||||||
|
<Label htmlFor="name">Experiment Name</Label>
|
||||||
|
<Input
|
||||||
|
id="name"
|
||||||
|
value={formData.name}
|
||||||
|
onChange={e => setFormData({ ...formData, name: e.target.value })}
|
||||||
|
placeholder="e.g., Homepage Button Color Test"
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<Label htmlFor="hypothesis">Hypothesis</Label>
|
||||||
|
<Textarea
|
||||||
|
id="hypothesis"
|
||||||
|
value={formData.hypothesis}
|
||||||
|
onChange={e => setFormData({ ...formData, hypothesis: e.target.value })}
|
||||||
|
placeholder="e.g., Changing the CTA button to green will increase click-through rate by 15% because green signals 'go' to users..."
|
||||||
|
className="mt-1"
|
||||||
|
rows={3}
|
||||||
|
/>
|
||||||
|
<p className="text-sm text-muted-foreground mt-1">
|
||||||
|
State what you expect to change and why.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<Label htmlFor="description">Description (optional)</Label>
|
||||||
|
<Textarea
|
||||||
|
id="description"
|
||||||
|
value={formData.description}
|
||||||
|
onChange={e => setFormData({ ...formData, description: e.target.value })}
|
||||||
|
placeholder="Additional context about this experiment..."
|
||||||
|
className="mt-1"
|
||||||
|
rows={2}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* AI Suggestions */}
|
||||||
|
<div className="border rounded-lg p-4 bg-muted/50">
|
||||||
|
<div className="flex items-center gap-2 mb-3">
|
||||||
|
<Sparkles className="h-4 w-4 text-primary" />
|
||||||
|
<h4 className="font-medium">AI-Generated Suggestions</h4>
|
||||||
|
<Button variant="ghost" size="sm" onClick={onLoadSuggestions} className="ml-auto">
|
||||||
|
Load Suggestions
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{aiSuggestions.length > 0 ? (
|
||||||
|
<div className="space-y-3">
|
||||||
|
{aiSuggestions.map((suggestion, index) => (
|
||||||
|
<Card key={index} className="cursor-pointer hover:border-primary" onClick={() => onApplySuggestion(suggestion)}>
|
||||||
|
<CardContent className="p-3">
|
||||||
|
<p className="text-sm font-medium line-clamp-2">{suggestion.hypothesis.primary}</p>
|
||||||
|
<div className="flex gap-2 mt-2">
|
||||||
|
<Badge variant="outline" className="text-xs">
|
||||||
|
Impact: {suggestion.hypothesis.impactScore}/100
|
||||||
|
</Badge>
|
||||||
|
<Badge variant="outline" className="text-xs">
|
||||||
|
Difficulty: {suggestion.hypothesis.difficultyScore}/100
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<p className="text-sm text-muted-foreground">
|
||||||
|
Click "Load Suggestions" to see AI-generated experiment ideas based on your product usage patterns.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function VariantsStep({
|
||||||
|
formData,
|
||||||
|
setFormData,
|
||||||
|
}: {
|
||||||
|
formData: Partial<CreateExperimentInput>;
|
||||||
|
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||||
|
}) {
|
||||||
|
const variants = formData.variants || [];
|
||||||
|
|
||||||
|
function addVariant() {
|
||||||
|
const newVariant = {
|
||||||
|
key: `variant_${String.fromCharCode(98 + variants.length - 1)}`,
|
||||||
|
name: `Variant ${String.fromCharCode(65 + variants.length - 1)}`,
|
||||||
|
description: '',
|
||||||
|
isControl: false,
|
||||||
|
flagConfig: {},
|
||||||
|
};
|
||||||
|
setFormData({ ...formData, variants: [...variants, newVariant] });
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateVariant(index: number, updates: Partial<(typeof variants)[0]>) {
|
||||||
|
const updated = variants.map((v, i) => (i === index ? { ...v, ...updates } : v));
|
||||||
|
setFormData({ ...formData, variants: updated });
|
||||||
|
}
|
||||||
|
|
||||||
|
function removeVariant(index: number) {
|
||||||
|
if (variants.length <= 2) return; // Minimum 2 variants
|
||||||
|
const updated = variants.filter((_, i) => i !== index);
|
||||||
|
setFormData({ ...formData, variants: updated });
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-6">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<h3 className="font-medium">Experiment Variants</h3>
|
||||||
|
<Button variant="outline" size="sm" onClick={addVariant}>
|
||||||
|
Add Variant
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{variants.map((variant, index) => (
|
||||||
|
<Card key={index} className={variant.isControl ? 'border-blue-200' : ''}>
|
||||||
|
<CardContent className="p-4 space-y-4">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{variant.isControl ? (
|
||||||
|
<Badge variant="outline" className="border-blue-300 text-blue-700">Control</Badge>
|
||||||
|
) : (
|
||||||
|
<Badge variant="outline">Variant</Badge>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
{!variant.isControl && (
|
||||||
|
<Button variant="ghost" size="sm" onClick={() => removeVariant(index)}>
|
||||||
|
Remove
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<Label>Name</Label>
|
||||||
|
<Input
|
||||||
|
value={variant.name}
|
||||||
|
onChange={e => updateVariant(index, { name: e.target.value })}
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<Label>Key (slug)</Label>
|
||||||
|
<Input
|
||||||
|
value={variant.key}
|
||||||
|
onChange={e => updateVariant(index, { key: e.target.value })}
|
||||||
|
className="mt-1"
|
||||||
|
disabled={variant.isControl}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<Label>Description</Label>
|
||||||
|
<Input
|
||||||
|
value={variant.description}
|
||||||
|
onChange={e => updateVariant(index, { description: e.target.value })}
|
||||||
|
placeholder="What makes this variant different?"
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
))}
|
||||||
|
|
||||||
|
<p className="text-sm text-muted-foreground">
|
||||||
|
You need at least 2 variants: a Control (current implementation) and at least one Treatment variant.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function MetricsStep({
|
||||||
|
formData,
|
||||||
|
setFormData,
|
||||||
|
}: {
|
||||||
|
formData: Partial<CreateExperimentInput>;
|
||||||
|
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div className="space-y-6">
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>Primary Metric</CardTitle>
|
||||||
|
<CardDescription>The main metric that determines experiment success.</CardDescription>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-4">
|
||||||
|
<div className="grid grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<Label>Metric Name</Label>
|
||||||
|
<Input
|
||||||
|
value={formData.primaryMetric?.name}
|
||||||
|
onChange={e =>
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
primaryMetric: { ...formData.primaryMetric!, name: e.target.value },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<Label>Event Name</Label>
|
||||||
|
<Input
|
||||||
|
value={formData.primaryMetric?.eventName}
|
||||||
|
onChange={e =>
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
primaryMetric: { ...formData.primaryMetric!, eventName: e.target.value },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
placeholder="e.g., purchase_clicked"
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<Label>Metric Type</Label>
|
||||||
|
<Select
|
||||||
|
value={formData.primaryMetric?.type}
|
||||||
|
onValueChange={v =>
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
primaryMetric: { ...formData.primaryMetric!, type: v as any },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<SelectTrigger className="mt-1">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="conversion">Conversion (binary)</SelectItem>
|
||||||
|
<SelectItem value="count">Count (integer)</SelectItem>
|
||||||
|
<SelectItem value="duration">Duration (time)</SelectItem>
|
||||||
|
<SelectItem value="revenue">Revenue (monetary)</SelectItem>
|
||||||
|
<SelectItem value="custom">Custom (numeric)</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<Label>Direction</Label>
|
||||||
|
<Select
|
||||||
|
value={formData.primaryMetric?.direction}
|
||||||
|
onValueChange={v =>
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
primaryMetric: { ...formData.primaryMetric!, direction: v as any },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<SelectTrigger className="mt-1">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="increase">Higher is better</SelectItem>
|
||||||
|
<SelectItem value="decrease">Lower is better</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>Allocation Strategy</CardTitle>
|
||||||
|
<CardDescription>How traffic is distributed between variants.</CardDescription>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-4">
|
||||||
|
<Select
|
||||||
|
value={formData.allocationStrategy}
|
||||||
|
onValueChange={v => setFormData({ ...formData, allocationStrategy: v as any })}
|
||||||
|
>
|
||||||
|
<SelectTrigger>
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="random">Random (fixed 50/50)</SelectItem>
|
||||||
|
<SelectItem value="thompson">Thompson Sampling (multi-armed bandit)</SelectItem>
|
||||||
|
<SelectItem value="epsilon_greedy">Epsilon-Greedy (explore/exploit)</SelectItem>
|
||||||
|
<SelectItem value="ucb">Upper Confidence Bound (UCB1)</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<Label>Target Traffic %</Label>
|
||||||
|
<Input
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
max={100}
|
||||||
|
value={formData.targetPercent}
|
||||||
|
onChange={e => setFormData({ ...formData, targetPercent: parseInt(e.target.value) })}
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
<p className="text-sm text-muted-foreground mt-1">
|
||||||
|
Percentage of eligible users to include in this experiment.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function TargetingStep({
|
||||||
|
formData,
|
||||||
|
setFormData,
|
||||||
|
}: {
|
||||||
|
formData: Partial<CreateExperimentInput>;
|
||||||
|
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div className="space-y-6">
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>Audience Targeting</CardTitle>
|
||||||
|
<CardDescription>Limit this experiment to specific user segments.</CardDescription>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-4">
|
||||||
|
<div>
|
||||||
|
<Label>Platforms</Label>
|
||||||
|
<div className="flex gap-2 mt-1">
|
||||||
|
{['ios', 'android', 'web'].map(platform => (
|
||||||
|
<Badge
|
||||||
|
key={platform}
|
||||||
|
variant={formData.targeting?.platforms?.includes(platform) ? 'default' : 'outline'}
|
||||||
|
className="cursor-pointer capitalize"
|
||||||
|
onClick={() => {
|
||||||
|
const current = formData.targeting?.platforms || [];
|
||||||
|
const updated = current.includes(platform)
|
||||||
|
? current.filter(p => p !== platform)
|
||||||
|
: [...current, platform];
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
targeting: { ...formData.targeting, platforms: updated },
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{platform}
|
||||||
|
</Badge>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<Label>User Segments</Label>
|
||||||
|
<div className="flex gap-2 mt-1">
|
||||||
|
{['free', 'pro', 'enterprise'].map(segment => (
|
||||||
|
<Badge
|
||||||
|
key={segment}
|
||||||
|
variant={formData.targeting?.userSegments?.includes(segment) ? 'default' : 'outline'}
|
||||||
|
className="cursor-pointer capitalize"
|
||||||
|
onClick={() => {
|
||||||
|
const current = formData.targeting?.userSegments || [];
|
||||||
|
const updated = current.includes(segment)
|
||||||
|
? current.filter(s => s !== segment)
|
||||||
|
: [...current, segment];
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
targeting: { ...formData.targeting, userSegments: updated },
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{segment}
|
||||||
|
</Badge>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>Guardrails & Safety</CardTitle>
|
||||||
|
<CardDescription>Configure automatic stopping rules.</CardDescription>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-4">
|
||||||
|
<div className="grid grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<Label>Min Sample Size per Variant</Label>
|
||||||
|
<Input
|
||||||
|
type="number"
|
||||||
|
value={formData.guardrails?.minSampleSizePerVariant}
|
||||||
|
onChange={e =>
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
guardrails: {
|
||||||
|
...formData.guardrails!,
|
||||||
|
minSampleSizePerVariant: parseInt(e.target.value),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<Label>Max Duration (days)</Label>
|
||||||
|
<Input
|
||||||
|
type="number"
|
||||||
|
value={formData.guardrails?.maxDurationDays}
|
||||||
|
onChange={e =>
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
guardrails: { ...formData.guardrails!, maxDurationDays: parseInt(e.target.value) },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<Label>Winner Threshold (%)</Label>
|
||||||
|
<Input
|
||||||
|
type="number"
|
||||||
|
min={80}
|
||||||
|
max={99}
|
||||||
|
value={formData.guardrails?.winnerThreshold}
|
||||||
|
onChange={e =>
|
||||||
|
setFormData({
|
||||||
|
...formData,
|
||||||
|
guardrails: { ...formData.guardrails!, winnerThreshold: parseInt(e.target.value) },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
className="mt-1"
|
||||||
|
/>
|
||||||
|
<p className="text-sm text-muted-foreground mt-1">
|
||||||
|
Probability threshold for declaring a winner (95% recommended).
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ReviewStep({ formData }: { formData: Partial<CreateExperimentInput> }) {
|
||||||
|
return (
|
||||||
|
<div className="space-y-6">
|
||||||
|
<h3 className="font-medium">Review Experiment Configuration</h3>
|
||||||
|
|
||||||
|
<div className="space-y-4">
|
||||||
|
<ReviewItem label="Name" value={formData.name} />
|
||||||
|
<ReviewItem label="Hypothesis" value={formData.hypothesis} />
|
||||||
|
<ReviewItem label="Variants" value={`${formData.variants?.length || 0} variants`} />
|
||||||
|
<ReviewItem label="Primary Metric" value={formData.primaryMetric?.name} />
|
||||||
|
<ReviewItem label="Allocation Strategy" value={formData.allocationStrategy} />
|
||||||
|
<ReviewItem label="Target Traffic" value={`${formData.targetPercent}%`} />
|
||||||
|
<ReviewItem label="Auto Stop" value={formData.guardrails?.autoStopEnabled ? 'Enabled' : 'Disabled'} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<Alert>
|
||||||
|
<AlertDescription>
|
||||||
|
Once created, you can start the experiment from the experiment details page.
|
||||||
|
</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ReviewItem({ label, value }: { label: string; value?: string | number | boolean }) {
|
||||||
|
return (
|
||||||
|
<div className="flex justify-between py-2 border-b">
|
||||||
|
<span className="text-muted-foreground">{label}</span>
|
||||||
|
<span className="font-medium">{value || '-'}</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@ -81,6 +81,14 @@ const CONTAINER_DEFS: Record<string, ContainerConfig> = {
|
|||||||
debug_traces: { partitionKeyPath: '/pk', defaultTtl: 7 * 86400 },
|
debug_traces: { partitionKeyPath: '/pk', defaultTtl: 7 * 86400 },
|
||||||
debug_logs: { partitionKeyPath: '/pk', defaultTtl: 3 * 86400 },
|
debug_logs: { partitionKeyPath: '/pk', defaultTtl: 3 * 86400 },
|
||||||
debug_screenshots: { partitionKeyPath: '/sessionId', defaultTtl: 7 * 86400 },
|
debug_screenshots: { partitionKeyPath: '/sessionId', defaultTtl: 7 * 86400 },
|
||||||
|
// Predictive Analytics
|
||||||
|
user_features: { partitionKeyPath: '/userId', defaultTtl: 90 * 86400 },
|
||||||
|
product_health: { partitionKeyPath: '/productId' },
|
||||||
|
feature_definitions: { partitionKeyPath: '/productId' },
|
||||||
|
churn_predictions: { partitionKeyPath: '/userId', defaultTtl: 120 * 86400 },
|
||||||
|
retention_campaigns: { partitionKeyPath: '/productId' },
|
||||||
|
campaign_deliveries: { partitionKeyPath: '/userId', defaultTtl: 90 * 86400 },
|
||||||
|
model_performance: { partitionKeyPath: '/id' },
|
||||||
// AI Diagnostics (see docs/roadmaps/AI_DIAGNOSTIC_ASSISTANT_ROADMAP.md)
|
// AI Diagnostics (see docs/roadmaps/AI_DIAGNOSTIC_ASSISTANT_ROADMAP.md)
|
||||||
error_clusters: { partitionKeyPath: '/productId', defaultTtl: 90 * 86400 },
|
error_clusters: { partitionKeyPath: '/productId', defaultTtl: 90 * 86400 },
|
||||||
error_fingerprints: { partitionKeyPath: '/fingerprintHash' },
|
error_fingerprints: { partitionKeyPath: '/fingerprintHash' },
|
||||||
|
|||||||
@ -0,0 +1,493 @@
|
|||||||
|
import { config } from '../../lib/config.js';
|
||||||
|
import type {
|
||||||
|
ErrorClusterDoc,
|
||||||
|
DiagnosticInsightDoc,
|
||||||
|
RootCauseCategory,
|
||||||
|
Evidence,
|
||||||
|
SimilarResolvedIssue,
|
||||||
|
} from './types.js';
|
||||||
|
import type { ClusterContextSummary } from './telemetry-linking.js';
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// LLM Prompts
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
interface RootCauseAnalysisPrompt {
|
||||||
|
cluster: ErrorClusterDoc;
|
||||||
|
context: ClusterContextSummary;
|
||||||
|
sampleStackTraces: string[];
|
||||||
|
relatedClusters: Array<{ id: string; errorType: string; status: string }>;
|
||||||
|
analysisType: 'root_cause' | 'pattern' | 'comparison' | 'trend';
|
||||||
|
}
|
||||||
|
|
||||||
|
const ROOT_CAUSE_ANALYSIS_PROMPT_TEMPLATE = `You are an expert software engineer diagnosing production errors. Analyze this error cluster and provide root cause insights.
|
||||||
|
|
||||||
|
## Error Cluster
|
||||||
|
- Error Type: {errorType}
|
||||||
|
- Message Pattern: {messagePattern}
|
||||||
|
- Stack Signature: {stackSignature}
|
||||||
|
- Occurrences: {occurrenceCount}
|
||||||
|
- Affected Users: {uniqueUsers}
|
||||||
|
- First Seen: {firstSeenAt}
|
||||||
|
- Last Seen: {lastSeenAt}
|
||||||
|
|
||||||
|
## Context Summary
|
||||||
|
- Total Occurrences: {totalOccurrences}
|
||||||
|
- Affected Users Count: {affectedUsersCount}
|
||||||
|
- Time Range: {timeRange}
|
||||||
|
- Most Common Screens: {commonScreens}
|
||||||
|
- Most Common Actions: {commonActions}
|
||||||
|
|
||||||
|
## Sample Stack Traces
|
||||||
|
{sampleStackTraces}
|
||||||
|
|
||||||
|
## Related Resolved Clusters
|
||||||
|
{relatedClusters}
|
||||||
|
|
||||||
|
## API Failure Pattern (if any)
|
||||||
|
{apiFailurePattern}
|
||||||
|
|
||||||
|
## Feature Flag Correlations
|
||||||
|
{featureFlagCorrelations}
|
||||||
|
|
||||||
|
Analyze this error and provide:
|
||||||
|
1. Root cause category (config, dependency, logic, resource, external, or unknown)
|
||||||
|
2. Specific hypothesis explaining WHY this error occurs
|
||||||
|
3. Your reasoning process
|
||||||
|
4. Evidence confidence level (high/medium/low)
|
||||||
|
5. Suggested investigation steps (3-5 specific actions)
|
||||||
|
6. Potential fix direction (if identifiable)
|
||||||
|
|
||||||
|
Respond in this JSON format:
|
||||||
|
{
|
||||||
|
"rootCauseCategory": "logic",
|
||||||
|
"hypothesis": "detailed explanation...",
|
||||||
|
"reasoning": "why you think this...",
|
||||||
|
"confidence": "high",
|
||||||
|
"suggestedInvestigation": ["step 1", "step 2", "step 3"],
|
||||||
|
"potentialFixDirection": "suggested fix approach...",
|
||||||
|
"evidence": [
|
||||||
|
{
|
||||||
|
"type": "stack_trace|telemetry_pattern|device_correlation|api_failure|similar_issue|config_mismatch|version_regression",
|
||||||
|
"description": "specific evidence...",
|
||||||
|
"strength": "strong|moderate|weak"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`;
|
||||||
|
|
||||||
|
const PATTERN_SUMMARY_PROMPT_TEMPLATE = `Summarize the pattern for this error cluster in 2-3 sentences. Focus on:
|
||||||
|
1. What triggers the error (user action, system state, timing)
|
||||||
|
2. The common characteristics across all occurrences
|
||||||
|
3. Why this error clusters together (same root cause)
|
||||||
|
|
||||||
|
Error: {errorType}
|
||||||
|
Message: {messagePattern}
|
||||||
|
Occurrences: {occurrenceCount}
|
||||||
|
Context: {contextSummary}
|
||||||
|
|
||||||
|
Provide a concise, engineer-friendly summary.`;
|
||||||
|
|
||||||
|
const COMPARATIVE_ANALYSIS_PROMPT_TEMPLATE = `Compare these two error clusters and identify:
|
||||||
|
1. Key differences in root cause
|
||||||
|
2. Whether they might be related variants
|
||||||
|
3. Which cluster is more urgent to fix
|
||||||
|
|
||||||
|
Cluster A:
|
||||||
|
- Type: {typeA}
|
||||||
|
- Message: {messageA}
|
||||||
|
- Occurrences: {occurrencesA}
|
||||||
|
- Status: {statusA}
|
||||||
|
|
||||||
|
Cluster B:
|
||||||
|
- Type: {typeB}
|
||||||
|
- Message: {messageB}
|
||||||
|
- Occurrences: {occurrencesB}
|
||||||
|
- Status: {statusB}
|
||||||
|
|
||||||
|
Respond in JSON format with comparison analysis.`;
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// LLM Client
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
interface LLMResponse {
|
||||||
|
rootCauseCategory: RootCauseCategory;
|
||||||
|
hypothesis: string;
|
||||||
|
reasoning: string;
|
||||||
|
confidence: 'high' | 'medium' | 'low';
|
||||||
|
suggestedInvestigation: string[];
|
||||||
|
potentialFixDirection?: string;
|
||||||
|
evidence: Array<{
|
||||||
|
type: string;
|
||||||
|
description: string;
|
||||||
|
strength: string;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LLMCallResult {
|
||||||
|
response: LLMResponse;
|
||||||
|
modelUsed: string;
|
||||||
|
promptTokens: number;
|
||||||
|
completionTokens: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calls Azure OpenAI GPT-4o for analysis
|
||||||
|
*/
|
||||||
|
async function callLLM(
|
||||||
|
prompt: string,
|
||||||
|
options: {
|
||||||
|
model?: 'gpt-4o' | 'gpt-4o-mini';
|
||||||
|
temperature?: number;
|
||||||
|
maxTokens?: number;
|
||||||
|
} = {}
|
||||||
|
): Promise<LLMCallResult> {
|
||||||
|
const model = options.model || 'gpt-4o-mini';
|
||||||
|
const apiKey = config.AZURE_OPENAI_KEY;
|
||||||
|
const endpoint = config.AZURE_OPENAI_ENDPOINT;
|
||||||
|
|
||||||
|
if (!apiKey || !endpoint) {
|
||||||
|
throw new Error('Azure OpenAI credentials not configured');
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = `${endpoint}/openai/deployments/${model}/chat/completions?api-version=2024-02-01`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'api-key': apiKey,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model,
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: 'system',
|
||||||
|
content:
|
||||||
|
'You are an expert software engineer specializing in debugging production systems. Provide structured, evidence-based analysis.',
|
||||||
|
},
|
||||||
|
{ role: 'user', content: prompt },
|
||||||
|
],
|
||||||
|
temperature: options.temperature ?? 0.3,
|
||||||
|
max_tokens: options.maxTokens ?? 2000,
|
||||||
|
response_format: { type: 'json_object' },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const error = await response.text();
|
||||||
|
throw new Error(`Azure OpenAI API error: ${response.status} ${error}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await response.json()) as {
|
||||||
|
choices: Array<{
|
||||||
|
message: { content: string };
|
||||||
|
}>;
|
||||||
|
usage: {
|
||||||
|
prompt_tokens: number;
|
||||||
|
completion_tokens: number;
|
||||||
|
};
|
||||||
|
model: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const content = data.choices[0]?.message?.content;
|
||||||
|
if (!content) {
|
||||||
|
throw new Error('Empty response from LLM');
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = JSON.parse(content) as LLMResponse;
|
||||||
|
|
||||||
|
return {
|
||||||
|
response: parsed,
|
||||||
|
modelUsed: data.model || model,
|
||||||
|
promptTokens: data.usage?.prompt_tokens || 0,
|
||||||
|
completionTokens: data.usage?.completion_tokens || 0,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error('LLM call failed:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Analysis Functions
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Performs root cause analysis on an error cluster
|
||||||
|
*/
|
||||||
|
export async function analyzeRootCause(
|
||||||
|
params: RootCauseAnalysisPrompt
|
||||||
|
): Promise<Partial<DiagnosticInsightDoc>> {
|
||||||
|
const {
|
||||||
|
cluster,
|
||||||
|
context,
|
||||||
|
sampleStackTraces,
|
||||||
|
relatedClusters,
|
||||||
|
analysisType,
|
||||||
|
} = params;
|
||||||
|
|
||||||
|
// Build prompt
|
||||||
|
const prompt = ROOT_CAUSE_ANALYSIS_PROMPT_TEMPLATE.replace(
|
||||||
|
'{errorType}',
|
||||||
|
cluster.errorType
|
||||||
|
)
|
||||||
|
.replace('{messagePattern}', cluster.messageTemplate)
|
||||||
|
.replace('{stackSignature}', cluster.stackSignature.slice(0, 500))
|
||||||
|
.replace('{occurrenceCount}', cluster.occurrenceCount.toString())
|
||||||
|
.replace('{uniqueUsers}', cluster.uniqueUsers.toString())
|
||||||
|
.replace('{firstSeenAt}', cluster.firstSeenAt)
|
||||||
|
.replace('{lastSeenAt}', cluster.lastSeenAt)
|
||||||
|
.replace('{totalOccurrences}', context.totalOccurrences.toString())
|
||||||
|
.replace('{affectedUsersCount}', context.affectedUsers.length.toString())
|
||||||
|
.replace(
|
||||||
|
'{timeRange}',
|
||||||
|
`${context.timeRange.start} to ${context.timeRange.end}`
|
||||||
|
)
|
||||||
|
.replace(
|
||||||
|
'{commonScreens}',
|
||||||
|
context.mostCommonScreens.map((s) => `${s.screen} (${s.count})`).join(', ')
|
||||||
|
)
|
||||||
|
.replace(
|
||||||
|
'{commonActions}',
|
||||||
|
context.mostCommonActions.map((a) => `${a.action} (${a.count})`).join(', ')
|
||||||
|
)
|
||||||
|
.replace('{sampleStackTraces}', sampleStackTraces.join('\n\n'))
|
||||||
|
.replace(
|
||||||
|
'{relatedClusters}',
|
||||||
|
relatedClusters
|
||||||
|
.filter((c) => c.status === 'resolved')
|
||||||
|
.map((c) => `- ${c.errorType} (${c.id})`)
|
||||||
|
.join('\n') || 'None found'
|
||||||
|
)
|
||||||
|
.replace(
|
||||||
|
'{apiFailurePattern}',
|
||||||
|
context.apiFailurePattern
|
||||||
|
? `${context.apiFailurePattern.endpoint}: ${context.apiFailurePattern.errorPattern} (${context.apiFailurePattern.occurrenceCount} occurrences)`
|
||||||
|
: 'No clear API failure pattern'
|
||||||
|
)
|
||||||
|
.replace(
|
||||||
|
'{featureFlagCorrelations}',
|
||||||
|
context.featureFlagCorrelations
|
||||||
|
.map((f) => `- ${f.flag}: ${f.errorCorrelation.toFixed(2)} correlation`)
|
||||||
|
.join('\n') || 'No strong correlations found'
|
||||||
|
);
|
||||||
|
|
||||||
|
// Call LLM
|
||||||
|
const llmResult = await callLLM(prompt, {
|
||||||
|
model: 'gpt-4o-mini',
|
||||||
|
temperature: 0.3,
|
||||||
|
maxTokens: 2000,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Map evidence types
|
||||||
|
const evidence: Evidence[] = llmResult.response.evidence.map((e) => ({
|
||||||
|
type: validateEvidenceType(e.type),
|
||||||
|
description: e.description,
|
||||||
|
strength: validateEvidenceStrength(e.strength),
|
||||||
|
data: {},
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Calculate confidence score
|
||||||
|
const confidenceScore = calculateConfidenceScore(
|
||||||
|
llmResult.response.confidence,
|
||||||
|
evidence,
|
||||||
|
context
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
analysisType,
|
||||||
|
rootCauseCategory: llmResult.response.rootCauseCategory,
|
||||||
|
hypothesis: llmResult.response.hypothesis,
|
||||||
|
reasoning: llmResult.response.reasoning,
|
||||||
|
confidence: llmResult.response.confidence,
|
||||||
|
confidenceScore,
|
||||||
|
evidence,
|
||||||
|
suggestedInvestigation: llmResult.response.suggestedInvestigation,
|
||||||
|
potentialFixDirection: llmResult.response.potentialFixDirection,
|
||||||
|
modelUsed: llmResult.modelUsed,
|
||||||
|
promptTokens: llmResult.promptTokens,
|
||||||
|
completionTokens: llmResult.completionTokens,
|
||||||
|
feedbackStats: { helpful: 0, notHelpful: 0, engineerNotes: [] },
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates a pattern summary for a cluster
|
||||||
|
*/
|
||||||
|
export async function generatePatternSummary(
|
||||||
|
cluster: ErrorClusterDoc,
|
||||||
|
context: ClusterContextSummary
|
||||||
|
): Promise<string> {
|
||||||
|
const prompt = PATTERN_SUMMARY_PROMPT_TEMPLATE.replace('{errorType}', cluster.errorType)
|
||||||
|
.replace('{messagePattern}', cluster.messageTemplate)
|
||||||
|
.replace('{occurrenceCount}', cluster.occurrenceCount.toString())
|
||||||
|
.replace(
|
||||||
|
'{contextSummary}',
|
||||||
|
`Screens: ${context.mostCommonScreens.map((s) => s.screen).join(', ')}
|
||||||
|
Actions: ${context.mostCommonActions.map((a) => a.action).join(', ')}`
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await callLLM(prompt, {
|
||||||
|
model: 'gpt-4o-mini',
|
||||||
|
temperature: 0.4,
|
||||||
|
maxTokens: 300,
|
||||||
|
});
|
||||||
|
|
||||||
|
return result.response.hypothesis || 'Pattern analysis pending.';
|
||||||
|
} catch {
|
||||||
|
return 'Pattern analysis failed. Please review manually.';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compares two clusters
|
||||||
|
*/
|
||||||
|
export async function compareClusters(
|
||||||
|
clusterA: ErrorClusterDoc,
|
||||||
|
clusterB: ErrorClusterDoc,
|
||||||
|
contextA: ClusterContextSummary,
|
||||||
|
contextB: ClusterContextSummary
|
||||||
|
): Promise<{
|
||||||
|
differences: string[];
|
||||||
|
related: boolean;
|
||||||
|
urgencyComparison: string;
|
||||||
|
}> {
|
||||||
|
const prompt = COMPARATIVE_ANALYSIS_PROMPT_TEMPLATE.replace('{typeA}', clusterA.errorType)
|
||||||
|
.replace('{messageA}', clusterA.messageTemplate)
|
||||||
|
.replace('{occurrencesA}', clusterA.occurrenceCount.toString())
|
||||||
|
.replace('{statusA}', clusterA.status)
|
||||||
|
.replace('{typeB}', clusterB.errorType)
|
||||||
|
.replace('{messageB}', clusterB.messageTemplate)
|
||||||
|
.replace('{occurrencesB}', clusterB.occurrenceCount.toString())
|
||||||
|
.replace('{statusB}', clusterB.status);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await callLLM(prompt, {
|
||||||
|
model: 'gpt-4o-mini',
|
||||||
|
temperature: 0.3,
|
||||||
|
maxTokens: 800,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
differences: result.response.suggestedInvestigation || [],
|
||||||
|
related: result.response.confidence === 'high',
|
||||||
|
urgencyComparison: result.response.potentialFixDirection || 'Compare occurrence counts',
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return {
|
||||||
|
differences: [],
|
||||||
|
related: false,
|
||||||
|
urgencyComparison: 'Unable to compare',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Helper Functions
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
function validateEvidenceType(type: string): Evidence['type'] {
|
||||||
|
const validTypes: Evidence['type'][] = [
|
||||||
|
'stack_trace',
|
||||||
|
'telemetry_pattern',
|
||||||
|
'device_correlation',
|
||||||
|
'api_failure',
|
||||||
|
'similar_issue',
|
||||||
|
'config_mismatch',
|
||||||
|
'version_regression',
|
||||||
|
];
|
||||||
|
return validTypes.includes(type as Evidence['type'])
|
||||||
|
? (type as Evidence['type'])
|
||||||
|
: 'stack_trace';
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateEvidenceStrength(strength: string): 'strong' | 'moderate' | 'weak' {
|
||||||
|
const validStrengths = ['strong', 'moderate', 'weak'];
|
||||||
|
return validStrengths.includes(strength) ? (strength as 'strong' | 'moderate' | 'weak') : 'moderate';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates a numeric confidence score based on multiple factors
|
||||||
|
*/
|
||||||
|
function calculateConfidenceScore(
|
||||||
|
llmConfidence: string,
|
||||||
|
evidence: Evidence[],
|
||||||
|
context: ClusterContextSummary
|
||||||
|
): number {
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
// Base score from LLM confidence
|
||||||
|
switch (llmConfidence) {
|
||||||
|
case 'high':
|
||||||
|
score += 0.6;
|
||||||
|
break;
|
||||||
|
case 'medium':
|
||||||
|
score += 0.4;
|
||||||
|
break;
|
||||||
|
case 'low':
|
||||||
|
score += 0.2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evidence strength bonus
|
||||||
|
const strongEvidence = evidence.filter((e) => e.strength === 'strong').length;
|
||||||
|
const moderateEvidence = evidence.filter((e) => e.strength === 'moderate').length;
|
||||||
|
score += strongEvidence * 0.1 + moderateEvidence * 0.05;
|
||||||
|
|
||||||
|
// Data volume bonus (more data = higher confidence)
|
||||||
|
if (context.totalOccurrences >= 100) score += 0.1;
|
||||||
|
else if (context.totalOccurrences >= 10) score += 0.05;
|
||||||
|
|
||||||
|
// Affected users bonus
|
||||||
|
if (context.affectedUsers.length >= 50) score += 0.1;
|
||||||
|
else if (context.affectedUsers.length >= 10) score += 0.05;
|
||||||
|
|
||||||
|
return Math.min(1.0, score);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Retry Logic
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
interface RetryOptions {
|
||||||
|
maxRetries?: number;
|
||||||
|
baseDelayMs?: number;
|
||||||
|
maxDelayMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function callLLMWithRetry<T>(
|
||||||
|
fn: () => Promise<T>,
|
||||||
|
options: RetryOptions = {}
|
||||||
|
): Promise<T> {
|
||||||
|
const { maxRetries = 3, baseDelayMs = 1000, maxDelayMs = 10000 } = options;
|
||||||
|
|
||||||
|
let lastError: Error | undefined;
|
||||||
|
|
||||||
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||||
|
try {
|
||||||
|
return await fn();
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error instanceof Error ? error : new Error(String(error));
|
||||||
|
|
||||||
|
// Don't retry on authentication errors
|
||||||
|
if (lastError.message.includes('401') || lastError.message.includes('403')) {
|
||||||
|
throw lastError;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt < maxRetries - 1) {
|
||||||
|
// Exponential backoff with jitter
|
||||||
|
const delay = Math.min(
|
||||||
|
baseDelayMs * Math.pow(2, attempt) + Math.random() * 1000,
|
||||||
|
maxDelayMs
|
||||||
|
);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError;
|
||||||
|
}
|
||||||
@ -350,7 +350,7 @@ export async function enrichErrorContext(
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
errorEvent,
|
errorEvent,
|
||||||
telemetryContext,
|
telemetryContext: telemetryContext || undefined,
|
||||||
sessionState,
|
sessionState,
|
||||||
recentActions,
|
recentActions,
|
||||||
apiFailures,
|
apiFailures,
|
||||||
@ -507,7 +507,7 @@ export function generateBreadcrumbTrail(
|
|||||||
// Cluster Context Aggregation
|
// Cluster Context Aggregation
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
interface ClusterContextSummary {
|
export interface ClusterContextSummary {
|
||||||
totalOccurrences: number;
|
totalOccurrences: number;
|
||||||
affectedUsers: string[];
|
affectedUsers: string[];
|
||||||
timeRange: { start: string; end: string };
|
timeRange: { start: string; end: string };
|
||||||
|
|||||||
@ -0,0 +1,634 @@
|
|||||||
|
/**
|
||||||
|
* Predictive Analytics Module Tests
|
||||||
|
* [Target 20+ tests]
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||||
|
import type { TelemetryEventDoc } from '../telemetry/types.js';
|
||||||
|
import {
|
||||||
|
extractFeaturesFromTelemetry,
|
||||||
|
extractProductSpecificFeatures,
|
||||||
|
CompleteFeatureVector,
|
||||||
|
} from './feature-extractor.js';
|
||||||
|
import { ChurnModel } from './churn-model.js';
|
||||||
|
import { HealthScoringEngine, HealthMetricsInput } from './health-scoring.js';
|
||||||
|
import { AnomalyDetectionEngine } from './anomaly-detection.js';
|
||||||
|
import {
|
||||||
|
ChurnPredictionInput,
|
||||||
|
RiskSegmentEnum,
|
||||||
|
ModelTypeEnum,
|
||||||
|
PredictionHorizonEnum,
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Test Fixtures
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
function createMockTelemetryEvent(overrides: Partial<TelemetryEventDoc> = {}): TelemetryEventDoc {
|
||||||
|
return {
|
||||||
|
id: crypto.randomUUID(),
|
||||||
|
pk: 'test:202601:ios',
|
||||||
|
productId: 'test',
|
||||||
|
userId: 'user_123',
|
||||||
|
sessionId: 'session_456',
|
||||||
|
platform: 'ios',
|
||||||
|
channel: 'mobile_app',
|
||||||
|
osFamily: 'ios',
|
||||||
|
appVersion: '1.0.0',
|
||||||
|
buildNumber: '100',
|
||||||
|
releaseChannel: 'prod',
|
||||||
|
eventType: 'info',
|
||||||
|
module: 'test',
|
||||||
|
feature: 'core',
|
||||||
|
eventName: 'test_event',
|
||||||
|
occurredAt: new Date().toISOString(),
|
||||||
|
receivedAt: new Date().toISOString(),
|
||||||
|
ttl: 30 * 24 * 60 * 60,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMockEventSeries(userId: string, days: number): TelemetryEventDoc[] {
|
||||||
|
const events: TelemetryEventDoc[] = [];
|
||||||
|
const now = new Date();
|
||||||
|
|
||||||
|
for (let d = 0; d < days; d++) {
|
||||||
|
const date = new Date(now);
|
||||||
|
date.setDate(date.getDate() - d);
|
||||||
|
|
||||||
|
// Create 2-5 sessions per day
|
||||||
|
const sessions = 2 + Math.floor(Math.random() * 3);
|
||||||
|
for (let s = 0; s < sessions; s++) {
|
||||||
|
events.push(
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
userId,
|
||||||
|
sessionId: `session_${d}_${s}`,
|
||||||
|
occurredAt: date.toISOString(),
|
||||||
|
feature: ['core', 'advanced', 'settings'][Math.floor(Math.random() * 3)],
|
||||||
|
eventName: 'action_completed',
|
||||||
|
metrics: { duration: 1000 + Math.random() * 5000 },
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return events;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Feature Extraction Tests
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
describe('Feature Extractor', () => {
|
||||||
|
it('should extract features from telemetry events', () => {
|
||||||
|
const events = createMockEventSeries('user_123', 14);
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||||
|
|
||||||
|
expect(features.userId).toBe('user_123');
|
||||||
|
expect(features.productId).toBe('test');
|
||||||
|
expect(features.featureSchemaVersion).toBe('1.0.0');
|
||||||
|
expect(features.dataQualityScore).toBeGreaterThan(0);
|
||||||
|
expect(features.dataQualityScore).toBeLessThanOrEqual(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate behavior features correctly', () => {
|
||||||
|
const now = new Date();
|
||||||
|
const yesterday = new Date(now);
|
||||||
|
yesterday.setDate(yesterday.getDate() - 1);
|
||||||
|
|
||||||
|
const events: TelemetryEventDoc[] = [
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
sessionId: 'session_1',
|
||||||
|
occurredAt: yesterday.toISOString(),
|
||||||
|
feature: 'core',
|
||||||
|
eventName: 'session_start',
|
||||||
|
metrics: { duration: 300000 },
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', 'test', events, now);
|
||||||
|
|
||||||
|
expect(features.behavior.daysSinceLastSession).toBeGreaterThanOrEqual(1);
|
||||||
|
expect(features.behavior.sessionsLast24Hours).toBe(0);
|
||||||
|
expect(features.behavior.sessionsLast7Days).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate engagement features correctly', () => {
|
||||||
|
const events: TelemetryEventDoc[] = [
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
feature: 'core',
|
||||||
|
eventName: 'core_action_completed',
|
||||||
|
}),
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
feature: 'advanced',
|
||||||
|
eventName: 'feature_used',
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||||
|
|
||||||
|
expect(features.engagement.featureUsageDiversity).toBeGreaterThan(0);
|
||||||
|
expect(features.engagement.uniqueFeaturesUsed).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate rolling window features', () => {
|
||||||
|
const events = createMockEventSeries('user_123', 30);
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||||
|
|
||||||
|
expect(features.rolling.rollingAvgSessions7d).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(features.rolling.cohortSessionPercentile).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(features.rolling.cohortSessionPercentile).toBeLessThanOrEqual(100);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should extract product-specific features for NomGap', () => {
|
||||||
|
const events: TelemetryEventDoc[] = [
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
feature: 'fasting',
|
||||||
|
eventName: 'fast_started',
|
||||||
|
}),
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
feature: 'fasting',
|
||||||
|
eventName: 'fast_completed',
|
||||||
|
context: { adhered: true },
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const features = extractProductSpecificFeatures(events, 'nomgap');
|
||||||
|
|
||||||
|
expect(features.fastCompletionRate).toBe(1);
|
||||||
|
expect(features.protocolAdherenceScore).toBeGreaterThanOrEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should extract product-specific features for JarvisJr', () => {
|
||||||
|
const events: TelemetryEventDoc[] = [
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
feature: 'agent',
|
||||||
|
eventName: 'session_started',
|
||||||
|
context: { mode: 'voice', agentId: 'agent_1' },
|
||||||
|
}),
|
||||||
|
createMockTelemetryEvent({
|
||||||
|
feature: 'agent',
|
||||||
|
eventName: 'session_completed',
|
||||||
|
context: { mode: 'text', agentId: 'agent_2' },
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const features = extractProductSpecificFeatures(events, 'jarvisjr');
|
||||||
|
|
||||||
|
expect(features.agentDiversityScore).toBeGreaterThan(0);
|
||||||
|
expect(features.voiceSessionRatio).toBe(0.5);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle empty telemetry gracefully', () => {
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', 'test', []);
|
||||||
|
|
||||||
|
expect(features.behavior.sessionsLast30Days).toBe(0);
|
||||||
|
expect(features.engagement.featureUsageDiversity).toBe(0);
|
||||||
|
expect(features.dataQualityScore).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate data quality score based on completeness', () => {
|
||||||
|
const richEvents = createMockEventSeries('user_123', 30);
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', 'test', richEvents);
|
||||||
|
|
||||||
|
expect(features.dataQualityScore).toBeGreaterThan(0.3);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Churn Model Tests
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
describe('Churn Model', () => {
|
||||||
|
let model: ChurnModel;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
model = new ChurnModel();
|
||||||
|
});
|
||||||
|
|
||||||
|
function createMockFeatures(): CompleteFeatureVector {
|
||||||
|
const now = new Date();
|
||||||
|
return {
|
||||||
|
userId: 'user_123',
|
||||||
|
productId: 'test',
|
||||||
|
computedAt: now,
|
||||||
|
observationWindow: { start: now, end: now },
|
||||||
|
behavior: {
|
||||||
|
daysSinceLastSession: 3,
|
||||||
|
daysSinceLastCoreAction: 2,
|
||||||
|
hoursSinceLastLogin: 72,
|
||||||
|
sessionsLast24Hours: 1,
|
||||||
|
sessionsLast7Days: 5,
|
||||||
|
sessionsLast30Days: 20,
|
||||||
|
avgSessionsPerWeek: 4,
|
||||||
|
avgSessionsPerDay: 0.6,
|
||||||
|
avgSessionDurationMinutes: 15,
|
||||||
|
totalSessionDurationMinutes: 300,
|
||||||
|
actionsPerSession: 8,
|
||||||
|
uniqueFeaturesUsed: 5,
|
||||||
|
sessionFrequencyTrend: 0.1,
|
||||||
|
engagementDepthTrend: 0.05,
|
||||||
|
},
|
||||||
|
engagement: {
|
||||||
|
featureUsageDiversity: 0.5,
|
||||||
|
coreActionCompletionRate: 0.8,
|
||||||
|
featureAdoptionVelocity: 1.2,
|
||||||
|
powerUserScore: 0.3,
|
||||||
|
onboardingCompletionRate: 0.9,
|
||||||
|
firstValueMomentAchieved: true,
|
||||||
|
timeToFirstValueHours: 2,
|
||||||
|
},
|
||||||
|
performance: {
|
||||||
|
errorRateLast7Days: 0.01,
|
||||||
|
errorRateLast30Days: 0.02,
|
||||||
|
crashCountLast7Days: 0,
|
||||||
|
crashCountLast30Days: 1,
|
||||||
|
avgLatencyMs: 200,
|
||||||
|
slowRequestCount: 2,
|
||||||
|
timeoutCount: 0,
|
||||||
|
errorRecoveryRate: 0.95,
|
||||||
|
supportTicketCount: 0,
|
||||||
|
},
|
||||||
|
social: {
|
||||||
|
shareCount: 3,
|
||||||
|
inviteCount: 1,
|
||||||
|
collaborationScore: 0.2,
|
||||||
|
teamMemberCount: 2,
|
||||||
|
integrationsConnected: 1,
|
||||||
|
externalSharesLast30Days: 2,
|
||||||
|
},
|
||||||
|
revenue: {
|
||||||
|
planTier: 1,
|
||||||
|
lifetimeValue: 120,
|
||||||
|
mrrContribution: 10,
|
||||||
|
upgradeCount: 1,
|
||||||
|
downgradeCount: 0,
|
||||||
|
daysSinceLastPayment: 5,
|
||||||
|
daysSincePlanChange: 60,
|
||||||
|
supportTicketCount: 0,
|
||||||
|
supportSatisfactionScore: 4.5,
|
||||||
|
escalatedTicketCount: 0,
|
||||||
|
},
|
||||||
|
rolling: {
|
||||||
|
rollingAvgSessions7d: 0.7,
|
||||||
|
rollingAvgDuration7d: 15,
|
||||||
|
rollingAvgActions7d: 8,
|
||||||
|
wowSessionChange: 0.1,
|
||||||
|
wowDurationChange: 0.05,
|
||||||
|
wowActionsChange: 0.08,
|
||||||
|
cohortSessionPercentile: 60,
|
||||||
|
cohortEngagementPercentile: 55,
|
||||||
|
cohortRetentionPercentile: 65,
|
||||||
|
},
|
||||||
|
productSpecific: {},
|
||||||
|
timeWindows: {
|
||||||
|
recent: { sessionCount: 1, totalDuration: 900, actionCount: 8, errorCount: 0, uniqueFeatures: ['core'] },
|
||||||
|
weekly: { sessionCount: 5, totalDuration: 4500, actionCount: 40, errorCount: 0, uniqueFeatures: ['core', 'advanced'], daysActive: 4 },
|
||||||
|
monthly: { sessionCount: 20, totalDuration: 18000, actionCount: 160, errorCount: 2, uniqueFeatures: ['core', 'advanced', 'settings'], daysActive: 15 },
|
||||||
|
lifetime: { totalSessions: 100, totalDuration: 90000, totalActions: 800, totalErrors: 5, allFeaturesUsed: ['core', 'advanced', 'settings'], accountAgeDays: 90 },
|
||||||
|
},
|
||||||
|
featureSchemaVersion: '1.0.0',
|
||||||
|
dataQualityScore: 0.85,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it('should predict churn probability', () => {
|
||||||
|
const features = createMockFeatures();
|
||||||
|
const prediction = model.predict(features);
|
||||||
|
|
||||||
|
expect(prediction.churnProbability).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(prediction.churnProbability).toBeLessThanOrEqual(1);
|
||||||
|
expect(prediction.userId).toBe('user_123');
|
||||||
|
expect(prediction.productId).toBe('test');
|
||||||
|
expect(prediction.modelType).toBe('xgboost');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should categorize risk segments correctly', () => {
|
||||||
|
const features = createMockFeatures();
|
||||||
|
|
||||||
|
// Low engagement should increase churn risk
|
||||||
|
features.engagement.featureUsageDiversity = 0.1;
|
||||||
|
features.behavior.daysSinceLastSession = 20;
|
||||||
|
|
||||||
|
const prediction = model.predict(features);
|
||||||
|
|
||||||
|
expect(['critical', 'high', 'medium', 'low']).toContain(prediction.riskSegment);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should generate explanations with top risk factors', () => {
|
||||||
|
const features = createMockFeatures();
|
||||||
|
const prediction = model.predict(features);
|
||||||
|
|
||||||
|
expect(prediction.explanation.topRiskFactors).toHaveLength(5);
|
||||||
|
expect(prediction.explanation.nlExplanation).toContain('churn risk');
|
||||||
|
expect(prediction.explanation.suggestedActions.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate confidence score based on data quality', () => {
|
||||||
|
const features = createMockFeatures();
|
||||||
|
features.dataQualityScore = 0.3;
|
||||||
|
|
||||||
|
const prediction = model.predict(features);
|
||||||
|
|
||||||
|
expect(prediction.confidenceScore).toBeLessThanOrEqual(features.dataQualityScore + 0.1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should predict batch correctly', () => {
|
||||||
|
const features = [createMockFeatures(), createMockFeatures()];
|
||||||
|
features[1].userId = 'user_456';
|
||||||
|
|
||||||
|
const predictions = model.predictBatch(features);
|
||||||
|
|
||||||
|
expect(predictions).toHaveLength(2);
|
||||||
|
expect(predictions[0].userId).toBe('user_123');
|
||||||
|
expect(predictions[1].userId).toBe('user_456');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle different prediction horizons', () => {
|
||||||
|
const features = createMockFeatures();
|
||||||
|
|
||||||
|
const p7 = model.predict(features, 7);
|
||||||
|
const p30 = model.predict(features, 30);
|
||||||
|
|
||||||
|
// Longer horizon should generally have different probability
|
||||||
|
expect(p7.predictionHorizon).toBe(7);
|
||||||
|
expect(p30.predictionHorizon).toBe(30);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should evaluate model performance', () => {
|
||||||
|
const predictions = [
|
||||||
|
{ actual: true, predicted: 0.8 },
|
||||||
|
{ actual: false, predicted: 0.2 },
|
||||||
|
{ actual: true, predicted: 0.9 },
|
||||||
|
{ actual: false, predicted: 0.1 },
|
||||||
|
];
|
||||||
|
|
||||||
|
const metrics = model.evaluateModel(predictions);
|
||||||
|
|
||||||
|
expect(metrics.auc).toBeGreaterThan(0.5);
|
||||||
|
expect(metrics.modelVersion).toBeDefined();
|
||||||
|
expect(metrics.featureImportance.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Health Scoring Tests
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
describe('Health Scoring Engine', () => {
|
||||||
|
let engine: HealthScoringEngine;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
engine = new HealthScoringEngine();
|
||||||
|
});
|
||||||
|
|
||||||
|
function createMockHealthInput(): HealthMetricsInput {
|
||||||
|
return {
|
||||||
|
productId: 'test',
|
||||||
|
date: new Date(),
|
||||||
|
newUsers: 100,
|
||||||
|
activationRateDay1: 0.6,
|
||||||
|
activationRateDay7: 0.45,
|
||||||
|
cac: 50,
|
||||||
|
firstValueMomentRate: 0.55,
|
||||||
|
timeToFirstAction: 120,
|
||||||
|
onboardingCompletionRate: 0.75,
|
||||||
|
dau: 1000,
|
||||||
|
mau: 5000,
|
||||||
|
day7Retention: 0.35,
|
||||||
|
day30Retention: 0.25,
|
||||||
|
avgSessionLength: 300,
|
||||||
|
sessionsPerUser: 3,
|
||||||
|
featureAdoption: { feature1: 0.5, feature2: 0.3 },
|
||||||
|
mrr: 10000,
|
||||||
|
arpu: 20,
|
||||||
|
churnRate: 0.05,
|
||||||
|
upgradeRate: 0.1,
|
||||||
|
crashFreeRate: 0.99,
|
||||||
|
errorRate: 0.01,
|
||||||
|
avgLatency: 150,
|
||||||
|
uptimePercent: 99.9,
|
||||||
|
baselines: {
|
||||||
|
dau: 900,
|
||||||
|
newUsers: 90,
|
||||||
|
activationRateDay1: 0.55,
|
||||||
|
day7Retention: 0.33,
|
||||||
|
avgSessionLength: 280,
|
||||||
|
mrr: 9500,
|
||||||
|
errorRate: 0.015,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it('should calculate overall health score', () => {
|
||||||
|
const input = createMockHealthInput();
|
||||||
|
const score = engine.calculateHealthScore(input);
|
||||||
|
|
||||||
|
expect(score.overallHealthScore).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(score.overallHealthScore).toBeLessThanOrEqual(100);
|
||||||
|
expect(score.productId).toBe('test');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate all 6 dimension scores', () => {
|
||||||
|
const input = createMockHealthInput();
|
||||||
|
const score = engine.calculateHealthScore(input);
|
||||||
|
|
||||||
|
expect(score.dimensions.acquisition.score).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(score.dimensions.activation.score).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(score.dimensions.retention.score).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(score.dimensions.engagement.score).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(score.dimensions.revenue.score).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(score.dimensions.stability.score).toBeGreaterThanOrEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect health status correctly', () => {
|
||||||
|
const input = createMockHealthInput();
|
||||||
|
|
||||||
|
// Healthy case
|
||||||
|
const healthyScore = engine.calculateHealthScore(input);
|
||||||
|
expect(['healthy', 'warning', 'critical']).toContain(healthyScore.healthStatus);
|
||||||
|
|
||||||
|
// Critical case
|
||||||
|
input.day7Retention = 0.05;
|
||||||
|
input.churnRate = 0.5;
|
||||||
|
input.crashFreeRate = 0.8;
|
||||||
|
const criticalScore = engine.calculateHealthScore(input);
|
||||||
|
expect(criticalScore.healthStatus).toBe('critical');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect anomalies', () => {
|
||||||
|
const input = createMockHealthInput();
|
||||||
|
input.dau = 500; // 50% drop from baseline
|
||||||
|
|
||||||
|
const score = engine.calculateHealthScore(input);
|
||||||
|
|
||||||
|
expect(score.anomalies.length).toBeGreaterThan(0);
|
||||||
|
expect(score.anomalies.some((a) => a.metric === 'dau')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should generate forecasts', () => {
|
||||||
|
const input = createMockHealthInput();
|
||||||
|
const score = engine.calculateHealthScore(input);
|
||||||
|
|
||||||
|
expect(score.forecasts.next7Days.expectedHealthScore).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(score.forecasts.next7Days.confidenceInterval).toHaveLength(2);
|
||||||
|
expect(score.forecasts.next30Days.expectedHealthScore).toBeGreaterThanOrEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate baseline comparisons', () => {
|
||||||
|
const input = createMockHealthInput();
|
||||||
|
const score = engine.calculateHealthScore(input);
|
||||||
|
|
||||||
|
expect(typeof score.vsBaseline7Day).toBe('number');
|
||||||
|
expect(typeof score.vsBaseline30Day).toBe('number');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should determine trends correctly', () => {
|
||||||
|
const input = createMockHealthInput();
|
||||||
|
|
||||||
|
// Improving trend
|
||||||
|
input.dau = 1200;
|
||||||
|
const improving = engine.calculateHealthScore(input);
|
||||||
|
expect(improving.dimensions.retention.trend).toBe('improving');
|
||||||
|
|
||||||
|
// Declining trend
|
||||||
|
input.dau = 600;
|
||||||
|
const declining = engine.calculateHealthScore(input);
|
||||||
|
expect(declining.dimensions.retention.trend).toBe('declining');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Anomaly Detection Tests
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
describe('Anomaly Detection Engine', () => {
|
||||||
|
let engine: AnomalyDetectionEngine;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
engine = new AnomalyDetectionEngine();
|
||||||
|
});
|
||||||
|
|
||||||
|
function createTimeSeries(length: number, baseValue: number, noise: number): Array<{ timestamp: Date; value: number }> {
|
||||||
|
const series: Array<{ timestamp: Date; value: number }> = [];
|
||||||
|
const now = new Date();
|
||||||
|
|
||||||
|
for (let i = 0; i < length; i++) {
|
||||||
|
const date = new Date(now);
|
||||||
|
date.setDate(date.getDate() - (length - i));
|
||||||
|
series.push({
|
||||||
|
timestamp: date,
|
||||||
|
value: baseValue + (Math.random() - 0.5) * noise,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return series;
|
||||||
|
}
|
||||||
|
|
||||||
|
it('should detect anomalies in time series', () => {
|
||||||
|
const series = createTimeSeries(30, 100, 10);
|
||||||
|
|
||||||
|
// Add anomaly
|
||||||
|
series[series.length - 1].value = 200;
|
||||||
|
|
||||||
|
const anomalies = engine.detectAnomalies(series, 'test_metric');
|
||||||
|
|
||||||
|
expect(anomalies.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not detect anomalies in stable series', () => {
|
||||||
|
const series = createTimeSeries(30, 100, 5);
|
||||||
|
|
||||||
|
const anomalies = engine.detectAnomalies(series, 'test_metric');
|
||||||
|
|
||||||
|
expect(anomalies.length).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should require minimum data for detection', () => {
|
||||||
|
const shortSeries = createTimeSeries(5, 100, 10);
|
||||||
|
|
||||||
|
const anomalies = engine.detectAnomalies(shortSeries, 'test_metric');
|
||||||
|
|
||||||
|
expect(anomalies.length).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should forecast future values', () => {
|
||||||
|
const series = createTimeSeries(30, 100, 10);
|
||||||
|
|
||||||
|
const forecast = engine.forecast(series, 7);
|
||||||
|
|
||||||
|
expect(forecast.forecast.length).toBe(7);
|
||||||
|
expect(forecast.confidenceIntervals.length).toBe(7);
|
||||||
|
expect(forecast.forecast[0].value).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return empty forecast for insufficient data', () => {
|
||||||
|
const shortSeries = createTimeSeries(5, 100, 10);
|
||||||
|
|
||||||
|
const forecast = engine.forecast(shortSeries, 7);
|
||||||
|
|
||||||
|
expect(forecast.forecast.length).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect multi-dimensional anomalies', () => {
|
||||||
|
const metrics: Record<string, Array<{ timestamp: Date; value: number }>> = {
|
||||||
|
metric1: createTimeSeries(30, 100, 10),
|
||||||
|
metric2: createTimeSeries(30, 50, 5),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Add correlated anomalies
|
||||||
|
metrics.metric1[metrics.metric1.length - 1].value = 180;
|
||||||
|
metrics.metric2[metrics.metric2.length - 1].value = 90;
|
||||||
|
|
||||||
|
const anomalies = engine.detectMultiDimensionalAnomaly(metrics);
|
||||||
|
|
||||||
|
expect(anomalies.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Integration Tests
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
describe('Predictive Analytics Integration', () => {
|
||||||
|
it('should complete full prediction pipeline', () => {
|
||||||
|
// 1. Extract features from telemetry
|
||||||
|
const events = createMockEventSeries('user_123', 30);
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||||
|
|
||||||
|
// 2. Run churn prediction
|
||||||
|
const model = new ChurnModel();
|
||||||
|
const prediction = model.predict(features);
|
||||||
|
|
||||||
|
// 3. Verify prediction structure
|
||||||
|
expect(prediction.churnProbability).toBeDefined();
|
||||||
|
expect(prediction.riskSegment).toBeDefined();
|
||||||
|
expect(prediction.explanation).toBeDefined();
|
||||||
|
expect(prediction.explanation.topRiskFactors).toHaveLength(5);
|
||||||
|
expect(prediction.explanation.suggestedActions.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle different product types', () => {
|
||||||
|
const products = ['nomgap', 'jarvisjr', 'chronomind', 'mindlyst', 'peakpulse', 'lysnrai'];
|
||||||
|
|
||||||
|
for (const productId of products) {
|
||||||
|
const events = createMockEventSeries('user_123', 14);
|
||||||
|
const features = extractFeaturesFromTelemetry('user_123', productId, events);
|
||||||
|
const productSpecific = extractProductSpecificFeatures(events, productId);
|
||||||
|
|
||||||
|
// Should complete without errors
|
||||||
|
expect(features.productId).toBe(productId);
|
||||||
|
expect(productSpecific).toBeDefined();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Summary
|
||||||
|
// ============================================================================
|
||||||
|
// Total tests: 26+
|
||||||
|
// - Feature extraction: 7 tests
|
||||||
|
// - Churn model: 7 tests
|
||||||
|
// - Health scoring: 7 tests
|
||||||
|
// - Anomaly detection: 6 tests
|
||||||
|
// - Integration: 2 tests
|
||||||
@ -0,0 +1,471 @@
|
|||||||
|
/**
|
||||||
|
* Predictive Analytics Routes - REST API endpoints
|
||||||
|
* [2.2] Real-time scoring API
|
||||||
|
* [4.3] CS team dashboard backend
|
||||||
|
* [5] Admin API routes
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
|
||||||
|
import { z } from 'zod';
|
||||||
|
import { requireAuth, requireAdmin } from '@bytelyst/auth';
|
||||||
|
import { extractFeaturesFromTelemetry } from './feature-extractor.js';
|
||||||
|
import { featureStore } from './feature-store.js';
|
||||||
|
import { churnModel } from './churn-model.js';
|
||||||
|
import { healthScoringEngine } from './health-scoring.js';
|
||||||
|
import { campaignEngine } from './campaign-engine.js';
|
||||||
|
import { predictiveAnalyticsRepo } from './repository.js';
|
||||||
|
import {
|
||||||
|
ChurnScoreRequestSchema,
|
||||||
|
ChurnBatchRequestSchema,
|
||||||
|
AtRiskUsersQuerySchema,
|
||||||
|
CreateCampaignSchema,
|
||||||
|
CampaignTriggerSchema,
|
||||||
|
RiskSegmentEnum,
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
|
// Get telemetry repository for fetching user events
|
||||||
|
async function getUserTelemetryEvents(
|
||||||
|
userId: string,
|
||||||
|
productId: string,
|
||||||
|
days: number = 30
|
||||||
|
): Promise<Array<Record<string, unknown>>> {
|
||||||
|
const { getRegisteredContainer } = await import('@bytelyst/cosmos');
|
||||||
|
const container = getRegisteredContainer('telemetry_events');
|
||||||
|
|
||||||
|
const cutoff = new Date();
|
||||||
|
cutoff.setDate(cutoff.getDate() - days);
|
||||||
|
|
||||||
|
const pk = `${productId}:${cutoff.getFullYear()}${String(cutoff.getMonth() + 1).padStart(2, '0')}`;
|
||||||
|
|
||||||
|
const query = {
|
||||||
|
query: 'SELECT * FROM c WHERE c.pk = @pk AND c.userId = @userId ORDER BY c.occurredAt DESC',
|
||||||
|
parameters: [
|
||||||
|
{ name: '@pk', value: pk },
|
||||||
|
{ name: '@userId', value: userId },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const { resources } = await container.items.query(query).fetchAll();
|
||||||
|
return resources as Array<Record<string, unknown>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function predictiveAnalyticsRoutes(fastify: FastifyInstance): Promise<void> {
|
||||||
|
// ==================== Churn Prediction Routes ====================
|
||||||
|
|
||||||
|
// POST /predictive/churn-score - Single user prediction
|
||||||
|
fastify.post('/predictive/churn-score', {
|
||||||
|
preHandler: [requireAuth],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const body = ChurnScoreRequestSchema.parse(request.body);
|
||||||
|
const { userId, productId, horizon } = body;
|
||||||
|
|
||||||
|
request.log.info({ userId, productId }, 'Churn score request');
|
||||||
|
|
||||||
|
// Get cached prediction if fresh (< 24h)
|
||||||
|
const cached = await predictiveAnalyticsRepo.getChurnPrediction(
|
||||||
|
userId,
|
||||||
|
productId,
|
||||||
|
parseInt(horizon, 10)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (cached) {
|
||||||
|
const age = Date.now() - new Date(cached.predictionTimestamp).getTime();
|
||||||
|
if (age < 24 * 60 * 60 * 1000) {
|
||||||
|
return reply.send({
|
||||||
|
userId: cached.userId,
|
||||||
|
churnProbability: cached.churnProbability,
|
||||||
|
riskSegment: cached.riskSegment,
|
||||||
|
confidenceScore: cached.confidenceScore,
|
||||||
|
explanation: cached.explanation,
|
||||||
|
modelVersion: cached.modelVersion,
|
||||||
|
cached: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetch telemetry events
|
||||||
|
const events = await getUserTelemetryEvents(userId, productId);
|
||||||
|
|
||||||
|
// Extract features
|
||||||
|
const features = extractFeaturesFromTelemetry(
|
||||||
|
userId,
|
||||||
|
productId,
|
||||||
|
events as Parameters<typeof extractFeaturesFromTelemetry>[2],
|
||||||
|
new Date()
|
||||||
|
);
|
||||||
|
|
||||||
|
// Store features
|
||||||
|
await featureStore.saveFeatureVector(userId, productId, features);
|
||||||
|
|
||||||
|
// Run prediction
|
||||||
|
const prediction = churnModel.predict(features, parseInt(horizon, 10));
|
||||||
|
|
||||||
|
// Save prediction
|
||||||
|
const doc = {
|
||||||
|
id: `cp_${crypto.randomUUID()}`,
|
||||||
|
pk: `${userId}:${productId}`,
|
||||||
|
userId,
|
||||||
|
productId,
|
||||||
|
...prediction,
|
||||||
|
actualChurned: undefined,
|
||||||
|
validationDate: undefined,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
ttl: (parseInt(horizon, 10) + 90) * 24 * 60 * 60,
|
||||||
|
};
|
||||||
|
|
||||||
|
await predictiveAnalyticsRepo.saveChurnPrediction(doc);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
userId,
|
||||||
|
churnProbability: prediction.churnProbability,
|
||||||
|
riskSegment: prediction.riskSegment,
|
||||||
|
confidenceScore: prediction.confidenceScore,
|
||||||
|
explanation: prediction.explanation,
|
||||||
|
modelVersion: prediction.modelVersion,
|
||||||
|
cached: false,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// POST /predictive/churn-batch - Batch scoring
|
||||||
|
fastify.post('/predictive/churn-batch', {
|
||||||
|
preHandler: [requireAuth],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const body = ChurnBatchRequestSchema.parse(request.body);
|
||||||
|
const { userIds, productId, horizon } = body;
|
||||||
|
|
||||||
|
request.log.info({ userCount: userIds.length, productId }, 'Batch churn score request');
|
||||||
|
|
||||||
|
const results = await Promise.all(
|
||||||
|
userIds.map(async (userId) => {
|
||||||
|
const events = await getUserTelemetryEvents(userId, productId);
|
||||||
|
const features = extractFeaturesFromTelemetry(
|
||||||
|
userId,
|
||||||
|
productId,
|
||||||
|
events as Parameters<typeof extractFeaturesFromTelemetry>[2],
|
||||||
|
new Date()
|
||||||
|
);
|
||||||
|
const prediction = churnModel.predict(features, parseInt(horizon, 10));
|
||||||
|
return {
|
||||||
|
userId,
|
||||||
|
churnProbability: prediction.churnProbability,
|
||||||
|
riskSegment: prediction.riskSegment,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
productId,
|
||||||
|
horizon: parseInt(horizon, 10),
|
||||||
|
results,
|
||||||
|
summary: {
|
||||||
|
total: results.length,
|
||||||
|
critical: results.filter((r) => r.riskSegment === 'critical').length,
|
||||||
|
high: results.filter((r) => r.riskSegment === 'high').length,
|
||||||
|
medium: results.filter((r) => r.riskSegment === 'medium').length,
|
||||||
|
low: results.filter((r) => r.riskSegment === 'low').length,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /predictive/at-risk-users - List at-risk users
|
||||||
|
fastify.get('/predictive/at-risk-users', {
|
||||||
|
preHandler: [requireAuth],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const query = AtRiskUsersQuerySchema.parse(request.query);
|
||||||
|
const { productId, segment, limit, offset } = query;
|
||||||
|
|
||||||
|
const predictions = await predictiveAnalyticsRepo.getAtRiskUsers(
|
||||||
|
productId || '',
|
||||||
|
segment,
|
||||||
|
limit,
|
||||||
|
offset
|
||||||
|
);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
users: predictions.map((p) => ({
|
||||||
|
userId: p.userId,
|
||||||
|
productId: p.productId,
|
||||||
|
churnProbability: p.churnProbability,
|
||||||
|
riskSegment: p.riskSegment,
|
||||||
|
confidenceScore: p.confidenceScore,
|
||||||
|
predictionTimestamp: p.predictionTimestamp,
|
||||||
|
topRiskFactors: p.explanation?.topRiskFactors?.slice(0, 3) || [],
|
||||||
|
})),
|
||||||
|
pagination: {
|
||||||
|
limit,
|
||||||
|
offset,
|
||||||
|
total: predictions.length, // Simplified - should use count query
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /predictive/users/:id/risk-profile - User risk profile
|
||||||
|
fastify.get('/predictive/users/:id/risk-profile', {
|
||||||
|
preHandler: [requireAuth],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { id } = request.params as { id: string };
|
||||||
|
const { productId } = request.query as { productId?: string };
|
||||||
|
|
||||||
|
if (!productId) {
|
||||||
|
return reply.status(400).send({ error: 'productId query parameter required' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const predictions = await predictiveAnalyticsRepo.getUserRiskProfile(id, productId);
|
||||||
|
const latest = predictions[0];
|
||||||
|
|
||||||
|
if (!latest) {
|
||||||
|
return reply.status(404).send({ error: 'No predictions found for user' });
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
userId: id,
|
||||||
|
productId,
|
||||||
|
currentRisk: {
|
||||||
|
churnProbability: latest.churnProbability,
|
||||||
|
riskSegment: latest.riskSegment,
|
||||||
|
confidenceScore: latest.confidenceScore,
|
||||||
|
predictionTimestamp: latest.predictionTimestamp,
|
||||||
|
},
|
||||||
|
history: predictions.map((p) => ({
|
||||||
|
timestamp: p.predictionTimestamp,
|
||||||
|
churnProbability: p.churnProbability,
|
||||||
|
riskSegment: p.riskSegment,
|
||||||
|
})),
|
||||||
|
explanation: latest.explanation,
|
||||||
|
suggestedActions: latest.explanation?.suggestedActions || [],
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// ==================== Health Score Routes ====================
|
||||||
|
|
||||||
|
// GET /predictive/health - All products health
|
||||||
|
fastify.get('/predictive/health', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (_request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const scores = await predictiveAnalyticsRepo.getAllProductHealthScores();
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
scores: scores.map((s) => ({
|
||||||
|
productId: s.productId,
|
||||||
|
date: s.date,
|
||||||
|
overallHealthScore: s.overallHealthScore,
|
||||||
|
healthStatus: s.healthStatus,
|
||||||
|
dimensions: s.dimensions,
|
||||||
|
anomalies: s.anomalies.slice(0, 3),
|
||||||
|
forecasts: s.forecasts,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /predictive/health/:productId - Product health detail
|
||||||
|
fastify.get('/predictive/health/:productId', {
|
||||||
|
preHandler: [requireAuth],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { productId } = request.params as { productId: string };
|
||||||
|
const today = new Date().toISOString().split('T')[0];
|
||||||
|
|
||||||
|
const score = await predictiveAnalyticsRepo.getHealthScore(productId, today);
|
||||||
|
|
||||||
|
if (!score) {
|
||||||
|
return reply.status(404).send({ error: 'Health score not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send(score);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /predictive/health/:productId/trends - Health trends
|
||||||
|
fastify.get('/predictive/health/:productId/trends', {
|
||||||
|
preHandler: [requireAuth],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { productId } = request.params as { productId: string };
|
||||||
|
const { days = '30' } = request.query as { days?: string };
|
||||||
|
|
||||||
|
const history = await predictiveAnalyticsRepo.getHealthHistory(
|
||||||
|
productId,
|
||||||
|
parseInt(days, 10)
|
||||||
|
);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
productId,
|
||||||
|
days: parseInt(days, 10),
|
||||||
|
trends: history.map((h) => ({
|
||||||
|
date: h.date,
|
||||||
|
overallHealthScore: h.overallHealthScore,
|
||||||
|
healthStatus: h.healthStatus,
|
||||||
|
dimensions: {
|
||||||
|
acquisition: h.dimensions.acquisition.score,
|
||||||
|
activation: h.dimensions.activation.score,
|
||||||
|
retention: h.dimensions.retention.score,
|
||||||
|
engagement: h.dimensions.engagement.score,
|
||||||
|
revenue: h.dimensions.revenue.score,
|
||||||
|
stability: h.dimensions.stability.score,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// ==================== Model Performance Routes ====================
|
||||||
|
|
||||||
|
// GET /predictive/model/performance - Model metrics
|
||||||
|
fastify.get('/predictive/model/performance', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (_request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const latest = await predictiveAnalyticsRepo.getLatestModelPerformance();
|
||||||
|
const history = await predictiveAnalyticsRepo.getModelPerformanceHistory(5);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
current: latest,
|
||||||
|
history: history.map((h) => ({
|
||||||
|
modelVersion: h.metrics.modelVersion,
|
||||||
|
modelType: h.metrics.modelType,
|
||||||
|
trainedAt: h.metrics.trainedAt,
|
||||||
|
auc: h.metrics.auc,
|
||||||
|
precisionAt10: h.metrics.precisionAt10,
|
||||||
|
recallAt10: h.metrics.recallAt10,
|
||||||
|
calibrationSlope: h.metrics.calibrationSlope,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /predictive/model/features - Feature importance
|
||||||
|
fastify.get('/predictive/model/features', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (_request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const latest = await predictiveAnalyticsRepo.getLatestModelPerformance();
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
modelVersion: latest?.metrics.modelVersion || '1.0.0',
|
||||||
|
featureImportance: latest?.metrics.featureImportance || [],
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// ==================== Campaign Routes ====================
|
||||||
|
|
||||||
|
// GET /predictive/campaigns - List campaigns
|
||||||
|
fastify.get('/predictive/campaigns', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { productId, status } = request.query as {
|
||||||
|
productId?: string;
|
||||||
|
status?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const campaigns = await predictiveAnalyticsRepo.listCampaigns(productId, status);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
campaigns: campaigns.map((c) => ({
|
||||||
|
id: c.id,
|
||||||
|
name: c.name,
|
||||||
|
description: c.description,
|
||||||
|
productId: c.productId,
|
||||||
|
status: c.status,
|
||||||
|
trigger: c.trigger,
|
||||||
|
audience: c.audience,
|
||||||
|
stats: c.stats,
|
||||||
|
createdAt: c.createdAt,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// POST /predictive/campaigns - Create campaign
|
||||||
|
fastify.post('/predictive/campaigns', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const body = CreateCampaignSchema.parse(request.body);
|
||||||
|
|
||||||
|
const campaign = await campaignEngine.createCampaign(body);
|
||||||
|
|
||||||
|
return reply.status(201).send({
|
||||||
|
id: campaign.id,
|
||||||
|
name: campaign.name,
|
||||||
|
status: campaign.status,
|
||||||
|
createdAt: campaign.createdAt,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// PATCH /predictive/campaigns/:id - Update campaign
|
||||||
|
fastify.patch('/predictive/campaigns/:id', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { id } = request.params as { id: string };
|
||||||
|
const updates = request.body as Partial<{
|
||||||
|
status: 'active' | 'paused' | 'completed';
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
if (updates.status === 'active') {
|
||||||
|
const campaign = await campaignEngine.activateCampaign(id);
|
||||||
|
return reply.send({ success: true, campaign });
|
||||||
|
}
|
||||||
|
|
||||||
|
const updated = await predictiveAnalyticsRepo.updateCampaign(id, updates);
|
||||||
|
|
||||||
|
if (!updated) {
|
||||||
|
return reply.status(404).send({ error: 'Campaign not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send({ success: true, campaign: updated });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /predictive/campaigns/:id/stats - Campaign stats
|
||||||
|
fastify.get('/predictive/campaigns/:id/stats', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { id } = request.params as { id: string };
|
||||||
|
|
||||||
|
const stats = await campaignEngine.getCampaignStats(id);
|
||||||
|
|
||||||
|
if (!stats) {
|
||||||
|
return reply.status(404).send({ error: 'Campaign not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send(stats);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// POST /predictive/campaigns/:id/trigger - Manual trigger
|
||||||
|
fastify.post('/predictive/campaigns/:id/trigger', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { id } = request.params as { id: string };
|
||||||
|
const body = CampaignTriggerSchema.parse(request.body);
|
||||||
|
|
||||||
|
const results = await campaignEngine.manualTrigger(id, body.testUserId || 'test_user');
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
success: true,
|
||||||
|
testMode: true,
|
||||||
|
results,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// DELETE /predictive/campaigns/:id - Delete campaign
|
||||||
|
fastify.delete('/predictive/campaigns/:id', {
|
||||||
|
preHandler: [requireAdmin],
|
||||||
|
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const { id } = request.params as { id: string };
|
||||||
|
|
||||||
|
const success = await predictiveAnalyticsRepo.deleteCampaign(id);
|
||||||
|
|
||||||
|
if (!success) {
|
||||||
|
return reply.status(404).send({ error: 'Campaign not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send({ success: true });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user