feat(ai-diagnostics): add LLM analyzer with prompts and insight generation [2.1-2.2]
This commit is contained in:
saravanakumardb1
parent
bfa3d088a4
commit
97b3ffb21d
@ -7,7 +7,7 @@ description: Refresh the Windsurf chat history archive (re-scan all repos, updat
|
||||
Refreshes the centralized Windsurf chat history archive at `__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/`.
|
||||
Auto-discovers new repos, updates symlinks, and re-copies docs + workflows.
|
||||
|
||||
## Covered Repos (All 7 workspaces)
|
||||
## Covered Repos (All 8 workspaces)
|
||||
|
||||
| Repo | Product | Workflows | Docs |
|
||||
|------|---------|-----------|------|
|
||||
@ -18,6 +18,7 @@ Auto-discovers new repos, updates symlinks, and re-copies docs + workflows.
|
||||
| `learning_ai_fastgap` | NomGap | ✅ | — |
|
||||
| `learning_ai_jarvis_jr` | JarvisJr | ✅ | — |
|
||||
| `learning_ai_common_plat` | Common Platform | ✅ | — |
|
||||
| `learning_agent_monitoring_fx` | Agent Monitoring | ✅ | — |
|
||||
|
||||
## Steps
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
Last refresh: 2026-03-03T19:50:04Z (2026-03-03 11:50:04 PST)
|
||||
Last refresh: 2026-03-03T19:50:43Z (2026-03-03 11:50:43 PST)
|
||||
Cascade conversations: 50 (297M)
|
||||
Memories: 65
|
||||
Implicit context: 20
|
||||
Code tracker dirs: 188
|
||||
Code tracker dirs: 190
|
||||
File edit history: 2343 entries
|
||||
Workspace storage: 28 workspaces
|
||||
Repo docs: 7 files across 2 repos
|
||||
Repo workflows: 35 files across 6 repos
|
||||
Repo workflows: 37 files across 8 repos
|
||||
|
||||
@ -0,0 +1,5 @@
|
||||
# NomGap Workflows
|
||||
|
||||
Workflows for NomGap fasting app.
|
||||
|
||||
See `/refresh-chat-history` for archive refresh.
|
||||
@ -0,0 +1,5 @@
|
||||
# JarvisJr Workflows
|
||||
|
||||
Workflows for JarvisJr voice coaching app.
|
||||
|
||||
See `/refresh-chat-history` for archive refresh.
|
||||
@ -0,0 +1,479 @@
|
||||
/**
|
||||
* Experiment Detail Page — Live Dashboard with Bayesian Statistics
|
||||
* Shows real-time metrics, variant comparisons, and statistical results.
|
||||
*/
|
||||
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useParams } from 'next/navigation';
|
||||
import Link from 'next/link';
|
||||
import {
|
||||
ArrowLeft,
|
||||
Play,
|
||||
Pause,
|
||||
Square,
|
||||
Trophy,
|
||||
AlertTriangle,
|
||||
Users,
|
||||
BarChart3,
|
||||
TrendingUp,
|
||||
Clock,
|
||||
Sparkles,
|
||||
Download,
|
||||
} from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||
import { Progress } from '@/components/ui/progress';
|
||||
import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
|
||||
import type { ExperimentDoc, VariantDoc, ExperimentResult } from '@/lib/experiments-types';
|
||||
|
||||
interface ExperimentData {
|
||||
experiment: ExperimentDoc;
|
||||
variants: VariantDoc[];
|
||||
results?: ExperimentResult;
|
||||
}
|
||||
|
||||
export default function ExperimentDetailPage() {
|
||||
const params = useParams();
|
||||
const experimentId = params.id as string;
|
||||
const [data, setData] = useState<ExperimentData | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
fetchExperimentData();
|
||||
const interval = setInterval(fetchExperimentData, 30000); // Auto-refresh every 30s
|
||||
return () => clearInterval(interval);
|
||||
}, [experimentId]);
|
||||
|
||||
async function fetchExperimentData() {
|
||||
try {
|
||||
const [expResponse, resultsResponse] = await Promise.all([
|
||||
fetch(`/api/experiments/${experimentId}`),
|
||||
fetch(`/api/experiments/${experimentId}/results`),
|
||||
]);
|
||||
|
||||
if (!expResponse.ok) throw new Error('Failed to fetch experiment');
|
||||
|
||||
const experiment = await expResponse.json();
|
||||
const results = resultsResponse.ok ? await resultsResponse.json() : null;
|
||||
|
||||
setData({ experiment, variants: experiment.variants, results });
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function updateStatus(status: string) {
|
||||
try {
|
||||
const response = await fetch(`/api/experiments/${experimentId}/${status}`, {
|
||||
method: 'POST',
|
||||
});
|
||||
if (!response.ok) throw new Error('Failed to update status');
|
||||
fetchExperimentData();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to update');
|
||||
}
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="container mx-auto py-8">
|
||||
<div className="flex items-center justify-center h-64">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary" />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return (
|
||||
<div className="container mx-auto py-8">
|
||||
<Alert variant="destructive">
|
||||
<AlertTriangle className="h-4 w-4" />
|
||||
<AlertTitle>Error</AlertTitle>
|
||||
<AlertDescription>{error || 'Experiment not found'}</AlertDescription>
|
||||
</Alert>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const { experiment, variants, results } = data;
|
||||
const daysRunning = experiment.startedAt
|
||||
? Math.floor((Date.now() - new Date(experiment.startedAt).getTime()) / (1000 * 60 * 60 * 24))
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<div className="container mx-auto py-8">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between mb-6">
|
||||
<div className="flex items-center gap-4">
|
||||
<Button variant="ghost" size="sm" asChild>
|
||||
<Link href="/experiments">
|
||||
<ArrowLeft className="h-4 w-4 mr-2" />
|
||||
Back
|
||||
</Link>
|
||||
</Button>
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold">{experiment.name}</h1>
|
||||
<p className="text-muted-foreground text-sm">{experiment.hypothesis}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
{experiment.status === 'draft' && (
|
||||
<Button onClick={() => updateStatus('start')}>
|
||||
<Play className="h-4 w-4 mr-2" />
|
||||
Start Experiment
|
||||
</Button>
|
||||
)}
|
||||
{experiment.status === 'running' && (
|
||||
<>
|
||||
<Button variant="outline" onClick={() => updateStatus('pause')}>
|
||||
<Pause className="h-4 w-4 mr-2" />
|
||||
Pause
|
||||
</Button>
|
||||
<Button variant="destructive" onClick={() => updateStatus('stop')}>
|
||||
<Square className="h-4 w-4 mr-2" />
|
||||
Stop
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
{experiment.status === 'paused' && (
|
||||
<Button onClick={() => updateStatus('start')}>
|
||||
<Play className="h-4 w-4 mr-2" />
|
||||
Resume
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Status Banner */}
|
||||
{results?.earlyStopped && (
|
||||
<Alert className="mb-6 bg-yellow-50 border-yellow-200">
|
||||
<AlertTriangle className="h-4 w-4 text-yellow-600" />
|
||||
<AlertTitle>Early Stopping Triggered</AlertTitle>
|
||||
<AlertDescription>{results.stopReason}</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{results?.winnerVariantId && (
|
||||
<Alert className="mb-6 bg-green-50 border-green-200">
|
||||
<Trophy className="h-4 w-4 text-green-600" />
|
||||
<AlertTitle>Winner Found!</AlertTitle>
|
||||
<AlertDescription>
|
||||
Variant has {((results.winnerProbability || 0) * 100).toFixed(1)}% probability of being best.
|
||||
Recommended action: {results.statisticalSummary.recommendedAction}.
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{/* Stats Overview */}
|
||||
<div className="grid grid-cols-1 md:grid-cols-4 gap-4 mb-8">
|
||||
<StatCard
|
||||
title="Status"
|
||||
value={experiment.status}
|
||||
badge={getStatusBadge(experiment.status)}
|
||||
/>
|
||||
<StatCard
|
||||
title="Participants"
|
||||
value={experiment.totalParticipants?.toLocaleString() || '0'}
|
||||
icon={<Users className="h-4 w-4" />}
|
||||
/>
|
||||
<StatCard
|
||||
title="Days Running"
|
||||
value={daysRunning.toString()}
|
||||
icon={<Clock className="h-4 w-4" />}
|
||||
/>
|
||||
<StatCard
|
||||
title="Total Events"
|
||||
value={experiment.totalEvents?.toLocaleString() || '0'}
|
||||
icon={<BarChart3 className="h-4 w-4" />}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Main Content */}
|
||||
<Tabs defaultValue="variants" className="space-y-6">
|
||||
<TabsList>
|
||||
<TabsTrigger value="variants">Variant Performance</TabsTrigger>
|
||||
<TabsTrigger value="statistics">Statistical Results</TabsTrigger>
|
||||
<TabsTrigger value="ai-insights">AI Insights</TabsTrigger>
|
||||
<TabsTrigger value="settings">Settings</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="variants" className="space-y-4">
|
||||
{variants.map(variant => (
|
||||
<VariantCard
|
||||
key={variant.id}
|
||||
variant={variant}
|
||||
isControl={variant.isControl}
|
||||
experiment={experiment}
|
||||
result={results?.variantResults.find(vr => vr.variantId === variant.id)}
|
||||
/>
|
||||
))}
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="statistics">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Statistical Summary</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-6">
|
||||
{results ? (
|
||||
<>
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
<div className="p-4 bg-muted rounded-lg">
|
||||
<div className="text-sm text-muted-foreground mb-1">
|
||||
Probability Any Beats Control
|
||||
</div>
|
||||
<div className="text-2xl font-bold">
|
||||
{(results.statisticalSummary.probabilityAnyBeatsControl * 100).toFixed(1)}%
|
||||
</div>
|
||||
</div>
|
||||
<div className="p-4 bg-muted rounded-lg">
|
||||
<div className="text-sm text-muted-foreground mb-1">
|
||||
Expected Loss If Shipped
|
||||
</div>
|
||||
<div className="text-2xl font-bold">
|
||||
{(results.statisticalSummary.expectedLossIfShipped * 100).toFixed(2)}%
|
||||
</div>
|
||||
</div>
|
||||
<div className="p-4 bg-muted rounded-lg">
|
||||
<div className="text-sm text-muted-foreground mb-1">
|
||||
Recommended Action
|
||||
</div>
|
||||
<div className="text-2xl font-bold capitalize">
|
||||
{results.statisticalSummary.recommendedAction}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="font-semibold mb-4">Variant Comparison</h4>
|
||||
<div className="space-y-3">
|
||||
{results.variantResults.map(vr => (
|
||||
<div key={vr.variantId} className="flex items-center gap-4 p-3 border rounded-lg">
|
||||
<div className="w-32 font-medium">{vr.variantName}</div>
|
||||
<div className="flex-1">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className="text-sm text-muted-foreground">
|
||||
P(beats control): {(vr.probabilityBeatsControl * 100).toFixed(1)}%
|
||||
</span>
|
||||
</div>
|
||||
<Progress value={vr.probabilityBeatsControl * 100} className="h-2" />
|
||||
</div>
|
||||
<div className="w-24 text-right">
|
||||
<div className={`text-sm font-medium ${vr.expectedLiftPercent > 0 ? 'text-green-600' : 'text-red-600'}`}>
|
||||
{vr.expectedLiftPercent > 0 ? '+' : ''}{vr.expectedLiftPercent.toFixed(1)}%
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<p className="text-muted-foreground">No statistical results available yet.</p>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="ai-insights">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-2">
|
||||
<Sparkles className="h-5 w-5" />
|
||||
AI-Generated Insights
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="p-4 bg-blue-50 border border-blue-200 rounded-lg">
|
||||
<h4 className="font-semibold mb-2">Summary</h4>
|
||||
<p className="text-sm">
|
||||
{results?.winnerVariantId
|
||||
? `The ${results.variantResults.find(v => v.variantId === results.winnerVariantId)?.variantName || 'winning variant'} significantly outperformed control with ${((results.winnerProbability || 0) * 100).toFixed(1)}% probability of superiority.`
|
||||
: 'Results are still inconclusive. Consider running the experiment longer or testing more distinct variations.'}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="font-semibold mb-2">Recommended Follow-up Experiments</h4>
|
||||
<ul className="space-y-2">
|
||||
<li className="flex items-center gap-2 text-sm">
|
||||
<TrendingUp className="h-4 w-4 text-muted-foreground" />
|
||||
Test the winning variant against additional success metrics
|
||||
</li>
|
||||
<li className="flex items-center gap-2 text-sm">
|
||||
<Users className="h-4 w-4 text-muted-foreground" />
|
||||
Run a validation experiment with a new user cohort
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="settings">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Experiment Configuration</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label className="text-sm font-medium">Allocation Strategy</label>
|
||||
<p className="text-muted-foreground capitalize">{experiment.allocationStrategy}</p>
|
||||
</div>
|
||||
<div>
|
||||
<label className="text-sm font-medium">Target Traffic</label>
|
||||
<p className="text-muted-foreground">{experiment.targetPercent}%</p>
|
||||
</div>
|
||||
<div>
|
||||
<label className="text-sm font-medium">Primary Metric</label>
|
||||
<p className="text-muted-foreground">{experiment.primaryMetric?.name}</p>
|
||||
</div>
|
||||
<div>
|
||||
<label className="text-sm font-medium">Min Sample Size</label>
|
||||
<p className="text-muted-foreground">{experiment.guardrails?.minSampleSizePerVariant} per variant</p>
|
||||
</div>
|
||||
<div>
|
||||
<label className="text-sm font-medium">Max Duration</label>
|
||||
<p className="text-muted-foreground">{experiment.guardrails?.maxDurationDays} days</p>
|
||||
</div>
|
||||
<div>
|
||||
<label className="text-sm font-medium">Auto Stop</label>
|
||||
<p className="text-muted-foreground">{experiment.guardrails?.autoStopEnabled ? 'Enabled' : 'Disabled'}</p>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StatCard({
|
||||
title,
|
||||
value,
|
||||
badge,
|
||||
icon,
|
||||
}: {
|
||||
title: string;
|
||||
value: string;
|
||||
badge?: React.ReactNode;
|
||||
icon?: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<Card>
|
||||
<CardContent className="p-4">
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<span className="text-sm text-muted-foreground">{title}</span>
|
||||
{icon}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-2xl font-bold capitalize">{value}</span>
|
||||
{badge}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
function VariantCard({
|
||||
variant,
|
||||
isControl,
|
||||
experiment,
|
||||
result,
|
||||
}: {
|
||||
variant: VariantDoc;
|
||||
isControl: boolean;
|
||||
experiment: ExperimentDoc;
|
||||
result?: { probabilityBeatsControl: number; expectedLiftPercent: number; credibleInterval: { lower: number; mean: number; upper: number } };
|
||||
}) {
|
||||
const conversionRate = variant.stats?.conversionRate || 0;
|
||||
const participants = variant.stats?.participants || 0;
|
||||
const events = variant.stats?.events || 0;
|
||||
|
||||
return (
|
||||
<Card className={isControl ? 'border-blue-200' : ''}>
|
||||
<CardContent className="p-6">
|
||||
<div className="flex items-start justify-between">
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<h3 className="text-lg font-semibold">{variant.name}</h3>
|
||||
{isControl && (
|
||||
<Badge variant="outline" className="border-blue-300 text-blue-700">
|
||||
Control
|
||||
</Badge>
|
||||
)}
|
||||
{variant.bayesianResults?.probabilityBeatsControl && variant.bayesianResults.probabilityBeatsControl > 0.95 && (
|
||||
<Badge className="bg-green-500 text-white">
|
||||
<Trophy className="h-3 w-3 mr-1" />
|
||||
Winner
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-sm text-muted-foreground">{variant.description}</p>
|
||||
</div>
|
||||
<div className="text-right">
|
||||
<div className="text-2xl font-bold">{(conversionRate * 100).toFixed(1)}%</div>
|
||||
<div className="text-sm text-muted-foreground">conversion rate</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-3 gap-4 mt-4 pt-4 border-t">
|
||||
<div>
|
||||
<div className="text-2xl font-semibold">{participants.toLocaleString()}</div>
|
||||
<div className="text-sm text-muted-foreground">participants</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-2xl font-semibold">{events.toLocaleString()}</div>
|
||||
<div className="text-sm text-muted-foreground">events</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-2xl font-semibold">
|
||||
{result ? `${(result.probabilityBeatsControl * 100).toFixed(0)}%` : '-'}
|
||||
</div>
|
||||
<div className="text-sm text-muted-foreground">P(beats control)</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{result && result.credibleInterval && (
|
||||
<div className="mt-4 pt-4 border-t">
|
||||
<div className="text-sm text-muted-foreground mb-2">95% Credible Interval</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm">{(result.credibleInterval.lower * 100).toFixed(1)}%</span>
|
||||
<div className="flex-1 h-2 bg-muted rounded-full relative">
|
||||
<div
|
||||
className="absolute h-full bg-primary rounded-full"
|
||||
style={{
|
||||
left: `${(result.credibleInterval.lower / result.credibleInterval.upper) * 50}%`,
|
||||
width: `${((result.credibleInterval.upper - result.credibleInterval.lower) / result.credibleInterval.upper) * 50}%`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<span className="text-sm">{(result.credibleInterval.upper * 100).toFixed(1)}%</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
function getStatusBadge(status: string) {
|
||||
const colors: Record<string, string> = {
|
||||
draft: 'bg-gray-500',
|
||||
running: 'bg-green-500',
|
||||
paused: 'bg-yellow-500',
|
||||
stopped: 'bg-red-500',
|
||||
completed: 'bg-blue-500',
|
||||
};
|
||||
return <Badge className={`${colors[status] || colors.draft} text-white capitalize`}>{status}</Badge>;
|
||||
}
|
||||
@ -0,0 +1,735 @@
|
||||
/**
|
||||
* New Experiment Wizard — Admin Dashboard
|
||||
* Multi-step wizard for creating experiments with AI suggestions.
|
||||
*/
|
||||
|
||||
'use client';
|
||||
|
||||
import { useState } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import Link from 'next/link';
|
||||
import {
|
||||
ArrowLeft,
|
||||
ArrowRight,
|
||||
Sparkles,
|
||||
FlaskConical,
|
||||
Target,
|
||||
Sliders,
|
||||
Check,
|
||||
Lightbulb,
|
||||
} from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Alert, AlertDescription } from '@/components/ui/alert';
|
||||
import { Progress } from '@/components/ui/progress';
|
||||
import type { CreateExperimentInput, ExperimentSuggestion } from '@/lib/experiments-types';
|
||||
|
||||
const steps = [
|
||||
{ id: 'hypothesis', title: 'Hypothesis', icon: Lightbulb },
|
||||
{ id: 'variants', title: 'Variants', icon: FlaskConical },
|
||||
{ id: 'metrics', title: 'Metrics', icon: Target },
|
||||
{ id: 'targeting', title: 'Targeting', icon: Sliders },
|
||||
{ id: 'review', title: 'Review', icon: Check },
|
||||
];
|
||||
|
||||
export default function NewExperimentPage() {
|
||||
const router = useRouter();
|
||||
const [currentStep, setCurrentStep] = useState(0);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [aiSuggestions, setAiSuggestions] = useState<ExperimentSuggestion[]>([]);
|
||||
|
||||
const [formData, setFormData] = useState<Partial<CreateExperimentInput>>({
|
||||
name: '',
|
||||
description: '',
|
||||
hypothesis: '',
|
||||
variants: [
|
||||
{ key: 'control', name: 'Control', description: 'Current implementation', isControl: true, flagConfig: {} },
|
||||
{ key: 'variant_a', name: 'Variant A', description: '', isControl: false, flagConfig: {} },
|
||||
],
|
||||
allocationStrategy: 'random',
|
||||
targetPercent: 100,
|
||||
targeting: {},
|
||||
primaryMetric: {
|
||||
name: 'conversion',
|
||||
type: 'conversion',
|
||||
eventName: 'purchase',
|
||||
aggregation: 'count',
|
||||
direction: 'increase',
|
||||
minimumDetectableEffect: 5,
|
||||
},
|
||||
guardrails: {
|
||||
minSampleSizePerVariant: 100,
|
||||
maxDurationDays: 30,
|
||||
autoStopEnabled: true,
|
||||
winnerThreshold: 95,
|
||||
requireApprovalFor: 'none',
|
||||
},
|
||||
});
|
||||
|
||||
async function loadAiSuggestions() {
|
||||
try {
|
||||
const response = await fetch('/api/ab-testing/suggestions');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setAiSuggestions(data.slice(0, 3));
|
||||
}
|
||||
} catch {
|
||||
// Silently fail - AI suggestions are optional
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSubmit() {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/ab-testing/experiments', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(formData),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
throw new Error(errorData.message || 'Failed to create experiment');
|
||||
}
|
||||
|
||||
const experiment = await response.json();
|
||||
router.push(`/experiments/${experiment.id}`);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to create experiment');
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
function applyAiSuggestion(suggestion: ExperimentSuggestion) {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
name: prev.name || `Experiment: ${suggestion.hypothesis.primary.slice(0, 50)}...`,
|
||||
hypothesis: suggestion.hypothesis.primary,
|
||||
description: suggestion.hypothesis.alternatives[0] || '',
|
||||
primaryMetric: suggestion.suggestedMetrics[0] || prev.primaryMetric,
|
||||
}));
|
||||
}
|
||||
|
||||
const progress = ((currentStep + 1) / steps.length) * 100;
|
||||
|
||||
return (
|
||||
<div className="container mx-auto py-8 max-w-4xl">
|
||||
{/* Header */}
|
||||
<div className="flex items-center gap-4 mb-8">
|
||||
<Button variant="ghost" size="sm" asChild>
|
||||
<Link href="/experiments">
|
||||
<ArrowLeft className="h-4 w-4 mr-2" />
|
||||
Back
|
||||
</Link>
|
||||
</Button>
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold">Create New Experiment</h1>
|
||||
<p className="text-muted-foreground text-sm">
|
||||
Step {currentStep + 1} of {steps.length}: {steps[currentStep].title}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Progress */}
|
||||
<Progress value={progress} className="mb-8" />
|
||||
|
||||
{/* Step Indicators */}
|
||||
<div className="flex justify-between mb-8">
|
||||
{steps.map((step, index) => {
|
||||
const Icon = step.icon;
|
||||
const isActive = index === currentStep;
|
||||
const isCompleted = index < currentStep;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={step.id}
|
||||
className={`flex flex-col items-center ${
|
||||
isActive ? 'text-primary' : isCompleted ? 'text-green-600' : 'text-muted-foreground'
|
||||
}`}
|
||||
>
|
||||
<div
|
||||
className={`w-10 h-10 rounded-full flex items-center justify-center mb-2 ${
|
||||
isActive
|
||||
? 'bg-primary text-white'
|
||||
: isCompleted
|
||||
? 'bg-green-100 text-green-600'
|
||||
: 'bg-muted'
|
||||
}`}
|
||||
>
|
||||
<Icon className="h-5 w-5" />
|
||||
</div>
|
||||
<span className="text-xs font-medium">{step.title}</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Error */}
|
||||
{error && (
|
||||
<Alert variant="destructive" className="mb-6">
|
||||
<AlertDescription>{error}</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{/* Step Content */}
|
||||
<Card className="mb-6">
|
||||
<CardContent className="p-6">
|
||||
{currentStep === 0 && (
|
||||
<HypothesisStep
|
||||
formData={formData}
|
||||
setFormData={setFormData}
|
||||
aiSuggestions={aiSuggestions}
|
||||
onLoadSuggestions={loadAiSuggestions}
|
||||
onApplySuggestion={applyAiSuggestion}
|
||||
/>
|
||||
)}
|
||||
{currentStep === 1 && (
|
||||
<VariantsStep formData={formData} setFormData={setFormData} />
|
||||
)}
|
||||
{currentStep === 2 && (
|
||||
<MetricsStep formData={formData} setFormData={setFormData} />
|
||||
)}
|
||||
{currentStep === 3 && (
|
||||
<TargetingStep formData={formData} setFormData={setFormData} />
|
||||
)}
|
||||
{currentStep === 4 && (
|
||||
<ReviewStep formData={formData} />
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Navigation */}
|
||||
<div className="flex justify-between">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => setCurrentStep(Math.max(0, currentStep - 1))}
|
||||
disabled={currentStep === 0}
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4 mr-2" />
|
||||
Previous
|
||||
</Button>
|
||||
|
||||
{currentStep < steps.length - 1 ? (
|
||||
<Button onClick={() => setCurrentStep(currentStep + 1)}>
|
||||
Next
|
||||
<ArrowRight className="h-4 w-4 ml-2" />
|
||||
</Button>
|
||||
) : (
|
||||
<Button onClick={handleSubmit} disabled={loading}>
|
||||
{loading ? (
|
||||
<>
|
||||
<div className="animate-spin mr-2 h-4 w-4 border-2 border-current border-t-transparent rounded-full" />
|
||||
Creating...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Check className="h-4 w-4 mr-2" />
|
||||
Create Experiment
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Step Components
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function HypothesisStep({
|
||||
formData,
|
||||
setFormData,
|
||||
aiSuggestions,
|
||||
onLoadSuggestions,
|
||||
onApplySuggestion,
|
||||
}: {
|
||||
formData: Partial<CreateExperimentInput>;
|
||||
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||
aiSuggestions: ExperimentSuggestion[];
|
||||
onLoadSuggestions: () => void;
|
||||
onApplySuggestion: (s: ExperimentSuggestion) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<Label htmlFor="name">Experiment Name</Label>
|
||||
<Input
|
||||
id="name"
|
||||
value={formData.name}
|
||||
onChange={e => setFormData({ ...formData, name: e.target.value })}
|
||||
placeholder="e.g., Homepage Button Color Test"
|
||||
className="mt-1"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Label htmlFor="hypothesis">Hypothesis</Label>
|
||||
<Textarea
|
||||
id="hypothesis"
|
||||
value={formData.hypothesis}
|
||||
onChange={e => setFormData({ ...formData, hypothesis: e.target.value })}
|
||||
placeholder="e.g., Changing the CTA button to green will increase click-through rate by 15% because green signals 'go' to users..."
|
||||
className="mt-1"
|
||||
rows={3}
|
||||
/>
|
||||
<p className="text-sm text-muted-foreground mt-1">
|
||||
State what you expect to change and why.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Label htmlFor="description">Description (optional)</Label>
|
||||
<Textarea
|
||||
id="description"
|
||||
value={formData.description}
|
||||
onChange={e => setFormData({ ...formData, description: e.target.value })}
|
||||
placeholder="Additional context about this experiment..."
|
||||
className="mt-1"
|
||||
rows={2}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* AI Suggestions */}
|
||||
<div className="border rounded-lg p-4 bg-muted/50">
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<Sparkles className="h-4 w-4 text-primary" />
|
||||
<h4 className="font-medium">AI-Generated Suggestions</h4>
|
||||
<Button variant="ghost" size="sm" onClick={onLoadSuggestions} className="ml-auto">
|
||||
Load Suggestions
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{aiSuggestions.length > 0 ? (
|
||||
<div className="space-y-3">
|
||||
{aiSuggestions.map((suggestion, index) => (
|
||||
<Card key={index} className="cursor-pointer hover:border-primary" onClick={() => onApplySuggestion(suggestion)}>
|
||||
<CardContent className="p-3">
|
||||
<p className="text-sm font-medium line-clamp-2">{suggestion.hypothesis.primary}</p>
|
||||
<div className="flex gap-2 mt-2">
|
||||
<Badge variant="outline" className="text-xs">
|
||||
Impact: {suggestion.hypothesis.impactScore}/100
|
||||
</Badge>
|
||||
<Badge variant="outline" className="text-xs">
|
||||
Difficulty: {suggestion.hypothesis.difficultyScore}/100
|
||||
</Badge>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Click "Load Suggestions" to see AI-generated experiment ideas based on your product usage patterns.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function VariantsStep({
|
||||
formData,
|
||||
setFormData,
|
||||
}: {
|
||||
formData: Partial<CreateExperimentInput>;
|
||||
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||
}) {
|
||||
const variants = formData.variants || [];
|
||||
|
||||
function addVariant() {
|
||||
const newVariant = {
|
||||
key: `variant_${String.fromCharCode(98 + variants.length - 1)}`,
|
||||
name: `Variant ${String.fromCharCode(65 + variants.length - 1)}`,
|
||||
description: '',
|
||||
isControl: false,
|
||||
flagConfig: {},
|
||||
};
|
||||
setFormData({ ...formData, variants: [...variants, newVariant] });
|
||||
}
|
||||
|
||||
function updateVariant(index: number, updates: Partial<(typeof variants)[0]>) {
|
||||
const updated = variants.map((v, i) => (i === index ? { ...v, ...updates } : v));
|
||||
setFormData({ ...formData, variants: updated });
|
||||
}
|
||||
|
||||
function removeVariant(index: number) {
|
||||
if (variants.length <= 2) return; // Minimum 2 variants
|
||||
const updated = variants.filter((_, i) => i !== index);
|
||||
setFormData({ ...formData, variants: updated });
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<h3 className="font-medium">Experiment Variants</h3>
|
||||
<Button variant="outline" size="sm" onClick={addVariant}>
|
||||
Add Variant
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{variants.map((variant, index) => (
|
||||
<Card key={index} className={variant.isControl ? 'border-blue-200' : ''}>
|
||||
<CardContent className="p-4 space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
{variant.isControl ? (
|
||||
<Badge variant="outline" className="border-blue-300 text-blue-700">Control</Badge>
|
||||
) : (
|
||||
<Badge variant="outline">Variant</Badge>
|
||||
)}
|
||||
</div>
|
||||
{!variant.isControl && (
|
||||
<Button variant="ghost" size="sm" onClick={() => removeVariant(index)}>
|
||||
Remove
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label>Name</Label>
|
||||
<Input
|
||||
value={variant.name}
|
||||
onChange={e => updateVariant(index, { name: e.target.value })}
|
||||
className="mt-1"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label>Key (slug)</Label>
|
||||
<Input
|
||||
value={variant.key}
|
||||
onChange={e => updateVariant(index, { key: e.target.value })}
|
||||
className="mt-1"
|
||||
disabled={variant.isControl}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Label>Description</Label>
|
||||
<Input
|
||||
value={variant.description}
|
||||
onChange={e => updateVariant(index, { description: e.target.value })}
|
||||
placeholder="What makes this variant different?"
|
||||
className="mt-1"
|
||||
/>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
|
||||
<p className="text-sm text-muted-foreground">
|
||||
You need at least 2 variants: a Control (current implementation) and at least one Treatment variant.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MetricsStep({
|
||||
formData,
|
||||
setFormData,
|
||||
}: {
|
||||
formData: Partial<CreateExperimentInput>;
|
||||
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Primary Metric</CardTitle>
|
||||
<CardDescription>The main metric that determines experiment success.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label>Metric Name</Label>
|
||||
<Input
|
||||
value={formData.primaryMetric?.name}
|
||||
onChange={e =>
|
||||
setFormData({
|
||||
...formData,
|
||||
primaryMetric: { ...formData.primaryMetric!, name: e.target.value },
|
||||
})
|
||||
}
|
||||
className="mt-1"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label>Event Name</Label>
|
||||
<Input
|
||||
value={formData.primaryMetric?.eventName}
|
||||
onChange={e =>
|
||||
setFormData({
|
||||
...formData,
|
||||
primaryMetric: { ...formData.primaryMetric!, eventName: e.target.value },
|
||||
})
|
||||
}
|
||||
placeholder="e.g., purchase_clicked"
|
||||
className="mt-1"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label>Metric Type</Label>
|
||||
<Select
|
||||
value={formData.primaryMetric?.type}
|
||||
onValueChange={v =>
|
||||
setFormData({
|
||||
...formData,
|
||||
primaryMetric: { ...formData.primaryMetric!, type: v as any },
|
||||
})
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="mt-1">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="conversion">Conversion (binary)</SelectItem>
|
||||
<SelectItem value="count">Count (integer)</SelectItem>
|
||||
<SelectItem value="duration">Duration (time)</SelectItem>
|
||||
<SelectItem value="revenue">Revenue (monetary)</SelectItem>
|
||||
<SelectItem value="custom">Custom (numeric)</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div>
|
||||
<Label>Direction</Label>
|
||||
<Select
|
||||
value={formData.primaryMetric?.direction}
|
||||
onValueChange={v =>
|
||||
setFormData({
|
||||
...formData,
|
||||
primaryMetric: { ...formData.primaryMetric!, direction: v as any },
|
||||
})
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="mt-1">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="increase">Higher is better</SelectItem>
|
||||
<SelectItem value="decrease">Lower is better</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Allocation Strategy</CardTitle>
|
||||
<CardDescription>How traffic is distributed between variants.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<Select
|
||||
value={formData.allocationStrategy}
|
||||
onValueChange={v => setFormData({ ...formData, allocationStrategy: v as any })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="random">Random (fixed 50/50)</SelectItem>
|
||||
<SelectItem value="thompson">Thompson Sampling (multi-armed bandit)</SelectItem>
|
||||
<SelectItem value="epsilon_greedy">Epsilon-Greedy (explore/exploit)</SelectItem>
|
||||
<SelectItem value="ucb">Upper Confidence Bound (UCB1)</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
<div>
|
||||
<Label>Target Traffic %</Label>
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={100}
|
||||
value={formData.targetPercent}
|
||||
onChange={e => setFormData({ ...formData, targetPercent: parseInt(e.target.value) })}
|
||||
className="mt-1"
|
||||
/>
|
||||
<p className="text-sm text-muted-foreground mt-1">
|
||||
Percentage of eligible users to include in this experiment.
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function TargetingStep({
|
||||
formData,
|
||||
setFormData,
|
||||
}: {
|
||||
formData: Partial<CreateExperimentInput>;
|
||||
setFormData: (data: Partial<CreateExperimentInput>) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Audience Targeting</CardTitle>
|
||||
<CardDescription>Limit this experiment to specific user segments.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div>
|
||||
<Label>Platforms</Label>
|
||||
<div className="flex gap-2 mt-1">
|
||||
{['ios', 'android', 'web'].map(platform => (
|
||||
<Badge
|
||||
key={platform}
|
||||
variant={formData.targeting?.platforms?.includes(platform) ? 'default' : 'outline'}
|
||||
className="cursor-pointer capitalize"
|
||||
onClick={() => {
|
||||
const current = formData.targeting?.platforms || [];
|
||||
const updated = current.includes(platform)
|
||||
? current.filter(p => p !== platform)
|
||||
: [...current, platform];
|
||||
setFormData({
|
||||
...formData,
|
||||
targeting: { ...formData.targeting, platforms: updated },
|
||||
});
|
||||
}}
|
||||
>
|
||||
{platform}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Label>User Segments</Label>
|
||||
<div className="flex gap-2 mt-1">
|
||||
{['free', 'pro', 'enterprise'].map(segment => (
|
||||
<Badge
|
||||
key={segment}
|
||||
variant={formData.targeting?.userSegments?.includes(segment) ? 'default' : 'outline'}
|
||||
className="cursor-pointer capitalize"
|
||||
onClick={() => {
|
||||
const current = formData.targeting?.userSegments || [];
|
||||
const updated = current.includes(segment)
|
||||
? current.filter(s => s !== segment)
|
||||
: [...current, segment];
|
||||
setFormData({
|
||||
...formData,
|
||||
targeting: { ...formData.targeting, userSegments: updated },
|
||||
});
|
||||
}}
|
||||
>
|
||||
{segment}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Guardrails & Safety</CardTitle>
|
||||
<CardDescription>Configure automatic stopping rules.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label>Min Sample Size per Variant</Label>
|
||||
<Input
|
||||
type="number"
|
||||
value={formData.guardrails?.minSampleSizePerVariant}
|
||||
onChange={e =>
|
||||
setFormData({
|
||||
...formData,
|
||||
guardrails: {
|
||||
...formData.guardrails!,
|
||||
minSampleSizePerVariant: parseInt(e.target.value),
|
||||
},
|
||||
})
|
||||
}
|
||||
className="mt-1"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label>Max Duration (days)</Label>
|
||||
<Input
|
||||
type="number"
|
||||
value={formData.guardrails?.maxDurationDays}
|
||||
onChange={e =>
|
||||
setFormData({
|
||||
...formData,
|
||||
guardrails: { ...formData.guardrails!, maxDurationDays: parseInt(e.target.value) },
|
||||
})
|
||||
}
|
||||
className="mt-1"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Label>Winner Threshold (%)</Label>
|
||||
<Input
|
||||
type="number"
|
||||
min={80}
|
||||
max={99}
|
||||
value={formData.guardrails?.winnerThreshold}
|
||||
onChange={e =>
|
||||
setFormData({
|
||||
...formData,
|
||||
guardrails: { ...formData.guardrails!, winnerThreshold: parseInt(e.target.value) },
|
||||
})
|
||||
}
|
||||
className="mt-1"
|
||||
/>
|
||||
<p className="text-sm text-muted-foreground mt-1">
|
||||
Probability threshold for declaring a winner (95% recommended).
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ReviewStep({ formData }: { formData: Partial<CreateExperimentInput> }) {
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<h3 className="font-medium">Review Experiment Configuration</h3>
|
||||
|
||||
<div className="space-y-4">
|
||||
<ReviewItem label="Name" value={formData.name} />
|
||||
<ReviewItem label="Hypothesis" value={formData.hypothesis} />
|
||||
<ReviewItem label="Variants" value={`${formData.variants?.length || 0} variants`} />
|
||||
<ReviewItem label="Primary Metric" value={formData.primaryMetric?.name} />
|
||||
<ReviewItem label="Allocation Strategy" value={formData.allocationStrategy} />
|
||||
<ReviewItem label="Target Traffic" value={`${formData.targetPercent}%`} />
|
||||
<ReviewItem label="Auto Stop" value={formData.guardrails?.autoStopEnabled ? 'Enabled' : 'Disabled'} />
|
||||
</div>
|
||||
|
||||
<Alert>
|
||||
<AlertDescription>
|
||||
Once created, you can start the experiment from the experiment details page.
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ReviewItem({ label, value }: { label: string; value?: string | number | boolean }) {
|
||||
return (
|
||||
<div className="flex justify-between py-2 border-b">
|
||||
<span className="text-muted-foreground">{label}</span>
|
||||
<span className="font-medium">{value || '-'}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -81,6 +81,14 @@ const CONTAINER_DEFS: Record<string, ContainerConfig> = {
|
||||
debug_traces: { partitionKeyPath: '/pk', defaultTtl: 7 * 86400 },
|
||||
debug_logs: { partitionKeyPath: '/pk', defaultTtl: 3 * 86400 },
|
||||
debug_screenshots: { partitionKeyPath: '/sessionId', defaultTtl: 7 * 86400 },
|
||||
// Predictive Analytics
|
||||
user_features: { partitionKeyPath: '/userId', defaultTtl: 90 * 86400 },
|
||||
product_health: { partitionKeyPath: '/productId' },
|
||||
feature_definitions: { partitionKeyPath: '/productId' },
|
||||
churn_predictions: { partitionKeyPath: '/userId', defaultTtl: 120 * 86400 },
|
||||
retention_campaigns: { partitionKeyPath: '/productId' },
|
||||
campaign_deliveries: { partitionKeyPath: '/userId', defaultTtl: 90 * 86400 },
|
||||
model_performance: { partitionKeyPath: '/id' },
|
||||
// AI Diagnostics (see docs/roadmaps/AI_DIAGNOSTIC_ASSISTANT_ROADMAP.md)
|
||||
error_clusters: { partitionKeyPath: '/productId', defaultTtl: 90 * 86400 },
|
||||
error_fingerprints: { partitionKeyPath: '/fingerprintHash' },
|
||||
|
||||
@ -0,0 +1,493 @@
|
||||
import { config } from '../../lib/config.js';
|
||||
import type {
|
||||
ErrorClusterDoc,
|
||||
DiagnosticInsightDoc,
|
||||
RootCauseCategory,
|
||||
Evidence,
|
||||
SimilarResolvedIssue,
|
||||
} from './types.js';
|
||||
import type { ClusterContextSummary } from './telemetry-linking.js';
|
||||
|
||||
// ============================================================================
|
||||
// LLM Prompts
|
||||
// ============================================================================
|
||||
|
||||
interface RootCauseAnalysisPrompt {
|
||||
cluster: ErrorClusterDoc;
|
||||
context: ClusterContextSummary;
|
||||
sampleStackTraces: string[];
|
||||
relatedClusters: Array<{ id: string; errorType: string; status: string }>;
|
||||
analysisType: 'root_cause' | 'pattern' | 'comparison' | 'trend';
|
||||
}
|
||||
|
||||
const ROOT_CAUSE_ANALYSIS_PROMPT_TEMPLATE = `You are an expert software engineer diagnosing production errors. Analyze this error cluster and provide root cause insights.
|
||||
|
||||
## Error Cluster
|
||||
- Error Type: {errorType}
|
||||
- Message Pattern: {messagePattern}
|
||||
- Stack Signature: {stackSignature}
|
||||
- Occurrences: {occurrenceCount}
|
||||
- Affected Users: {uniqueUsers}
|
||||
- First Seen: {firstSeenAt}
|
||||
- Last Seen: {lastSeenAt}
|
||||
|
||||
## Context Summary
|
||||
- Total Occurrences: {totalOccurrences}
|
||||
- Affected Users Count: {affectedUsersCount}
|
||||
- Time Range: {timeRange}
|
||||
- Most Common Screens: {commonScreens}
|
||||
- Most Common Actions: {commonActions}
|
||||
|
||||
## Sample Stack Traces
|
||||
{sampleStackTraces}
|
||||
|
||||
## Related Resolved Clusters
|
||||
{relatedClusters}
|
||||
|
||||
## API Failure Pattern (if any)
|
||||
{apiFailurePattern}
|
||||
|
||||
## Feature Flag Correlations
|
||||
{featureFlagCorrelations}
|
||||
|
||||
Analyze this error and provide:
|
||||
1. Root cause category (config, dependency, logic, resource, external, or unknown)
|
||||
2. Specific hypothesis explaining WHY this error occurs
|
||||
3. Your reasoning process
|
||||
4. Evidence confidence level (high/medium/low)
|
||||
5. Suggested investigation steps (3-5 specific actions)
|
||||
6. Potential fix direction (if identifiable)
|
||||
|
||||
Respond in this JSON format:
|
||||
{
|
||||
"rootCauseCategory": "logic",
|
||||
"hypothesis": "detailed explanation...",
|
||||
"reasoning": "why you think this...",
|
||||
"confidence": "high",
|
||||
"suggestedInvestigation": ["step 1", "step 2", "step 3"],
|
||||
"potentialFixDirection": "suggested fix approach...",
|
||||
"evidence": [
|
||||
{
|
||||
"type": "stack_trace|telemetry_pattern|device_correlation|api_failure|similar_issue|config_mismatch|version_regression",
|
||||
"description": "specific evidence...",
|
||||
"strength": "strong|moderate|weak"
|
||||
}
|
||||
]
|
||||
}`;
|
||||
|
||||
const PATTERN_SUMMARY_PROMPT_TEMPLATE = `Summarize the pattern for this error cluster in 2-3 sentences. Focus on:
|
||||
1. What triggers the error (user action, system state, timing)
|
||||
2. The common characteristics across all occurrences
|
||||
3. Why this error clusters together (same root cause)
|
||||
|
||||
Error: {errorType}
|
||||
Message: {messagePattern}
|
||||
Occurrences: {occurrenceCount}
|
||||
Context: {contextSummary}
|
||||
|
||||
Provide a concise, engineer-friendly summary.`;
|
||||
|
||||
const COMPARATIVE_ANALYSIS_PROMPT_TEMPLATE = `Compare these two error clusters and identify:
|
||||
1. Key differences in root cause
|
||||
2. Whether they might be related variants
|
||||
3. Which cluster is more urgent to fix
|
||||
|
||||
Cluster A:
|
||||
- Type: {typeA}
|
||||
- Message: {messageA}
|
||||
- Occurrences: {occurrencesA}
|
||||
- Status: {statusA}
|
||||
|
||||
Cluster B:
|
||||
- Type: {typeB}
|
||||
- Message: {messageB}
|
||||
- Occurrences: {occurrencesB}
|
||||
- Status: {statusB}
|
||||
|
||||
Respond in JSON format with comparison analysis.`;
|
||||
|
||||
// ============================================================================
|
||||
// LLM Client
|
||||
// ============================================================================
|
||||
|
||||
interface LLMResponse {
|
||||
rootCauseCategory: RootCauseCategory;
|
||||
hypothesis: string;
|
||||
reasoning: string;
|
||||
confidence: 'high' | 'medium' | 'low';
|
||||
suggestedInvestigation: string[];
|
||||
potentialFixDirection?: string;
|
||||
evidence: Array<{
|
||||
type: string;
|
||||
description: string;
|
||||
strength: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface LLMCallResult {
|
||||
response: LLMResponse;
|
||||
modelUsed: string;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls Azure OpenAI GPT-4o for analysis
|
||||
*/
|
||||
async function callLLM(
|
||||
prompt: string,
|
||||
options: {
|
||||
model?: 'gpt-4o' | 'gpt-4o-mini';
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
} = {}
|
||||
): Promise<LLMCallResult> {
|
||||
const model = options.model || 'gpt-4o-mini';
|
||||
const apiKey = config.AZURE_OPENAI_KEY;
|
||||
const endpoint = config.AZURE_OPENAI_ENDPOINT;
|
||||
|
||||
if (!apiKey || !endpoint) {
|
||||
throw new Error('Azure OpenAI credentials not configured');
|
||||
}
|
||||
|
||||
const url = `${endpoint}/openai/deployments/${model}/chat/completions?api-version=2024-02-01`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'api-key': apiKey,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content:
|
||||
'You are an expert software engineer specializing in debugging production systems. Provide structured, evidence-based analysis.',
|
||||
},
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
temperature: options.temperature ?? 0.3,
|
||||
max_tokens: options.maxTokens ?? 2000,
|
||||
response_format: { type: 'json_object' },
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`Azure OpenAI API error: ${response.status} ${error}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
choices: Array<{
|
||||
message: { content: string };
|
||||
}>;
|
||||
usage: {
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
};
|
||||
model: string;
|
||||
};
|
||||
|
||||
const content = data.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('Empty response from LLM');
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(content) as LLMResponse;
|
||||
|
||||
return {
|
||||
response: parsed,
|
||||
modelUsed: data.model || model,
|
||||
promptTokens: data.usage?.prompt_tokens || 0,
|
||||
completionTokens: data.usage?.completion_tokens || 0,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('LLM call failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Analysis Functions
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Performs root cause analysis on an error cluster
|
||||
*/
|
||||
export async function analyzeRootCause(
|
||||
params: RootCauseAnalysisPrompt
|
||||
): Promise<Partial<DiagnosticInsightDoc>> {
|
||||
const {
|
||||
cluster,
|
||||
context,
|
||||
sampleStackTraces,
|
||||
relatedClusters,
|
||||
analysisType,
|
||||
} = params;
|
||||
|
||||
// Build prompt
|
||||
const prompt = ROOT_CAUSE_ANALYSIS_PROMPT_TEMPLATE.replace(
|
||||
'{errorType}',
|
||||
cluster.errorType
|
||||
)
|
||||
.replace('{messagePattern}', cluster.messageTemplate)
|
||||
.replace('{stackSignature}', cluster.stackSignature.slice(0, 500))
|
||||
.replace('{occurrenceCount}', cluster.occurrenceCount.toString())
|
||||
.replace('{uniqueUsers}', cluster.uniqueUsers.toString())
|
||||
.replace('{firstSeenAt}', cluster.firstSeenAt)
|
||||
.replace('{lastSeenAt}', cluster.lastSeenAt)
|
||||
.replace('{totalOccurrences}', context.totalOccurrences.toString())
|
||||
.replace('{affectedUsersCount}', context.affectedUsers.length.toString())
|
||||
.replace(
|
||||
'{timeRange}',
|
||||
`${context.timeRange.start} to ${context.timeRange.end}`
|
||||
)
|
||||
.replace(
|
||||
'{commonScreens}',
|
||||
context.mostCommonScreens.map((s) => `${s.screen} (${s.count})`).join(', ')
|
||||
)
|
||||
.replace(
|
||||
'{commonActions}',
|
||||
context.mostCommonActions.map((a) => `${a.action} (${a.count})`).join(', ')
|
||||
)
|
||||
.replace('{sampleStackTraces}', sampleStackTraces.join('\n\n'))
|
||||
.replace(
|
||||
'{relatedClusters}',
|
||||
relatedClusters
|
||||
.filter((c) => c.status === 'resolved')
|
||||
.map((c) => `- ${c.errorType} (${c.id})`)
|
||||
.join('\n') || 'None found'
|
||||
)
|
||||
.replace(
|
||||
'{apiFailurePattern}',
|
||||
context.apiFailurePattern
|
||||
? `${context.apiFailurePattern.endpoint}: ${context.apiFailurePattern.errorPattern} (${context.apiFailurePattern.occurrenceCount} occurrences)`
|
||||
: 'No clear API failure pattern'
|
||||
)
|
||||
.replace(
|
||||
'{featureFlagCorrelations}',
|
||||
context.featureFlagCorrelations
|
||||
.map((f) => `- ${f.flag}: ${f.errorCorrelation.toFixed(2)} correlation`)
|
||||
.join('\n') || 'No strong correlations found'
|
||||
);
|
||||
|
||||
// Call LLM
|
||||
const llmResult = await callLLM(prompt, {
|
||||
model: 'gpt-4o-mini',
|
||||
temperature: 0.3,
|
||||
maxTokens: 2000,
|
||||
});
|
||||
|
||||
// Map evidence types
|
||||
const evidence: Evidence[] = llmResult.response.evidence.map((e) => ({
|
||||
type: validateEvidenceType(e.type),
|
||||
description: e.description,
|
||||
strength: validateEvidenceStrength(e.strength),
|
||||
data: {},
|
||||
}));
|
||||
|
||||
// Calculate confidence score
|
||||
const confidenceScore = calculateConfidenceScore(
|
||||
llmResult.response.confidence,
|
||||
evidence,
|
||||
context
|
||||
);
|
||||
|
||||
return {
|
||||
analysisType,
|
||||
rootCauseCategory: llmResult.response.rootCauseCategory,
|
||||
hypothesis: llmResult.response.hypothesis,
|
||||
reasoning: llmResult.response.reasoning,
|
||||
confidence: llmResult.response.confidence,
|
||||
confidenceScore,
|
||||
evidence,
|
||||
suggestedInvestigation: llmResult.response.suggestedInvestigation,
|
||||
potentialFixDirection: llmResult.response.potentialFixDirection,
|
||||
modelUsed: llmResult.modelUsed,
|
||||
promptTokens: llmResult.promptTokens,
|
||||
completionTokens: llmResult.completionTokens,
|
||||
feedbackStats: { helpful: 0, notHelpful: 0, engineerNotes: [] },
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a pattern summary for a cluster
|
||||
*/
|
||||
export async function generatePatternSummary(
|
||||
cluster: ErrorClusterDoc,
|
||||
context: ClusterContextSummary
|
||||
): Promise<string> {
|
||||
const prompt = PATTERN_SUMMARY_PROMPT_TEMPLATE.replace('{errorType}', cluster.errorType)
|
||||
.replace('{messagePattern}', cluster.messageTemplate)
|
||||
.replace('{occurrenceCount}', cluster.occurrenceCount.toString())
|
||||
.replace(
|
||||
'{contextSummary}',
|
||||
`Screens: ${context.mostCommonScreens.map((s) => s.screen).join(', ')}
|
||||
Actions: ${context.mostCommonActions.map((a) => a.action).join(', ')}`
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await callLLM(prompt, {
|
||||
model: 'gpt-4o-mini',
|
||||
temperature: 0.4,
|
||||
maxTokens: 300,
|
||||
});
|
||||
|
||||
return result.response.hypothesis || 'Pattern analysis pending.';
|
||||
} catch {
|
||||
return 'Pattern analysis failed. Please review manually.';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares two clusters
|
||||
*/
|
||||
export async function compareClusters(
|
||||
clusterA: ErrorClusterDoc,
|
||||
clusterB: ErrorClusterDoc,
|
||||
contextA: ClusterContextSummary,
|
||||
contextB: ClusterContextSummary
|
||||
): Promise<{
|
||||
differences: string[];
|
||||
related: boolean;
|
||||
urgencyComparison: string;
|
||||
}> {
|
||||
const prompt = COMPARATIVE_ANALYSIS_PROMPT_TEMPLATE.replace('{typeA}', clusterA.errorType)
|
||||
.replace('{messageA}', clusterA.messageTemplate)
|
||||
.replace('{occurrencesA}', clusterA.occurrenceCount.toString())
|
||||
.replace('{statusA}', clusterA.status)
|
||||
.replace('{typeB}', clusterB.errorType)
|
||||
.replace('{messageB}', clusterB.messageTemplate)
|
||||
.replace('{occurrencesB}', clusterB.occurrenceCount.toString())
|
||||
.replace('{statusB}', clusterB.status);
|
||||
|
||||
try {
|
||||
const result = await callLLM(prompt, {
|
||||
model: 'gpt-4o-mini',
|
||||
temperature: 0.3,
|
||||
maxTokens: 800,
|
||||
});
|
||||
|
||||
return {
|
||||
differences: result.response.suggestedInvestigation || [],
|
||||
related: result.response.confidence === 'high',
|
||||
urgencyComparison: result.response.potentialFixDirection || 'Compare occurrence counts',
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
differences: [],
|
||||
related: false,
|
||||
urgencyComparison: 'Unable to compare',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
function validateEvidenceType(type: string): Evidence['type'] {
|
||||
const validTypes: Evidence['type'][] = [
|
||||
'stack_trace',
|
||||
'telemetry_pattern',
|
||||
'device_correlation',
|
||||
'api_failure',
|
||||
'similar_issue',
|
||||
'config_mismatch',
|
||||
'version_regression',
|
||||
];
|
||||
return validTypes.includes(type as Evidence['type'])
|
||||
? (type as Evidence['type'])
|
||||
: 'stack_trace';
|
||||
}
|
||||
|
||||
function validateEvidenceStrength(strength: string): 'strong' | 'moderate' | 'weak' {
|
||||
const validStrengths = ['strong', 'moderate', 'weak'];
|
||||
return validStrengths.includes(strength) ? (strength as 'strong' | 'moderate' | 'weak') : 'moderate';
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates a numeric confidence score based on multiple factors
|
||||
*/
|
||||
function calculateConfidenceScore(
|
||||
llmConfidence: string,
|
||||
evidence: Evidence[],
|
||||
context: ClusterContextSummary
|
||||
): number {
|
||||
let score = 0;
|
||||
|
||||
// Base score from LLM confidence
|
||||
switch (llmConfidence) {
|
||||
case 'high':
|
||||
score += 0.6;
|
||||
break;
|
||||
case 'medium':
|
||||
score += 0.4;
|
||||
break;
|
||||
case 'low':
|
||||
score += 0.2;
|
||||
break;
|
||||
}
|
||||
|
||||
// Evidence strength bonus
|
||||
const strongEvidence = evidence.filter((e) => e.strength === 'strong').length;
|
||||
const moderateEvidence = evidence.filter((e) => e.strength === 'moderate').length;
|
||||
score += strongEvidence * 0.1 + moderateEvidence * 0.05;
|
||||
|
||||
// Data volume bonus (more data = higher confidence)
|
||||
if (context.totalOccurrences >= 100) score += 0.1;
|
||||
else if (context.totalOccurrences >= 10) score += 0.05;
|
||||
|
||||
// Affected users bonus
|
||||
if (context.affectedUsers.length >= 50) score += 0.1;
|
||||
else if (context.affectedUsers.length >= 10) score += 0.05;
|
||||
|
||||
return Math.min(1.0, score);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Retry Logic
|
||||
// ============================================================================
|
||||
|
||||
interface RetryOptions {
|
||||
maxRetries?: number;
|
||||
baseDelayMs?: number;
|
||||
maxDelayMs?: number;
|
||||
}
|
||||
|
||||
export async function callLLMWithRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: RetryOptions = {}
|
||||
): Promise<T> {
|
||||
const { maxRetries = 3, baseDelayMs = 1000, maxDelayMs = 10000 } = options;
|
||||
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
|
||||
// Don't retry on authentication errors
|
||||
if (lastError.message.includes('401') || lastError.message.includes('403')) {
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
if (attempt < maxRetries - 1) {
|
||||
// Exponential backoff with jitter
|
||||
const delay = Math.min(
|
||||
baseDelayMs * Math.pow(2, attempt) + Math.random() * 1000,
|
||||
maxDelayMs
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
@ -350,7 +350,7 @@ export async function enrichErrorContext(
|
||||
|
||||
return {
|
||||
errorEvent,
|
||||
telemetryContext,
|
||||
telemetryContext: telemetryContext || undefined,
|
||||
sessionState,
|
||||
recentActions,
|
||||
apiFailures,
|
||||
@ -507,7 +507,7 @@ export function generateBreadcrumbTrail(
|
||||
// Cluster Context Aggregation
|
||||
// ============================================================================
|
||||
|
||||
interface ClusterContextSummary {
|
||||
export interface ClusterContextSummary {
|
||||
totalOccurrences: number;
|
||||
affectedUsers: string[];
|
||||
timeRange: { start: string; end: string };
|
||||
|
||||
@ -0,0 +1,634 @@
|
||||
/**
|
||||
* Predictive Analytics Module Tests
|
||||
* [Target 20+ tests]
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import type { TelemetryEventDoc } from '../telemetry/types.js';
|
||||
import {
|
||||
extractFeaturesFromTelemetry,
|
||||
extractProductSpecificFeatures,
|
||||
CompleteFeatureVector,
|
||||
} from './feature-extractor.js';
|
||||
import { ChurnModel } from './churn-model.js';
|
||||
import { HealthScoringEngine, HealthMetricsInput } from './health-scoring.js';
|
||||
import { AnomalyDetectionEngine } from './anomaly-detection.js';
|
||||
import {
|
||||
ChurnPredictionInput,
|
||||
RiskSegmentEnum,
|
||||
ModelTypeEnum,
|
||||
PredictionHorizonEnum,
|
||||
} from './types.js';
|
||||
|
||||
// ============================================================================
|
||||
// Test Fixtures
|
||||
// ============================================================================
|
||||
|
||||
function createMockTelemetryEvent(overrides: Partial<TelemetryEventDoc> = {}): TelemetryEventDoc {
|
||||
return {
|
||||
id: crypto.randomUUID(),
|
||||
pk: 'test:202601:ios',
|
||||
productId: 'test',
|
||||
userId: 'user_123',
|
||||
sessionId: 'session_456',
|
||||
platform: 'ios',
|
||||
channel: 'mobile_app',
|
||||
osFamily: 'ios',
|
||||
appVersion: '1.0.0',
|
||||
buildNumber: '100',
|
||||
releaseChannel: 'prod',
|
||||
eventType: 'info',
|
||||
module: 'test',
|
||||
feature: 'core',
|
||||
eventName: 'test_event',
|
||||
occurredAt: new Date().toISOString(),
|
||||
receivedAt: new Date().toISOString(),
|
||||
ttl: 30 * 24 * 60 * 60,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createMockEventSeries(userId: string, days: number): TelemetryEventDoc[] {
|
||||
const events: TelemetryEventDoc[] = [];
|
||||
const now = new Date();
|
||||
|
||||
for (let d = 0; d < days; d++) {
|
||||
const date = new Date(now);
|
||||
date.setDate(date.getDate() - d);
|
||||
|
||||
// Create 2-5 sessions per day
|
||||
const sessions = 2 + Math.floor(Math.random() * 3);
|
||||
for (let s = 0; s < sessions; s++) {
|
||||
events.push(
|
||||
createMockTelemetryEvent({
|
||||
userId,
|
||||
sessionId: `session_${d}_${s}`,
|
||||
occurredAt: date.toISOString(),
|
||||
feature: ['core', 'advanced', 'settings'][Math.floor(Math.random() * 3)],
|
||||
eventName: 'action_completed',
|
||||
metrics: { duration: 1000 + Math.random() * 5000 },
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Feature Extraction Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Feature Extractor', () => {
|
||||
it('should extract features from telemetry events', () => {
|
||||
const events = createMockEventSeries('user_123', 14);
|
||||
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||
|
||||
expect(features.userId).toBe('user_123');
|
||||
expect(features.productId).toBe('test');
|
||||
expect(features.featureSchemaVersion).toBe('1.0.0');
|
||||
expect(features.dataQualityScore).toBeGreaterThan(0);
|
||||
expect(features.dataQualityScore).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it('should calculate behavior features correctly', () => {
|
||||
const now = new Date();
|
||||
const yesterday = new Date(now);
|
||||
yesterday.setDate(yesterday.getDate() - 1);
|
||||
|
||||
const events: TelemetryEventDoc[] = [
|
||||
createMockTelemetryEvent({
|
||||
sessionId: 'session_1',
|
||||
occurredAt: yesterday.toISOString(),
|
||||
feature: 'core',
|
||||
eventName: 'session_start',
|
||||
metrics: { duration: 300000 },
|
||||
}),
|
||||
];
|
||||
|
||||
const features = extractFeaturesFromTelemetry('user_123', 'test', events, now);
|
||||
|
||||
expect(features.behavior.daysSinceLastSession).toBeGreaterThanOrEqual(1);
|
||||
expect(features.behavior.sessionsLast24Hours).toBe(0);
|
||||
expect(features.behavior.sessionsLast7Days).toBe(1);
|
||||
});
|
||||
|
||||
it('should calculate engagement features correctly', () => {
|
||||
const events: TelemetryEventDoc[] = [
|
||||
createMockTelemetryEvent({
|
||||
feature: 'core',
|
||||
eventName: 'core_action_completed',
|
||||
}),
|
||||
createMockTelemetryEvent({
|
||||
feature: 'advanced',
|
||||
eventName: 'feature_used',
|
||||
}),
|
||||
];
|
||||
|
||||
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||
|
||||
expect(features.engagement.featureUsageDiversity).toBeGreaterThan(0);
|
||||
expect(features.engagement.uniqueFeaturesUsed).toBe(2);
|
||||
});
|
||||
|
||||
it('should calculate rolling window features', () => {
|
||||
const events = createMockEventSeries('user_123', 30);
|
||||
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||
|
||||
expect(features.rolling.rollingAvgSessions7d).toBeGreaterThanOrEqual(0);
|
||||
expect(features.rolling.cohortSessionPercentile).toBeGreaterThanOrEqual(0);
|
||||
expect(features.rolling.cohortSessionPercentile).toBeLessThanOrEqual(100);
|
||||
});
|
||||
|
||||
it('should extract product-specific features for NomGap', () => {
|
||||
const events: TelemetryEventDoc[] = [
|
||||
createMockTelemetryEvent({
|
||||
feature: 'fasting',
|
||||
eventName: 'fast_started',
|
||||
}),
|
||||
createMockTelemetryEvent({
|
||||
feature: 'fasting',
|
||||
eventName: 'fast_completed',
|
||||
context: { adhered: true },
|
||||
}),
|
||||
];
|
||||
|
||||
const features = extractProductSpecificFeatures(events, 'nomgap');
|
||||
|
||||
expect(features.fastCompletionRate).toBe(1);
|
||||
expect(features.protocolAdherenceScore).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('should extract product-specific features for JarvisJr', () => {
|
||||
const events: TelemetryEventDoc[] = [
|
||||
createMockTelemetryEvent({
|
||||
feature: 'agent',
|
||||
eventName: 'session_started',
|
||||
context: { mode: 'voice', agentId: 'agent_1' },
|
||||
}),
|
||||
createMockTelemetryEvent({
|
||||
feature: 'agent',
|
||||
eventName: 'session_completed',
|
||||
context: { mode: 'text', agentId: 'agent_2' },
|
||||
}),
|
||||
];
|
||||
|
||||
const features = extractProductSpecificFeatures(events, 'jarvisjr');
|
||||
|
||||
expect(features.agentDiversityScore).toBeGreaterThan(0);
|
||||
expect(features.voiceSessionRatio).toBe(0.5);
|
||||
});
|
||||
|
||||
it('should handle empty telemetry gracefully', () => {
|
||||
const features = extractFeaturesFromTelemetry('user_123', 'test', []);
|
||||
|
||||
expect(features.behavior.sessionsLast30Days).toBe(0);
|
||||
expect(features.engagement.featureUsageDiversity).toBe(0);
|
||||
expect(features.dataQualityScore).toBe(0);
|
||||
});
|
||||
|
||||
it('should calculate data quality score based on completeness', () => {
|
||||
const richEvents = createMockEventSeries('user_123', 30);
|
||||
const features = extractFeaturesFromTelemetry('user_123', 'test', richEvents);
|
||||
|
||||
expect(features.dataQualityScore).toBeGreaterThan(0.3);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Churn Model Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Churn Model', () => {
|
||||
let model: ChurnModel;
|
||||
|
||||
beforeEach(() => {
|
||||
model = new ChurnModel();
|
||||
});
|
||||
|
||||
function createMockFeatures(): CompleteFeatureVector {
|
||||
const now = new Date();
|
||||
return {
|
||||
userId: 'user_123',
|
||||
productId: 'test',
|
||||
computedAt: now,
|
||||
observationWindow: { start: now, end: now },
|
||||
behavior: {
|
||||
daysSinceLastSession: 3,
|
||||
daysSinceLastCoreAction: 2,
|
||||
hoursSinceLastLogin: 72,
|
||||
sessionsLast24Hours: 1,
|
||||
sessionsLast7Days: 5,
|
||||
sessionsLast30Days: 20,
|
||||
avgSessionsPerWeek: 4,
|
||||
avgSessionsPerDay: 0.6,
|
||||
avgSessionDurationMinutes: 15,
|
||||
totalSessionDurationMinutes: 300,
|
||||
actionsPerSession: 8,
|
||||
uniqueFeaturesUsed: 5,
|
||||
sessionFrequencyTrend: 0.1,
|
||||
engagementDepthTrend: 0.05,
|
||||
},
|
||||
engagement: {
|
||||
featureUsageDiversity: 0.5,
|
||||
coreActionCompletionRate: 0.8,
|
||||
featureAdoptionVelocity: 1.2,
|
||||
powerUserScore: 0.3,
|
||||
onboardingCompletionRate: 0.9,
|
||||
firstValueMomentAchieved: true,
|
||||
timeToFirstValueHours: 2,
|
||||
},
|
||||
performance: {
|
||||
errorRateLast7Days: 0.01,
|
||||
errorRateLast30Days: 0.02,
|
||||
crashCountLast7Days: 0,
|
||||
crashCountLast30Days: 1,
|
||||
avgLatencyMs: 200,
|
||||
slowRequestCount: 2,
|
||||
timeoutCount: 0,
|
||||
errorRecoveryRate: 0.95,
|
||||
supportTicketCount: 0,
|
||||
},
|
||||
social: {
|
||||
shareCount: 3,
|
||||
inviteCount: 1,
|
||||
collaborationScore: 0.2,
|
||||
teamMemberCount: 2,
|
||||
integrationsConnected: 1,
|
||||
externalSharesLast30Days: 2,
|
||||
},
|
||||
revenue: {
|
||||
planTier: 1,
|
||||
lifetimeValue: 120,
|
||||
mrrContribution: 10,
|
||||
upgradeCount: 1,
|
||||
downgradeCount: 0,
|
||||
daysSinceLastPayment: 5,
|
||||
daysSincePlanChange: 60,
|
||||
supportTicketCount: 0,
|
||||
supportSatisfactionScore: 4.5,
|
||||
escalatedTicketCount: 0,
|
||||
},
|
||||
rolling: {
|
||||
rollingAvgSessions7d: 0.7,
|
||||
rollingAvgDuration7d: 15,
|
||||
rollingAvgActions7d: 8,
|
||||
wowSessionChange: 0.1,
|
||||
wowDurationChange: 0.05,
|
||||
wowActionsChange: 0.08,
|
||||
cohortSessionPercentile: 60,
|
||||
cohortEngagementPercentile: 55,
|
||||
cohortRetentionPercentile: 65,
|
||||
},
|
||||
productSpecific: {},
|
||||
timeWindows: {
|
||||
recent: { sessionCount: 1, totalDuration: 900, actionCount: 8, errorCount: 0, uniqueFeatures: ['core'] },
|
||||
weekly: { sessionCount: 5, totalDuration: 4500, actionCount: 40, errorCount: 0, uniqueFeatures: ['core', 'advanced'], daysActive: 4 },
|
||||
monthly: { sessionCount: 20, totalDuration: 18000, actionCount: 160, errorCount: 2, uniqueFeatures: ['core', 'advanced', 'settings'], daysActive: 15 },
|
||||
lifetime: { totalSessions: 100, totalDuration: 90000, totalActions: 800, totalErrors: 5, allFeaturesUsed: ['core', 'advanced', 'settings'], accountAgeDays: 90 },
|
||||
},
|
||||
featureSchemaVersion: '1.0.0',
|
||||
dataQualityScore: 0.85,
|
||||
};
|
||||
}
|
||||
|
||||
it('should predict churn probability', () => {
|
||||
const features = createMockFeatures();
|
||||
const prediction = model.predict(features);
|
||||
|
||||
expect(prediction.churnProbability).toBeGreaterThanOrEqual(0);
|
||||
expect(prediction.churnProbability).toBeLessThanOrEqual(1);
|
||||
expect(prediction.userId).toBe('user_123');
|
||||
expect(prediction.productId).toBe('test');
|
||||
expect(prediction.modelType).toBe('xgboost');
|
||||
});
|
||||
|
||||
it('should categorize risk segments correctly', () => {
|
||||
const features = createMockFeatures();
|
||||
|
||||
// Low engagement should increase churn risk
|
||||
features.engagement.featureUsageDiversity = 0.1;
|
||||
features.behavior.daysSinceLastSession = 20;
|
||||
|
||||
const prediction = model.predict(features);
|
||||
|
||||
expect(['critical', 'high', 'medium', 'low']).toContain(prediction.riskSegment);
|
||||
});
|
||||
|
||||
it('should generate explanations with top risk factors', () => {
|
||||
const features = createMockFeatures();
|
||||
const prediction = model.predict(features);
|
||||
|
||||
expect(prediction.explanation.topRiskFactors).toHaveLength(5);
|
||||
expect(prediction.explanation.nlExplanation).toContain('churn risk');
|
||||
expect(prediction.explanation.suggestedActions.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should calculate confidence score based on data quality', () => {
|
||||
const features = createMockFeatures();
|
||||
features.dataQualityScore = 0.3;
|
||||
|
||||
const prediction = model.predict(features);
|
||||
|
||||
expect(prediction.confidenceScore).toBeLessThanOrEqual(features.dataQualityScore + 0.1);
|
||||
});
|
||||
|
||||
it('should predict batch correctly', () => {
|
||||
const features = [createMockFeatures(), createMockFeatures()];
|
||||
features[1].userId = 'user_456';
|
||||
|
||||
const predictions = model.predictBatch(features);
|
||||
|
||||
expect(predictions).toHaveLength(2);
|
||||
expect(predictions[0].userId).toBe('user_123');
|
||||
expect(predictions[1].userId).toBe('user_456');
|
||||
});
|
||||
|
||||
it('should handle different prediction horizons', () => {
|
||||
const features = createMockFeatures();
|
||||
|
||||
const p7 = model.predict(features, 7);
|
||||
const p30 = model.predict(features, 30);
|
||||
|
||||
// Longer horizon should generally have different probability
|
||||
expect(p7.predictionHorizon).toBe(7);
|
||||
expect(p30.predictionHorizon).toBe(30);
|
||||
});
|
||||
|
||||
it('should evaluate model performance', () => {
|
||||
const predictions = [
|
||||
{ actual: true, predicted: 0.8 },
|
||||
{ actual: false, predicted: 0.2 },
|
||||
{ actual: true, predicted: 0.9 },
|
||||
{ actual: false, predicted: 0.1 },
|
||||
];
|
||||
|
||||
const metrics = model.evaluateModel(predictions);
|
||||
|
||||
expect(metrics.auc).toBeGreaterThan(0.5);
|
||||
expect(metrics.modelVersion).toBeDefined();
|
||||
expect(metrics.featureImportance.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Health Scoring Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Health Scoring Engine', () => {
|
||||
let engine: HealthScoringEngine;
|
||||
|
||||
beforeEach(() => {
|
||||
engine = new HealthScoringEngine();
|
||||
});
|
||||
|
||||
function createMockHealthInput(): HealthMetricsInput {
|
||||
return {
|
||||
productId: 'test',
|
||||
date: new Date(),
|
||||
newUsers: 100,
|
||||
activationRateDay1: 0.6,
|
||||
activationRateDay7: 0.45,
|
||||
cac: 50,
|
||||
firstValueMomentRate: 0.55,
|
||||
timeToFirstAction: 120,
|
||||
onboardingCompletionRate: 0.75,
|
||||
dau: 1000,
|
||||
mau: 5000,
|
||||
day7Retention: 0.35,
|
||||
day30Retention: 0.25,
|
||||
avgSessionLength: 300,
|
||||
sessionsPerUser: 3,
|
||||
featureAdoption: { feature1: 0.5, feature2: 0.3 },
|
||||
mrr: 10000,
|
||||
arpu: 20,
|
||||
churnRate: 0.05,
|
||||
upgradeRate: 0.1,
|
||||
crashFreeRate: 0.99,
|
||||
errorRate: 0.01,
|
||||
avgLatency: 150,
|
||||
uptimePercent: 99.9,
|
||||
baselines: {
|
||||
dau: 900,
|
||||
newUsers: 90,
|
||||
activationRateDay1: 0.55,
|
||||
day7Retention: 0.33,
|
||||
avgSessionLength: 280,
|
||||
mrr: 9500,
|
||||
errorRate: 0.015,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it('should calculate overall health score', () => {
|
||||
const input = createMockHealthInput();
|
||||
const score = engine.calculateHealthScore(input);
|
||||
|
||||
expect(score.overallHealthScore).toBeGreaterThanOrEqual(0);
|
||||
expect(score.overallHealthScore).toBeLessThanOrEqual(100);
|
||||
expect(score.productId).toBe('test');
|
||||
});
|
||||
|
||||
it('should calculate all 6 dimension scores', () => {
|
||||
const input = createMockHealthInput();
|
||||
const score = engine.calculateHealthScore(input);
|
||||
|
||||
expect(score.dimensions.acquisition.score).toBeGreaterThanOrEqual(0);
|
||||
expect(score.dimensions.activation.score).toBeGreaterThanOrEqual(0);
|
||||
expect(score.dimensions.retention.score).toBeGreaterThanOrEqual(0);
|
||||
expect(score.dimensions.engagement.score).toBeGreaterThanOrEqual(0);
|
||||
expect(score.dimensions.revenue.score).toBeGreaterThanOrEqual(0);
|
||||
expect(score.dimensions.stability.score).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('should detect health status correctly', () => {
|
||||
const input = createMockHealthInput();
|
||||
|
||||
// Healthy case
|
||||
const healthyScore = engine.calculateHealthScore(input);
|
||||
expect(['healthy', 'warning', 'critical']).toContain(healthyScore.healthStatus);
|
||||
|
||||
// Critical case
|
||||
input.day7Retention = 0.05;
|
||||
input.churnRate = 0.5;
|
||||
input.crashFreeRate = 0.8;
|
||||
const criticalScore = engine.calculateHealthScore(input);
|
||||
expect(criticalScore.healthStatus).toBe('critical');
|
||||
});
|
||||
|
||||
it('should detect anomalies', () => {
|
||||
const input = createMockHealthInput();
|
||||
input.dau = 500; // 50% drop from baseline
|
||||
|
||||
const score = engine.calculateHealthScore(input);
|
||||
|
||||
expect(score.anomalies.length).toBeGreaterThan(0);
|
||||
expect(score.anomalies.some((a) => a.metric === 'dau')).toBe(true);
|
||||
});
|
||||
|
||||
it('should generate forecasts', () => {
|
||||
const input = createMockHealthInput();
|
||||
const score = engine.calculateHealthScore(input);
|
||||
|
||||
expect(score.forecasts.next7Days.expectedHealthScore).toBeGreaterThanOrEqual(0);
|
||||
expect(score.forecasts.next7Days.confidenceInterval).toHaveLength(2);
|
||||
expect(score.forecasts.next30Days.expectedHealthScore).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('should calculate baseline comparisons', () => {
|
||||
const input = createMockHealthInput();
|
||||
const score = engine.calculateHealthScore(input);
|
||||
|
||||
expect(typeof score.vsBaseline7Day).toBe('number');
|
||||
expect(typeof score.vsBaseline30Day).toBe('number');
|
||||
});
|
||||
|
||||
it('should determine trends correctly', () => {
|
||||
const input = createMockHealthInput();
|
||||
|
||||
// Improving trend
|
||||
input.dau = 1200;
|
||||
const improving = engine.calculateHealthScore(input);
|
||||
expect(improving.dimensions.retention.trend).toBe('improving');
|
||||
|
||||
// Declining trend
|
||||
input.dau = 600;
|
||||
const declining = engine.calculateHealthScore(input);
|
||||
expect(declining.dimensions.retention.trend).toBe('declining');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Anomaly Detection Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Anomaly Detection Engine', () => {
|
||||
let engine: AnomalyDetectionEngine;
|
||||
|
||||
beforeEach(() => {
|
||||
engine = new AnomalyDetectionEngine();
|
||||
});
|
||||
|
||||
function createTimeSeries(length: number, baseValue: number, noise: number): Array<{ timestamp: Date; value: number }> {
|
||||
const series: Array<{ timestamp: Date; value: number }> = [];
|
||||
const now = new Date();
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
const date = new Date(now);
|
||||
date.setDate(date.getDate() - (length - i));
|
||||
series.push({
|
||||
timestamp: date,
|
||||
value: baseValue + (Math.random() - 0.5) * noise,
|
||||
});
|
||||
}
|
||||
|
||||
return series;
|
||||
}
|
||||
|
||||
it('should detect anomalies in time series', () => {
|
||||
const series = createTimeSeries(30, 100, 10);
|
||||
|
||||
// Add anomaly
|
||||
series[series.length - 1].value = 200;
|
||||
|
||||
const anomalies = engine.detectAnomalies(series, 'test_metric');
|
||||
|
||||
expect(anomalies.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should not detect anomalies in stable series', () => {
|
||||
const series = createTimeSeries(30, 100, 5);
|
||||
|
||||
const anomalies = engine.detectAnomalies(series, 'test_metric');
|
||||
|
||||
expect(anomalies.length).toBe(0);
|
||||
});
|
||||
|
||||
it('should require minimum data for detection', () => {
|
||||
const shortSeries = createTimeSeries(5, 100, 10);
|
||||
|
||||
const anomalies = engine.detectAnomalies(shortSeries, 'test_metric');
|
||||
|
||||
expect(anomalies.length).toBe(0);
|
||||
});
|
||||
|
||||
it('should forecast future values', () => {
|
||||
const series = createTimeSeries(30, 100, 10);
|
||||
|
||||
const forecast = engine.forecast(series, 7);
|
||||
|
||||
expect(forecast.forecast.length).toBe(7);
|
||||
expect(forecast.confidenceIntervals.length).toBe(7);
|
||||
expect(forecast.forecast[0].value).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should return empty forecast for insufficient data', () => {
|
||||
const shortSeries = createTimeSeries(5, 100, 10);
|
||||
|
||||
const forecast = engine.forecast(shortSeries, 7);
|
||||
|
||||
expect(forecast.forecast.length).toBe(0);
|
||||
});
|
||||
|
||||
it('should detect multi-dimensional anomalies', () => {
|
||||
const metrics: Record<string, Array<{ timestamp: Date; value: number }>> = {
|
||||
metric1: createTimeSeries(30, 100, 10),
|
||||
metric2: createTimeSeries(30, 50, 5),
|
||||
};
|
||||
|
||||
// Add correlated anomalies
|
||||
metrics.metric1[metrics.metric1.length - 1].value = 180;
|
||||
metrics.metric2[metrics.metric2.length - 1].value = 90;
|
||||
|
||||
const anomalies = engine.detectMultiDimensionalAnomaly(metrics);
|
||||
|
||||
expect(anomalies.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Integration Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Predictive Analytics Integration', () => {
|
||||
it('should complete full prediction pipeline', () => {
|
||||
// 1. Extract features from telemetry
|
||||
const events = createMockEventSeries('user_123', 30);
|
||||
const features = extractFeaturesFromTelemetry('user_123', 'test', events);
|
||||
|
||||
// 2. Run churn prediction
|
||||
const model = new ChurnModel();
|
||||
const prediction = model.predict(features);
|
||||
|
||||
// 3. Verify prediction structure
|
||||
expect(prediction.churnProbability).toBeDefined();
|
||||
expect(prediction.riskSegment).toBeDefined();
|
||||
expect(prediction.explanation).toBeDefined();
|
||||
expect(prediction.explanation.topRiskFactors).toHaveLength(5);
|
||||
expect(prediction.explanation.suggestedActions.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should handle different product types', () => {
|
||||
const products = ['nomgap', 'jarvisjr', 'chronomind', 'mindlyst', 'peakpulse', 'lysnrai'];
|
||||
|
||||
for (const productId of products) {
|
||||
const events = createMockEventSeries('user_123', 14);
|
||||
const features = extractFeaturesFromTelemetry('user_123', productId, events);
|
||||
const productSpecific = extractProductSpecificFeatures(events, productId);
|
||||
|
||||
// Should complete without errors
|
||||
expect(features.productId).toBe(productId);
|
||||
expect(productSpecific).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Summary
|
||||
// ============================================================================
|
||||
// Total tests: 26+
|
||||
// - Feature extraction: 7 tests
|
||||
// - Churn model: 7 tests
|
||||
// - Health scoring: 7 tests
|
||||
// - Anomaly detection: 6 tests
|
||||
// - Integration: 2 tests
|
||||
@ -0,0 +1,471 @@
|
||||
/**
|
||||
* Predictive Analytics Routes - REST API endpoints
|
||||
* [2.2] Real-time scoring API
|
||||
* [4.3] CS team dashboard backend
|
||||
* [5] Admin API routes
|
||||
*/
|
||||
|
||||
import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
|
||||
import { z } from 'zod';
|
||||
import { requireAuth, requireAdmin } from '@bytelyst/auth';
|
||||
import { extractFeaturesFromTelemetry } from './feature-extractor.js';
|
||||
import { featureStore } from './feature-store.js';
|
||||
import { churnModel } from './churn-model.js';
|
||||
import { healthScoringEngine } from './health-scoring.js';
|
||||
import { campaignEngine } from './campaign-engine.js';
|
||||
import { predictiveAnalyticsRepo } from './repository.js';
|
||||
import {
|
||||
ChurnScoreRequestSchema,
|
||||
ChurnBatchRequestSchema,
|
||||
AtRiskUsersQuerySchema,
|
||||
CreateCampaignSchema,
|
||||
CampaignTriggerSchema,
|
||||
RiskSegmentEnum,
|
||||
} from './types.js';
|
||||
|
||||
// Get telemetry repository for fetching user events
|
||||
async function getUserTelemetryEvents(
|
||||
userId: string,
|
||||
productId: string,
|
||||
days: number = 30
|
||||
): Promise<Array<Record<string, unknown>>> {
|
||||
const { getRegisteredContainer } = await import('@bytelyst/cosmos');
|
||||
const container = getRegisteredContainer('telemetry_events');
|
||||
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - days);
|
||||
|
||||
const pk = `${productId}:${cutoff.getFullYear()}${String(cutoff.getMonth() + 1).padStart(2, '0')}`;
|
||||
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.pk = @pk AND c.userId = @userId ORDER BY c.occurredAt DESC',
|
||||
parameters: [
|
||||
{ name: '@pk', value: pk },
|
||||
{ name: '@userId', value: userId },
|
||||
],
|
||||
};
|
||||
|
||||
const { resources } = await container.items.query(query).fetchAll();
|
||||
return resources as Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export async function predictiveAnalyticsRoutes(fastify: FastifyInstance): Promise<void> {
|
||||
// ==================== Churn Prediction Routes ====================
|
||||
|
||||
// POST /predictive/churn-score - Single user prediction
|
||||
fastify.post('/predictive/churn-score', {
|
||||
preHandler: [requireAuth],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const body = ChurnScoreRequestSchema.parse(request.body);
|
||||
const { userId, productId, horizon } = body;
|
||||
|
||||
request.log.info({ userId, productId }, 'Churn score request');
|
||||
|
||||
// Get cached prediction if fresh (< 24h)
|
||||
const cached = await predictiveAnalyticsRepo.getChurnPrediction(
|
||||
userId,
|
||||
productId,
|
||||
parseInt(horizon, 10)
|
||||
);
|
||||
|
||||
if (cached) {
|
||||
const age = Date.now() - new Date(cached.predictionTimestamp).getTime();
|
||||
if (age < 24 * 60 * 60 * 1000) {
|
||||
return reply.send({
|
||||
userId: cached.userId,
|
||||
churnProbability: cached.churnProbability,
|
||||
riskSegment: cached.riskSegment,
|
||||
confidenceScore: cached.confidenceScore,
|
||||
explanation: cached.explanation,
|
||||
modelVersion: cached.modelVersion,
|
||||
cached: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch telemetry events
|
||||
const events = await getUserTelemetryEvents(userId, productId);
|
||||
|
||||
// Extract features
|
||||
const features = extractFeaturesFromTelemetry(
|
||||
userId,
|
||||
productId,
|
||||
events as Parameters<typeof extractFeaturesFromTelemetry>[2],
|
||||
new Date()
|
||||
);
|
||||
|
||||
// Store features
|
||||
await featureStore.saveFeatureVector(userId, productId, features);
|
||||
|
||||
// Run prediction
|
||||
const prediction = churnModel.predict(features, parseInt(horizon, 10));
|
||||
|
||||
// Save prediction
|
||||
const doc = {
|
||||
id: `cp_${crypto.randomUUID()}`,
|
||||
pk: `${userId}:${productId}`,
|
||||
userId,
|
||||
productId,
|
||||
...prediction,
|
||||
actualChurned: undefined,
|
||||
validationDate: undefined,
|
||||
createdAt: new Date().toISOString(),
|
||||
ttl: (parseInt(horizon, 10) + 90) * 24 * 60 * 60,
|
||||
};
|
||||
|
||||
await predictiveAnalyticsRepo.saveChurnPrediction(doc);
|
||||
|
||||
return reply.send({
|
||||
userId,
|
||||
churnProbability: prediction.churnProbability,
|
||||
riskSegment: prediction.riskSegment,
|
||||
confidenceScore: prediction.confidenceScore,
|
||||
explanation: prediction.explanation,
|
||||
modelVersion: prediction.modelVersion,
|
||||
cached: false,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// POST /predictive/churn-batch - Batch scoring
|
||||
fastify.post('/predictive/churn-batch', {
|
||||
preHandler: [requireAuth],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const body = ChurnBatchRequestSchema.parse(request.body);
|
||||
const { userIds, productId, horizon } = body;
|
||||
|
||||
request.log.info({ userCount: userIds.length, productId }, 'Batch churn score request');
|
||||
|
||||
const results = await Promise.all(
|
||||
userIds.map(async (userId) => {
|
||||
const events = await getUserTelemetryEvents(userId, productId);
|
||||
const features = extractFeaturesFromTelemetry(
|
||||
userId,
|
||||
productId,
|
||||
events as Parameters<typeof extractFeaturesFromTelemetry>[2],
|
||||
new Date()
|
||||
);
|
||||
const prediction = churnModel.predict(features, parseInt(horizon, 10));
|
||||
return {
|
||||
userId,
|
||||
churnProbability: prediction.churnProbability,
|
||||
riskSegment: prediction.riskSegment,
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
return reply.send({
|
||||
productId,
|
||||
horizon: parseInt(horizon, 10),
|
||||
results,
|
||||
summary: {
|
||||
total: results.length,
|
||||
critical: results.filter((r) => r.riskSegment === 'critical').length,
|
||||
high: results.filter((r) => r.riskSegment === 'high').length,
|
||||
medium: results.filter((r) => r.riskSegment === 'medium').length,
|
||||
low: results.filter((r) => r.riskSegment === 'low').length,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// GET /predictive/at-risk-users - List at-risk users
|
||||
fastify.get('/predictive/at-risk-users', {
|
||||
preHandler: [requireAuth],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const query = AtRiskUsersQuerySchema.parse(request.query);
|
||||
const { productId, segment, limit, offset } = query;
|
||||
|
||||
const predictions = await predictiveAnalyticsRepo.getAtRiskUsers(
|
||||
productId || '',
|
||||
segment,
|
||||
limit,
|
||||
offset
|
||||
);
|
||||
|
||||
return reply.send({
|
||||
users: predictions.map((p) => ({
|
||||
userId: p.userId,
|
||||
productId: p.productId,
|
||||
churnProbability: p.churnProbability,
|
||||
riskSegment: p.riskSegment,
|
||||
confidenceScore: p.confidenceScore,
|
||||
predictionTimestamp: p.predictionTimestamp,
|
||||
topRiskFactors: p.explanation?.topRiskFactors?.slice(0, 3) || [],
|
||||
})),
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
total: predictions.length, // Simplified - should use count query
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// GET /predictive/users/:id/risk-profile - User risk profile
|
||||
fastify.get('/predictive/users/:id/risk-profile', {
|
||||
preHandler: [requireAuth],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { id } = request.params as { id: string };
|
||||
const { productId } = request.query as { productId?: string };
|
||||
|
||||
if (!productId) {
|
||||
return reply.status(400).send({ error: 'productId query parameter required' });
|
||||
}
|
||||
|
||||
const predictions = await predictiveAnalyticsRepo.getUserRiskProfile(id, productId);
|
||||
const latest = predictions[0];
|
||||
|
||||
if (!latest) {
|
||||
return reply.status(404).send({ error: 'No predictions found for user' });
|
||||
}
|
||||
|
||||
return reply.send({
|
||||
userId: id,
|
||||
productId,
|
||||
currentRisk: {
|
||||
churnProbability: latest.churnProbability,
|
||||
riskSegment: latest.riskSegment,
|
||||
confidenceScore: latest.confidenceScore,
|
||||
predictionTimestamp: latest.predictionTimestamp,
|
||||
},
|
||||
history: predictions.map((p) => ({
|
||||
timestamp: p.predictionTimestamp,
|
||||
churnProbability: p.churnProbability,
|
||||
riskSegment: p.riskSegment,
|
||||
})),
|
||||
explanation: latest.explanation,
|
||||
suggestedActions: latest.explanation?.suggestedActions || [],
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// ==================== Health Score Routes ====================
|
||||
|
||||
// GET /predictive/health - All products health
|
||||
fastify.get('/predictive/health', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (_request: FastifyRequest, reply: FastifyReply) => {
|
||||
const scores = await predictiveAnalyticsRepo.getAllProductHealthScores();
|
||||
|
||||
return reply.send({
|
||||
scores: scores.map((s) => ({
|
||||
productId: s.productId,
|
||||
date: s.date,
|
||||
overallHealthScore: s.overallHealthScore,
|
||||
healthStatus: s.healthStatus,
|
||||
dimensions: s.dimensions,
|
||||
anomalies: s.anomalies.slice(0, 3),
|
||||
forecasts: s.forecasts,
|
||||
})),
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// GET /predictive/health/:productId - Product health detail
|
||||
fastify.get('/predictive/health/:productId', {
|
||||
preHandler: [requireAuth],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { productId } = request.params as { productId: string };
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
const score = await predictiveAnalyticsRepo.getHealthScore(productId, today);
|
||||
|
||||
if (!score) {
|
||||
return reply.status(404).send({ error: 'Health score not found' });
|
||||
}
|
||||
|
||||
return reply.send(score);
|
||||
},
|
||||
});
|
||||
|
||||
// GET /predictive/health/:productId/trends - Health trends
|
||||
fastify.get('/predictive/health/:productId/trends', {
|
||||
preHandler: [requireAuth],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { productId } = request.params as { productId: string };
|
||||
const { days = '30' } = request.query as { days?: string };
|
||||
|
||||
const history = await predictiveAnalyticsRepo.getHealthHistory(
|
||||
productId,
|
||||
parseInt(days, 10)
|
||||
);
|
||||
|
||||
return reply.send({
|
||||
productId,
|
||||
days: parseInt(days, 10),
|
||||
trends: history.map((h) => ({
|
||||
date: h.date,
|
||||
overallHealthScore: h.overallHealthScore,
|
||||
healthStatus: h.healthStatus,
|
||||
dimensions: {
|
||||
acquisition: h.dimensions.acquisition.score,
|
||||
activation: h.dimensions.activation.score,
|
||||
retention: h.dimensions.retention.score,
|
||||
engagement: h.dimensions.engagement.score,
|
||||
revenue: h.dimensions.revenue.score,
|
||||
stability: h.dimensions.stability.score,
|
||||
},
|
||||
})),
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// ==================== Model Performance Routes ====================
|
||||
|
||||
// GET /predictive/model/performance - Model metrics
|
||||
fastify.get('/predictive/model/performance', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (_request: FastifyRequest, reply: FastifyReply) => {
|
||||
const latest = await predictiveAnalyticsRepo.getLatestModelPerformance();
|
||||
const history = await predictiveAnalyticsRepo.getModelPerformanceHistory(5);
|
||||
|
||||
return reply.send({
|
||||
current: latest,
|
||||
history: history.map((h) => ({
|
||||
modelVersion: h.metrics.modelVersion,
|
||||
modelType: h.metrics.modelType,
|
||||
trainedAt: h.metrics.trainedAt,
|
||||
auc: h.metrics.auc,
|
||||
precisionAt10: h.metrics.precisionAt10,
|
||||
recallAt10: h.metrics.recallAt10,
|
||||
calibrationSlope: h.metrics.calibrationSlope,
|
||||
})),
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// GET /predictive/model/features - Feature importance
|
||||
fastify.get('/predictive/model/features', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (_request: FastifyRequest, reply: FastifyReply) => {
|
||||
const latest = await predictiveAnalyticsRepo.getLatestModelPerformance();
|
||||
|
||||
return reply.send({
|
||||
modelVersion: latest?.metrics.modelVersion || '1.0.0',
|
||||
featureImportance: latest?.metrics.featureImportance || [],
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// ==================== Campaign Routes ====================
|
||||
|
||||
// GET /predictive/campaigns - List campaigns
|
||||
fastify.get('/predictive/campaigns', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { productId, status } = request.query as {
|
||||
productId?: string;
|
||||
status?: string;
|
||||
};
|
||||
|
||||
const campaigns = await predictiveAnalyticsRepo.listCampaigns(productId, status);
|
||||
|
||||
return reply.send({
|
||||
campaigns: campaigns.map((c) => ({
|
||||
id: c.id,
|
||||
name: c.name,
|
||||
description: c.description,
|
||||
productId: c.productId,
|
||||
status: c.status,
|
||||
trigger: c.trigger,
|
||||
audience: c.audience,
|
||||
stats: c.stats,
|
||||
createdAt: c.createdAt,
|
||||
})),
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// POST /predictive/campaigns - Create campaign
|
||||
fastify.post('/predictive/campaigns', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const body = CreateCampaignSchema.parse(request.body);
|
||||
|
||||
const campaign = await campaignEngine.createCampaign(body);
|
||||
|
||||
return reply.status(201).send({
|
||||
id: campaign.id,
|
||||
name: campaign.name,
|
||||
status: campaign.status,
|
||||
createdAt: campaign.createdAt,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// PATCH /predictive/campaigns/:id - Update campaign
|
||||
fastify.patch('/predictive/campaigns/:id', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { id } = request.params as { id: string };
|
||||
const updates = request.body as Partial<{
|
||||
status: 'active' | 'paused' | 'completed';
|
||||
name: string;
|
||||
description: string;
|
||||
}>;
|
||||
|
||||
if (updates.status === 'active') {
|
||||
const campaign = await campaignEngine.activateCampaign(id);
|
||||
return reply.send({ success: true, campaign });
|
||||
}
|
||||
|
||||
const updated = await predictiveAnalyticsRepo.updateCampaign(id, updates);
|
||||
|
||||
if (!updated) {
|
||||
return reply.status(404).send({ error: 'Campaign not found' });
|
||||
}
|
||||
|
||||
return reply.send({ success: true, campaign: updated });
|
||||
},
|
||||
});
|
||||
|
||||
// GET /predictive/campaigns/:id/stats - Campaign stats
|
||||
fastify.get('/predictive/campaigns/:id/stats', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { id } = request.params as { id: string };
|
||||
|
||||
const stats = await campaignEngine.getCampaignStats(id);
|
||||
|
||||
if (!stats) {
|
||||
return reply.status(404).send({ error: 'Campaign not found' });
|
||||
}
|
||||
|
||||
return reply.send(stats);
|
||||
},
|
||||
});
|
||||
|
||||
// POST /predictive/campaigns/:id/trigger - Manual trigger
|
||||
fastify.post('/predictive/campaigns/:id/trigger', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { id } = request.params as { id: string };
|
||||
const body = CampaignTriggerSchema.parse(request.body);
|
||||
|
||||
const results = await campaignEngine.manualTrigger(id, body.testUserId || 'test_user');
|
||||
|
||||
return reply.send({
|
||||
success: true,
|
||||
testMode: true,
|
||||
results,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// DELETE /predictive/campaigns/:id - Delete campaign
|
||||
fastify.delete('/predictive/campaigns/:id', {
|
||||
preHandler: [requireAdmin],
|
||||
handler: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { id } = request.params as { id: string };
|
||||
|
||||
const success = await predictiveAnalyticsRepo.deleteCampaign(id);
|
||||
|
||||
if (!success) {
|
||||
return reply.status(404).send({ error: 'Campaign not found' });
|
||||
}
|
||||
|
||||
return reply.send({ success: true });
|
||||
},
|
||||
});
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user