feat(ai-diagnostics): add telemetry linking and context enrichment [1.3]
This commit is contained in:
saravanakumardb1
parent
a9e1486a09
commit
1ff02934fa
@ -7,6 +7,18 @@ description: Refresh the Windsurf chat history archive (re-scan all repos, updat
|
||||
Refreshes the centralized Windsurf chat history archive at `__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/`.
|
||||
Auto-discovers new repos, updates symlinks, and re-copies docs + workflows.
|
||||
|
||||
## Covered Repos (All 7 workspaces)
|
||||
|
||||
| Repo | Product | Workflows | Docs |
|
||||
|------|---------|-----------|------|
|
||||
| `learning_voice_ai_agent` | LysnrAI | ✅ | ✅ |
|
||||
| `learning_multimodal_memory_agents` | MindLyst | ✅ | ✅ |
|
||||
| `learning_ai_clock` | ChronoMind | ✅ | — |
|
||||
| `learning_ai_peakpulse` | PeakPulse | ✅ | — |
|
||||
| `learning_ai_fastgap` | NomGap | ✅ | — |
|
||||
| `learning_ai_jarvis_jr` | JarvisJr | ✅ | — |
|
||||
| `learning_ai_common_plat` | Common Platform | ✅ | — |
|
||||
|
||||
## Steps
|
||||
|
||||
// turbo
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
Last refresh: 2026-03-03T07:00:03Z (2026-03-02 23:00:03 PST)
|
||||
Cascade conversations: 50 (348M)
|
||||
Last refresh: 2026-03-03T19:50:04Z (2026-03-03 11:50:04 PST)
|
||||
Cascade conversations: 50 (297M)
|
||||
Memories: 65
|
||||
Implicit context: 20
|
||||
Code tracker dirs: 149
|
||||
File edit history: 2278 entries
|
||||
Code tracker dirs: 188
|
||||
File edit history: 2343 entries
|
||||
Workspace storage: 28 workspaces
|
||||
Repo docs: 7 files across 2 repos
|
||||
Repo workflows: 35 files across 6 repos
|
||||
|
||||
@ -7,6 +7,18 @@ description: Refresh the Windsurf chat history archive (re-scan all repos, updat
|
||||
Refreshes the centralized Windsurf chat history archive at `__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/`.
|
||||
Auto-discovers new repos, updates symlinks, and re-copies docs + workflows.
|
||||
|
||||
## Covered Repos (All 7 workspaces)
|
||||
|
||||
| Repo | Product | Workflows | Docs |
|
||||
|------|---------|-----------|------|
|
||||
| `learning_voice_ai_agent` | LysnrAI | ✅ | ✅ |
|
||||
| `learning_multimodal_memory_agents` | MindLyst | ✅ | ✅ |
|
||||
| `learning_ai_clock` | ChronoMind | ✅ | — |
|
||||
| `learning_ai_peakpulse` | PeakPulse | ✅ | — |
|
||||
| `learning_ai_fastgap` | NomGap | ✅ | — |
|
||||
| `learning_ai_jarvis_jr` | JarvisJr | ✅ | — |
|
||||
| `learning_ai_common_plat` | Common Platform | ✅ | — |
|
||||
|
||||
## Steps
|
||||
|
||||
// turbo
|
||||
|
||||
@ -18,7 +18,7 @@ Run `bash scripts/backup-main.sh` from any repository root
|
||||
// turbo
|
||||
|
||||
```bash
|
||||
for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap; do
|
||||
for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap learning_ai_jarvis_jr learning_ai_peakpulse; do
|
||||
echo "━━━ Pushing $repo ━━━"
|
||||
(cd ~/code/mygh/$repo && git push origin main 2>&1)
|
||||
done
|
||||
@ -29,7 +29,7 @@ echo "✨ All repos pushed!"
|
||||
## What it does:
|
||||
|
||||
1. **Backup** — creates timestamped backup branches, cleans up old ones (7 days), skips duplicates
|
||||
2. **Push** — pushes `main` to `origin/main` for all 5 repos
|
||||
2. **Push** — pushes `main` to `origin/main` for all 7 repos
|
||||
|
||||
## Repositories:
|
||||
|
||||
@ -38,6 +38,8 @@ echo "✨ All repos pushed!"
|
||||
- learning_multimodal_memory_agents
|
||||
- learning_ai_clock
|
||||
- learning_ai_fastgap
|
||||
- learning_ai_jarvis_jr
|
||||
- learning_ai_peakpulse
|
||||
|
||||
## When to use:
|
||||
|
||||
|
||||
@ -24,6 +24,8 @@ Run `bash scripts/backup-main.sh` from any repository root
|
||||
- learning_multimodal_memory_agents
|
||||
- learning_ai_clock
|
||||
- learning_ai_fastgap
|
||||
- learning_ai_jarvis_jr
|
||||
- learning_ai_peakpulse
|
||||
|
||||
## Features:
|
||||
|
||||
|
||||
@ -12,11 +12,14 @@ Scans all repositories for pending changes and commits them in logical order wit
|
||||
|
||||
## What it does:
|
||||
|
||||
1. **Scans** all 4 repos for changes:
|
||||
1. **Scans** all 7 repos for changes:
|
||||
- learning_ai_common_plat
|
||||
- learning_voice_ai_agent
|
||||
- learning_multimodal_memory_agents
|
||||
- learning_ai_clock
|
||||
- learning_ai_fastgap
|
||||
- learning_ai_jarvis_jr
|
||||
- learning_ai_peakpulse
|
||||
|
||||
2. **Analyzes** changed files to determine:
|
||||
- Commit scope (auth, ci, docs, feat, chore, etc.)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
---
|
||||
description: Push local main branch to origin for all 5 workspace repos
|
||||
description: Push local main branch to origin for all 7 workspace repos
|
||||
---
|
||||
|
||||
# Push Repos
|
||||
@ -9,7 +9,7 @@ Pushes local `main` to `origin/main` for all workspace repositories.
|
||||
// turbo
|
||||
|
||||
```bash
|
||||
for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap; do
|
||||
for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap learning_ai_jarvis_jr learning_ai_peakpulse; do
|
||||
echo "━━━ $repo ━━━"
|
||||
(cd ~/code/mygh/$repo && git push origin main)
|
||||
done
|
||||
@ -17,7 +17,7 @@ done
|
||||
|
||||
## What it does:
|
||||
|
||||
1. Iterates over all 5 workspace repos
|
||||
1. Iterates over all 7 workspace repos
|
||||
2. Runs `git push origin main` in each
|
||||
3. Fails fast if a repo has diverged from remote (resolve with rebase manually)
|
||||
|
||||
@ -28,6 +28,8 @@ done
|
||||
- learning_multimodal_memory_agents
|
||||
- learning_ai_clock
|
||||
- learning_ai_fastgap
|
||||
- learning_ai_jarvis_jr
|
||||
- learning_ai_peakpulse
|
||||
|
||||
## When to use:
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
---
|
||||
description: Pull latest from origin main across all 5 workspace repos
|
||||
description: Pull latest from origin main across all 7 workspace repos
|
||||
---
|
||||
|
||||
# Sync Repos
|
||||
@ -9,7 +9,7 @@ Pulls the latest changes from `origin/main` for all workspace repositories.
|
||||
// turbo
|
||||
|
||||
```bash
|
||||
for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap; do
|
||||
for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap learning_ai_jarvis_jr learning_ai_peakpulse; do
|
||||
echo "━━━ $repo ━━━"
|
||||
(cd ~/code/mygh/$repo && git pull --ff-only origin main)
|
||||
done
|
||||
@ -17,7 +17,7 @@ done
|
||||
|
||||
## What it does:
|
||||
|
||||
1. Iterates over all 5 workspace repos
|
||||
1. Iterates over all 7 workspace repos
|
||||
2. Runs `git pull --ff-only origin main` in each
|
||||
3. Fails fast if there are local divergent commits (use `git pull --rebase` manually in that case)
|
||||
|
||||
@ -28,6 +28,8 @@ done
|
||||
- learning_multimodal_memory_agents
|
||||
- learning_ai_clock
|
||||
- learning_ai_fastgap
|
||||
- learning_ai_jarvis_jr
|
||||
- learning_ai_peakpulse
|
||||
|
||||
## When to use:
|
||||
|
||||
|
||||
257
dashboards/admin-web/src/app/(dashboard)/experiments/page.tsx
Normal file
257
dashboards/admin-web/src/app/(dashboard)/experiments/page.tsx
Normal file
@ -0,0 +1,257 @@
|
||||
/**
|
||||
* Experiments List Page — Admin Dashboard
|
||||
* Shows all A/B experiments with status, metrics, and quick actions.
|
||||
*/
|
||||
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import Link from 'next/link';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import {
|
||||
FlaskConical,
|
||||
Plus,
|
||||
Play,
|
||||
Pause,
|
||||
Square,
|
||||
BarChart3,
|
||||
Users,
|
||||
Clock,
|
||||
CheckCircle,
|
||||
AlertCircle,
|
||||
Sparkles,
|
||||
} from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||
import type { ExperimentDoc } from '@/lib/experiments-types';
|
||||
|
||||
const statusConfig: Record<string, { color: string; icon: typeof Play; label: string }> = {
|
||||
draft: { color: 'bg-gray-500', icon: AlertCircle, label: 'Draft' },
|
||||
running: { color: 'bg-green-500', icon: Play, label: 'Running' },
|
||||
paused: { color: 'bg-yellow-500', icon: Pause, label: 'Paused' },
|
||||
stopped: { color: 'bg-red-500', icon: Square, label: 'Stopped' },
|
||||
completed: { color: 'bg-blue-500', icon: CheckCircle, label: 'Completed' },
|
||||
};
|
||||
|
||||
export default function ExperimentsPage() {
|
||||
const router = useRouter();
|
||||
const [experiments, setExperiments] = useState<ExperimentDoc[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
fetchExperiments();
|
||||
}, []);
|
||||
|
||||
async function fetchExperiments() {
|
||||
try {
|
||||
const response = await fetch('/api/experiments');
|
||||
if (!response.ok) throw new Error('Failed to fetch experiments');
|
||||
const data = await response.json();
|
||||
setExperiments(data);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
const runningCount = experiments.filter(e => e.status === 'running').length;
|
||||
const completedCount = experiments.filter(e => e.status === 'completed').length;
|
||||
const totalParticipants = experiments.reduce((sum, e) => sum + (e.totalParticipants || 0), 0);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="container mx-auto py-8">
|
||||
<div className="flex items-center justify-center h-64">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary" />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="container mx-auto py-8">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between mb-8">
|
||||
<div>
|
||||
<h1 className="text-3xl font-bold flex items-center gap-3">
|
||||
<FlaskConical className="h-8 w-8" />
|
||||
A/B Experiments
|
||||
</h1>
|
||||
<p className="text-muted-foreground mt-1">
|
||||
Intelligent experimentation with Bayesian statistics and auto-allocation
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-3">
|
||||
<Button variant="outline" asChild>
|
||||
<Link href="/experiments/suggestions">
|
||||
<Sparkles className="h-4 w-4 mr-2" />
|
||||
AI Suggestions
|
||||
</Link>
|
||||
</Button>
|
||||
<Button onClick={() => router.push('/experiments/new')}>
|
||||
<Plus className="h-4 w-4 mr-2" />
|
||||
New Experiment
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Stats Cards */}
|
||||
<div className="grid grid-cols-1 md:grid-cols-4 gap-4 mb-8">
|
||||
<Card>
|
||||
<CardHeader className="pb-2">
|
||||
<CardTitle className="text-sm font-medium text-muted-foreground">
|
||||
Total Experiments
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="text-3xl font-bold">{experiments.length}</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<CardHeader className="pb-2">
|
||||
<CardTitle className="text-sm font-medium text-muted-foreground">
|
||||
Running
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="text-3xl font-bold text-green-600">{runningCount}</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<CardHeader className="pb-2">
|
||||
<CardTitle className="text-sm font-medium text-muted-foreground">
|
||||
Completed
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="text-3xl font-bold text-blue-600">{completedCount}</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<CardHeader className="pb-2">
|
||||
<CardTitle className="text-sm font-medium text-muted-foreground">
|
||||
Total Participants
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="text-3xl font-bold">{totalParticipants.toLocaleString()}</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
{/* Tabs */}
|
||||
<Tabs defaultValue="all" className="space-y-6">
|
||||
<TabsList>
|
||||
<TabsTrigger value="all">All ({experiments.length})</TabsTrigger>
|
||||
<TabsTrigger value="running">Running ({runningCount})</TabsTrigger>
|
||||
<TabsTrigger value="completed">Completed ({completedCount})</TabsTrigger>
|
||||
<TabsTrigger value="draft">Drafts ({experiments.filter(e => e.status === 'draft').length})</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="all" className="space-y-4">
|
||||
{experiments.map(experiment => (
|
||||
<ExperimentCard key={experiment.id} experiment={experiment} />
|
||||
))}
|
||||
{experiments.length === 0 && (
|
||||
<div className="text-center py-12 text-muted-foreground">
|
||||
<FlaskConical className="h-12 w-12 mx-auto mb-4 opacity-50" />
|
||||
<p>No experiments yet. Create your first experiment to get started.</p>
|
||||
</div>
|
||||
)}
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="running" className="space-y-4">
|
||||
{experiments
|
||||
.filter(e => e.status === 'running')
|
||||
.map(experiment => (
|
||||
<ExperimentCard key={experiment.id} experiment={experiment} />
|
||||
))}
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="completed" className="space-y-4">
|
||||
{experiments
|
||||
.filter(e => e.status === 'completed')
|
||||
.map(experiment => (
|
||||
<ExperimentCard key={experiment.id} experiment={experiment} />
|
||||
))}
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="draft" className="space-y-4">
|
||||
{experiments
|
||||
.filter(e => e.status === 'draft')
|
||||
.map(experiment => (
|
||||
<ExperimentCard key={experiment.id} experiment={experiment} />
|
||||
))}
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ExperimentCard({ experiment }: { experiment: ExperimentDoc }) {
|
||||
const status = statusConfig[experiment.status] || statusConfig.draft;
|
||||
const StatusIcon = status.icon;
|
||||
|
||||
const daysRunning = experiment.startedAt
|
||||
? Math.floor((Date.now() - new Date(experiment.startedAt).getTime()) / (1000 * 60 * 60 * 24))
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<Card className="hover:shadow-md transition-shadow">
|
||||
<CardContent className="p-6">
|
||||
<div className="flex items-start justify-between">
|
||||
<div className="flex-1">
|
||||
<div className="flex items-center gap-3 mb-2">
|
||||
<h3 className="text-lg font-semibold">
|
||||
<Link href={`/experiments/${experiment.id}`} className="hover:underline">
|
||||
{experiment.name}
|
||||
</Link>
|
||||
</h3>
|
||||
<Badge className={`${status.color} text-white`}>
|
||||
<StatusIcon className="h-3 w-3 mr-1" />
|
||||
{status.label}
|
||||
</Badge>
|
||||
{experiment.aiGeneratedHypothesis && (
|
||||
<Badge variant="outline">
|
||||
<Sparkles className="h-3 w-3 mr-1" />
|
||||
AI Generated
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-muted-foreground text-sm mb-4">{experiment.hypothesis}</p>
|
||||
|
||||
<div className="flex items-center gap-6 text-sm">
|
||||
<div className="flex items-center gap-2 text-muted-foreground">
|
||||
<BarChart3 className="h-4 w-4" />
|
||||
<span>Metric: {experiment.primaryMetric?.name || 'Not set'}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-muted-foreground">
|
||||
<Users className="h-4 w-4" />
|
||||
<span>{experiment.totalParticipants?.toLocaleString() || 0} participants</span>
|
||||
</div>
|
||||
{experiment.status === 'running' && (
|
||||
<div className="flex items-center gap-2 text-muted-foreground">
|
||||
<Clock className="h-4 w-4" />
|
||||
<span>{daysRunning} days running</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex gap-2">
|
||||
<Button variant="outline" size="sm" asChild>
|
||||
<Link href={`/experiments/${experiment.id}`}>
|
||||
<BarChart3 className="h-4 w-4 mr-2" />
|
||||
Results
|
||||
</Link>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@ -0,0 +1,445 @@
|
||||
/**
|
||||
* Intelligent A/B Testing — AI Hypothesis Generation.
|
||||
* Pattern detection from telemetry, LLM-powered hypothesis generation, auto-suggestions.
|
||||
*/
|
||||
|
||||
import type { ExperimentSuggestion, GeneratedHypothesis, HypothesisInput, PrimaryMetric, ExperimentDoc } from './types.js';
|
||||
import { createSuggestion } from './repository.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Pattern Detection
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface UsagePattern {
|
||||
featureName: string;
|
||||
totalUsage: number;
|
||||
uniqueUsers: number;
|
||||
adoptionRate: number; // 0–1
|
||||
trendDirection: 'up' | 'down' | 'stable';
|
||||
trendMagnitude: number; // % change
|
||||
segmentPerformance: Record<string, number>; // platform/segment -> rate
|
||||
temporalPatterns: {
|
||||
dayOfWeek: Record<string, number>;
|
||||
hourOfDay: Record<string, number>;
|
||||
};
|
||||
funnelDropOffs?: Array<{ stage: string; dropRate: number }>;
|
||||
}
|
||||
|
||||
export interface AnomalyDetection {
|
||||
featureName: string;
|
||||
anomalyType: 'drop' | 'spike' | 'regression' | 'segment_mismatch';
|
||||
severity: 'low' | 'medium' | 'high';
|
||||
expectedValue: number;
|
||||
actualValue: number;
|
||||
deviation: number; // standard deviations from mean
|
||||
affectedSegment?: string;
|
||||
}
|
||||
|
||||
export interface Opportunity {
|
||||
featureName: string;
|
||||
opportunityType: 'low_adoption' | 'high_dropoff' | 'competitor_gap' | 'unused_potential';
|
||||
impactScore: number; // 0–100
|
||||
currentMetric: number;
|
||||
targetMetric: number;
|
||||
estimatedLift: number;
|
||||
supportingEvidence: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze telemetry data for usage patterns.
|
||||
* Placeholder: In production, this queries telemetry_events container.
|
||||
*/
|
||||
export async function detectUsagePatterns(
|
||||
productId: string,
|
||||
featureNames: string[]
|
||||
): Promise<UsagePattern[]> {
|
||||
// Placeholder implementation
|
||||
// In production: query telemetry_events with aggregation queries
|
||||
return featureNames.map(name => ({
|
||||
featureName: name,
|
||||
totalUsage: Math.floor(Math.random() * 10000) + 1000,
|
||||
uniqueUsers: Math.floor(Math.random() * 5000) + 500,
|
||||
adoptionRate: Math.random() * 0.5 + 0.1,
|
||||
trendDirection: Math.random() > 0.5 ? 'up' : 'down',
|
||||
trendMagnitude: Math.random() * 0.3,
|
||||
segmentPerformance: {
|
||||
ios: Math.random() * 0.4 + 0.1,
|
||||
android: Math.random() * 0.4 + 0.1,
|
||||
web: Math.random() * 0.4 + 0.1,
|
||||
},
|
||||
temporalPatterns: {
|
||||
dayOfWeek: { Mon: 0.2, Tue: 0.25, Wed: 0.25, Thu: 0.2, Fri: 0.1 },
|
||||
hourOfDay: { '9': 0.15, '12': 0.2, '15': 0.25, '18': 0.2, '21': 0.1 },
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect anomalies in feature usage.
|
||||
*/
|
||||
export async function detectAnomalies(
|
||||
productId: string,
|
||||
baselineDays = 30
|
||||
): Promise<AnomalyDetection[]> {
|
||||
// Placeholder: In production, compare current metrics against rolling averages
|
||||
const anomalies: AnomalyDetection[] = [];
|
||||
|
||||
// Simulate anomaly detection
|
||||
const randomFeatures = ['voice_dictation', 'keyboard_shortcuts', 'cloud_sync', 'analytics_view'];
|
||||
for (const feature of randomFeatures) {
|
||||
if (Math.random() > 0.7) {
|
||||
anomalies.push({
|
||||
featureName: feature,
|
||||
anomalyType: Math.random() > 0.5 ? 'drop' : 'regression',
|
||||
severity: Math.random() > 0.5 ? 'high' : 'medium',
|
||||
expectedValue: 0.25,
|
||||
actualValue: 0.15,
|
||||
deviation: 2.5,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return anomalies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Identify experiment opportunities from usage patterns.
|
||||
*/
|
||||
export function identifyOpportunities(patterns: UsagePattern[]): Opportunity[] {
|
||||
const opportunities: Opportunity[] = [];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
// Low adoption opportunity
|
||||
if (pattern.adoptionRate < 0.2) {
|
||||
opportunities.push({
|
||||
featureName: pattern.featureName,
|
||||
opportunityType: 'low_adoption',
|
||||
impactScore: Math.floor((0.2 - pattern.adoptionRate) * 500),
|
||||
currentMetric: pattern.adoptionRate,
|
||||
targetMetric: 0.2,
|
||||
estimatedLift: (0.2 - pattern.adoptionRate) / pattern.adoptionRate,
|
||||
supportingEvidence: [
|
||||
`Current adoption rate is ${(pattern.adoptionRate * 100).toFixed(1)}%`,
|
||||
`Trend is ${pattern.trendDirection} by ${(pattern.trendMagnitude * 100).toFixed(1)}%`,
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
// Segment mismatch opportunity
|
||||
const segmentRates = Object.values(pattern.segmentPerformance);
|
||||
const maxSegment = Math.max(...segmentRates);
|
||||
const minSegment = Math.min(...segmentRates);
|
||||
if (maxSegment > minSegment * 2) {
|
||||
opportunities.push({
|
||||
featureName: pattern.featureName,
|
||||
opportunityType: 'unused_potential',
|
||||
impactScore: Math.floor((maxSegment - minSegment) * 100),
|
||||
currentMetric: minSegment,
|
||||
targetMetric: maxSegment,
|
||||
estimatedLift: (maxSegment - minSegment) / minSegment,
|
||||
supportingEvidence: [
|
||||
`Best performing segment: ${(maxSegment * 100).toFixed(1)}%`,
|
||||
`Worst performing segment: ${(minSegment * 100).toFixed(1)}%`,
|
||||
`Potential to lift underperforming segments to best-in-class`,
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return opportunities.sort((a, b) => b.impactScore - a.impactScore);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// LLM Hypothesis Generation
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Generate experiment hypothesis using LLM.
|
||||
* Placeholder: In production, this calls Azure OpenAI.
|
||||
*/
|
||||
export async function generateHypothesis(
|
||||
input: HypothesisInput
|
||||
): Promise<GeneratedHypothesis> {
|
||||
// Build prompt for LLM
|
||||
const prompt = buildHypothesisPrompt(input);
|
||||
|
||||
// Placeholder: Simulate LLM response
|
||||
// In production: const response = await callAzureOpenAI(prompt);
|
||||
return simulateLLMHypothesis(input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build structured prompt for hypothesis generation.
|
||||
*/
|
||||
function buildHypothesisPrompt(input: HypothesisInput): string {
|
||||
return `
|
||||
You are an expert product analyst and experimentation strategist.
|
||||
|
||||
Analyze this feature usage data and generate experiment hypotheses:
|
||||
|
||||
Feature: ${input.featureName}
|
||||
Current Adoption Rate: ${(input.adoptionRate * 100).toFixed(1)}%
|
||||
Baseline Rate: ${(input.baselineRate * 100).toFixed(1)}%
|
||||
|
||||
Segment Performance:
|
||||
${Object.entries(input.segmentData)
|
||||
.map(([segment, rate]) => `- ${segment}: ${(rate * 100).toFixed(1)}%`)
|
||||
.join('\n')}
|
||||
|
||||
User Feedback Samples:
|
||||
${input.feedbackSamples.map(f => `- "${f}"`).join('\n')}
|
||||
|
||||
Generate:
|
||||
1. Primary hypothesis: "Changing X will improve Y because..."
|
||||
2. 2-3 alternative hypotheses
|
||||
3. Expected effect size (conservative)
|
||||
4. Success metric recommendation
|
||||
5. Risk assessment (low/medium/high)
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulate LLM hypothesis response (placeholder).
|
||||
*/
|
||||
function simulateLLMHypothesis(input: HypothesisInput): GeneratedHypothesis {
|
||||
const effectSize = Math.abs(input.adoptionRate - input.baselineRate) * 0.5;
|
||||
const riskLevel = effectSize > 0.1 ? 'medium' : 'low';
|
||||
|
||||
return {
|
||||
primary: `Redesigning the ${input.featureName} onboarding flow to highlight key benefits will increase adoption by ${(effectSize * 100).toFixed(0)}% because users currently don't understand the value proposition.`,
|
||||
alternatives: [
|
||||
`Adding tooltips and contextual help to ${input.featureName} will reduce confusion and increase usage by ${(effectSize * 0.7 * 100).toFixed(0)}%.`,
|
||||
`Simplifying the ${input.featureName} UI to match competitor patterns will improve familiarity and adoption by ${(effectSize * 0.5 * 100).toFixed(0)}%.`,
|
||||
],
|
||||
expectedEffectSize: Math.max(effectSize, 0.05),
|
||||
successMetric: 'adoption_rate',
|
||||
riskAssessment: riskLevel,
|
||||
impactScore: Math.floor((input.baselineRate - input.adoptionRate) * 200),
|
||||
difficultyScore: Math.floor(Math.random() * 30) + 20,
|
||||
powerPrediction: 85,
|
||||
};
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Hypothesis Ranking
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface RankedHypothesis extends GeneratedHypothesis {
|
||||
rankScore: number;
|
||||
estimatedImpact: number;
|
||||
estimatedEffort: number;
|
||||
estimatedPower: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank hypotheses by expected value.
|
||||
*/
|
||||
export function rankHypotheses(
|
||||
hypotheses: GeneratedHypothesis[],
|
||||
baseTraffic: number
|
||||
): RankedHypothesis[] {
|
||||
return hypotheses.map(h => {
|
||||
// Expected value calculation
|
||||
const impact = h.impactScore;
|
||||
const effort = h.difficultyScore;
|
||||
const power = h.powerPrediction;
|
||||
|
||||
// Risk-adjusted expected value
|
||||
const riskMultiplier = h.riskAssessment === 'low' ? 1.0 : h.riskAssessment === 'medium' ? 0.8 : 0.6;
|
||||
|
||||
// Rank score: higher impact, lower effort, higher power = better
|
||||
const rankScore = (impact * power * riskMultiplier) / (effort + 10);
|
||||
|
||||
return {
|
||||
...h,
|
||||
rankScore,
|
||||
estimatedImpact: impact,
|
||||
estimatedEffort: effort,
|
||||
estimatedPower: power,
|
||||
};
|
||||
}).sort((a, b) => b.rankScore - a.rankScore);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Auto-Experiment Suggestions
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SuggestedExperiment {
|
||||
name: string;
|
||||
hypothesis: string;
|
||||
description: string;
|
||||
variants: Array<{
|
||||
name: string;
|
||||
description: string;
|
||||
flagConfig: Record<string, unknown>;
|
||||
}>;
|
||||
primaryMetric: PrimaryMetric;
|
||||
suggestedDuration: number;
|
||||
suggestedSampleSize: number;
|
||||
priority: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate complete experiment suggestion from opportunity.
|
||||
*/
|
||||
export function generateExperimentSuggestion(
|
||||
opportunity: Opportunity,
|
||||
hypothesis: GeneratedHypothesis,
|
||||
productId: string
|
||||
): Omit<ExperimentSuggestion, 'id' | 'createdAt'> {
|
||||
const variantNames = ['Control', 'Treatment A', 'Treatment B'];
|
||||
if (hypothesis.alternatives.length < 2) {
|
||||
variantNames.pop();
|
||||
}
|
||||
|
||||
const suggestedVariants = variantNames.map((name, i) => ({
|
||||
name,
|
||||
description: i === 0
|
||||
? 'Current implementation (control)'
|
||||
: hypothesis.alternatives[i - 1] || `Alternative approach ${i}`,
|
||||
}));
|
||||
|
||||
const sampleSize = Math.ceil(
|
||||
(hypothesis.expectedEffectSize > 0 ? 16 / (hypothesis.expectedEffectSize ** 2) : 1000)
|
||||
);
|
||||
|
||||
return {
|
||||
hypothesis,
|
||||
suggestedVariants,
|
||||
suggestedMetrics: [
|
||||
{
|
||||
name: opportunity.opportunityType === 'low_adoption' ? 'adoption_rate' : 'conversion_rate',
|
||||
type: 'conversion',
|
||||
eventName: `${opportunity.featureName}_used`,
|
||||
aggregation: 'unique',
|
||||
direction: 'increase',
|
||||
minimumDetectableEffect: hypothesis.expectedEffectSize * 100,
|
||||
} as PrimaryMetric,
|
||||
],
|
||||
suggestedDuration: Math.max(7, Math.ceil(sampleSize / 100)),
|
||||
suggestedSampleSize: sampleSize,
|
||||
priority: opportunity.impactScore,
|
||||
aiGenerated: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate weekly AI report with top experiment opportunities.
|
||||
*/
|
||||
export async function generateWeeklyReport(
|
||||
productId: string
|
||||
): Promise<{
|
||||
topOpportunities: Opportunity[];
|
||||
suggestedExperiments: Array<Omit<ExperimentSuggestion, 'id' | 'createdAt'>>;
|
||||
anomalies: AnomalyDetection[];
|
||||
generatedAt: string;
|
||||
}> {
|
||||
// Detect patterns and opportunities
|
||||
const patterns = await detectUsagePatterns(productId, [
|
||||
'voice_dictation',
|
||||
'keyboard_shortcuts',
|
||||
'cloud_sync',
|
||||
'analytics_view',
|
||||
'settings_customization',
|
||||
]);
|
||||
|
||||
const opportunities = identifyOpportunities(patterns);
|
||||
const anomalies = await detectAnomalies(productId);
|
||||
|
||||
// Generate hypotheses for top opportunities
|
||||
const topOpportunities = opportunities.slice(0, 5);
|
||||
const suggestedExperiments: Array<Omit<ExperimentSuggestion, 'id' | 'createdAt'>> = [];
|
||||
|
||||
for (const opp of topOpportunities) {
|
||||
const hypothesisInput: HypothesisInput = {
|
||||
featureName: opp.featureName,
|
||||
adoptionRate: opp.currentMetric,
|
||||
baselineRate: opp.targetMetric,
|
||||
segmentData: {},
|
||||
feedbackSamples: opp.supportingEvidence,
|
||||
};
|
||||
|
||||
const hypothesis = await generateHypothesis(hypothesisInput);
|
||||
const suggestion = generateExperimentSuggestion(opp, hypothesis, productId);
|
||||
suggestedExperiments.push(suggestion);
|
||||
}
|
||||
|
||||
return {
|
||||
topOpportunities,
|
||||
suggestedExperiments,
|
||||
anomalies,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// AI Insights for Results
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ExperimentInsights {
|
||||
summary: string;
|
||||
unexpectedFindings: string[];
|
||||
segmentAnalysis: Record<string, string>;
|
||||
followUpSuggestions: Array<{
|
||||
experimentName: string;
|
||||
hypothesis: string;
|
||||
priority: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate AI insights for completed experiment.
|
||||
* Placeholder: In production, calls Azure OpenAI with full experiment data.
|
||||
*/
|
||||
export function generateExperimentInsights(
|
||||
experiment: ExperimentDoc,
|
||||
result: { variantResults: Array<{ variantName: string; probabilityBeatsControl: number; expectedLiftPercent: number }> },
|
||||
winnerVariantId?: string
|
||||
): ExperimentInsights {
|
||||
const winner = result.variantResults.find(v => v.probabilityBeatsControl > 0.95);
|
||||
const loser = result.variantResults.find(v => v.probabilityBeatsControl < 0.05);
|
||||
|
||||
const insights: ExperimentInsights = {
|
||||
summary: '',
|
||||
unexpectedFindings: [],
|
||||
segmentAnalysis: {},
|
||||
followUpSuggestions: [],
|
||||
};
|
||||
|
||||
if (winner) {
|
||||
insights.summary = `The ${winner.variantName} variant significantly outperformed control with ${winner.probabilityBeatsControl.toFixed(1)}% probability of superiority and a ${winner.expectedLiftPercent.toFixed(1)}% lift. This validates the hypothesis that ${experiment.hypothesis}`;
|
||||
} else if (loser) {
|
||||
insights.summary = `The experiment did not produce a winning variant. The control performed better than expected, suggesting the current implementation is already optimized for this metric.`;
|
||||
} else {
|
||||
insights.summary = `Results were inconclusive. No variant demonstrated statistically significant improvement over control. Consider running the experiment longer or testing more distinct variations.`;
|
||||
}
|
||||
|
||||
// Add unexpected findings based on result patterns
|
||||
const allPositive = result.variantResults.every(v => v.expectedLiftPercent > 0);
|
||||
const allNegative = result.variantResults.every(v => v.expectedLiftPercent < 0);
|
||||
|
||||
if (allNegative) {
|
||||
insights.unexpectedFindings.push('All variants underperformed control, suggesting possible confounding factors or measurement issues.');
|
||||
}
|
||||
|
||||
if (Math.abs(result.variantResults[0]?.expectedLiftPercent || 0) > 50) {
|
||||
insights.unexpectedFindings.push('Extremely large effect size detected. Verify data quality and consider running a validation experiment.');
|
||||
}
|
||||
|
||||
// Generate follow-up suggestions
|
||||
if (winner) {
|
||||
insights.followUpSuggestions.push({
|
||||
experimentName: `${experiment.name} - Follow-up Validation`,
|
||||
hypothesis: `Validating the ${winner.variantName} win in a new user cohort to confirm generalizability`,
|
||||
priority: 90,
|
||||
});
|
||||
}
|
||||
|
||||
insights.followUpSuggestions.push({
|
||||
experimentName: `${experiment.name} - Secondary Metrics`,
|
||||
hypothesis: `Testing the winning variant against additional success metrics to ensure no trade-offs`,
|
||||
priority: 70,
|
||||
});
|
||||
|
||||
return insights;
|
||||
}
|
||||
@ -0,0 +1,632 @@
|
||||
import { getRegisteredContainer } from '@bytelyst/cosmos';
|
||||
import type { Container } from '@azure/cosmos';
|
||||
import type { ErrorEvent, ErrorClusterDoc } from './types.js';
|
||||
|
||||
// ============================================================================
|
||||
// Telemetry Linking Service
|
||||
// ============================================================================
|
||||
|
||||
interface TelemetryEvent {
|
||||
id: string;
|
||||
correlationId?: string;
|
||||
productId: string;
|
||||
userId?: string;
|
||||
sessionId?: string;
|
||||
eventType: string;
|
||||
eventName: string;
|
||||
timestamp: string;
|
||||
platform?: string;
|
||||
osVersion?: string;
|
||||
appVersion?: string;
|
||||
deviceModel?: string;
|
||||
screen?: string;
|
||||
properties?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface TelemetryContext {
|
||||
correlationId: string;
|
||||
precedingEvents: TelemetryEvent[];
|
||||
followingEvents: TelemetryEvent[];
|
||||
sessionEvents: TelemetryEvent[];
|
||||
userJourney: UserJourneyStep[];
|
||||
deviceContext: DeviceContext;
|
||||
apiCalls: ApiCall[];
|
||||
}
|
||||
|
||||
interface UserJourneyStep {
|
||||
timestamp: string;
|
||||
screen: string;
|
||||
action: string;
|
||||
durationMs?: number;
|
||||
}
|
||||
|
||||
interface DeviceContext {
|
||||
platform?: string;
|
||||
osVersion?: string;
|
||||
appVersion?: string;
|
||||
deviceModel?: string;
|
||||
screenContext?: string;
|
||||
memory?: number;
|
||||
batteryLevel?: number;
|
||||
networkType?: string;
|
||||
}
|
||||
|
||||
interface ApiCall {
|
||||
endpoint: string;
|
||||
method: string;
|
||||
statusCode?: number;
|
||||
durationMs?: number;
|
||||
timestamp: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface EnrichedErrorContext {
|
||||
errorEvent: ErrorEvent;
|
||||
telemetryContext?: TelemetryContext;
|
||||
sessionState: SessionState;
|
||||
recentActions: string[];
|
||||
apiFailures: ApiCall[];
|
||||
featureFlags: Record<string, boolean>;
|
||||
configSnapshot: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface SessionState {
|
||||
screen: string;
|
||||
durationOnScreen: number;
|
||||
previousScreen?: string;
|
||||
userActions: string[];
|
||||
formData?: Record<string, unknown>;
|
||||
scrollPosition?: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Container Access
|
||||
// ============================================================================
|
||||
|
||||
function getTelemetryContainer(): Container {
|
||||
return getRegisteredContainer('telemetry_events');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Correlation ID Propagation
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Links errors to telemetry events via correlation ID
|
||||
* Returns telemetry events within 5-minute window of the error
|
||||
*/
|
||||
export async function linkErrorToTelemetry(
|
||||
errorEvent: ErrorEvent,
|
||||
options: {
|
||||
windowMinutes?: number;
|
||||
maxEvents?: number;
|
||||
} = {}
|
||||
): Promise<TelemetryContext | null> {
|
||||
const windowMinutes = options.windowMinutes || 5;
|
||||
const maxEvents = options.maxEvents || 50;
|
||||
|
||||
if (!errorEvent.correlationId) {
|
||||
// Try to find by user + timestamp if no correlation ID
|
||||
return await linkByUserAndTime(errorEvent, windowMinutes, maxEvents);
|
||||
}
|
||||
|
||||
const container = getTelemetryContainer();
|
||||
|
||||
// Fetch events with same correlation ID
|
||||
const query = `
|
||||
SELECT * FROM c
|
||||
WHERE c.correlationId = @correlationId
|
||||
AND c.productId = @productId
|
||||
ORDER BY c.timestamp ASC
|
||||
`;
|
||||
|
||||
const { resources } = await container.items
|
||||
.query({
|
||||
query,
|
||||
parameters: [
|
||||
{ name: '@correlationId', value: errorEvent.correlationId },
|
||||
{ name: '@productId', value: errorEvent.productId },
|
||||
],
|
||||
})
|
||||
.fetchAll();
|
||||
|
||||
const events = resources as TelemetryEvent[];
|
||||
|
||||
if (events.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Find the error event position
|
||||
const errorIndex = events.findIndex(
|
||||
(e) => e.timestamp === errorEvent.timestamp
|
||||
);
|
||||
|
||||
const windowStart = errorIndex >= 0
|
||||
? Math.max(0, errorIndex - Math.floor(maxEvents / 2))
|
||||
: 0;
|
||||
const windowEnd = Math.min(events.length, windowStart + maxEvents);
|
||||
|
||||
const windowEvents = events.slice(windowStart, windowEnd);
|
||||
const errorPosition = errorIndex >= 0 ? errorIndex - windowStart : windowEvents.length / 2;
|
||||
|
||||
// Split into preceding and following events
|
||||
const precedingEvents = windowEvents.slice(0, errorPosition);
|
||||
const followingEvents = windowEvents.slice(errorPosition + 1);
|
||||
|
||||
// Extract user journey
|
||||
const userJourney = extractUserJourney(windowEvents);
|
||||
|
||||
// Extract device context from the latest event
|
||||
const deviceContext = extractDeviceContext(windowEvents[windowEvents.length - 1]);
|
||||
|
||||
// Extract API calls
|
||||
const apiCalls = extractApiCalls(windowEvents);
|
||||
|
||||
return {
|
||||
correlationId: errorEvent.correlationId,
|
||||
precedingEvents,
|
||||
followingEvents,
|
||||
sessionEvents: windowEvents,
|
||||
userJourney,
|
||||
deviceContext,
|
||||
apiCalls,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback linking when correlation ID is not available
|
||||
* Uses user ID + time window
|
||||
*/
|
||||
async function linkByUserAndTime(
|
||||
errorEvent: ErrorEvent,
|
||||
windowMinutes: number,
|
||||
maxEvents: number
|
||||
): Promise<TelemetryContext | null> {
|
||||
if (!errorEvent.userId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const container = getTelemetryContainer();
|
||||
|
||||
const errorTime = new Date(errorEvent.timestamp);
|
||||
const windowStart = new Date(errorTime.getTime() - windowMinutes * 60 * 1000).toISOString();
|
||||
const windowEnd = new Date(errorTime.getTime() + windowMinutes * 60 * 1000).toISOString();
|
||||
|
||||
const query = `
|
||||
SELECT * FROM c
|
||||
WHERE c.userId = @userId
|
||||
AND c.productId = @productId
|
||||
AND c.timestamp >= @windowStart
|
||||
AND c.timestamp <= @windowEnd
|
||||
ORDER BY c.timestamp ASC
|
||||
`;
|
||||
|
||||
const { resources } = await container.items
|
||||
.query({
|
||||
query,
|
||||
parameters: [
|
||||
{ name: '@userId', value: errorEvent.userId },
|
||||
{ name: '@productId', value: errorEvent.productId },
|
||||
{ name: '@windowStart', value: windowStart },
|
||||
{ name: '@windowEnd', value: windowEnd },
|
||||
],
|
||||
}, { maxItemCount: maxEvents })
|
||||
.fetchAll();
|
||||
|
||||
const events = resources as TelemetryEvent[];
|
||||
|
||||
if (events.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Generate a synthetic correlation ID
|
||||
const correlationId = `synth_${errorEvent.userId}_${errorTime.getTime()}`;
|
||||
|
||||
// Find events before and after error timestamp
|
||||
const errorTimeMs = errorTime.getTime();
|
||||
const errorIndex = events.findIndex(
|
||||
(e) => new Date(e.timestamp).getTime() >= errorTimeMs
|
||||
);
|
||||
|
||||
const precedingEvents = errorIndex > 0 ? events.slice(0, errorIndex) : [];
|
||||
const followingEvents = errorIndex >= 0 ? events.slice(errorIndex + 1) : events;
|
||||
|
||||
return {
|
||||
correlationId,
|
||||
precedingEvents,
|
||||
followingEvents,
|
||||
sessionEvents: events,
|
||||
userJourney: extractUserJourney(events),
|
||||
deviceContext: extractDeviceContext(events[events.length - 1]),
|
||||
apiCalls: extractApiCalls(events),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Context Extraction
|
||||
// ============================================================================
|
||||
|
||||
function extractUserJourney(events: TelemetryEvent[]): UserJourneyStep[] {
|
||||
const journey: UserJourneyStep[] = [];
|
||||
let lastScreen: string | null = null;
|
||||
let lastTimestamp: string | null = null;
|
||||
|
||||
for (const event of events) {
|
||||
if (event.eventType === 'screen_view' || event.eventName.includes('screen')) {
|
||||
const durationMs = lastTimestamp
|
||||
? new Date(event.timestamp).getTime() - new Date(lastTimestamp).getTime()
|
||||
: undefined;
|
||||
|
||||
journey.push({
|
||||
timestamp: event.timestamp,
|
||||
screen: event.screen || event.properties?.screen as string || 'unknown',
|
||||
action: event.eventName,
|
||||
durationMs,
|
||||
});
|
||||
|
||||
lastScreen = event.screen || null;
|
||||
lastTimestamp = event.timestamp;
|
||||
}
|
||||
|
||||
if (event.eventType === 'action' || event.eventType === 'interaction') {
|
||||
journey.push({
|
||||
timestamp: event.timestamp,
|
||||
screen: lastScreen || 'unknown',
|
||||
action: event.eventName,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return journey;
|
||||
}
|
||||
|
||||
function extractDeviceContext(event?: TelemetryEvent): DeviceContext {
|
||||
if (!event) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return {
|
||||
platform: event.platform,
|
||||
osVersion: event.osVersion,
|
||||
appVersion: event.appVersion,
|
||||
deviceModel: event.deviceModel,
|
||||
screenContext: event.screen,
|
||||
memory: event.properties?.memory as number,
|
||||
batteryLevel: event.properties?.batteryLevel as number,
|
||||
networkType: event.properties?.networkType as string,
|
||||
};
|
||||
}
|
||||
|
||||
function extractApiCalls(events: TelemetryEvent[]): ApiCall[] {
|
||||
return events
|
||||
.filter(
|
||||
(e) =>
|
||||
e.eventType === 'api_call' ||
|
||||
e.eventName.includes('api') ||
|
||||
e.eventName.includes('request')
|
||||
)
|
||||
.map((e) => ({
|
||||
endpoint: (e.properties?.endpoint as string) || 'unknown',
|
||||
method: (e.properties?.method as string) || 'GET',
|
||||
statusCode: e.properties?.statusCode as number,
|
||||
durationMs: e.properties?.durationMs as number,
|
||||
timestamp: e.timestamp,
|
||||
error: e.properties?.error as string,
|
||||
}));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Context Enrichment
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Enriches error event with full telemetry context
|
||||
*/
|
||||
export async function enrichErrorContext(
|
||||
errorEvent: ErrorEvent
|
||||
): Promise<EnrichedErrorContext> {
|
||||
// Link to telemetry
|
||||
const telemetryContext = await linkErrorToTelemetry(errorEvent, {
|
||||
windowMinutes: 5,
|
||||
maxEvents: 100,
|
||||
});
|
||||
|
||||
// Build session state
|
||||
const sessionState = buildSessionState(telemetryContext);
|
||||
|
||||
// Extract recent actions
|
||||
const recentActions = extractRecentActions(telemetryContext);
|
||||
|
||||
// Extract API failures
|
||||
const apiFailures = telemetryContext?.apiCalls.filter((call) =>
|
||||
call.error || (call.statusCode && call.statusCode >= 400)
|
||||
) || [];
|
||||
|
||||
// Extract feature flags from telemetry
|
||||
const featureFlags = extractFeatureFlags(telemetryContext);
|
||||
|
||||
// Build config snapshot
|
||||
const configSnapshot = buildConfigSnapshot(telemetryContext);
|
||||
|
||||
return {
|
||||
errorEvent,
|
||||
telemetryContext,
|
||||
sessionState,
|
||||
recentActions,
|
||||
apiFailures,
|
||||
featureFlags,
|
||||
configSnapshot,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSessionState(telemetryContext: TelemetryContext | null): SessionState {
|
||||
if (!telemetryContext || telemetryContext.sessionEvents.length === 0) {
|
||||
return {
|
||||
screen: 'unknown',
|
||||
durationOnScreen: 0,
|
||||
userActions: [],
|
||||
};
|
||||
}
|
||||
|
||||
const events = telemetryContext.sessionEvents;
|
||||
const lastEvent = events[events.length - 1];
|
||||
|
||||
// Find last screen view
|
||||
const screenViews = events.filter(
|
||||
(e) => e.eventType === 'screen_view' || e.eventName.includes('screen')
|
||||
);
|
||||
|
||||
const currentScreen = screenViews.length > 0
|
||||
? screenViews[screenViews.length - 1].screen ||
|
||||
(screenViews[screenViews.length - 1].properties?.screen as string)
|
||||
: 'unknown';
|
||||
|
||||
const previousScreen = screenViews.length > 1
|
||||
? screenViews[screenViews.length - 2].screen ||
|
||||
(screenViews[screenViews.length - 2].properties?.screen as string)
|
||||
: undefined;
|
||||
|
||||
// Calculate duration on current screen
|
||||
let durationOnScreen = 0;
|
||||
if (screenViews.length >= 1) {
|
||||
const screenEnterTime = new Date(screenViews[screenViews.length - 1].timestamp).getTime();
|
||||
const lastEventTime = new Date(lastEvent.timestamp).getTime();
|
||||
durationOnScreen = lastEventTime - screenEnterTime;
|
||||
}
|
||||
|
||||
// Extract user actions
|
||||
const userActions = events
|
||||
.filter((e) => e.eventType === 'action' || e.eventType === 'interaction')
|
||||
.map((e) => e.eventName);
|
||||
|
||||
return {
|
||||
screen: currentScreen || 'unknown',
|
||||
durationOnScreen,
|
||||
previousScreen,
|
||||
userActions,
|
||||
scrollPosition: lastEvent.properties?.scrollPosition as number,
|
||||
formData: lastEvent.properties?.formData as Record<string, unknown>,
|
||||
};
|
||||
}
|
||||
|
||||
function extractRecentActions(telemetryContext: TelemetryContext | null): string[] {
|
||||
if (!telemetryContext) return [];
|
||||
|
||||
return telemetryContext.precedingEvents
|
||||
.slice(-10) // Last 10 actions before error
|
||||
.filter((e) => e.eventType === 'action' || e.eventType === 'interaction')
|
||||
.map((e) => e.eventName);
|
||||
}
|
||||
|
||||
function extractFeatureFlags(
|
||||
telemetryContext: TelemetryContext | null
|
||||
): Record<string, boolean> {
|
||||
if (!telemetryContext) return {};
|
||||
|
||||
const flags: Record<string, boolean> = {};
|
||||
|
||||
for (const event of telemetryContext.sessionEvents) {
|
||||
if (event.properties?.featureFlags) {
|
||||
Object.assign(flags, event.properties.featureFlags);
|
||||
}
|
||||
}
|
||||
|
||||
return flags;
|
||||
}
|
||||
|
||||
function buildConfigSnapshot(
|
||||
telemetryContext: TelemetryContext | null
|
||||
): Record<string, unknown> {
|
||||
if (!telemetryContext) return {};
|
||||
|
||||
const config: Record<string, unknown> = {};
|
||||
|
||||
for (const event of telemetryContext.sessionEvents) {
|
||||
if (event.properties?.config) {
|
||||
Object.assign(config, event.properties.config);
|
||||
}
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Breadcrumb Trail Generation
|
||||
// ============================================================================
|
||||
|
||||
export interface Breadcrumb {
|
||||
timestamp: string;
|
||||
category: 'navigation' | 'action' | 'api' | 'error' | 'state';
|
||||
message: string;
|
||||
data?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a breadcrumb trail from telemetry context
|
||||
*/
|
||||
export function generateBreadcrumbTrail(
|
||||
telemetryContext: TelemetryContext | null
|
||||
): Breadcrumb[] {
|
||||
if (!telemetryContext) return [];
|
||||
|
||||
const breadcrumbs: Breadcrumb[] = [];
|
||||
|
||||
// Add navigation breadcrumbs
|
||||
for (const step of telemetryContext.userJourney) {
|
||||
breadcrumbs.push({
|
||||
timestamp: step.timestamp,
|
||||
category: 'navigation',
|
||||
message: `Screen: ${step.screen}`,
|
||||
data: { durationMs: step.durationMs },
|
||||
});
|
||||
}
|
||||
|
||||
// Add API call breadcrumbs
|
||||
for (const call of telemetryContext.apiCalls) {
|
||||
breadcrumbs.push({
|
||||
timestamp: call.timestamp,
|
||||
category: 'api',
|
||||
message: `${call.method} ${call.endpoint}`,
|
||||
data: {
|
||||
statusCode: call.statusCode,
|
||||
durationMs: call.durationMs,
|
||||
error: call.error,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by timestamp
|
||||
breadcrumbs.sort(
|
||||
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
||||
);
|
||||
|
||||
return breadcrumbs;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Cluster Context Aggregation
|
||||
// ============================================================================
|
||||
|
||||
interface ClusterContextSummary {
|
||||
totalOccurrences: number;
|
||||
affectedUsers: string[];
|
||||
timeRange: { start: string; end: string };
|
||||
mostCommonScreens: Array<{ screen: string; count: number }>;
|
||||
mostCommonActions: Array<{ action: string; count: number }>;
|
||||
apiFailurePattern?: {
|
||||
endpoint: string;
|
||||
errorPattern: string;
|
||||
occurrenceCount: number;
|
||||
};
|
||||
featureFlagCorrelations: Array<{
|
||||
flag: string;
|
||||
enabled: boolean;
|
||||
errorCorrelation: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregates context across all errors in a cluster
|
||||
*/
|
||||
export async function aggregateClusterContext(
|
||||
cluster: ErrorClusterDoc,
|
||||
errorEvents: ErrorEvent[]
|
||||
): Promise<ClusterContextSummary> {
|
||||
const affectedUsers = new Set<string>();
|
||||
const screenCounts = new Map<string, number>();
|
||||
const actionCounts = new Map<string, number>();
|
||||
const apiErrors = new Map<string, { count: number; errors: string[] }>();
|
||||
const flagCorrelations = new Map<string, { enabled: number; total: number }>();
|
||||
|
||||
let earliestTime = new Date(cluster.firstSeenAt);
|
||||
let latestTime = new Date(cluster.lastSeenAt);
|
||||
|
||||
for (const error of errorEvents) {
|
||||
if (error.userId) {
|
||||
affectedUsers.add(error.userId);
|
||||
}
|
||||
|
||||
// Track screens and actions
|
||||
if (error.screen) {
|
||||
screenCounts.set(error.screen, (screenCounts.get(error.screen) || 0) + 1);
|
||||
}
|
||||
|
||||
// Enrich and extract more context
|
||||
const enriched = await enrichErrorContext(error);
|
||||
|
||||
for (const action of enriched.recentActions) {
|
||||
actionCounts.set(action, (actionCounts.get(action) || 0) + 1);
|
||||
}
|
||||
|
||||
// Track API failures
|
||||
for (const apiFailure of enriched.apiFailures) {
|
||||
const key = `${apiFailure.method} ${apiFailure.endpoint}`;
|
||||
const existing = apiErrors.get(key) || { count: 0, errors: [] };
|
||||
existing.count++;
|
||||
if (apiFailure.error) {
|
||||
existing.errors.push(apiFailure.error);
|
||||
}
|
||||
apiErrors.set(key, existing);
|
||||
}
|
||||
|
||||
// Track feature flag correlations
|
||||
for (const [flag, enabled] of Object.entries(enriched.featureFlags)) {
|
||||
const existing = flagCorrelations.get(flag) || { enabled: 0, total: 0 };
|
||||
existing.total++;
|
||||
if (enabled) existing.enabled++;
|
||||
flagCorrelations.set(flag, existing);
|
||||
}
|
||||
}
|
||||
|
||||
// Find most common API error pattern
|
||||
let apiFailurePattern: ClusterContextSummary['apiFailurePattern'] | undefined;
|
||||
let maxApiErrors = 0;
|
||||
|
||||
for (const [endpoint, data] of apiErrors.entries()) {
|
||||
if (data.count > maxApiErrors) {
|
||||
maxApiErrors = data.count;
|
||||
// Find common error pattern
|
||||
const errorFrequency = new Map<string, number>();
|
||||
for (const err of data.errors) {
|
||||
errorFrequency.set(err, (errorFrequency.get(err) || 0) + 1);
|
||||
}
|
||||
const mostCommonError = Array.from(errorFrequency.entries()).sort((a, b) => b[1] - a[1])[0];
|
||||
|
||||
apiFailurePattern = {
|
||||
endpoint,
|
||||
errorPattern: mostCommonError?.[0] || 'unknown',
|
||||
occurrenceCount: data.count,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate feature flag correlations
|
||||
const featureFlagCorrelations = Array.from(flagCorrelations.entries())
|
||||
.map(([flag, data]) => ({
|
||||
flag,
|
||||
enabled: data.enabled > 0,
|
||||
errorCorrelation: data.enabled / data.total,
|
||||
}))
|
||||
.filter((f) => f.errorCorrelation > 0.5) // Only include flags with >50% correlation
|
||||
.sort((a, b) => b.errorCorrelation - a.errorCorrelation)
|
||||
.slice(0, 5);
|
||||
|
||||
return {
|
||||
totalOccurrences: errorEvents.length,
|
||||
affectedUsers: Array.from(affectedUsers),
|
||||
timeRange: {
|
||||
start: earliestTime.toISOString(),
|
||||
end: latestTime.toISOString(),
|
||||
},
|
||||
mostCommonScreens: Array.from(screenCounts.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 5)
|
||||
.map(([screen, count]) => ({ screen, count })),
|
||||
mostCommonActions: Array.from(actionCounts.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 5)
|
||||
.map(([action, count]) => ({ action, count })),
|
||||
apiFailurePattern,
|
||||
featureFlagCorrelations,
|
||||
};
|
||||
}
|
||||
@ -0,0 +1,570 @@
|
||||
/**
|
||||
* Retention Campaign Engine - Automated intervention system
|
||||
* [4.1] Campaign triggers and personalized messaging
|
||||
* [4.2] Platform integrations (email/push/Slack)
|
||||
*/
|
||||
|
||||
import { getRegisteredContainer } from '@bytelyst/cosmos';
|
||||
import { bus as eventBus } from '../../lib/event-bus.js';
|
||||
import type {
|
||||
RetentionCampaignDoc,
|
||||
CreateCampaignInput,
|
||||
ChurnPredictionInput,
|
||||
UserChurnPredictionDoc,
|
||||
} from './types.js';
|
||||
|
||||
export interface CampaignTriggerContext {
|
||||
userId: string;
|
||||
productId: string;
|
||||
churnProbability: number;
|
||||
riskSegment: string;
|
||||
topRiskFactors: Array<{ feature: string; description: string }>;
|
||||
suggestedActions: string[];
|
||||
}
|
||||
|
||||
export interface CampaignDeliveryResult {
|
||||
success: boolean;
|
||||
channel: string;
|
||||
messageId?: string;
|
||||
error?: string;
|
||||
sentAt: string;
|
||||
}
|
||||
|
||||
export class CampaignEngine {
|
||||
/**
|
||||
* Create a new retention campaign
|
||||
*/
|
||||
async createCampaign(input: CreateCampaignInput): Promise<RetentionCampaignDoc> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
|
||||
const doc: RetentionCampaignDoc = {
|
||||
id: `rc_${crypto.randomUUID()}`,
|
||||
...input,
|
||||
status: 'draft',
|
||||
stats: {
|
||||
triggered: 0,
|
||||
sent: 0,
|
||||
opened: 0,
|
||||
clicked: 0,
|
||||
converted: 0,
|
||||
controlGroupSize: 0,
|
||||
controlChurnRate: 0,
|
||||
treatmentChurnRate: 0,
|
||||
},
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
ttl: 365 * 24 * 60 * 60, // 1 year
|
||||
};
|
||||
|
||||
await container.items.create(doc);
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Activate a campaign
|
||||
*/
|
||||
async activateCampaign(campaignId: string): Promise<RetentionCampaignDoc | null> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
|
||||
try {
|
||||
const { resource: doc } = await container.item(campaignId).read<RetentionCampaignDoc>();
|
||||
if (!doc) return null;
|
||||
|
||||
doc.status = 'active';
|
||||
doc.updatedAt = new Date().toISOString();
|
||||
|
||||
await container.item(campaignId).replace(doc);
|
||||
return doc;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger campaign for a user based on churn prediction
|
||||
*/
|
||||
async triggerForUser(
|
||||
prediction: ChurnPredictionInput & { explanation: { topRiskFactors: Array<{ feature: string; description: string }>; suggestedActions: string[] } },
|
||||
testMode: boolean = false
|
||||
): Promise<CampaignDeliveryResult[]> {
|
||||
const campaigns = await this.getActiveCampaignsForProduct(prediction.productId);
|
||||
const results: CampaignDeliveryResult[] = [];
|
||||
|
||||
const context: CampaignTriggerContext = {
|
||||
userId: prediction.userId,
|
||||
productId: prediction.productId,
|
||||
churnProbability: prediction.churnProbability,
|
||||
riskSegment: prediction.riskSegment,
|
||||
topRiskFactors: prediction.explanation.topRiskFactors.slice(0, 3),
|
||||
suggestedActions: prediction.explanation.suggestedActions,
|
||||
};
|
||||
|
||||
for (const campaign of campaigns) {
|
||||
// Check if user matches campaign conditions
|
||||
if (!this.matchesCampaignConditions(context, campaign)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check frequency capping
|
||||
if (await this.isFrequencyCapped(context.userId, campaign)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Deliver messages
|
||||
for (const message of campaign.messages) {
|
||||
const result = await this.deliverMessage(context, message, testMode);
|
||||
results.push(result);
|
||||
|
||||
if (result.success && !testMode) {
|
||||
await this.recordDelivery(campaign.id, context.userId, message.channel);
|
||||
}
|
||||
}
|
||||
|
||||
// Emit event for tracking
|
||||
if (!testMode) {
|
||||
eventBus.emit({
|
||||
type: 'predictive.campaign.triggered',
|
||||
payload: {
|
||||
campaignId: campaign.id,
|
||||
userId: context.userId,
|
||||
productId: context.productId,
|
||||
riskSegment: context.riskSegment,
|
||||
channels: campaign.messages.map((m) => m.channel),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Manual trigger for testing
|
||||
*/
|
||||
async manualTrigger(
|
||||
campaignId: string,
|
||||
testUserId: string
|
||||
): Promise<CampaignDeliveryResult[]> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
|
||||
try {
|
||||
const { resource: campaign } = await container
|
||||
.item(campaignId)
|
||||
.read<RetentionCampaignDoc>();
|
||||
if (!campaign) return [];
|
||||
|
||||
const context: CampaignTriggerContext = {
|
||||
userId: testUserId,
|
||||
productId: campaign.productId,
|
||||
churnProbability: 0.75,
|
||||
riskSegment: 'high',
|
||||
topRiskFactors: [
|
||||
{ feature: 'daysSinceLastSession', description: 'Session recency declined' },
|
||||
],
|
||||
suggestedActions: ['Send re-engagement email'],
|
||||
};
|
||||
|
||||
const results: CampaignDeliveryResult[] = [];
|
||||
|
||||
for (const message of campaign.messages) {
|
||||
const result = await this.deliverMessage(context, message, true);
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
return results;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get active campaigns for a product
|
||||
*/
|
||||
private async getActiveCampaignsForProduct(
|
||||
productId: string
|
||||
): Promise<RetentionCampaignDoc[]> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.productId = @productId AND c.status = @status',
|
||||
parameters: [
|
||||
{ name: '@productId', value: productId },
|
||||
{ name: '@status', value: 'active' },
|
||||
],
|
||||
};
|
||||
|
||||
const { resources } = await container.items.query<RetentionCampaignDoc>(query).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if user matches campaign conditions
|
||||
*/
|
||||
private matchesCampaignConditions(
|
||||
context: CampaignTriggerContext,
|
||||
campaign: RetentionCampaignDoc
|
||||
): boolean {
|
||||
// Check risk segment match
|
||||
if (
|
||||
campaign.audience.riskSegments?.length &&
|
||||
!campaign.audience.riskSegments.includes(context.riskSegment)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check trigger conditions
|
||||
for (const condition of campaign.trigger.conditions) {
|
||||
const value = this.getContextValue(context, condition.field);
|
||||
|
||||
switch (condition.operator) {
|
||||
case 'gt':
|
||||
if (!(value > (condition.value as number))) return false;
|
||||
break;
|
||||
case 'lt':
|
||||
if (!(value < (condition.value as number))) return false;
|
||||
break;
|
||||
case 'eq':
|
||||
if (value !== condition.value) return false;
|
||||
break;
|
||||
case 'in':
|
||||
if (!(condition.value as unknown[]).includes(value)) return false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get value from context by field path
|
||||
*/
|
||||
private getContextValue(context: CampaignTriggerContext, field: string): unknown {
|
||||
const parts = field.split('.');
|
||||
let value: unknown = context;
|
||||
|
||||
for (const part of parts) {
|
||||
if (value && typeof value === 'object') {
|
||||
value = (value as Record<string, unknown>)[part];
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if user is frequency capped
|
||||
*/
|
||||
private async isFrequencyCapped(
|
||||
userId: string,
|
||||
campaign: RetentionCampaignDoc
|
||||
): Promise<boolean> {
|
||||
if (!campaign.audience.excludeRecentContact) return false;
|
||||
|
||||
const container = getRegisteredContainer('campaign_deliveries');
|
||||
const hoursAgo = new Date();
|
||||
hoursAgo.setHours(hoursAgo.getHours() - campaign.audience.excludeRecentContact);
|
||||
|
||||
const query = {
|
||||
query:
|
||||
'SELECT VALUE COUNT(1) FROM c WHERE c.userId = @userId AND c.sentAt >= @cutoff',
|
||||
parameters: [
|
||||
{ name: '@userId', value: userId },
|
||||
{ name: '@cutoff', value: hoursAgo.toISOString() },
|
||||
],
|
||||
};
|
||||
|
||||
try {
|
||||
const { resources } = await container.items.query<number>(query).fetchAll();
|
||||
return (resources[0] || 0) > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deliver message through appropriate channel
|
||||
*/
|
||||
private async deliverMessage(
|
||||
context: CampaignTriggerContext,
|
||||
message: { channel: string; templateId: string; variant?: string; delayHours?: number },
|
||||
testMode: boolean
|
||||
): Promise<CampaignDeliveryResult> {
|
||||
const sentAt = new Date().toISOString();
|
||||
|
||||
// Simulate delay
|
||||
if (message.delayHours && !testMode) {
|
||||
// In production, this would schedule for later delivery
|
||||
}
|
||||
|
||||
switch (message.channel) {
|
||||
case 'email':
|
||||
return this.deliverEmail(context, message.templateId, testMode, sentAt);
|
||||
case 'push':
|
||||
return this.deliverPush(context, message.templateId, testMode, sentAt);
|
||||
case 'in_app':
|
||||
return this.deliverInApp(context, message.templateId, testMode, sentAt);
|
||||
case 'slack_cs':
|
||||
return this.deliverSlackCS(context, message.templateId, testMode, sentAt);
|
||||
default:
|
||||
return {
|
||||
success: false,
|
||||
channel: message.channel,
|
||||
error: 'Unknown channel',
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deliver email via delivery module
|
||||
*/
|
||||
private async deliverEmail(
|
||||
context: CampaignTriggerContext,
|
||||
templateId: string,
|
||||
testMode: boolean,
|
||||
sentAt: string
|
||||
): Promise<CampaignDeliveryResult> {
|
||||
if (testMode) {
|
||||
return {
|
||||
success: true,
|
||||
channel: 'email',
|
||||
messageId: `test_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// In production, call delivery module
|
||||
// await deliveryService.sendEmail({...})
|
||||
|
||||
eventBus.emit({
|
||||
type: 'delivery.email.requested',
|
||||
payload: {
|
||||
userId: context.userId,
|
||||
productId: context.productId,
|
||||
templateId,
|
||||
context: {
|
||||
churnProbability: context.churnProbability,
|
||||
riskSegment: context.riskSegment,
|
||||
suggestedActions: context.suggestedActions,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
channel: 'email',
|
||||
messageId: `email_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
channel: 'email',
|
||||
error: error instanceof Error ? error.message : 'Email delivery failed',
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deliver push notification
|
||||
*/
|
||||
private async deliverPush(
|
||||
context: CampaignTriggerContext,
|
||||
templateId: string,
|
||||
testMode: boolean,
|
||||
sentAt: string
|
||||
): Promise<CampaignDeliveryResult> {
|
||||
if (testMode) {
|
||||
return {
|
||||
success: true,
|
||||
channel: 'push',
|
||||
messageId: `test_push_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
|
||||
eventBus.emit({
|
||||
type: 'notifications.push.requested',
|
||||
payload: {
|
||||
userId: context.userId,
|
||||
productId: context.productId,
|
||||
title: 'We miss you!',
|
||||
body: 'Come back and explore new features.',
|
||||
data: { campaign: 'retention', riskSegment: context.riskSegment },
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
channel: 'push',
|
||||
messageId: `push_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Deliver in-app message
|
||||
*/
|
||||
private async deliverInApp(
|
||||
context: CampaignTriggerContext,
|
||||
templateId: string,
|
||||
testMode: boolean,
|
||||
sentAt: string
|
||||
): Promise<CampaignDeliveryResult> {
|
||||
if (testMode) {
|
||||
return {
|
||||
success: true,
|
||||
channel: 'in_app',
|
||||
messageId: `test_inapp_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
|
||||
eventBus.emit({
|
||||
type: 'notifications.inapp.create',
|
||||
payload: {
|
||||
userId: context.userId,
|
||||
productId: context.productId,
|
||||
title: 'Personalized for you',
|
||||
content: 'Based on your interests, check out these features.',
|
||||
priority: context.riskSegment === 'critical' ? 'high' : 'normal',
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
channel: 'in_app',
|
||||
messageId: `inapp_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Deliver Slack notification to CS team
|
||||
*/
|
||||
private async deliverSlackCS(
|
||||
context: CampaignTriggerContext,
|
||||
templateId: string,
|
||||
testMode: boolean,
|
||||
sentAt: string
|
||||
): Promise<CampaignDeliveryResult> {
|
||||
if (testMode) {
|
||||
return {
|
||||
success: true,
|
||||
channel: 'slack_cs',
|
||||
messageId: `test_slack_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
|
||||
const message = {
|
||||
text: `🚨 High Risk User Alert`,
|
||||
blocks: [
|
||||
{
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: `*High Risk User Detected*\nUser: ${context.userId}\nProduct: ${context.productId}\nChurn Probability: ${Math.round(context.churnProbability * 100)}%\nRisk Segment: ${context.riskSegment}`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: `*Top Risk Factors:*\n${context.topRiskFactors.map((f) => `- ${f.description}`).join('\n')}`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: `*Suggested Actions:*\n${context.suggestedActions.map((a) => `- ${a}`).join('\n')}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
eventBus.emit({
|
||||
type: 'integrations.slack.notify',
|
||||
payload: {
|
||||
channel: '#customer-success',
|
||||
message,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
channel: 'slack_cs',
|
||||
messageId: `slack_${crypto.randomUUID()}`,
|
||||
sentAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Record campaign delivery
|
||||
*/
|
||||
private async recordDelivery(
|
||||
campaignId: string,
|
||||
userId: string,
|
||||
channel: string
|
||||
): Promise<void> {
|
||||
const container = getRegisteredContainer('campaign_deliveries');
|
||||
|
||||
await container.items.create({
|
||||
id: `cd_${crypto.randomUUID()}`,
|
||||
campaignId,
|
||||
userId,
|
||||
channel,
|
||||
sentAt: new Date().toISOString(),
|
||||
openedAt: null,
|
||||
clickedAt: null,
|
||||
convertedAt: null,
|
||||
ttl: 90 * 24 * 60 * 60,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get campaign statistics
|
||||
*/
|
||||
async getCampaignStats(campaignId: string): Promise<{
|
||||
triggered: number;
|
||||
sent: number;
|
||||
opened: number;
|
||||
clicked: number;
|
||||
converted: number;
|
||||
conversionRate: number;
|
||||
} | null> {
|
||||
const container = getRegisteredContainer('campaign_deliveries');
|
||||
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.campaignId = @campaignId',
|
||||
parameters: [{ name: '@campaignId', value: campaignId }],
|
||||
};
|
||||
|
||||
try {
|
||||
const { resources } = await container.items.query(query).fetchAll();
|
||||
|
||||
const sent = resources.length;
|
||||
const opened = resources.filter((r) => r.openedAt).length;
|
||||
const clicked = resources.filter((r) => r.clickedAt).length;
|
||||
const converted = resources.filter((r) => r.convertedAt).length;
|
||||
|
||||
return {
|
||||
triggered: sent,
|
||||
sent,
|
||||
opened,
|
||||
clicked,
|
||||
converted,
|
||||
conversionRate: sent > 0 ? converted / sent : 0,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const campaignEngine = new CampaignEngine();
|
||||
@ -0,0 +1,235 @@
|
||||
/**
|
||||
* Predictive Analytics Repository - Data access layer
|
||||
*/
|
||||
|
||||
import { getRegisteredContainer } from '@bytelyst/cosmos';
|
||||
import type {
|
||||
UserChurnPredictionDoc,
|
||||
ProductHealthScoreDoc,
|
||||
RetentionCampaignDoc,
|
||||
UserFeatureVectorDoc,
|
||||
FeatureDefinition,
|
||||
ModelPerformanceDoc,
|
||||
} from './types.js';
|
||||
|
||||
export class PredictiveAnalyticsRepository {
|
||||
// ==================== Churn Predictions ====================
|
||||
|
||||
async saveChurnPrediction(doc: UserChurnPredictionDoc): Promise<UserChurnPredictionDoc> {
|
||||
const container = getRegisteredContainer('churn_predictions');
|
||||
await container.items.create(doc);
|
||||
return doc;
|
||||
}
|
||||
|
||||
async getChurnPrediction(userId: string, productId: string, horizon: number = 30): Promise<UserChurnPredictionDoc | null> {
|
||||
const container = getRegisteredContainer('churn_predictions');
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.userId = @userId AND c.productId = @productId AND c.predictionHorizon = @horizon ORDER BY c.predictionTimestamp DESC OFFSET 0 LIMIT 1',
|
||||
parameters: [
|
||||
{ name: '@userId', value: userId },
|
||||
{ name: '@productId', value: productId },
|
||||
{ name: '@horizon', value: horizon },
|
||||
],
|
||||
};
|
||||
const { resources } = await container.items.query<UserChurnPredictionDoc>(query).fetchAll();
|
||||
return resources[0] || null;
|
||||
}
|
||||
|
||||
async getAtRiskUsers(
|
||||
productId: string,
|
||||
segment?: string,
|
||||
limit: number = 50,
|
||||
offset: number = 0
|
||||
): Promise<UserChurnPredictionDoc[]> {
|
||||
const container = getRegisteredContainer('churn_predictions');
|
||||
let queryStr = 'SELECT * FROM c WHERE c.productId = @productId';
|
||||
const parameters: Array<{ name: string; value: unknown }> = [
|
||||
{ name: '@productId', value: productId },
|
||||
];
|
||||
|
||||
if (segment) {
|
||||
queryStr += ' AND c.riskSegment = @segment';
|
||||
parameters.push({ name: '@segment', value: segment });
|
||||
}
|
||||
|
||||
queryStr += ' ORDER BY c.churnProbability DESC OFFSET @offset LIMIT @limit';
|
||||
parameters.push({ name: '@offset', value: offset });
|
||||
parameters.push({ name: '@limit', value: limit });
|
||||
|
||||
const { resources } = await container.items.query<UserChurnPredictionDoc>({
|
||||
query: queryStr,
|
||||
parameters,
|
||||
}).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
|
||||
async getUserRiskProfile(userId: string, productId: string): Promise<UserChurnPredictionDoc[]> {
|
||||
const container = getRegisteredContainer('churn_predictions');
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.userId = @userId AND c.productId = @productId ORDER BY c.predictionTimestamp DESC',
|
||||
parameters: [
|
||||
{ name: '@userId', value: userId },
|
||||
{ name: '@productId', value: productId },
|
||||
],
|
||||
};
|
||||
const { resources } = await container.items.query<UserChurnPredictionDoc>(query).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
|
||||
// ==================== Health Scores ====================
|
||||
|
||||
async saveHealthScore(doc: ProductHealthScoreDoc): Promise<ProductHealthScoreDoc> {
|
||||
const container = getRegisteredContainer('product_health');
|
||||
await container.items.create(doc);
|
||||
return doc;
|
||||
}
|
||||
|
||||
async getHealthScore(productId: string, date: string): Promise<ProductHealthScoreDoc | null> {
|
||||
const container = getRegisteredContainer('product_health');
|
||||
try {
|
||||
const { resource } = await container.item(`${productId}:${date}`, productId).read<ProductHealthScoreDoc>();
|
||||
return resource || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getHealthHistory(productId: string, days: number = 30): Promise<ProductHealthScoreDoc[]> {
|
||||
const container = getRegisteredContainer('product_health');
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - days);
|
||||
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.productId = @productId AND c.date >= @cutoff ORDER BY c.date DESC',
|
||||
parameters: [
|
||||
{ name: '@productId', value: productId },
|
||||
{ name: '@cutoff', value: cutoff.toISOString().split('T')[0] },
|
||||
],
|
||||
};
|
||||
const { resources } = await container.items.query<ProductHealthScoreDoc>(query).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
|
||||
async getAllProductHealthScores(): Promise<ProductHealthScoreDoc[]> {
|
||||
const container = getRegisteredContainer('product_health');
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.date = @today',
|
||||
parameters: [{ name: '@today', value: today }],
|
||||
};
|
||||
const { resources } = await container.items.query<ProductHealthScoreDoc>(query).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
|
||||
// ==================== Campaigns ====================
|
||||
|
||||
async getCampaign(campaignId: string): Promise<RetentionCampaignDoc | null> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
try {
|
||||
const { resource } = await container.item(campaignId, campaignId).read<RetentionCampaignDoc>();
|
||||
return resource || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async listCampaigns(productId?: string, status?: string): Promise<RetentionCampaignDoc[]> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
let queryStr = 'SELECT * FROM c WHERE 1=1';
|
||||
const parameters: Array<{ name: string; value: unknown }> = [];
|
||||
|
||||
if (productId) {
|
||||
queryStr += ' AND c.productId = @productId';
|
||||
parameters.push({ name: '@productId', value: productId });
|
||||
}
|
||||
|
||||
if (status) {
|
||||
queryStr += ' AND c.status = @status';
|
||||
parameters.push({ name: '@status', value: status });
|
||||
}
|
||||
|
||||
queryStr += ' ORDER BY c.createdAt DESC';
|
||||
|
||||
const { resources } = await container.items.query<RetentionCampaignDoc>({
|
||||
query: queryStr,
|
||||
parameters,
|
||||
}).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
|
||||
async updateCampaign(
|
||||
campaignId: string,
|
||||
updates: Partial<RetentionCampaignDoc>
|
||||
): Promise<RetentionCampaignDoc | null> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
|
||||
try {
|
||||
const { resource: doc } = await container.item(campaignId).read<RetentionCampaignDoc>();
|
||||
if (!doc) return null;
|
||||
|
||||
const updated = { ...doc, ...updates, updatedAt: new Date().toISOString() };
|
||||
await container.item(campaignId).replace(updated);
|
||||
return updated;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async deleteCampaign(campaignId: string): Promise<boolean> {
|
||||
const container = getRegisteredContainer('retention_campaigns');
|
||||
try {
|
||||
await container.item(campaignId).delete();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Feature Definitions ====================
|
||||
|
||||
async saveFeatureDefinition(doc: FeatureDefinition): Promise<FeatureDefinition> {
|
||||
const container = getRegisteredContainer('feature_definitions');
|
||||
await container.items.create(doc);
|
||||
return doc;
|
||||
}
|
||||
|
||||
async getFeatureDefinitions(productId: string): Promise<FeatureDefinition[]> {
|
||||
const container = getRegisteredContainer('feature_definitions');
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.productId = @productId',
|
||||
parameters: [{ name: '@productId', value: productId }],
|
||||
};
|
||||
const { resources } = await container.items.query<FeatureDefinition>(query).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
|
||||
// ==================== Model Performance ====================
|
||||
|
||||
async saveModelPerformance(doc: ModelPerformanceDoc): Promise<ModelPerformanceDoc> {
|
||||
const container = getRegisteredContainer('model_performance');
|
||||
await container.items.create(doc);
|
||||
return doc;
|
||||
}
|
||||
|
||||
async getLatestModelPerformance(): Promise<ModelPerformanceDoc | null> {
|
||||
const container = getRegisteredContainer('model_performance');
|
||||
const query = {
|
||||
query: 'SELECT * FROM c WHERE c.isActive = true ORDER BY c.createdAt DESC OFFSET 0 LIMIT 1',
|
||||
};
|
||||
const { resources } = await container.items.query<ModelPerformanceDoc>(query).fetchAll();
|
||||
return resources[0] || null;
|
||||
}
|
||||
|
||||
async getModelPerformanceHistory(limit: number = 10): Promise<ModelPerformanceDoc[]> {
|
||||
const container = getRegisteredContainer('model_performance');
|
||||
const query = {
|
||||
query: 'SELECT * FROM c ORDER BY c.createdAt DESC OFFSET 0 LIMIT @limit',
|
||||
parameters: [{ name: '@limit', value: limit }],
|
||||
};
|
||||
const { resources } = await container.items.query<ModelPerformanceDoc>(query).fetchAll();
|
||||
return resources;
|
||||
}
|
||||
}
|
||||
|
||||
export const predictiveAnalyticsRepo = new PredictiveAnalyticsRepository();
|
||||
Loading…
Reference in New Issue
Block a user