From 1ff02934faad24a30afab48ac3bb363d8908ec01 Mon Sep 17 00:00:00 2001 From: saravanakumardb1 Date: Tue, 3 Mar 2026 11:50:28 -0800 Subject: [PATCH] feat(ai-diagnostics): add telemetry linking and context enrichment [1.3] --- .windsurf/workflows/refresh-chat-history.md | 12 + .../WINDSURF/.last-refresh.log | 8 +- .../refresh-chat-history.md | 12 + .../repo_backup-and-push.md | 6 +- .../repo_backup-main-branch.md | 2 + .../repo_commit-workspace.md | 5 +- .../repo_push-repos.md | 8 +- .../repo_sync-repos.md | 8 +- .../src/app/(dashboard)/experiments/page.tsx | 257 +++++++ .../ab-testing/hypothesis-generator.ts | 445 ++++++++++++ .../ai-diagnostics/telemetry-linking.ts | 632 ++++++++++++++++++ .../predictive-analytics/campaign-engine.ts | 570 ++++++++++++++++ .../predictive-analytics/repository.ts | 235 +++++++ 13 files changed, 2187 insertions(+), 13 deletions(-) create mode 100644 dashboards/admin-web/src/app/(dashboard)/experiments/page.tsx create mode 100644 services/platform-service/src/modules/ab-testing/hypothesis-generator.ts create mode 100644 services/platform-service/src/modules/ai-diagnostics/telemetry-linking.ts create mode 100644 services/platform-service/src/modules/predictive-analytics/campaign-engine.ts create mode 100644 services/platform-service/src/modules/predictive-analytics/repository.ts diff --git a/.windsurf/workflows/refresh-chat-history.md b/.windsurf/workflows/refresh-chat-history.md index b266b3f6..28156ebf 100644 --- a/.windsurf/workflows/refresh-chat-history.md +++ b/.windsurf/workflows/refresh-chat-history.md @@ -7,6 +7,18 @@ description: Refresh the Windsurf chat history archive (re-scan all repos, updat Refreshes the centralized Windsurf chat history archive at `__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/`. Auto-discovers new repos, updates symlinks, and re-copies docs + workflows. +## Covered Repos (All 7 workspaces) + +| Repo | Product | Workflows | Docs | +|------|---------|-----------|------| +| `learning_voice_ai_agent` | LysnrAI | ✅ | ✅ | +| `learning_multimodal_memory_agents` | MindLyst | ✅ | ✅ | +| `learning_ai_clock` | ChronoMind | ✅ | — | +| `learning_ai_peakpulse` | PeakPulse | ✅ | — | +| `learning_ai_fastgap` | NomGap | ✅ | — | +| `learning_ai_jarvis_jr` | JarvisJr | ✅ | — | +| `learning_ai_common_plat` | Common Platform | ✅ | — | + ## Steps // turbo diff --git a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/.last-refresh.log b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/.last-refresh.log index 9b60f29a..de57d24b 100644 --- a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/.last-refresh.log +++ b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/.last-refresh.log @@ -1,9 +1,9 @@ -Last refresh: 2026-03-03T07:00:03Z (2026-03-02 23:00:03 PST) -Cascade conversations: 50 (348M) +Last refresh: 2026-03-03T19:50:04Z (2026-03-03 11:50:04 PST) +Cascade conversations: 50 (297M) Memories: 65 Implicit context: 20 -Code tracker dirs: 149 -File edit history: 2278 entries +Code tracker dirs: 188 +File edit history: 2343 entries Workspace storage: 28 workspaces Repo docs: 7 files across 2 repos Repo workflows: 35 files across 6 repos diff --git a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/refresh-chat-history.md b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/refresh-chat-history.md index b266b3f6..28156ebf 100644 --- a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/refresh-chat-history.md +++ b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/refresh-chat-history.md @@ -7,6 +7,18 @@ description: Refresh the Windsurf chat history archive (re-scan all repos, updat Refreshes the centralized Windsurf chat history archive at `__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/`. Auto-discovers new repos, updates symlinks, and re-copies docs + workflows. +## Covered Repos (All 7 workspaces) + +| Repo | Product | Workflows | Docs | +|------|---------|-----------|------| +| `learning_voice_ai_agent` | LysnrAI | ✅ | ✅ | +| `learning_multimodal_memory_agents` | MindLyst | ✅ | ✅ | +| `learning_ai_clock` | ChronoMind | ✅ | — | +| `learning_ai_peakpulse` | PeakPulse | ✅ | — | +| `learning_ai_fastgap` | NomGap | ✅ | — | +| `learning_ai_jarvis_jr` | JarvisJr | ✅ | — | +| `learning_ai_common_plat` | Common Platform | ✅ | — | + ## Steps // turbo diff --git a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-and-push.md b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-and-push.md index 33d47a1e..e05b7634 100644 --- a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-and-push.md +++ b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-and-push.md @@ -18,7 +18,7 @@ Run `bash scripts/backup-main.sh` from any repository root // turbo ```bash -for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap; do +for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap learning_ai_jarvis_jr learning_ai_peakpulse; do echo "━━━ Pushing $repo ━━━" (cd ~/code/mygh/$repo && git push origin main 2>&1) done @@ -29,7 +29,7 @@ echo "✨ All repos pushed!" ## What it does: 1. **Backup** — creates timestamped backup branches, cleans up old ones (7 days), skips duplicates -2. **Push** — pushes `main` to `origin/main` for all 5 repos +2. **Push** — pushes `main` to `origin/main` for all 7 repos ## Repositories: @@ -38,6 +38,8 @@ echo "✨ All repos pushed!" - learning_multimodal_memory_agents - learning_ai_clock - learning_ai_fastgap +- learning_ai_jarvis_jr +- learning_ai_peakpulse ## When to use: diff --git a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-main-branch.md b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-main-branch.md index 8e81418d..1bae9e93 100644 --- a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-main-branch.md +++ b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_backup-main-branch.md @@ -24,6 +24,8 @@ Run `bash scripts/backup-main.sh` from any repository root - learning_multimodal_memory_agents - learning_ai_clock - learning_ai_fastgap +- learning_ai_jarvis_jr +- learning_ai_peakpulse ## Features: diff --git a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_commit-workspace.md b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_commit-workspace.md index a717422d..2cce84ca 100644 --- a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_commit-workspace.md +++ b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_commit-workspace.md @@ -12,11 +12,14 @@ Scans all repositories for pending changes and commits them in logical order wit ## What it does: -1. **Scans** all 4 repos for changes: +1. **Scans** all 7 repos for changes: - learning_ai_common_plat - learning_voice_ai_agent - learning_multimodal_memory_agents - learning_ai_clock + - learning_ai_fastgap + - learning_ai_jarvis_jr + - learning_ai_peakpulse 2. **Analyzes** changed files to determine: - Commit scope (auth, ci, docs, feat, chore, etc.) diff --git a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_push-repos.md b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_push-repos.md index 3238edf7..7caa060c 100644 --- a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_push-repos.md +++ b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_push-repos.md @@ -1,5 +1,5 @@ --- -description: Push local main branch to origin for all 5 workspace repos +description: Push local main branch to origin for all 7 workspace repos --- # Push Repos @@ -9,7 +9,7 @@ Pushes local `main` to `origin/main` for all workspace repositories. // turbo ```bash -for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap; do +for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap learning_ai_jarvis_jr learning_ai_peakpulse; do echo "━━━ $repo ━━━" (cd ~/code/mygh/$repo && git push origin main) done @@ -17,7 +17,7 @@ done ## What it does: -1. Iterates over all 5 workspace repos +1. Iterates over all 7 workspace repos 2. Runs `git push origin main` in each 3. Fails fast if a repo has diverged from remote (resolve with rebase manually) @@ -28,6 +28,8 @@ done - learning_multimodal_memory_agents - learning_ai_clock - learning_ai_fastgap +- learning_ai_jarvis_jr +- learning_ai_peakpulse ## When to use: diff --git a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_sync-repos.md b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_sync-repos.md index 8fd52aed..2f513bbb 100644 --- a/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_sync-repos.md +++ b/__LOCAL_LLMs/AI_IDE_CHAT_HISTORY/WINDSURF/repo_workflows/learning_ai_common_plat/repo_sync-repos.md @@ -1,5 +1,5 @@ --- -description: Pull latest from origin main across all 5 workspace repos +description: Pull latest from origin main across all 7 workspace repos --- # Sync Repos @@ -9,7 +9,7 @@ Pulls the latest changes from `origin/main` for all workspace repositories. // turbo ```bash -for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap; do +for repo in learning_ai_common_plat learning_voice_ai_agent learning_multimodal_memory_agents learning_ai_clock learning_ai_fastgap learning_ai_jarvis_jr learning_ai_peakpulse; do echo "━━━ $repo ━━━" (cd ~/code/mygh/$repo && git pull --ff-only origin main) done @@ -17,7 +17,7 @@ done ## What it does: -1. Iterates over all 5 workspace repos +1. Iterates over all 7 workspace repos 2. Runs `git pull --ff-only origin main` in each 3. Fails fast if there are local divergent commits (use `git pull --rebase` manually in that case) @@ -28,6 +28,8 @@ done - learning_multimodal_memory_agents - learning_ai_clock - learning_ai_fastgap +- learning_ai_jarvis_jr +- learning_ai_peakpulse ## When to use: diff --git a/dashboards/admin-web/src/app/(dashboard)/experiments/page.tsx b/dashboards/admin-web/src/app/(dashboard)/experiments/page.tsx new file mode 100644 index 00000000..8a364c27 --- /dev/null +++ b/dashboards/admin-web/src/app/(dashboard)/experiments/page.tsx @@ -0,0 +1,257 @@ +/** + * Experiments List Page — Admin Dashboard + * Shows all A/B experiments with status, metrics, and quick actions. + */ + +'use client'; + +import { useState, useEffect } from 'react'; +import Link from 'next/link'; +import { useRouter } from 'next/navigation'; +import { + FlaskConical, + Plus, + Play, + Pause, + Square, + BarChart3, + Users, + Clock, + CheckCircle, + AlertCircle, + Sparkles, +} from 'lucide-react'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import type { ExperimentDoc } from '@/lib/experiments-types'; + +const statusConfig: Record = { + draft: { color: 'bg-gray-500', icon: AlertCircle, label: 'Draft' }, + running: { color: 'bg-green-500', icon: Play, label: 'Running' }, + paused: { color: 'bg-yellow-500', icon: Pause, label: 'Paused' }, + stopped: { color: 'bg-red-500', icon: Square, label: 'Stopped' }, + completed: { color: 'bg-blue-500', icon: CheckCircle, label: 'Completed' }, +}; + +export default function ExperimentsPage() { + const router = useRouter(); + const [experiments, setExperiments] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + fetchExperiments(); + }, []); + + async function fetchExperiments() { + try { + const response = await fetch('/api/experiments'); + if (!response.ok) throw new Error('Failed to fetch experiments'); + const data = await response.json(); + setExperiments(data); + } catch (err) { + setError(err instanceof Error ? err.message : 'Unknown error'); + } finally { + setLoading(false); + } + } + + const runningCount = experiments.filter(e => e.status === 'running').length; + const completedCount = experiments.filter(e => e.status === 'completed').length; + const totalParticipants = experiments.reduce((sum, e) => sum + (e.totalParticipants || 0), 0); + + if (loading) { + return ( +
+
+
+
+
+ ); + } + + return ( +
+ {/* Header */} +
+
+

+ + A/B Experiments +

+

+ Intelligent experimentation with Bayesian statistics and auto-allocation +

+
+
+ + +
+
+ + {/* Stats Cards */} +
+ + + + Total Experiments + + + +
{experiments.length}
+ + + + + + Running + + + +
{runningCount}
+ + + + + + Completed + + + +
{completedCount}
+ + + + + + Total Participants + + + +
{totalParticipants.toLocaleString()}
+ + +
+ + {/* Tabs */} + + + All ({experiments.length}) + Running ({runningCount}) + Completed ({completedCount}) + Drafts ({experiments.filter(e => e.status === 'draft').length}) + + + + {experiments.map(experiment => ( + + ))} + {experiments.length === 0 && ( +
+ +

No experiments yet. Create your first experiment to get started.

+
+ )} + + + + {experiments + .filter(e => e.status === 'running') + .map(experiment => ( + + ))} + + + + {experiments + .filter(e => e.status === 'completed') + .map(experiment => ( + + ))} + + + + {experiments + .filter(e => e.status === 'draft') + .map(experiment => ( + + ))} + + +
+ ); +} + +function ExperimentCard({ experiment }: { experiment: ExperimentDoc }) { + const status = statusConfig[experiment.status] || statusConfig.draft; + const StatusIcon = status.icon; + + const daysRunning = experiment.startedAt + ? Math.floor((Date.now() - new Date(experiment.startedAt).getTime()) / (1000 * 60 * 60 * 24)) + : 0; + + return ( + + +
+
+
+

+ + {experiment.name} + +

+ + + {status.label} + + {experiment.aiGeneratedHypothesis && ( + + + AI Generated + + )} +
+

{experiment.hypothesis}

+ +
+
+ + Metric: {experiment.primaryMetric?.name || 'Not set'} +
+
+ + {experiment.totalParticipants?.toLocaleString() || 0} participants +
+ {experiment.status === 'running' && ( +
+ + {daysRunning} days running +
+ )} +
+
+ +
+ +
+
+ + + ); +} diff --git a/services/platform-service/src/modules/ab-testing/hypothesis-generator.ts b/services/platform-service/src/modules/ab-testing/hypothesis-generator.ts new file mode 100644 index 00000000..0252829f --- /dev/null +++ b/services/platform-service/src/modules/ab-testing/hypothesis-generator.ts @@ -0,0 +1,445 @@ +/** + * Intelligent A/B Testing — AI Hypothesis Generation. + * Pattern detection from telemetry, LLM-powered hypothesis generation, auto-suggestions. + */ + +import type { ExperimentSuggestion, GeneratedHypothesis, HypothesisInput, PrimaryMetric, ExperimentDoc } from './types.js'; +import { createSuggestion } from './repository.js'; + +// ───────────────────────────────────────────────────────────────────────────── +// Pattern Detection +// ───────────────────────────────────────────────────────────────────────────── + +export interface UsagePattern { + featureName: string; + totalUsage: number; + uniqueUsers: number; + adoptionRate: number; // 0–1 + trendDirection: 'up' | 'down' | 'stable'; + trendMagnitude: number; // % change + segmentPerformance: Record; // platform/segment -> rate + temporalPatterns: { + dayOfWeek: Record; + hourOfDay: Record; + }; + funnelDropOffs?: Array<{ stage: string; dropRate: number }>; +} + +export interface AnomalyDetection { + featureName: string; + anomalyType: 'drop' | 'spike' | 'regression' | 'segment_mismatch'; + severity: 'low' | 'medium' | 'high'; + expectedValue: number; + actualValue: number; + deviation: number; // standard deviations from mean + affectedSegment?: string; +} + +export interface Opportunity { + featureName: string; + opportunityType: 'low_adoption' | 'high_dropoff' | 'competitor_gap' | 'unused_potential'; + impactScore: number; // 0–100 + currentMetric: number; + targetMetric: number; + estimatedLift: number; + supportingEvidence: string[]; +} + +/** + * Analyze telemetry data for usage patterns. + * Placeholder: In production, this queries telemetry_events container. + */ +export async function detectUsagePatterns( + productId: string, + featureNames: string[] +): Promise { + // Placeholder implementation + // In production: query telemetry_events with aggregation queries + return featureNames.map(name => ({ + featureName: name, + totalUsage: Math.floor(Math.random() * 10000) + 1000, + uniqueUsers: Math.floor(Math.random() * 5000) + 500, + adoptionRate: Math.random() * 0.5 + 0.1, + trendDirection: Math.random() > 0.5 ? 'up' : 'down', + trendMagnitude: Math.random() * 0.3, + segmentPerformance: { + ios: Math.random() * 0.4 + 0.1, + android: Math.random() * 0.4 + 0.1, + web: Math.random() * 0.4 + 0.1, + }, + temporalPatterns: { + dayOfWeek: { Mon: 0.2, Tue: 0.25, Wed: 0.25, Thu: 0.2, Fri: 0.1 }, + hourOfDay: { '9': 0.15, '12': 0.2, '15': 0.25, '18': 0.2, '21': 0.1 }, + }, + })); +} + +/** + * Detect anomalies in feature usage. + */ +export async function detectAnomalies( + productId: string, + baselineDays = 30 +): Promise { + // Placeholder: In production, compare current metrics against rolling averages + const anomalies: AnomalyDetection[] = []; + + // Simulate anomaly detection + const randomFeatures = ['voice_dictation', 'keyboard_shortcuts', 'cloud_sync', 'analytics_view']; + for (const feature of randomFeatures) { + if (Math.random() > 0.7) { + anomalies.push({ + featureName: feature, + anomalyType: Math.random() > 0.5 ? 'drop' : 'regression', + severity: Math.random() > 0.5 ? 'high' : 'medium', + expectedValue: 0.25, + actualValue: 0.15, + deviation: 2.5, + }); + } + } + + return anomalies; +} + +/** + * Identify experiment opportunities from usage patterns. + */ +export function identifyOpportunities(patterns: UsagePattern[]): Opportunity[] { + const opportunities: Opportunity[] = []; + + for (const pattern of patterns) { + // Low adoption opportunity + if (pattern.adoptionRate < 0.2) { + opportunities.push({ + featureName: pattern.featureName, + opportunityType: 'low_adoption', + impactScore: Math.floor((0.2 - pattern.adoptionRate) * 500), + currentMetric: pattern.adoptionRate, + targetMetric: 0.2, + estimatedLift: (0.2 - pattern.adoptionRate) / pattern.adoptionRate, + supportingEvidence: [ + `Current adoption rate is ${(pattern.adoptionRate * 100).toFixed(1)}%`, + `Trend is ${pattern.trendDirection} by ${(pattern.trendMagnitude * 100).toFixed(1)}%`, + ], + }); + } + + // Segment mismatch opportunity + const segmentRates = Object.values(pattern.segmentPerformance); + const maxSegment = Math.max(...segmentRates); + const minSegment = Math.min(...segmentRates); + if (maxSegment > minSegment * 2) { + opportunities.push({ + featureName: pattern.featureName, + opportunityType: 'unused_potential', + impactScore: Math.floor((maxSegment - minSegment) * 100), + currentMetric: minSegment, + targetMetric: maxSegment, + estimatedLift: (maxSegment - minSegment) / minSegment, + supportingEvidence: [ + `Best performing segment: ${(maxSegment * 100).toFixed(1)}%`, + `Worst performing segment: ${(minSegment * 100).toFixed(1)}%`, + `Potential to lift underperforming segments to best-in-class`, + ], + }); + } + } + + return opportunities.sort((a, b) => b.impactScore - a.impactScore); +} + +// ───────────────────────────────────────────────────────────────────────────── +// LLM Hypothesis Generation +// ───────────────────────────────────────────────────────────────────────────── + +/** + * Generate experiment hypothesis using LLM. + * Placeholder: In production, this calls Azure OpenAI. + */ +export async function generateHypothesis( + input: HypothesisInput +): Promise { + // Build prompt for LLM + const prompt = buildHypothesisPrompt(input); + + // Placeholder: Simulate LLM response + // In production: const response = await callAzureOpenAI(prompt); + return simulateLLMHypothesis(input); +} + +/** + * Build structured prompt for hypothesis generation. + */ +function buildHypothesisPrompt(input: HypothesisInput): string { + return ` +You are an expert product analyst and experimentation strategist. + +Analyze this feature usage data and generate experiment hypotheses: + +Feature: ${input.featureName} +Current Adoption Rate: ${(input.adoptionRate * 100).toFixed(1)}% +Baseline Rate: ${(input.baselineRate * 100).toFixed(1)}% + +Segment Performance: +${Object.entries(input.segmentData) + .map(([segment, rate]) => `- ${segment}: ${(rate * 100).toFixed(1)}%`) + .join('\n')} + +User Feedback Samples: +${input.feedbackSamples.map(f => `- "${f}"`).join('\n')} + +Generate: +1. Primary hypothesis: "Changing X will improve Y because..." +2. 2-3 alternative hypotheses +3. Expected effect size (conservative) +4. Success metric recommendation +5. Risk assessment (low/medium/high) +`; +} + +/** + * Simulate LLM hypothesis response (placeholder). + */ +function simulateLLMHypothesis(input: HypothesisInput): GeneratedHypothesis { + const effectSize = Math.abs(input.adoptionRate - input.baselineRate) * 0.5; + const riskLevel = effectSize > 0.1 ? 'medium' : 'low'; + + return { + primary: `Redesigning the ${input.featureName} onboarding flow to highlight key benefits will increase adoption by ${(effectSize * 100).toFixed(0)}% because users currently don't understand the value proposition.`, + alternatives: [ + `Adding tooltips and contextual help to ${input.featureName} will reduce confusion and increase usage by ${(effectSize * 0.7 * 100).toFixed(0)}%.`, + `Simplifying the ${input.featureName} UI to match competitor patterns will improve familiarity and adoption by ${(effectSize * 0.5 * 100).toFixed(0)}%.`, + ], + expectedEffectSize: Math.max(effectSize, 0.05), + successMetric: 'adoption_rate', + riskAssessment: riskLevel, + impactScore: Math.floor((input.baselineRate - input.adoptionRate) * 200), + difficultyScore: Math.floor(Math.random() * 30) + 20, + powerPrediction: 85, + }; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Hypothesis Ranking +// ───────────────────────────────────────────────────────────────────────────── + +export interface RankedHypothesis extends GeneratedHypothesis { + rankScore: number; + estimatedImpact: number; + estimatedEffort: number; + estimatedPower: number; +} + +/** + * Rank hypotheses by expected value. + */ +export function rankHypotheses( + hypotheses: GeneratedHypothesis[], + baseTraffic: number +): RankedHypothesis[] { + return hypotheses.map(h => { + // Expected value calculation + const impact = h.impactScore; + const effort = h.difficultyScore; + const power = h.powerPrediction; + + // Risk-adjusted expected value + const riskMultiplier = h.riskAssessment === 'low' ? 1.0 : h.riskAssessment === 'medium' ? 0.8 : 0.6; + + // Rank score: higher impact, lower effort, higher power = better + const rankScore = (impact * power * riskMultiplier) / (effort + 10); + + return { + ...h, + rankScore, + estimatedImpact: impact, + estimatedEffort: effort, + estimatedPower: power, + }; + }).sort((a, b) => b.rankScore - a.rankScore); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Auto-Experiment Suggestions +// ───────────────────────────────────────────────────────────────────────────── + +export interface SuggestedExperiment { + name: string; + hypothesis: string; + description: string; + variants: Array<{ + name: string; + description: string; + flagConfig: Record; + }>; + primaryMetric: PrimaryMetric; + suggestedDuration: number; + suggestedSampleSize: number; + priority: number; +} + +/** + * Generate complete experiment suggestion from opportunity. + */ +export function generateExperimentSuggestion( + opportunity: Opportunity, + hypothesis: GeneratedHypothesis, + productId: string +): Omit { + const variantNames = ['Control', 'Treatment A', 'Treatment B']; + if (hypothesis.alternatives.length < 2) { + variantNames.pop(); + } + + const suggestedVariants = variantNames.map((name, i) => ({ + name, + description: i === 0 + ? 'Current implementation (control)' + : hypothesis.alternatives[i - 1] || `Alternative approach ${i}`, + })); + + const sampleSize = Math.ceil( + (hypothesis.expectedEffectSize > 0 ? 16 / (hypothesis.expectedEffectSize ** 2) : 1000) + ); + + return { + hypothesis, + suggestedVariants, + suggestedMetrics: [ + { + name: opportunity.opportunityType === 'low_adoption' ? 'adoption_rate' : 'conversion_rate', + type: 'conversion', + eventName: `${opportunity.featureName}_used`, + aggregation: 'unique', + direction: 'increase', + minimumDetectableEffect: hypothesis.expectedEffectSize * 100, + } as PrimaryMetric, + ], + suggestedDuration: Math.max(7, Math.ceil(sampleSize / 100)), + suggestedSampleSize: sampleSize, + priority: opportunity.impactScore, + aiGenerated: true, + }; +} + +/** + * Generate weekly AI report with top experiment opportunities. + */ +export async function generateWeeklyReport( + productId: string +): Promise<{ + topOpportunities: Opportunity[]; + suggestedExperiments: Array>; + anomalies: AnomalyDetection[]; + generatedAt: string; +}> { + // Detect patterns and opportunities + const patterns = await detectUsagePatterns(productId, [ + 'voice_dictation', + 'keyboard_shortcuts', + 'cloud_sync', + 'analytics_view', + 'settings_customization', + ]); + + const opportunities = identifyOpportunities(patterns); + const anomalies = await detectAnomalies(productId); + + // Generate hypotheses for top opportunities + const topOpportunities = opportunities.slice(0, 5); + const suggestedExperiments: Array> = []; + + for (const opp of topOpportunities) { + const hypothesisInput: HypothesisInput = { + featureName: opp.featureName, + adoptionRate: opp.currentMetric, + baselineRate: opp.targetMetric, + segmentData: {}, + feedbackSamples: opp.supportingEvidence, + }; + + const hypothesis = await generateHypothesis(hypothesisInput); + const suggestion = generateExperimentSuggestion(opp, hypothesis, productId); + suggestedExperiments.push(suggestion); + } + + return { + topOpportunities, + suggestedExperiments, + anomalies, + generatedAt: new Date().toISOString(), + }; +} + +// ───────────────────────────────────────────────────────────────────────────── +// AI Insights for Results +// ───────────────────────────────────────────────────────────────────────────── + +export interface ExperimentInsights { + summary: string; + unexpectedFindings: string[]; + segmentAnalysis: Record; + followUpSuggestions: Array<{ + experimentName: string; + hypothesis: string; + priority: number; + }>; +} + +/** + * Generate AI insights for completed experiment. + * Placeholder: In production, calls Azure OpenAI with full experiment data. + */ +export function generateExperimentInsights( + experiment: ExperimentDoc, + result: { variantResults: Array<{ variantName: string; probabilityBeatsControl: number; expectedLiftPercent: number }> }, + winnerVariantId?: string +): ExperimentInsights { + const winner = result.variantResults.find(v => v.probabilityBeatsControl > 0.95); + const loser = result.variantResults.find(v => v.probabilityBeatsControl < 0.05); + + const insights: ExperimentInsights = { + summary: '', + unexpectedFindings: [], + segmentAnalysis: {}, + followUpSuggestions: [], + }; + + if (winner) { + insights.summary = `The ${winner.variantName} variant significantly outperformed control with ${winner.probabilityBeatsControl.toFixed(1)}% probability of superiority and a ${winner.expectedLiftPercent.toFixed(1)}% lift. This validates the hypothesis that ${experiment.hypothesis}`; + } else if (loser) { + insights.summary = `The experiment did not produce a winning variant. The control performed better than expected, suggesting the current implementation is already optimized for this metric.`; + } else { + insights.summary = `Results were inconclusive. No variant demonstrated statistically significant improvement over control. Consider running the experiment longer or testing more distinct variations.`; + } + + // Add unexpected findings based on result patterns + const allPositive = result.variantResults.every(v => v.expectedLiftPercent > 0); + const allNegative = result.variantResults.every(v => v.expectedLiftPercent < 0); + + if (allNegative) { + insights.unexpectedFindings.push('All variants underperformed control, suggesting possible confounding factors or measurement issues.'); + } + + if (Math.abs(result.variantResults[0]?.expectedLiftPercent || 0) > 50) { + insights.unexpectedFindings.push('Extremely large effect size detected. Verify data quality and consider running a validation experiment.'); + } + + // Generate follow-up suggestions + if (winner) { + insights.followUpSuggestions.push({ + experimentName: `${experiment.name} - Follow-up Validation`, + hypothesis: `Validating the ${winner.variantName} win in a new user cohort to confirm generalizability`, + priority: 90, + }); + } + + insights.followUpSuggestions.push({ + experimentName: `${experiment.name} - Secondary Metrics`, + hypothesis: `Testing the winning variant against additional success metrics to ensure no trade-offs`, + priority: 70, + }); + + return insights; +} diff --git a/services/platform-service/src/modules/ai-diagnostics/telemetry-linking.ts b/services/platform-service/src/modules/ai-diagnostics/telemetry-linking.ts new file mode 100644 index 00000000..11810f87 --- /dev/null +++ b/services/platform-service/src/modules/ai-diagnostics/telemetry-linking.ts @@ -0,0 +1,632 @@ +import { getRegisteredContainer } from '@bytelyst/cosmos'; +import type { Container } from '@azure/cosmos'; +import type { ErrorEvent, ErrorClusterDoc } from './types.js'; + +// ============================================================================ +// Telemetry Linking Service +// ============================================================================ + +interface TelemetryEvent { + id: string; + correlationId?: string; + productId: string; + userId?: string; + sessionId?: string; + eventType: string; + eventName: string; + timestamp: string; + platform?: string; + osVersion?: string; + appVersion?: string; + deviceModel?: string; + screen?: string; + properties?: Record; +} + +interface TelemetryContext { + correlationId: string; + precedingEvents: TelemetryEvent[]; + followingEvents: TelemetryEvent[]; + sessionEvents: TelemetryEvent[]; + userJourney: UserJourneyStep[]; + deviceContext: DeviceContext; + apiCalls: ApiCall[]; +} + +interface UserJourneyStep { + timestamp: string; + screen: string; + action: string; + durationMs?: number; +} + +interface DeviceContext { + platform?: string; + osVersion?: string; + appVersion?: string; + deviceModel?: string; + screenContext?: string; + memory?: number; + batteryLevel?: number; + networkType?: string; +} + +interface ApiCall { + endpoint: string; + method: string; + statusCode?: number; + durationMs?: number; + timestamp: string; + error?: string; +} + +interface EnrichedErrorContext { + errorEvent: ErrorEvent; + telemetryContext?: TelemetryContext; + sessionState: SessionState; + recentActions: string[]; + apiFailures: ApiCall[]; + featureFlags: Record; + configSnapshot: Record; +} + +interface SessionState { + screen: string; + durationOnScreen: number; + previousScreen?: string; + userActions: string[]; + formData?: Record; + scrollPosition?: number; +} + +// ============================================================================ +// Container Access +// ============================================================================ + +function getTelemetryContainer(): Container { + return getRegisteredContainer('telemetry_events'); +} + +// ============================================================================ +// Correlation ID Propagation +// ============================================================================ + +/** + * Links errors to telemetry events via correlation ID + * Returns telemetry events within 5-minute window of the error + */ +export async function linkErrorToTelemetry( + errorEvent: ErrorEvent, + options: { + windowMinutes?: number; + maxEvents?: number; + } = {} +): Promise { + const windowMinutes = options.windowMinutes || 5; + const maxEvents = options.maxEvents || 50; + + if (!errorEvent.correlationId) { + // Try to find by user + timestamp if no correlation ID + return await linkByUserAndTime(errorEvent, windowMinutes, maxEvents); + } + + const container = getTelemetryContainer(); + + // Fetch events with same correlation ID + const query = ` + SELECT * FROM c + WHERE c.correlationId = @correlationId + AND c.productId = @productId + ORDER BY c.timestamp ASC + `; + + const { resources } = await container.items + .query({ + query, + parameters: [ + { name: '@correlationId', value: errorEvent.correlationId }, + { name: '@productId', value: errorEvent.productId }, + ], + }) + .fetchAll(); + + const events = resources as TelemetryEvent[]; + + if (events.length === 0) { + return null; + } + + // Find the error event position + const errorIndex = events.findIndex( + (e) => e.timestamp === errorEvent.timestamp + ); + + const windowStart = errorIndex >= 0 + ? Math.max(0, errorIndex - Math.floor(maxEvents / 2)) + : 0; + const windowEnd = Math.min(events.length, windowStart + maxEvents); + + const windowEvents = events.slice(windowStart, windowEnd); + const errorPosition = errorIndex >= 0 ? errorIndex - windowStart : windowEvents.length / 2; + + // Split into preceding and following events + const precedingEvents = windowEvents.slice(0, errorPosition); + const followingEvents = windowEvents.slice(errorPosition + 1); + + // Extract user journey + const userJourney = extractUserJourney(windowEvents); + + // Extract device context from the latest event + const deviceContext = extractDeviceContext(windowEvents[windowEvents.length - 1]); + + // Extract API calls + const apiCalls = extractApiCalls(windowEvents); + + return { + correlationId: errorEvent.correlationId, + precedingEvents, + followingEvents, + sessionEvents: windowEvents, + userJourney, + deviceContext, + apiCalls, + }; +} + +/** + * Fallback linking when correlation ID is not available + * Uses user ID + time window + */ +async function linkByUserAndTime( + errorEvent: ErrorEvent, + windowMinutes: number, + maxEvents: number +): Promise { + if (!errorEvent.userId) { + return null; + } + + const container = getTelemetryContainer(); + + const errorTime = new Date(errorEvent.timestamp); + const windowStart = new Date(errorTime.getTime() - windowMinutes * 60 * 1000).toISOString(); + const windowEnd = new Date(errorTime.getTime() + windowMinutes * 60 * 1000).toISOString(); + + const query = ` + SELECT * FROM c + WHERE c.userId = @userId + AND c.productId = @productId + AND c.timestamp >= @windowStart + AND c.timestamp <= @windowEnd + ORDER BY c.timestamp ASC + `; + + const { resources } = await container.items + .query({ + query, + parameters: [ + { name: '@userId', value: errorEvent.userId }, + { name: '@productId', value: errorEvent.productId }, + { name: '@windowStart', value: windowStart }, + { name: '@windowEnd', value: windowEnd }, + ], + }, { maxItemCount: maxEvents }) + .fetchAll(); + + const events = resources as TelemetryEvent[]; + + if (events.length === 0) { + return null; + } + + // Generate a synthetic correlation ID + const correlationId = `synth_${errorEvent.userId}_${errorTime.getTime()}`; + + // Find events before and after error timestamp + const errorTimeMs = errorTime.getTime(); + const errorIndex = events.findIndex( + (e) => new Date(e.timestamp).getTime() >= errorTimeMs + ); + + const precedingEvents = errorIndex > 0 ? events.slice(0, errorIndex) : []; + const followingEvents = errorIndex >= 0 ? events.slice(errorIndex + 1) : events; + + return { + correlationId, + precedingEvents, + followingEvents, + sessionEvents: events, + userJourney: extractUserJourney(events), + deviceContext: extractDeviceContext(events[events.length - 1]), + apiCalls: extractApiCalls(events), + }; +} + +// ============================================================================ +// Context Extraction +// ============================================================================ + +function extractUserJourney(events: TelemetryEvent[]): UserJourneyStep[] { + const journey: UserJourneyStep[] = []; + let lastScreen: string | null = null; + let lastTimestamp: string | null = null; + + for (const event of events) { + if (event.eventType === 'screen_view' || event.eventName.includes('screen')) { + const durationMs = lastTimestamp + ? new Date(event.timestamp).getTime() - new Date(lastTimestamp).getTime() + : undefined; + + journey.push({ + timestamp: event.timestamp, + screen: event.screen || event.properties?.screen as string || 'unknown', + action: event.eventName, + durationMs, + }); + + lastScreen = event.screen || null; + lastTimestamp = event.timestamp; + } + + if (event.eventType === 'action' || event.eventType === 'interaction') { + journey.push({ + timestamp: event.timestamp, + screen: lastScreen || 'unknown', + action: event.eventName, + }); + } + } + + return journey; +} + +function extractDeviceContext(event?: TelemetryEvent): DeviceContext { + if (!event) { + return {}; + } + + return { + platform: event.platform, + osVersion: event.osVersion, + appVersion: event.appVersion, + deviceModel: event.deviceModel, + screenContext: event.screen, + memory: event.properties?.memory as number, + batteryLevel: event.properties?.batteryLevel as number, + networkType: event.properties?.networkType as string, + }; +} + +function extractApiCalls(events: TelemetryEvent[]): ApiCall[] { + return events + .filter( + (e) => + e.eventType === 'api_call' || + e.eventName.includes('api') || + e.eventName.includes('request') + ) + .map((e) => ({ + endpoint: (e.properties?.endpoint as string) || 'unknown', + method: (e.properties?.method as string) || 'GET', + statusCode: e.properties?.statusCode as number, + durationMs: e.properties?.durationMs as number, + timestamp: e.timestamp, + error: e.properties?.error as string, + })); +} + +// ============================================================================ +// Error Context Enrichment +// ============================================================================ + +/** + * Enriches error event with full telemetry context + */ +export async function enrichErrorContext( + errorEvent: ErrorEvent +): Promise { + // Link to telemetry + const telemetryContext = await linkErrorToTelemetry(errorEvent, { + windowMinutes: 5, + maxEvents: 100, + }); + + // Build session state + const sessionState = buildSessionState(telemetryContext); + + // Extract recent actions + const recentActions = extractRecentActions(telemetryContext); + + // Extract API failures + const apiFailures = telemetryContext?.apiCalls.filter((call) => + call.error || (call.statusCode && call.statusCode >= 400) + ) || []; + + // Extract feature flags from telemetry + const featureFlags = extractFeatureFlags(telemetryContext); + + // Build config snapshot + const configSnapshot = buildConfigSnapshot(telemetryContext); + + return { + errorEvent, + telemetryContext, + sessionState, + recentActions, + apiFailures, + featureFlags, + configSnapshot, + }; +} + +function buildSessionState(telemetryContext: TelemetryContext | null): SessionState { + if (!telemetryContext || telemetryContext.sessionEvents.length === 0) { + return { + screen: 'unknown', + durationOnScreen: 0, + userActions: [], + }; + } + + const events = telemetryContext.sessionEvents; + const lastEvent = events[events.length - 1]; + + // Find last screen view + const screenViews = events.filter( + (e) => e.eventType === 'screen_view' || e.eventName.includes('screen') + ); + + const currentScreen = screenViews.length > 0 + ? screenViews[screenViews.length - 1].screen || + (screenViews[screenViews.length - 1].properties?.screen as string) + : 'unknown'; + + const previousScreen = screenViews.length > 1 + ? screenViews[screenViews.length - 2].screen || + (screenViews[screenViews.length - 2].properties?.screen as string) + : undefined; + + // Calculate duration on current screen + let durationOnScreen = 0; + if (screenViews.length >= 1) { + const screenEnterTime = new Date(screenViews[screenViews.length - 1].timestamp).getTime(); + const lastEventTime = new Date(lastEvent.timestamp).getTime(); + durationOnScreen = lastEventTime - screenEnterTime; + } + + // Extract user actions + const userActions = events + .filter((e) => e.eventType === 'action' || e.eventType === 'interaction') + .map((e) => e.eventName); + + return { + screen: currentScreen || 'unknown', + durationOnScreen, + previousScreen, + userActions, + scrollPosition: lastEvent.properties?.scrollPosition as number, + formData: lastEvent.properties?.formData as Record, + }; +} + +function extractRecentActions(telemetryContext: TelemetryContext | null): string[] { + if (!telemetryContext) return []; + + return telemetryContext.precedingEvents + .slice(-10) // Last 10 actions before error + .filter((e) => e.eventType === 'action' || e.eventType === 'interaction') + .map((e) => e.eventName); +} + +function extractFeatureFlags( + telemetryContext: TelemetryContext | null +): Record { + if (!telemetryContext) return {}; + + const flags: Record = {}; + + for (const event of telemetryContext.sessionEvents) { + if (event.properties?.featureFlags) { + Object.assign(flags, event.properties.featureFlags); + } + } + + return flags; +} + +function buildConfigSnapshot( + telemetryContext: TelemetryContext | null +): Record { + if (!telemetryContext) return {}; + + const config: Record = {}; + + for (const event of telemetryContext.sessionEvents) { + if (event.properties?.config) { + Object.assign(config, event.properties.config); + } + } + + return config; +} + +// ============================================================================ +// Breadcrumb Trail Generation +// ============================================================================ + +export interface Breadcrumb { + timestamp: string; + category: 'navigation' | 'action' | 'api' | 'error' | 'state'; + message: string; + data?: Record; +} + +/** + * Generates a breadcrumb trail from telemetry context + */ +export function generateBreadcrumbTrail( + telemetryContext: TelemetryContext | null +): Breadcrumb[] { + if (!telemetryContext) return []; + + const breadcrumbs: Breadcrumb[] = []; + + // Add navigation breadcrumbs + for (const step of telemetryContext.userJourney) { + breadcrumbs.push({ + timestamp: step.timestamp, + category: 'navigation', + message: `Screen: ${step.screen}`, + data: { durationMs: step.durationMs }, + }); + } + + // Add API call breadcrumbs + for (const call of telemetryContext.apiCalls) { + breadcrumbs.push({ + timestamp: call.timestamp, + category: 'api', + message: `${call.method} ${call.endpoint}`, + data: { + statusCode: call.statusCode, + durationMs: call.durationMs, + error: call.error, + }, + }); + } + + // Sort by timestamp + breadcrumbs.sort( + (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime() + ); + + return breadcrumbs; +} + +// ============================================================================ +// Cluster Context Aggregation +// ============================================================================ + +interface ClusterContextSummary { + totalOccurrences: number; + affectedUsers: string[]; + timeRange: { start: string; end: string }; + mostCommonScreens: Array<{ screen: string; count: number }>; + mostCommonActions: Array<{ action: string; count: number }>; + apiFailurePattern?: { + endpoint: string; + errorPattern: string; + occurrenceCount: number; + }; + featureFlagCorrelations: Array<{ + flag: string; + enabled: boolean; + errorCorrelation: number; + }>; +} + +/** + * Aggregates context across all errors in a cluster + */ +export async function aggregateClusterContext( + cluster: ErrorClusterDoc, + errorEvents: ErrorEvent[] +): Promise { + const affectedUsers = new Set(); + const screenCounts = new Map(); + const actionCounts = new Map(); + const apiErrors = new Map(); + const flagCorrelations = new Map(); + + let earliestTime = new Date(cluster.firstSeenAt); + let latestTime = new Date(cluster.lastSeenAt); + + for (const error of errorEvents) { + if (error.userId) { + affectedUsers.add(error.userId); + } + + // Track screens and actions + if (error.screen) { + screenCounts.set(error.screen, (screenCounts.get(error.screen) || 0) + 1); + } + + // Enrich and extract more context + const enriched = await enrichErrorContext(error); + + for (const action of enriched.recentActions) { + actionCounts.set(action, (actionCounts.get(action) || 0) + 1); + } + + // Track API failures + for (const apiFailure of enriched.apiFailures) { + const key = `${apiFailure.method} ${apiFailure.endpoint}`; + const existing = apiErrors.get(key) || { count: 0, errors: [] }; + existing.count++; + if (apiFailure.error) { + existing.errors.push(apiFailure.error); + } + apiErrors.set(key, existing); + } + + // Track feature flag correlations + for (const [flag, enabled] of Object.entries(enriched.featureFlags)) { + const existing = flagCorrelations.get(flag) || { enabled: 0, total: 0 }; + existing.total++; + if (enabled) existing.enabled++; + flagCorrelations.set(flag, existing); + } + } + + // Find most common API error pattern + let apiFailurePattern: ClusterContextSummary['apiFailurePattern'] | undefined; + let maxApiErrors = 0; + + for (const [endpoint, data] of apiErrors.entries()) { + if (data.count > maxApiErrors) { + maxApiErrors = data.count; + // Find common error pattern + const errorFrequency = new Map(); + for (const err of data.errors) { + errorFrequency.set(err, (errorFrequency.get(err) || 0) + 1); + } + const mostCommonError = Array.from(errorFrequency.entries()).sort((a, b) => b[1] - a[1])[0]; + + apiFailurePattern = { + endpoint, + errorPattern: mostCommonError?.[0] || 'unknown', + occurrenceCount: data.count, + }; + } + } + + // Calculate feature flag correlations + const featureFlagCorrelations = Array.from(flagCorrelations.entries()) + .map(([flag, data]) => ({ + flag, + enabled: data.enabled > 0, + errorCorrelation: data.enabled / data.total, + })) + .filter((f) => f.errorCorrelation > 0.5) // Only include flags with >50% correlation + .sort((a, b) => b.errorCorrelation - a.errorCorrelation) + .slice(0, 5); + + return { + totalOccurrences: errorEvents.length, + affectedUsers: Array.from(affectedUsers), + timeRange: { + start: earliestTime.toISOString(), + end: latestTime.toISOString(), + }, + mostCommonScreens: Array.from(screenCounts.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([screen, count]) => ({ screen, count })), + mostCommonActions: Array.from(actionCounts.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([action, count]) => ({ action, count })), + apiFailurePattern, + featureFlagCorrelations, + }; +} diff --git a/services/platform-service/src/modules/predictive-analytics/campaign-engine.ts b/services/platform-service/src/modules/predictive-analytics/campaign-engine.ts new file mode 100644 index 00000000..b61c8c72 --- /dev/null +++ b/services/platform-service/src/modules/predictive-analytics/campaign-engine.ts @@ -0,0 +1,570 @@ +/** + * Retention Campaign Engine - Automated intervention system + * [4.1] Campaign triggers and personalized messaging + * [4.2] Platform integrations (email/push/Slack) + */ + +import { getRegisteredContainer } from '@bytelyst/cosmos'; +import { bus as eventBus } from '../../lib/event-bus.js'; +import type { + RetentionCampaignDoc, + CreateCampaignInput, + ChurnPredictionInput, + UserChurnPredictionDoc, +} from './types.js'; + +export interface CampaignTriggerContext { + userId: string; + productId: string; + churnProbability: number; + riskSegment: string; + topRiskFactors: Array<{ feature: string; description: string }>; + suggestedActions: string[]; +} + +export interface CampaignDeliveryResult { + success: boolean; + channel: string; + messageId?: string; + error?: string; + sentAt: string; +} + +export class CampaignEngine { + /** + * Create a new retention campaign + */ + async createCampaign(input: CreateCampaignInput): Promise { + const container = getRegisteredContainer('retention_campaigns'); + + const doc: RetentionCampaignDoc = { + id: `rc_${crypto.randomUUID()}`, + ...input, + status: 'draft', + stats: { + triggered: 0, + sent: 0, + opened: 0, + clicked: 0, + converted: 0, + controlGroupSize: 0, + controlChurnRate: 0, + treatmentChurnRate: 0, + }, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + ttl: 365 * 24 * 60 * 60, // 1 year + }; + + await container.items.create(doc); + return doc; + } + + /** + * Activate a campaign + */ + async activateCampaign(campaignId: string): Promise { + const container = getRegisteredContainer('retention_campaigns'); + + try { + const { resource: doc } = await container.item(campaignId).read(); + if (!doc) return null; + + doc.status = 'active'; + doc.updatedAt = new Date().toISOString(); + + await container.item(campaignId).replace(doc); + return doc; + } catch { + return null; + } + } + + /** + * Trigger campaign for a user based on churn prediction + */ + async triggerForUser( + prediction: ChurnPredictionInput & { explanation: { topRiskFactors: Array<{ feature: string; description: string }>; suggestedActions: string[] } }, + testMode: boolean = false + ): Promise { + const campaigns = await this.getActiveCampaignsForProduct(prediction.productId); + const results: CampaignDeliveryResult[] = []; + + const context: CampaignTriggerContext = { + userId: prediction.userId, + productId: prediction.productId, + churnProbability: prediction.churnProbability, + riskSegment: prediction.riskSegment, + topRiskFactors: prediction.explanation.topRiskFactors.slice(0, 3), + suggestedActions: prediction.explanation.suggestedActions, + }; + + for (const campaign of campaigns) { + // Check if user matches campaign conditions + if (!this.matchesCampaignConditions(context, campaign)) { + continue; + } + + // Check frequency capping + if (await this.isFrequencyCapped(context.userId, campaign)) { + continue; + } + + // Deliver messages + for (const message of campaign.messages) { + const result = await this.deliverMessage(context, message, testMode); + results.push(result); + + if (result.success && !testMode) { + await this.recordDelivery(campaign.id, context.userId, message.channel); + } + } + + // Emit event for tracking + if (!testMode) { + eventBus.emit({ + type: 'predictive.campaign.triggered', + payload: { + campaignId: campaign.id, + userId: context.userId, + productId: context.productId, + riskSegment: context.riskSegment, + channels: campaign.messages.map((m) => m.channel), + }, + }); + } + } + + return results; + } + + /** + * Manual trigger for testing + */ + async manualTrigger( + campaignId: string, + testUserId: string + ): Promise { + const container = getRegisteredContainer('retention_campaigns'); + + try { + const { resource: campaign } = await container + .item(campaignId) + .read(); + if (!campaign) return []; + + const context: CampaignTriggerContext = { + userId: testUserId, + productId: campaign.productId, + churnProbability: 0.75, + riskSegment: 'high', + topRiskFactors: [ + { feature: 'daysSinceLastSession', description: 'Session recency declined' }, + ], + suggestedActions: ['Send re-engagement email'], + }; + + const results: CampaignDeliveryResult[] = []; + + for (const message of campaign.messages) { + const result = await this.deliverMessage(context, message, true); + results.push(result); + } + + return results; + } catch { + return []; + } + } + + /** + * Get active campaigns for a product + */ + private async getActiveCampaignsForProduct( + productId: string + ): Promise { + const container = getRegisteredContainer('retention_campaigns'); + + const query = { + query: 'SELECT * FROM c WHERE c.productId = @productId AND c.status = @status', + parameters: [ + { name: '@productId', value: productId }, + { name: '@status', value: 'active' }, + ], + }; + + const { resources } = await container.items.query(query).fetchAll(); + return resources; + } + + /** + * Check if user matches campaign conditions + */ + private matchesCampaignConditions( + context: CampaignTriggerContext, + campaign: RetentionCampaignDoc + ): boolean { + // Check risk segment match + if ( + campaign.audience.riskSegments?.length && + !campaign.audience.riskSegments.includes(context.riskSegment) + ) { + return false; + } + + // Check trigger conditions + for (const condition of campaign.trigger.conditions) { + const value = this.getContextValue(context, condition.field); + + switch (condition.operator) { + case 'gt': + if (!(value > (condition.value as number))) return false; + break; + case 'lt': + if (!(value < (condition.value as number))) return false; + break; + case 'eq': + if (value !== condition.value) return false; + break; + case 'in': + if (!(condition.value as unknown[]).includes(value)) return false; + break; + } + } + + return true; + } + + /** + * Get value from context by field path + */ + private getContextValue(context: CampaignTriggerContext, field: string): unknown { + const parts = field.split('.'); + let value: unknown = context; + + for (const part of parts) { + if (value && typeof value === 'object') { + value = (value as Record)[part]; + } else { + return undefined; + } + } + + return value; + } + + /** + * Check if user is frequency capped + */ + private async isFrequencyCapped( + userId: string, + campaign: RetentionCampaignDoc + ): Promise { + if (!campaign.audience.excludeRecentContact) return false; + + const container = getRegisteredContainer('campaign_deliveries'); + const hoursAgo = new Date(); + hoursAgo.setHours(hoursAgo.getHours() - campaign.audience.excludeRecentContact); + + const query = { + query: + 'SELECT VALUE COUNT(1) FROM c WHERE c.userId = @userId AND c.sentAt >= @cutoff', + parameters: [ + { name: '@userId', value: userId }, + { name: '@cutoff', value: hoursAgo.toISOString() }, + ], + }; + + try { + const { resources } = await container.items.query(query).fetchAll(); + return (resources[0] || 0) > 0; + } catch { + return false; + } + } + + /** + * Deliver message through appropriate channel + */ + private async deliverMessage( + context: CampaignTriggerContext, + message: { channel: string; templateId: string; variant?: string; delayHours?: number }, + testMode: boolean + ): Promise { + const sentAt = new Date().toISOString(); + + // Simulate delay + if (message.delayHours && !testMode) { + // In production, this would schedule for later delivery + } + + switch (message.channel) { + case 'email': + return this.deliverEmail(context, message.templateId, testMode, sentAt); + case 'push': + return this.deliverPush(context, message.templateId, testMode, sentAt); + case 'in_app': + return this.deliverInApp(context, message.templateId, testMode, sentAt); + case 'slack_cs': + return this.deliverSlackCS(context, message.templateId, testMode, sentAt); + default: + return { + success: false, + channel: message.channel, + error: 'Unknown channel', + sentAt, + }; + } + } + + /** + * Deliver email via delivery module + */ + private async deliverEmail( + context: CampaignTriggerContext, + templateId: string, + testMode: boolean, + sentAt: string + ): Promise { + if (testMode) { + return { + success: true, + channel: 'email', + messageId: `test_${crypto.randomUUID()}`, + sentAt, + }; + } + + try { + // In production, call delivery module + // await deliveryService.sendEmail({...}) + + eventBus.emit({ + type: 'delivery.email.requested', + payload: { + userId: context.userId, + productId: context.productId, + templateId, + context: { + churnProbability: context.churnProbability, + riskSegment: context.riskSegment, + suggestedActions: context.suggestedActions, + }, + }, + }); + + return { + success: true, + channel: 'email', + messageId: `email_${crypto.randomUUID()}`, + sentAt, + }; + } catch (error) { + return { + success: false, + channel: 'email', + error: error instanceof Error ? error.message : 'Email delivery failed', + sentAt, + }; + } + } + + /** + * Deliver push notification + */ + private async deliverPush( + context: CampaignTriggerContext, + templateId: string, + testMode: boolean, + sentAt: string + ): Promise { + if (testMode) { + return { + success: true, + channel: 'push', + messageId: `test_push_${crypto.randomUUID()}`, + sentAt, + }; + } + + eventBus.emit({ + type: 'notifications.push.requested', + payload: { + userId: context.userId, + productId: context.productId, + title: 'We miss you!', + body: 'Come back and explore new features.', + data: { campaign: 'retention', riskSegment: context.riskSegment }, + }, + }); + + return { + success: true, + channel: 'push', + messageId: `push_${crypto.randomUUID()}`, + sentAt, + }; + } + + /** + * Deliver in-app message + */ + private async deliverInApp( + context: CampaignTriggerContext, + templateId: string, + testMode: boolean, + sentAt: string + ): Promise { + if (testMode) { + return { + success: true, + channel: 'in_app', + messageId: `test_inapp_${crypto.randomUUID()}`, + sentAt, + }; + } + + eventBus.emit({ + type: 'notifications.inapp.create', + payload: { + userId: context.userId, + productId: context.productId, + title: 'Personalized for you', + content: 'Based on your interests, check out these features.', + priority: context.riskSegment === 'critical' ? 'high' : 'normal', + }, + }); + + return { + success: true, + channel: 'in_app', + messageId: `inapp_${crypto.randomUUID()}`, + sentAt, + }; + } + + /** + * Deliver Slack notification to CS team + */ + private async deliverSlackCS( + context: CampaignTriggerContext, + templateId: string, + testMode: boolean, + sentAt: string + ): Promise { + if (testMode) { + return { + success: true, + channel: 'slack_cs', + messageId: `test_slack_${crypto.randomUUID()}`, + sentAt, + }; + } + + const message = { + text: `🚨 High Risk User Alert`, + blocks: [ + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*High Risk User Detected*\nUser: ${context.userId}\nProduct: ${context.productId}\nChurn Probability: ${Math.round(context.churnProbability * 100)}%\nRisk Segment: ${context.riskSegment}`, + }, + }, + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*Top Risk Factors:*\n${context.topRiskFactors.map((f) => `- ${f.description}`).join('\n')}`, + }, + }, + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*Suggested Actions:*\n${context.suggestedActions.map((a) => `- ${a}`).join('\n')}`, + }, + }, + ], + }; + + eventBus.emit({ + type: 'integrations.slack.notify', + payload: { + channel: '#customer-success', + message, + }, + }); + + return { + success: true, + channel: 'slack_cs', + messageId: `slack_${crypto.randomUUID()}`, + sentAt, + }; + } + + /** + * Record campaign delivery + */ + private async recordDelivery( + campaignId: string, + userId: string, + channel: string + ): Promise { + const container = getRegisteredContainer('campaign_deliveries'); + + await container.items.create({ + id: `cd_${crypto.randomUUID()}`, + campaignId, + userId, + channel, + sentAt: new Date().toISOString(), + openedAt: null, + clickedAt: null, + convertedAt: null, + ttl: 90 * 24 * 60 * 60, + }); + } + + /** + * Get campaign statistics + */ + async getCampaignStats(campaignId: string): Promise<{ + triggered: number; + sent: number; + opened: number; + clicked: number; + converted: number; + conversionRate: number; + } | null> { + const container = getRegisteredContainer('campaign_deliveries'); + + const query = { + query: 'SELECT * FROM c WHERE c.campaignId = @campaignId', + parameters: [{ name: '@campaignId', value: campaignId }], + }; + + try { + const { resources } = await container.items.query(query).fetchAll(); + + const sent = resources.length; + const opened = resources.filter((r) => r.openedAt).length; + const clicked = resources.filter((r) => r.clickedAt).length; + const converted = resources.filter((r) => r.convertedAt).length; + + return { + triggered: sent, + sent, + opened, + clicked, + converted, + conversionRate: sent > 0 ? converted / sent : 0, + }; + } catch { + return null; + } + } +} + +export const campaignEngine = new CampaignEngine(); diff --git a/services/platform-service/src/modules/predictive-analytics/repository.ts b/services/platform-service/src/modules/predictive-analytics/repository.ts new file mode 100644 index 00000000..0795842f --- /dev/null +++ b/services/platform-service/src/modules/predictive-analytics/repository.ts @@ -0,0 +1,235 @@ +/** + * Predictive Analytics Repository - Data access layer + */ + +import { getRegisteredContainer } from '@bytelyst/cosmos'; +import type { + UserChurnPredictionDoc, + ProductHealthScoreDoc, + RetentionCampaignDoc, + UserFeatureVectorDoc, + FeatureDefinition, + ModelPerformanceDoc, +} from './types.js'; + +export class PredictiveAnalyticsRepository { + // ==================== Churn Predictions ==================== + + async saveChurnPrediction(doc: UserChurnPredictionDoc): Promise { + const container = getRegisteredContainer('churn_predictions'); + await container.items.create(doc); + return doc; + } + + async getChurnPrediction(userId: string, productId: string, horizon: number = 30): Promise { + const container = getRegisteredContainer('churn_predictions'); + const query = { + query: 'SELECT * FROM c WHERE c.userId = @userId AND c.productId = @productId AND c.predictionHorizon = @horizon ORDER BY c.predictionTimestamp DESC OFFSET 0 LIMIT 1', + parameters: [ + { name: '@userId', value: userId }, + { name: '@productId', value: productId }, + { name: '@horizon', value: horizon }, + ], + }; + const { resources } = await container.items.query(query).fetchAll(); + return resources[0] || null; + } + + async getAtRiskUsers( + productId: string, + segment?: string, + limit: number = 50, + offset: number = 0 + ): Promise { + const container = getRegisteredContainer('churn_predictions'); + let queryStr = 'SELECT * FROM c WHERE c.productId = @productId'; + const parameters: Array<{ name: string; value: unknown }> = [ + { name: '@productId', value: productId }, + ]; + + if (segment) { + queryStr += ' AND c.riskSegment = @segment'; + parameters.push({ name: '@segment', value: segment }); + } + + queryStr += ' ORDER BY c.churnProbability DESC OFFSET @offset LIMIT @limit'; + parameters.push({ name: '@offset', value: offset }); + parameters.push({ name: '@limit', value: limit }); + + const { resources } = await container.items.query({ + query: queryStr, + parameters, + }).fetchAll(); + return resources; + } + + async getUserRiskProfile(userId: string, productId: string): Promise { + const container = getRegisteredContainer('churn_predictions'); + const query = { + query: 'SELECT * FROM c WHERE c.userId = @userId AND c.productId = @productId ORDER BY c.predictionTimestamp DESC', + parameters: [ + { name: '@userId', value: userId }, + { name: '@productId', value: productId }, + ], + }; + const { resources } = await container.items.query(query).fetchAll(); + return resources; + } + + // ==================== Health Scores ==================== + + async saveHealthScore(doc: ProductHealthScoreDoc): Promise { + const container = getRegisteredContainer('product_health'); + await container.items.create(doc); + return doc; + } + + async getHealthScore(productId: string, date: string): Promise { + const container = getRegisteredContainer('product_health'); + try { + const { resource } = await container.item(`${productId}:${date}`, productId).read(); + return resource || null; + } catch { + return null; + } + } + + async getHealthHistory(productId: string, days: number = 30): Promise { + const container = getRegisteredContainer('product_health'); + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - days); + + const query = { + query: 'SELECT * FROM c WHERE c.productId = @productId AND c.date >= @cutoff ORDER BY c.date DESC', + parameters: [ + { name: '@productId', value: productId }, + { name: '@cutoff', value: cutoff.toISOString().split('T')[0] }, + ], + }; + const { resources } = await container.items.query(query).fetchAll(); + return resources; + } + + async getAllProductHealthScores(): Promise { + const container = getRegisteredContainer('product_health'); + const today = new Date().toISOString().split('T')[0]; + + const query = { + query: 'SELECT * FROM c WHERE c.date = @today', + parameters: [{ name: '@today', value: today }], + }; + const { resources } = await container.items.query(query).fetchAll(); + return resources; + } + + // ==================== Campaigns ==================== + + async getCampaign(campaignId: string): Promise { + const container = getRegisteredContainer('retention_campaigns'); + try { + const { resource } = await container.item(campaignId, campaignId).read(); + return resource || null; + } catch { + return null; + } + } + + async listCampaigns(productId?: string, status?: string): Promise { + const container = getRegisteredContainer('retention_campaigns'); + let queryStr = 'SELECT * FROM c WHERE 1=1'; + const parameters: Array<{ name: string; value: unknown }> = []; + + if (productId) { + queryStr += ' AND c.productId = @productId'; + parameters.push({ name: '@productId', value: productId }); + } + + if (status) { + queryStr += ' AND c.status = @status'; + parameters.push({ name: '@status', value: status }); + } + + queryStr += ' ORDER BY c.createdAt DESC'; + + const { resources } = await container.items.query({ + query: queryStr, + parameters, + }).fetchAll(); + return resources; + } + + async updateCampaign( + campaignId: string, + updates: Partial + ): Promise { + const container = getRegisteredContainer('retention_campaigns'); + + try { + const { resource: doc } = await container.item(campaignId).read(); + if (!doc) return null; + + const updated = { ...doc, ...updates, updatedAt: new Date().toISOString() }; + await container.item(campaignId).replace(updated); + return updated; + } catch { + return null; + } + } + + async deleteCampaign(campaignId: string): Promise { + const container = getRegisteredContainer('retention_campaigns'); + try { + await container.item(campaignId).delete(); + return true; + } catch { + return false; + } + } + + // ==================== Feature Definitions ==================== + + async saveFeatureDefinition(doc: FeatureDefinition): Promise { + const container = getRegisteredContainer('feature_definitions'); + await container.items.create(doc); + return doc; + } + + async getFeatureDefinitions(productId: string): Promise { + const container = getRegisteredContainer('feature_definitions'); + const query = { + query: 'SELECT * FROM c WHERE c.productId = @productId', + parameters: [{ name: '@productId', value: productId }], + }; + const { resources } = await container.items.query(query).fetchAll(); + return resources; + } + + // ==================== Model Performance ==================== + + async saveModelPerformance(doc: ModelPerformanceDoc): Promise { + const container = getRegisteredContainer('model_performance'); + await container.items.create(doc); + return doc; + } + + async getLatestModelPerformance(): Promise { + const container = getRegisteredContainer('model_performance'); + const query = { + query: 'SELECT * FROM c WHERE c.isActive = true ORDER BY c.createdAt DESC OFFSET 0 LIMIT 1', + }; + const { resources } = await container.items.query(query).fetchAll(); + return resources[0] || null; + } + + async getModelPerformanceHistory(limit: number = 10): Promise { + const container = getRegisteredContainer('model_performance'); + const query = { + query: 'SELECT * FROM c ORDER BY c.createdAt DESC OFFSET 0 LIMIT @limit', + parameters: [{ name: '@limit', value: limit }], + }; + const { resources } = await container.items.query(query).fetchAll(); + return resources; + } +} + +export const predictiveAnalyticsRepo = new PredictiveAnalyticsRepository();