feat(mcp-server): A2A batch-3 — ReflectionSynthesisAgent (mindlyst) + KeyboardDiagnosticsAgent (lysnrai) + NLParserEvalAgent (chronomind)
reflection-synthesis-pipeline.ts: mindlyst.reflections.synthesize - ReflectionCollectorAgent -> ThemeExtractionAgent -> SynthesisReportAgent - Fetches recent reflections, runs reflection-enrichment extraction, surfaces themes by frequency - Proposes weekly summary text; max 52 reflections (1 year lookback) keyboard-diagnostics-pipeline.ts: lysnrai.keyboard.diagnose - KeyboardErrorScannerAgent -> DiagnosticsSessionAgent -> BugReportDraftAgent - Scans error telemetry filtered to keyboard surface, groups by anonymousInstallId - Opens diagnostics session per erroring install; drafts severity-ranked bug reports - 48h default window, configurable minErrors threshold nl-parser-eval-pipeline.ts: chronomind.nlParser.eval - PhraseSamplerAgent -> ParseEvalAgent -> RegressionReportAgent - 10-phrase canonical test suite + optional custom phrases - Submits to extraction-service timer-parse task, validates parsed field presence - Returns pass/fail/partial scorecard with regression list MCP server total: 102 tools
This commit is contained in:
saravanakumardb1
parent
b8e230f018
commit
7ed4a105b7
@ -0,0 +1,282 @@
|
||||
/**
|
||||
* KeyboardDiagnosticsAgent — A2A pipeline for LysnrAI iOS keyboard extension diagnostics.
|
||||
*
|
||||
* Agent roster (3 steps):
|
||||
* 1. KeyboardErrorScannerAgent — query telemetry for 'error' events from keyboard install IDs
|
||||
* 2. DiagnosticsSessionAgent — open a diagnostics session targeting each erroring installId
|
||||
* 3. BugReportDraftAgent — assemble structured bug report per install ID with error details
|
||||
*
|
||||
* MCP tools:
|
||||
* lysnrai.keyboard.diagnose(from?, to?, minErrors?) — run pipeline
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { z } from 'zod';
|
||||
import { registerTool } from '../tools/registry.js';
|
||||
import type { McpToolRequest } from '../tools/types.js';
|
||||
import {
|
||||
telemetryQuery,
|
||||
diagnosticsCreateSession,
|
||||
diagnosticsUpdateSession,
|
||||
type DebugSession,
|
||||
} from '../../lib/platform-client.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface KeyboardErrorGroup {
|
||||
anonymousInstallId: string;
|
||||
errorCount: number;
|
||||
errorTypes: string[];
|
||||
firstSeen: string;
|
||||
lastSeen: string;
|
||||
rawEvents: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
interface DiagnosticsResult {
|
||||
anonymousInstallId: string;
|
||||
session: DebugSession | null;
|
||||
skipReason?: string;
|
||||
}
|
||||
|
||||
interface BugReportDraft {
|
||||
anonymousInstallId: string;
|
||||
diagnosticsSessionId: string | null;
|
||||
errorCount: number;
|
||||
errorTypes: string[];
|
||||
firstSeen: string;
|
||||
lastSeen: string;
|
||||
reproSteps: string;
|
||||
severity: 'critical' | 'high' | 'medium';
|
||||
}
|
||||
|
||||
export interface KeyboardDiagnosticsReport {
|
||||
runId: string;
|
||||
productId: 'lysnrai';
|
||||
from: string;
|
||||
to: string;
|
||||
installsScanned: number;
|
||||
erroringInstalls: number;
|
||||
diagnosticsOpened: number;
|
||||
bugReports: BugReportDraft[];
|
||||
summary: string;
|
||||
generatedAt: string;
|
||||
}
|
||||
|
||||
// ── Step 1: KeyboardErrorScannerAgent ─────────────────────────────────────────
|
||||
|
||||
async function scanKeyboardErrors(
|
||||
fromTime: string,
|
||||
toTime: string,
|
||||
minErrors: number,
|
||||
opts: { token?: string; requestId?: string }
|
||||
): Promise<KeyboardErrorGroup[]> {
|
||||
const installMap = new Map<string, KeyboardErrorGroup>();
|
||||
|
||||
try {
|
||||
const result = await telemetryQuery(
|
||||
{
|
||||
productId: 'lysnrai',
|
||||
eventType: 'error',
|
||||
from: fromTime,
|
||||
to: toTime,
|
||||
limit: 500,
|
||||
},
|
||||
{ ...opts, productId: 'lysnrai' }
|
||||
);
|
||||
|
||||
const events = result.events as Array<Record<string, unknown>>;
|
||||
|
||||
for (const event of events) {
|
||||
const props = (event['properties'] as Record<string, unknown>) ?? {};
|
||||
const surface = String(props['surface'] ?? event['surface'] ?? '').toLowerCase();
|
||||
if (!surface.includes('keyboard')) continue;
|
||||
|
||||
const installId = String(
|
||||
props['anonymousInstallId'] ??
|
||||
event['anonymousId'] ??
|
||||
event['anonymousInstallId'] ??
|
||||
'unknown'
|
||||
);
|
||||
const errorType = String(
|
||||
props['errorType'] ?? props['error_type'] ?? event['eventType'] ?? 'unknown'
|
||||
);
|
||||
const timestamp = String(
|
||||
event['timestamp'] ?? event['createdAt'] ?? new Date().toISOString()
|
||||
);
|
||||
|
||||
const group = installMap.get(installId) ?? {
|
||||
anonymousInstallId: installId,
|
||||
errorCount: 0,
|
||||
errorTypes: [],
|
||||
firstSeen: timestamp,
|
||||
lastSeen: timestamp,
|
||||
rawEvents: [],
|
||||
};
|
||||
|
||||
group.errorCount++;
|
||||
if (!group.errorTypes.includes(errorType)) group.errorTypes.push(errorType);
|
||||
if (timestamp < group.firstSeen) group.firstSeen = timestamp;
|
||||
if (timestamp > group.lastSeen) group.lastSeen = timestamp;
|
||||
group.rawEvents.push(event);
|
||||
|
||||
installMap.set(installId, group);
|
||||
}
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
|
||||
return Array.from(installMap.values()).filter(g => g.errorCount >= minErrors);
|
||||
}
|
||||
|
||||
// ── Step 2: DiagnosticsSessionAgent ───────────────────────────────────────────
|
||||
|
||||
async function openDiagnosticsSession(
|
||||
group: KeyboardErrorGroup,
|
||||
opts: { token?: string; requestId?: string }
|
||||
): Promise<DiagnosticsResult> {
|
||||
try {
|
||||
const session = await diagnosticsCreateSession(
|
||||
{
|
||||
productId: 'lysnrai',
|
||||
targetAnonymousId: group.anonymousInstallId,
|
||||
collectionLevel: 'debug',
|
||||
captureLogs: true,
|
||||
},
|
||||
opts
|
||||
);
|
||||
return { anonymousInstallId: group.anonymousInstallId, session };
|
||||
} catch (err) {
|
||||
return {
|
||||
anonymousInstallId: group.anonymousInstallId,
|
||||
session: null,
|
||||
skipReason: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 3: BugReportDraftAgent ───────────────────────────────────────────────
|
||||
|
||||
async function closeDiagnosticsAndDraft(
|
||||
group: KeyboardErrorGroup,
|
||||
diagnosticsResult: DiagnosticsResult,
|
||||
opts: { requestId?: string }
|
||||
): Promise<BugReportDraft> {
|
||||
if (diagnosticsResult.session) {
|
||||
try {
|
||||
await diagnosticsUpdateSession(diagnosticsResult.session.id, { status: 'completed' }, opts);
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
|
||||
const severity: 'critical' | 'high' | 'medium' =
|
||||
group.errorCount >= 10 ? 'critical' : group.errorCount >= 5 ? 'high' : 'medium';
|
||||
|
||||
const reproSteps = [
|
||||
`1. Open LysnrAI iOS keyboard extension on device with installId ${group.anonymousInstallId}.`,
|
||||
`2. Trigger dictation — error types observed: ${group.errorTypes.join(', ')}.`,
|
||||
`3. First seen ${group.firstSeen}, last seen ${group.lastSeen} (${group.errorCount} occurrences).`,
|
||||
diagnosticsResult.session
|
||||
? `4. Diagnostics session ${diagnosticsResult.session.id} opened — check logs for stack traces.`
|
||||
: '4. Diagnostics session could not be opened automatically.',
|
||||
].join('\n');
|
||||
|
||||
return {
|
||||
anonymousInstallId: group.anonymousInstallId,
|
||||
diagnosticsSessionId: diagnosticsResult.session?.id ?? null,
|
||||
errorCount: group.errorCount,
|
||||
errorTypes: group.errorTypes,
|
||||
firstSeen: group.firstSeen,
|
||||
lastSeen: group.lastSeen,
|
||||
reproSteps,
|
||||
severity,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pipeline runner ────────────────────────────────────────────────────────────
|
||||
|
||||
async function runKeyboardDiagnosticsPipeline(
|
||||
fromTime: string,
|
||||
toTime: string,
|
||||
minErrors: number,
|
||||
req: McpToolRequest
|
||||
): Promise<KeyboardDiagnosticsReport> {
|
||||
const runId = randomUUID();
|
||||
const opts = { token: req.headers.authorization?.slice(7), requestId: req.id };
|
||||
|
||||
req.log.info(
|
||||
{ runId, stepId: 'scan', fromTime, toTime, minErrors },
|
||||
'KeyboardErrorScannerAgent start'
|
||||
);
|
||||
const errorGroups = await scanKeyboardErrors(fromTime, toTime, minErrors, opts);
|
||||
req.log.info(
|
||||
{ runId, stepId: 'scan', erroringInstalls: errorGroups.length },
|
||||
'KeyboardErrorScannerAgent done'
|
||||
);
|
||||
|
||||
req.log.info(
|
||||
{ runId, stepId: 'diagnostics', count: errorGroups.length },
|
||||
'DiagnosticsSessionAgent start'
|
||||
);
|
||||
const diagnosticsResults: DiagnosticsResult[] = [];
|
||||
for (const group of errorGroups) {
|
||||
const result = await openDiagnosticsSession(group, opts);
|
||||
diagnosticsResults.push(result);
|
||||
}
|
||||
const opened = diagnosticsResults.filter(r => r.session !== null).length;
|
||||
req.log.info({ runId, stepId: 'diagnostics', opened }, 'DiagnosticsSessionAgent done');
|
||||
|
||||
req.log.info({ runId, stepId: 'draft', count: errorGroups.length }, 'BugReportDraftAgent start');
|
||||
const bugReports: BugReportDraft[] = [];
|
||||
for (let i = 0; i < errorGroups.length; i++) {
|
||||
const draft = await closeDiagnosticsAndDraft(errorGroups[i]!, diagnosticsResults[i]!, {
|
||||
requestId: req.id,
|
||||
});
|
||||
bugReports.push(draft);
|
||||
}
|
||||
|
||||
const summary =
|
||||
errorGroups.length === 0
|
||||
? `No keyboard errors with >= ${minErrors} occurrence(s) found in the specified window.`
|
||||
: `${errorGroups.length} install(s) with keyboard errors detected. ${opened} diagnostics session(s) opened. ${bugReports.filter(r => r.severity === 'critical').length} critical, ${bugReports.filter(r => r.severity === 'high').length} high severity.`;
|
||||
|
||||
req.log.info({ runId, stepId: 'draft', summary }, 'BugReportDraftAgent done');
|
||||
|
||||
return {
|
||||
runId,
|
||||
productId: 'lysnrai',
|
||||
from: fromTime,
|
||||
to: toTime,
|
||||
installsScanned: errorGroups.length,
|
||||
erroringInstalls: errorGroups.length,
|
||||
diagnosticsOpened: opened,
|
||||
bugReports,
|
||||
summary,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── MCP tool registration ─────────────────────────────────────────────────────
|
||||
|
||||
registerTool({
|
||||
name: 'lysnrai.keyboard.diagnose',
|
||||
description:
|
||||
'A2A pipeline: scans LysnrAI telemetry for error events from the iOS keyboard extension, groups errors by anonymousInstallId, opens a diagnostics session per affected install, and drafts a structured bug report. Returns severity-ranked reports with repro steps. Requires admin role.',
|
||||
requiredRole: 'admin',
|
||||
inputSchema: z.object({
|
||||
from: z.string().datetime().optional().describe('Start of telemetry window (default: 48h ago)'),
|
||||
to: z.string().datetime().optional().describe('End of telemetry window (default: now)'),
|
||||
minErrors: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.default(3)
|
||||
.describe('Minimum error count per install to include in report (default 3)'),
|
||||
}),
|
||||
async execute(args, req) {
|
||||
const now = new Date();
|
||||
const toTime = args.to ?? now.toISOString();
|
||||
const fromTime = args.from ?? new Date(now.getTime() - 48 * 60 * 60 * 1000).toISOString();
|
||||
return runKeyboardDiagnosticsPipeline(fromTime, toTime, args.minErrors, req);
|
||||
},
|
||||
});
|
||||
266
services/mcp-server/src/modules/a2a/nl-parser-eval-pipeline.ts
Normal file
266
services/mcp-server/src/modules/a2a/nl-parser-eval-pipeline.ts
Normal file
@ -0,0 +1,266 @@
|
||||
/**
|
||||
* NLParserEvalAgent — A2A pipeline for ChronoMind natural-language timer parsing evaluation.
|
||||
*
|
||||
* Agent roster (3 steps):
|
||||
* 1. PhraseSamplerAgent — assemble canonical NL phrase test suite
|
||||
* 2. ParseEvalAgent — run each phrase through extraction service 'timer-parse' task
|
||||
* 3. RegressionReportAgent — validate parsed fields, flag failures, produce pass/fail scorecard
|
||||
*
|
||||
* MCP tools:
|
||||
* chronomind.nlParser.eval(customPhrases?) — run eval pipeline; returns pass/fail scorecard
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { z } from 'zod';
|
||||
import { registerTool } from '../tools/registry.js';
|
||||
import type { McpToolRequest } from '../tools/types.js';
|
||||
import { extractionRun } from '../../lib/extraction-client.js';
|
||||
|
||||
// ── Canonical test suite ──────────────────────────────────────────────────────
|
||||
|
||||
interface NLTestCase {
|
||||
phrase: string;
|
||||
expectedFields: string[];
|
||||
description: string;
|
||||
}
|
||||
|
||||
const CANONICAL_TEST_CASES: NLTestCase[] = [
|
||||
{
|
||||
phrase: 'Remind me in 30 minutes',
|
||||
expectedFields: ['duration'],
|
||||
description: 'Simple countdown',
|
||||
},
|
||||
{
|
||||
phrase: 'Wake me up at 7am tomorrow',
|
||||
expectedFields: ['targetTime'],
|
||||
description: 'Alarm at specific time',
|
||||
},
|
||||
{
|
||||
phrase: 'Meeting in 2 hours and 15 minutes',
|
||||
expectedFields: ['duration'],
|
||||
description: 'Compound duration',
|
||||
},
|
||||
{
|
||||
phrase: 'Call mom every day at 6pm',
|
||||
expectedFields: ['targetTime', 'recurrence'],
|
||||
description: 'Recurring alarm',
|
||||
},
|
||||
{
|
||||
phrase: 'Study session for 45 minutes then break for 10',
|
||||
expectedFields: ['duration'],
|
||||
description: 'Linked timers',
|
||||
},
|
||||
{
|
||||
phrase: 'Pomodoro 25 minutes focus',
|
||||
expectedFields: ['duration', 'timerType'],
|
||||
description: 'Pomodoro type',
|
||||
},
|
||||
{
|
||||
phrase: 'Take meds at 8am, noon, and 8pm',
|
||||
expectedFields: ['targetTime', 'recurrence'],
|
||||
description: 'Multi-time alarm',
|
||||
},
|
||||
{
|
||||
phrase: 'Boil eggs for 7 minutes',
|
||||
expectedFields: ['duration', 'label'],
|
||||
description: 'Labeled timer',
|
||||
},
|
||||
{
|
||||
phrase: 'Set a 1 hour countdown',
|
||||
expectedFields: ['duration'],
|
||||
description: 'Explicit countdown',
|
||||
},
|
||||
{
|
||||
phrase: 'Dentist appointment in 3 days',
|
||||
expectedFields: ['targetTime'],
|
||||
description: 'Days-ahead event',
|
||||
},
|
||||
];
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
type EvalOutcome = 'pass' | 'fail' | 'partial' | 'error';
|
||||
|
||||
interface ParseEvalResult {
|
||||
phrase: string;
|
||||
description: string;
|
||||
expectedFields: string[];
|
||||
parsedFields: string[];
|
||||
missingFields: string[];
|
||||
outcome: EvalOutcome;
|
||||
errorMessage?: string;
|
||||
rawExtraction: unknown;
|
||||
}
|
||||
|
||||
export interface NLParserEvalReport {
|
||||
runId: string;
|
||||
productId: 'chronomind';
|
||||
totalCases: number;
|
||||
passed: number;
|
||||
partial: number;
|
||||
failed: number;
|
||||
errored: number;
|
||||
passRate: number;
|
||||
results: ParseEvalResult[];
|
||||
regressions: string[];
|
||||
summary: string;
|
||||
generatedAt: string;
|
||||
}
|
||||
|
||||
// ── Step 1: PhraseSamplerAgent ────────────────────────────────────────────────
|
||||
|
||||
function assemblePhrases(customPhrases: string[]): NLTestCase[] {
|
||||
const custom: NLTestCase[] = customPhrases.map(phrase => ({
|
||||
phrase,
|
||||
expectedFields: ['duration'],
|
||||
description: 'Custom phrase',
|
||||
}));
|
||||
return [...CANONICAL_TEST_CASES, ...custom];
|
||||
}
|
||||
|
||||
// ── Step 2: ParseEvalAgent ────────────────────────────────────────────────────
|
||||
|
||||
async function evalPhrase(
|
||||
testCase: NLTestCase,
|
||||
opts: { token?: string; requestId?: string }
|
||||
): Promise<ParseEvalResult> {
|
||||
try {
|
||||
const result = await extractionRun({ text: testCase.phrase, taskId: 'timer-parse' }, opts);
|
||||
|
||||
// Extract field names from the extraction result
|
||||
const parsedFields: string[] = [];
|
||||
for (const item of result.extractions) {
|
||||
const itemRecord = item as unknown as Record<string, unknown>;
|
||||
const type = String(itemRecord['type'] ?? itemRecord['entity'] ?? '').toLowerCase();
|
||||
if (type) parsedFields.push(type);
|
||||
// Also check properties for known timer fields
|
||||
const props = itemRecord['properties'] as Record<string, unknown> | undefined;
|
||||
if (props) {
|
||||
for (const key of ['duration', 'targetTime', 'timerType', 'recurrence', 'label']) {
|
||||
if (key in props && !parsedFields.includes(key)) parsedFields.push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const missingFields = testCase.expectedFields.filter(
|
||||
f => !parsedFields.some(p => p.includes(f.toLowerCase()))
|
||||
);
|
||||
const outcome: EvalOutcome =
|
||||
missingFields.length === 0
|
||||
? 'pass'
|
||||
: missingFields.length < testCase.expectedFields.length
|
||||
? 'partial'
|
||||
: 'fail';
|
||||
|
||||
return {
|
||||
phrase: testCase.phrase,
|
||||
description: testCase.description,
|
||||
expectedFields: testCase.expectedFields,
|
||||
parsedFields,
|
||||
missingFields,
|
||||
outcome,
|
||||
rawExtraction: result.extractions,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
phrase: testCase.phrase,
|
||||
description: testCase.description,
|
||||
expectedFields: testCase.expectedFields,
|
||||
parsedFields: [],
|
||||
missingFields: testCase.expectedFields,
|
||||
outcome: 'error',
|
||||
errorMessage: err instanceof Error ? err.message : String(err),
|
||||
rawExtraction: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 3: RegressionReportAgent ─────────────────────────────────────────────
|
||||
|
||||
function buildEvalReport(runId: string, results: ParseEvalResult[]): NLParserEvalReport {
|
||||
const passed = results.filter(r => r.outcome === 'pass').length;
|
||||
const partial = results.filter(r => r.outcome === 'partial').length;
|
||||
const failed = results.filter(r => r.outcome === 'fail').length;
|
||||
const errored = results.filter(r => r.outcome === 'error').length;
|
||||
const passRate = results.length > 0 ? passed / results.length : 0;
|
||||
|
||||
const regressions = results
|
||||
.filter(r => r.outcome === 'fail' || r.outcome === 'error')
|
||||
.map(r => `"${r.phrase}" — missing: ${r.missingFields.join(', ') || r.errorMessage}`);
|
||||
|
||||
const summary =
|
||||
results.length === 0
|
||||
? 'No test cases evaluated.'
|
||||
: `${passed}/${results.length} passed (${(passRate * 100).toFixed(0)}%). ${partial} partial, ${failed} failed, ${errored} errored.${regressions.length > 0 ? ` Regressions: ${regressions.length}.` : ' No regressions detected.'}`;
|
||||
|
||||
return {
|
||||
runId,
|
||||
productId: 'chronomind',
|
||||
totalCases: results.length,
|
||||
passed,
|
||||
partial,
|
||||
failed,
|
||||
errored,
|
||||
passRate,
|
||||
results,
|
||||
regressions,
|
||||
summary,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pipeline runner ────────────────────────────────────────────────────────────
|
||||
|
||||
async function runNLParserEvalPipeline(
|
||||
customPhrases: string[],
|
||||
req: McpToolRequest
|
||||
): Promise<NLParserEvalReport> {
|
||||
const runId = randomUUID();
|
||||
const opts = { token: req.headers.authorization?.slice(7), requestId: req.id };
|
||||
|
||||
req.log.info(
|
||||
{ runId, stepId: 'sample', customCount: customPhrases.length },
|
||||
'PhraseSamplerAgent start'
|
||||
);
|
||||
const testCases = assemblePhrases(customPhrases);
|
||||
req.log.info(
|
||||
{ runId, stepId: 'sample', totalCases: testCases.length },
|
||||
'PhraseSamplerAgent done'
|
||||
);
|
||||
|
||||
req.log.info({ runId, stepId: 'eval', totalCases: testCases.length }, 'ParseEvalAgent start');
|
||||
const results: ParseEvalResult[] = [];
|
||||
for (const testCase of testCases) {
|
||||
const result = await evalPhrase(testCase, opts);
|
||||
results.push(result);
|
||||
}
|
||||
const passed = results.filter(r => r.outcome === 'pass').length;
|
||||
req.log.info({ runId, stepId: 'eval', passed, total: results.length }, 'ParseEvalAgent done');
|
||||
|
||||
req.log.info({ runId, stepId: 'report' }, 'RegressionReportAgent start');
|
||||
const report = buildEvalReport(runId, results);
|
||||
req.log.info(
|
||||
{ runId, stepId: 'report', passRate: report.passRate, regressions: report.regressions.length },
|
||||
'RegressionReportAgent done'
|
||||
);
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
// ── MCP tool registration ─────────────────────────────────────────────────────
|
||||
|
||||
registerTool({
|
||||
name: 'chronomind.nlParser.eval',
|
||||
description:
|
||||
'A2A pipeline: evaluates the ChronoMind natural-language timer parser by submitting a canonical phrase test suite (plus optional custom phrases) to the extraction service timer-parse task, validates parsed fields, and produces a pass/fail regression scorecard. Run after nl-parser changes to detect regressions. Requires admin role.',
|
||||
requiredRole: 'admin',
|
||||
inputSchema: z.object({
|
||||
customPhrases: z
|
||||
.array(z.string().min(1))
|
||||
.default([])
|
||||
.describe('Additional phrases to test beyond the built-in canonical suite'),
|
||||
}),
|
||||
async execute(args, req) {
|
||||
return runNLParserEvalPipeline(args.customPhrases, req);
|
||||
},
|
||||
});
|
||||
@ -0,0 +1,203 @@
|
||||
/**
|
||||
* ReflectionSynthesisAgent — A2A pipeline for MindLyst weekly reflection synthesis.
|
||||
*
|
||||
* Agent roster (3 steps):
|
||||
* 1. ReflectionCollectorAgent — fetch recent reflections
|
||||
* 2. ThemeExtractionAgent — run reflection-enrichment extraction on concatenated reflection text
|
||||
* 3. SynthesisReportAgent — surface recurring themes, propose weekly summary text
|
||||
*
|
||||
* MCP tools:
|
||||
* mindlyst.reflections.synthesize(limit?) — run pipeline
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { z } from 'zod';
|
||||
import { registerTool } from '../tools/registry.js';
|
||||
import type { McpToolRequest } from '../tools/types.js';
|
||||
import { mindlystReflectionsList, type ReflectionDoc } from '../../lib/mindlyst-client.js';
|
||||
import { extractionRun, type ExtractionItem } from '../../lib/extraction-client.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface ThemeFrequency {
|
||||
theme: string;
|
||||
occurrences: number;
|
||||
sources: string[];
|
||||
}
|
||||
|
||||
export interface ReflectionSynthesisReport {
|
||||
runId: string;
|
||||
productId: 'mindlyst';
|
||||
reflectionsAnalyzed: number;
|
||||
topThemes: ThemeFrequency[];
|
||||
highlights: string[];
|
||||
challenges: string[];
|
||||
extractedEntities: ExtractionItem[];
|
||||
proposedSummary: string;
|
||||
generatedAt: string;
|
||||
}
|
||||
|
||||
// ── Step 1: ReflectionCollectorAgent ──────────────────────────────────────────
|
||||
|
||||
async function collectReflections(
|
||||
limit: number,
|
||||
opts: { token?: string; requestId?: string }
|
||||
): Promise<ReflectionDoc[]> {
|
||||
try {
|
||||
const result = await mindlystReflectionsList({ limit }, opts);
|
||||
return result.items;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 2: ThemeExtractionAgent ──────────────────────────────────────────────
|
||||
|
||||
async function extractThemes(
|
||||
reflections: ReflectionDoc[],
|
||||
opts: { token?: string; requestId?: string }
|
||||
): Promise<{ entities: ExtractionItem[]; rawText: string }> {
|
||||
if (reflections.length === 0) return { entities: [], rawText: '' };
|
||||
|
||||
const allThemes = reflections.flatMap(r => r.themes);
|
||||
const allHighlights = reflections.flatMap(r => r.highlights);
|
||||
const allChallenges = reflections.flatMap(r => r.challenges);
|
||||
const summaries = reflections.map(r => r.summary).filter(Boolean) as string[];
|
||||
|
||||
const rawText = [
|
||||
'Themes: ' + allThemes.join(', '),
|
||||
'Highlights: ' + allHighlights.join('. '),
|
||||
'Challenges: ' + allChallenges.join('. '),
|
||||
summaries.length > 0 ? 'Summaries: ' + summaries.join(' ') : '',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
try {
|
||||
const result = await extractionRun({ text: rawText, taskId: 'reflection-enrichment' }, opts);
|
||||
return { entities: result.extractions, rawText };
|
||||
} catch {
|
||||
return { entities: [], rawText };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 3: SynthesisReportAgent ──────────────────────────────────────────────
|
||||
|
||||
function buildThemeFrequencies(reflections: ReflectionDoc[]): ThemeFrequency[] {
|
||||
const frequencyMap = new Map<string, { count: number; sources: string[] }>();
|
||||
|
||||
for (const reflection of reflections) {
|
||||
for (const theme of reflection.themes) {
|
||||
const entry = frequencyMap.get(theme) ?? { count: 0, sources: [] };
|
||||
entry.count++;
|
||||
entry.sources.push(reflection.weekStartDate);
|
||||
frequencyMap.set(theme, entry);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(frequencyMap.entries())
|
||||
.map(([theme, { count, sources }]) => ({ theme, occurrences: count, sources }))
|
||||
.sort((a, b) => b.occurrences - a.occurrences)
|
||||
.slice(0, 10);
|
||||
}
|
||||
|
||||
function buildProposedSummary(
|
||||
topThemes: ThemeFrequency[],
|
||||
highlights: string[],
|
||||
challenges: string[],
|
||||
reflectionCount: number
|
||||
): string {
|
||||
if (reflectionCount === 0) return 'No reflections found to synthesize.';
|
||||
|
||||
const topThemeNames = topThemes
|
||||
.slice(0, 3)
|
||||
.map(t => t.theme)
|
||||
.join(', ');
|
||||
const summaryParts: string[] = [
|
||||
`Across ${reflectionCount} reflection(s), recurring themes include: ${topThemeNames || 'none identified'}.`,
|
||||
];
|
||||
if (highlights.length > 0) {
|
||||
summaryParts.push(`Key highlights: ${highlights.slice(0, 3).join('; ')}.`);
|
||||
}
|
||||
if (challenges.length > 0) {
|
||||
summaryParts.push(`Challenges to address: ${challenges.slice(0, 2).join('; ')}.`);
|
||||
}
|
||||
return summaryParts.join(' ');
|
||||
}
|
||||
|
||||
// ── Pipeline runner ────────────────────────────────────────────────────────────
|
||||
|
||||
async function runReflectionSynthesisPipeline(
|
||||
limit: number,
|
||||
req: McpToolRequest
|
||||
): Promise<ReflectionSynthesisReport> {
|
||||
const runId = randomUUID();
|
||||
const opts = { token: req.headers.authorization?.slice(7), requestId: req.id };
|
||||
|
||||
req.log.info({ runId, stepId: 'collect', limit }, 'ReflectionCollectorAgent start');
|
||||
const reflections = await collectReflections(limit, opts);
|
||||
req.log.info(
|
||||
{ runId, stepId: 'collect', count: reflections.length },
|
||||
'ReflectionCollectorAgent done'
|
||||
);
|
||||
|
||||
req.log.info(
|
||||
{ runId, stepId: 'extract', count: reflections.length },
|
||||
'ThemeExtractionAgent start'
|
||||
);
|
||||
const { entities } = await extractThemes(reflections, opts);
|
||||
req.log.info(
|
||||
{ runId, stepId: 'extract', entityCount: entities.length },
|
||||
'ThemeExtractionAgent done'
|
||||
);
|
||||
|
||||
req.log.info({ runId, stepId: 'synthesize' }, 'SynthesisReportAgent start');
|
||||
const topThemes = buildThemeFrequencies(reflections);
|
||||
const allHighlights = reflections.flatMap(r => r.highlights);
|
||||
const allChallenges = reflections.flatMap(r => r.challenges);
|
||||
const proposedSummary = buildProposedSummary(
|
||||
topThemes,
|
||||
allHighlights,
|
||||
allChallenges,
|
||||
reflections.length
|
||||
);
|
||||
|
||||
const report: ReflectionSynthesisReport = {
|
||||
runId,
|
||||
productId: 'mindlyst',
|
||||
reflectionsAnalyzed: reflections.length,
|
||||
topThemes,
|
||||
highlights: allHighlights,
|
||||
challenges: allChallenges,
|
||||
extractedEntities: entities,
|
||||
proposedSummary,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
req.log.info(
|
||||
{ runId, stepId: 'synthesize', themeCount: topThemes.length },
|
||||
'SynthesisReportAgent done'
|
||||
);
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
// ── MCP tool registration ─────────────────────────────────────────────────────
|
||||
|
||||
registerTool({
|
||||
name: 'mindlyst.reflections.synthesize',
|
||||
description:
|
||||
'A2A pipeline: collects recent MindLyst reflections, runs reflection-enrichment extraction to surface entities and patterns, identifies recurring themes, and proposes a weekly summary text. Returns top themes by frequency, extracted entities, and a draft weekly summary. Requires admin role.',
|
||||
requiredRole: 'admin',
|
||||
inputSchema: z.object({
|
||||
limit: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(52)
|
||||
.default(8)
|
||||
.describe('Number of recent reflections to include (default 8, max 52 = 1 year)'),
|
||||
}),
|
||||
async execute(args, req) {
|
||||
return runReflectionSynthesisPipeline(args.limit, req);
|
||||
},
|
||||
});
|
||||
@ -48,6 +48,9 @@ import './modules/a2a/triage-quality-pipeline.js';
|
||||
import './modules/a2a/stt-fallback-monitor-pipeline.js';
|
||||
import './modules/a2a/progress-analyst-pipeline.js';
|
||||
import './modules/a2a/brain-overflow-pipeline.js';
|
||||
import './modules/a2a/reflection-synthesis-pipeline.js';
|
||||
import './modules/a2a/keyboard-diagnostics-pipeline.js';
|
||||
import './modules/a2a/nl-parser-eval-pipeline.js';
|
||||
import './modules/mindlyst/mindlyst-tools.js';
|
||||
import './modules/lysnrai/lysnrai-tools.js';
|
||||
import './modules/jarvis/jarvis-tools.js';
|
||||
|
||||
Loading…
Reference in New Issue
Block a user