Add Hermes token analytics dashboard
This commit is contained in:
Hermes VM
parent
fee43faf62
commit
a05702c315
@ -27,7 +27,7 @@ function setExec(handler: Handler) {
|
||||
);
|
||||
}
|
||||
|
||||
const { getHermesTelemetrySnapshot, clearHermesTelemetryCache } = await import('./repository.js');
|
||||
const { getHermesTelemetrySnapshot, getHermesTokenUsageSnapshot, clearHermesTelemetryCache } = await import('./repository.js');
|
||||
|
||||
describe('hermes-telemetry repository', () => {
|
||||
beforeEach(() => {
|
||||
@ -198,4 +198,43 @@ describe('hermes-telemetry repository', () => {
|
||||
expect(snapshot.sessionEvents.entries[1].summary).toBe('user message (content redacted)');
|
||||
expect(JSON.stringify(snapshot.sessionEvents.entries)).not.toContain('secret prompt');
|
||||
});
|
||||
|
||||
it('reads token usage analytics from the Hermes state database without message content', async () => {
|
||||
setExec((command, args) => {
|
||||
if (command === 'python3' && args[0] === '-c') {
|
||||
return {
|
||||
stdout: JSON.stringify({
|
||||
summary: {
|
||||
sessionCount: 3,
|
||||
messageCount: 42,
|
||||
toolCallCount: 7,
|
||||
inputTokens: 1000,
|
||||
outputTokens: 250,
|
||||
cacheReadTokens: 5000,
|
||||
cacheWriteTokens: 50,
|
||||
reasoningTokens: 25,
|
||||
totalTokens: 6325,
|
||||
estimatedCostUsd: 1.23,
|
||||
actualCostUsd: 0.75,
|
||||
firstStartedAt: '2026-06-01T00:00:00.000Z',
|
||||
lastActivityAt: '2026-06-06T00:00:00.000Z',
|
||||
},
|
||||
byModel: [{ model: 'gpt-5.5', provider: 'openai-codex', sessionCount: 2, totalTokens: 6000, estimatedCostUsd: 1.2 }],
|
||||
byDay: [{ day: '2026-06-06', sessionCount: 1, totalTokens: 4000, estimatedCostUsd: 0.8 }],
|
||||
recentSessions: [{ id: 's1', title: 'DevOps task', source: 'telegram', model: 'gpt-5.5', provider: 'openai-codex', startedAt: '2026-06-06T00:00:00.000Z', endedAt: null, totalTokens: 4000, messageCount: 12, toolCallCount: 3, estimatedCostUsd: 0.8 }],
|
||||
}),
|
||||
};
|
||||
}
|
||||
return { stdout: '' };
|
||||
});
|
||||
|
||||
const snapshot = await getHermesTokenUsageSnapshot('vijay');
|
||||
expect(snapshot.instanceId).toBe('vijay');
|
||||
expect(snapshot.status).toBe('up');
|
||||
expect(snapshot.summary.totalTokens).toBe(6325);
|
||||
expect(snapshot.byModel[0]).toMatchObject({ model: 'gpt-5.5', provider: 'openai-codex', totalTokens: 6000 });
|
||||
expect(snapshot.byDay[0]).toMatchObject({ day: '2026-06-06', sessionCount: 1 });
|
||||
expect(snapshot.recentSessions[0]).toMatchObject({ id: 's1', title: 'DevOps task', totalTokens: 4000 });
|
||||
expect(JSON.stringify(snapshot)).not.toContain('secret prompt');
|
||||
});
|
||||
});
|
||||
|
||||
@ -18,6 +18,7 @@ import type {
|
||||
HermesSessionStats,
|
||||
HermesSkillList,
|
||||
HermesTelemetrySnapshot,
|
||||
HermesTokenUsageSnapshot,
|
||||
HermesWatchdogAlert,
|
||||
HermesWatchdogFeed,
|
||||
HermesWatchdogSeverity,
|
||||
@ -37,6 +38,7 @@ interface InstanceConfig {
|
||||
watchdogLog: string;
|
||||
sessionsIndex: string;
|
||||
sessionsDir: string;
|
||||
stateDb: string;
|
||||
}
|
||||
|
||||
const INSTANCES: Record<HermesInstanceId, InstanceConfig> = {
|
||||
@ -47,6 +49,7 @@ const INSTANCES: Record<HermesInstanceId, InstanceConfig> = {
|
||||
watchdogLog: '/root/.hermes/logs/hermes-health-watchdog.log',
|
||||
sessionsIndex: '/root/.hermes/sessions/sessions.json',
|
||||
sessionsDir: '/root/.hermes/sessions',
|
||||
stateDb: '/root/.hermes/state.db',
|
||||
},
|
||||
bheem: {
|
||||
id: 'bheem',
|
||||
@ -55,6 +58,7 @@ const INSTANCES: Record<HermesInstanceId, InstanceConfig> = {
|
||||
watchdogLog: '/home/uma/.hermes/logs/hermes-health-watchdog.log',
|
||||
sessionsIndex: '/home/uma/.hermes/sessions/sessions.json',
|
||||
sessionsDir: '/home/uma/.hermes/sessions',
|
||||
stateDb: '/home/uma/.hermes/state.db',
|
||||
},
|
||||
};
|
||||
|
||||
@ -389,10 +393,200 @@ async function readBackupHistory(inst: InstanceConfig): Promise<HermesBackupHist
|
||||
return { entries, repoPath: inst.repoPath, status: 'up' };
|
||||
}
|
||||
|
||||
// --- Snapshot assembly ------------------------------------------------------
|
||||
// --- Token usage analytics --------------------------------------------------
|
||||
// Reads aggregate-only usage from Hermes' SQLite state database. This never
|
||||
// selects message content; the dashboard gets session totals, model/provider
|
||||
// buckets, day buckets, and recent-session metadata only.
|
||||
const TOKEN_USAGE_SCRIPT = String.raw`
|
||||
import json, sqlite3, sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
def iso(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
ts = float(value)
|
||||
return datetime.fromtimestamp(ts, timezone.utc).isoformat().replace('+00:00', 'Z')
|
||||
except Exception:
|
||||
return str(value)
|
||||
|
||||
def day(value):
|
||||
text = iso(value)
|
||||
return text[:10] if text else 'unknown'
|
||||
|
||||
db = sys.argv[1]
|
||||
con = sqlite3.connect(db)
|
||||
con.row_factory = sqlite3.Row
|
||||
|
||||
def total_expr():
|
||||
return "coalesce(input_tokens,0)+coalesce(output_tokens,0)+coalesce(cache_read_tokens,0)+coalesce(cache_write_tokens,0)+coalesce(reasoning_tokens,0)"
|
||||
|
||||
summary_row = con.execute(f'''
|
||||
select count(*) session_count,
|
||||
coalesce(sum(message_count),0) message_count,
|
||||
coalesce(sum(tool_call_count),0) tool_call_count,
|
||||
coalesce(sum(input_tokens),0) input_tokens,
|
||||
coalesce(sum(output_tokens),0) output_tokens,
|
||||
coalesce(sum(cache_read_tokens),0) cache_read_tokens,
|
||||
coalesce(sum(cache_write_tokens),0) cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens),0) reasoning_tokens,
|
||||
coalesce(sum({total_expr()}),0) total_tokens,
|
||||
coalesce(sum(estimated_cost_usd),0) estimated_cost_usd,
|
||||
coalesce(sum(actual_cost_usd),0) actual_cost_usd,
|
||||
min(started_at) first_started_at,
|
||||
max(coalesce(ended_at, started_at)) last_activity_at
|
||||
from sessions
|
||||
''').fetchone()
|
||||
|
||||
by_model = [
|
||||
{
|
||||
'model': row['model'] or '(unknown)',
|
||||
'provider': row['provider'] or '(unknown)',
|
||||
'sessionCount': row['session_count'],
|
||||
'totalTokens': row['total_tokens'],
|
||||
'estimatedCostUsd': row['estimated_cost_usd'],
|
||||
}
|
||||
for row in con.execute(f'''
|
||||
select coalesce(model, '(unknown)') model,
|
||||
coalesce(billing_provider, '(unknown)') provider,
|
||||
count(*) session_count,
|
||||
coalesce(sum({total_expr()}),0) total_tokens,
|
||||
coalesce(sum(estimated_cost_usd),0) estimated_cost_usd
|
||||
from sessions
|
||||
group by 1,2
|
||||
order by total_tokens desc
|
||||
limit 25
|
||||
''')
|
||||
]
|
||||
|
||||
raw_days = {}
|
||||
for row in con.execute(f'''
|
||||
select coalesce(ended_at, started_at) ts,
|
||||
count(*) session_count,
|
||||
coalesce(sum({total_expr()}),0) total_tokens,
|
||||
coalesce(sum(estimated_cost_usd),0) estimated_cost_usd
|
||||
from sessions
|
||||
group by ts
|
||||
'''):
|
||||
key = day(row['ts'])
|
||||
item = raw_days.setdefault(key, {'day': key, 'sessionCount': 0, 'totalTokens': 0, 'estimatedCostUsd': 0})
|
||||
item['sessionCount'] += row['session_count']
|
||||
item['totalTokens'] += row['total_tokens']
|
||||
item['estimatedCostUsd'] += row['estimated_cost_usd']
|
||||
by_day = sorted(raw_days.values(), key=lambda x: x['day'], reverse=True)[:90]
|
||||
|
||||
recent_sessions = [
|
||||
{
|
||||
'id': row['id'],
|
||||
'title': row['title'],
|
||||
'source': row['source'],
|
||||
'model': row['model'] or '(unknown)',
|
||||
'provider': row['provider'] or '(unknown)',
|
||||
'startedAt': iso(row['started_at']),
|
||||
'endedAt': iso(row['ended_at']),
|
||||
'totalTokens': row['total_tokens'],
|
||||
'messageCount': row['message_count'],
|
||||
'toolCallCount': row['tool_call_count'],
|
||||
'estimatedCostUsd': row['estimated_cost_usd'],
|
||||
}
|
||||
for row in con.execute(f'''
|
||||
select id, title, source, model, billing_provider provider, started_at, ended_at,
|
||||
coalesce({total_expr()},0) total_tokens,
|
||||
coalesce(message_count,0) message_count,
|
||||
coalesce(tool_call_count,0) tool_call_count,
|
||||
coalesce(estimated_cost_usd,0) estimated_cost_usd
|
||||
from sessions
|
||||
order by coalesce(ended_at, started_at) desc
|
||||
limit 100
|
||||
''')
|
||||
]
|
||||
|
||||
payload = {
|
||||
'summary': {
|
||||
'sessionCount': summary_row['session_count'],
|
||||
'messageCount': summary_row['message_count'],
|
||||
'toolCallCount': summary_row['tool_call_count'],
|
||||
'inputTokens': summary_row['input_tokens'],
|
||||
'outputTokens': summary_row['output_tokens'],
|
||||
'cacheReadTokens': summary_row['cache_read_tokens'],
|
||||
'cacheWriteTokens': summary_row['cache_write_tokens'],
|
||||
'reasoningTokens': summary_row['reasoning_tokens'],
|
||||
'totalTokens': summary_row['total_tokens'],
|
||||
'estimatedCostUsd': summary_row['estimated_cost_usd'],
|
||||
'actualCostUsd': summary_row['actual_cost_usd'],
|
||||
'firstStartedAt': iso(summary_row['first_started_at']),
|
||||
'lastActivityAt': iso(summary_row['last_activity_at']),
|
||||
},
|
||||
'byModel': by_model,
|
||||
'byDay': by_day,
|
||||
'recentSessions': recent_sessions,
|
||||
}
|
||||
print(json.dumps(payload))
|
||||
`;
|
||||
|
||||
const emptyTokenUsage = (instanceId: HermesInstanceId, source: string, warning: string): HermesTokenUsageSnapshot => ({
|
||||
generatedAt: new Date().toISOString(),
|
||||
cached: false,
|
||||
instanceId,
|
||||
status: 'unknown',
|
||||
source,
|
||||
summary: {
|
||||
sessionCount: 0,
|
||||
messageCount: 0,
|
||||
toolCallCount: 0,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
totalTokens: 0,
|
||||
estimatedCostUsd: 0,
|
||||
actualCostUsd: 0,
|
||||
firstStartedAt: null,
|
||||
lastActivityAt: null,
|
||||
},
|
||||
byModel: [],
|
||||
byDay: [],
|
||||
recentSessions: [],
|
||||
warnings: [warning],
|
||||
});
|
||||
|
||||
async function buildTokenUsageSnapshot(instanceId: HermesInstanceId): Promise<HermesTokenUsageSnapshot> {
|
||||
const inst = INSTANCES[instanceId];
|
||||
const result = await execAs(inst, 'python3', ['-c', TOKEN_USAGE_SCRIPT, inst.stateDb], 8000);
|
||||
if (!result.ran || !result.stdout) {
|
||||
const warning = `${instanceId}: Hermes state database not readable at ${inst.stateDb}`;
|
||||
await appendDashboardWarning({ severity: 'warn', instance: instanceId, message: warning });
|
||||
return emptyTokenUsage(instanceId, inst.stateDb, warning);
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(result.stdout) as Omit<HermesTokenUsageSnapshot, 'generatedAt' | 'cached' | 'instanceId' | 'status' | 'source' | 'warnings'>;
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
cached: false,
|
||||
instanceId,
|
||||
status: 'up',
|
||||
source: inst.stateDb,
|
||||
summary: parsed.summary,
|
||||
byModel: parsed.byModel,
|
||||
byDay: parsed.byDay,
|
||||
recentSessions: parsed.recentSessions,
|
||||
warnings: [],
|
||||
};
|
||||
} catch (err) {
|
||||
const warning = `${instanceId}: Hermes token analytics parse failed`;
|
||||
log.warn({ err, instance: instanceId, source: inst.stateDb }, 'failed to parse Hermes token usage');
|
||||
await appendDashboardWarning({ severity: 'warn', instance: instanceId, message: warning });
|
||||
return emptyTokenUsage(instanceId, inst.stateDb, warning);
|
||||
}
|
||||
}
|
||||
|
||||
const CACHE_TTL = 30000;
|
||||
const cache = new Map<HermesInstanceId, { snapshot: HermesTelemetrySnapshot; at: number }>();
|
||||
const inflight = new Map<HermesInstanceId, Promise<HermesTelemetrySnapshot>>();
|
||||
const tokenCache = new Map<HermesInstanceId, { snapshot: HermesTokenUsageSnapshot; at: number }>();
|
||||
const tokenInflight = new Map<HermesInstanceId, Promise<HermesTokenUsageSnapshot>>();
|
||||
|
||||
async function buildSnapshot(instanceId: HermesInstanceId): Promise<HermesTelemetrySnapshot> {
|
||||
const inst = INSTANCES[instanceId];
|
||||
@ -469,8 +663,39 @@ export async function getHermesTelemetrySnapshot(
|
||||
return promise;
|
||||
}
|
||||
|
||||
export async function getHermesTokenUsageSnapshot(
|
||||
instanceId: HermesInstanceId,
|
||||
options?: { force?: boolean },
|
||||
): Promise<HermesTokenUsageSnapshot> {
|
||||
const force = options?.force ?? false;
|
||||
|
||||
if (!force) {
|
||||
const cached = tokenCache.get(instanceId);
|
||||
if (cached && Date.now() - cached.at < CACHE_TTL) {
|
||||
return { ...cached.snapshot, cached: true };
|
||||
}
|
||||
|
||||
const pending = tokenInflight.get(instanceId);
|
||||
if (pending) return pending;
|
||||
}
|
||||
|
||||
const promise = buildTokenUsageSnapshot(instanceId)
|
||||
.then((snapshot) => {
|
||||
tokenCache.set(instanceId, { snapshot, at: Date.now() });
|
||||
return snapshot;
|
||||
})
|
||||
.finally(() => {
|
||||
if (tokenInflight.get(instanceId) === promise) tokenInflight.delete(instanceId);
|
||||
});
|
||||
|
||||
if (!force) tokenInflight.set(instanceId, promise);
|
||||
return promise;
|
||||
}
|
||||
|
||||
// Test hook so `vitest` cases don't bleed cached state across runs.
|
||||
export function clearHermesTelemetryCache(): void {
|
||||
cache.clear();
|
||||
inflight.clear();
|
||||
tokenCache.clear();
|
||||
tokenInflight.clear();
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { z } from 'zod';
|
||||
import { getHermesTelemetrySnapshot } from './repository.js';
|
||||
import { HermesInstanceIdSchema, HermesTelemetrySnapshotSchema } from './types.js';
|
||||
import { getHermesTelemetrySnapshot, getHermesTokenUsageSnapshot } from './repository.js';
|
||||
import { HermesInstanceIdSchema, HermesTelemetrySnapshotSchema, HermesTokenUsageSnapshotSchema } from './types.js';
|
||||
import { requireAdmin } from '../../lib/auth.js';
|
||||
|
||||
const ParamsSchema = z.object({ instance: HermesInstanceIdSchema });
|
||||
@ -33,4 +33,27 @@ export async function hermesTelemetryRoutes(fastify: FastifyInstance) {
|
||||
return reply.code(500).send({ error: 'Failed to build hermes telemetry snapshot' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/hermes/telemetry/:instance/token-usage
|
||||
// Admin-only: aggregate/session-level token analytics from Hermes state.db.
|
||||
// The repository deliberately avoids selecting raw message content.
|
||||
fastify.get('/hermes/telemetry/:instance/token-usage', {
|
||||
preHandler: async (req) => requireAdmin(req),
|
||||
}, async (req, reply) => {
|
||||
let params: z.infer<typeof ParamsSchema>;
|
||||
try {
|
||||
params = ParamsSchema.parse(req.params);
|
||||
} catch (err) {
|
||||
return reply.code(400).send({ error: 'Invalid instance', detail: (err as Error).message });
|
||||
}
|
||||
|
||||
try {
|
||||
const snapshot = await getHermesTokenUsageSnapshot(params.instance);
|
||||
const validated = HermesTokenUsageSnapshotSchema.parse(snapshot);
|
||||
return reply.send(validated);
|
||||
} catch (err) {
|
||||
fastify.log.error(err, 'failed to build hermes token usage snapshot');
|
||||
return reply.code(500).send({ error: 'Failed to build hermes token usage snapshot' });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@ -141,6 +141,69 @@ export const HermesBackupHistorySchema = z.object({
|
||||
});
|
||||
export type HermesBackupHistory = z.infer<typeof HermesBackupHistorySchema>;
|
||||
|
||||
export const HermesTokenSummarySchema = z.object({
|
||||
sessionCount: z.number(),
|
||||
messageCount: z.number(),
|
||||
toolCallCount: z.number(),
|
||||
inputTokens: z.number(),
|
||||
outputTokens: z.number(),
|
||||
cacheReadTokens: z.number(),
|
||||
cacheWriteTokens: z.number(),
|
||||
reasoningTokens: z.number(),
|
||||
totalTokens: z.number(),
|
||||
estimatedCostUsd: z.number(),
|
||||
actualCostUsd: z.number(),
|
||||
firstStartedAt: z.string().nullable(),
|
||||
lastActivityAt: z.string().nullable(),
|
||||
});
|
||||
export type HermesTokenSummary = z.infer<typeof HermesTokenSummarySchema>;
|
||||
|
||||
export const HermesTokenByModelSchema = z.object({
|
||||
model: z.string(),
|
||||
provider: z.string(),
|
||||
sessionCount: z.number(),
|
||||
totalTokens: z.number(),
|
||||
estimatedCostUsd: z.number(),
|
||||
});
|
||||
export type HermesTokenByModel = z.infer<typeof HermesTokenByModelSchema>;
|
||||
|
||||
export const HermesTokenByDaySchema = z.object({
|
||||
day: z.string(),
|
||||
sessionCount: z.number(),
|
||||
totalTokens: z.number(),
|
||||
estimatedCostUsd: z.number(),
|
||||
});
|
||||
export type HermesTokenByDay = z.infer<typeof HermesTokenByDaySchema>;
|
||||
|
||||
export const HermesTokenSessionSchema = z.object({
|
||||
id: z.string(),
|
||||
title: z.string().nullable(),
|
||||
source: z.string().nullable(),
|
||||
model: z.string(),
|
||||
provider: z.string(),
|
||||
startedAt: z.string().nullable(),
|
||||
endedAt: z.string().nullable(),
|
||||
totalTokens: z.number(),
|
||||
messageCount: z.number(),
|
||||
toolCallCount: z.number(),
|
||||
estimatedCostUsd: z.number(),
|
||||
});
|
||||
export type HermesTokenSession = z.infer<typeof HermesTokenSessionSchema>;
|
||||
|
||||
export const HermesTokenUsageSnapshotSchema = z.object({
|
||||
generatedAt: z.string(),
|
||||
cached: z.boolean(),
|
||||
instanceId: HermesInstanceIdSchema,
|
||||
status: ProbeStatusSchema,
|
||||
source: z.string().nullable(),
|
||||
summary: HermesTokenSummarySchema,
|
||||
byModel: z.array(HermesTokenByModelSchema),
|
||||
byDay: z.array(HermesTokenByDaySchema),
|
||||
recentSessions: z.array(HermesTokenSessionSchema),
|
||||
warnings: z.array(z.string()),
|
||||
});
|
||||
export type HermesTokenUsageSnapshot = z.infer<typeof HermesTokenUsageSnapshotSchema>;
|
||||
|
||||
export const HermesTelemetrySnapshotSchema = z.object({
|
||||
generatedAt: z.string(),
|
||||
// True when this payload was served from the short-TTL cache.
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import Link from 'next/link';
|
||||
import { ArrowRight, BadgeCheck, BellRing, Bot, CheckCircle2, Clock3, LayoutDashboard, OctagonAlert, Rocket, ShieldAlert, Sparkles, TriangleAlert } from 'lucide-react';
|
||||
import { ArrowRight, BadgeCheck, BarChart3, BellRing, Bot, CheckCircle2, Clock3, LayoutDashboard, OctagonAlert, Rocket, ShieldAlert, Sparkles, TriangleAlert } from 'lucide-react';
|
||||
import { Badge, Button } from '@/components/ui/Primitives';
|
||||
import { HermesShell, MetricCard, SectionCard } from '@/components/hermes-shell';
|
||||
import { HermesInstanceBadge } from '@/components/hermes-instance-switcher';
|
||||
@ -179,6 +179,7 @@ export default function HermesMissionControlPage() {
|
||||
actions={(
|
||||
<>
|
||||
<Button asChild variant="secondary"><Link href="/hermes/tasks"><LayoutDashboard className="mr-2 h-4 w-4" />Task Ledger</Link></Button>
|
||||
<Button asChild variant="secondary"><Link href="/hermes/tokens"><BarChart3 className="mr-2 h-4 w-4" />Token Analytics</Link></Button>
|
||||
<Button asChild variant="primary"><Link href="/hermes/products"><Rocket className="mr-2 h-4 w-4" />Product Portfolio</Link></Button>
|
||||
</>
|
||||
)}
|
||||
|
||||
241
dashboard/web/src/app/hermes/tokens/page.tsx
Normal file
241
dashboard/web/src/app/hermes/tokens/page.tsx
Normal file
@ -0,0 +1,241 @@
|
||||
'use client';
|
||||
|
||||
import Link from 'next/link';
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { ArrowLeft, BarChart3, Coins, Database, Filter, Gauge, MessageSquareText, Sigma } from 'lucide-react';
|
||||
import { Badge, Button } from '@/components/ui/Primitives';
|
||||
import { HermesShell, MetricCard, SectionCard } from '@/components/hermes-shell';
|
||||
import { HermesInstanceBadge } from '@/components/hermes-instance-switcher';
|
||||
import { useHermesInstance } from '@/lib/hermes-instance-context';
|
||||
import { api, type HermesTokenSession, type HermesTokenUsageSnapshot } from '@/lib/api';
|
||||
|
||||
type InstanceId = 'vijay' | 'bheem';
|
||||
const INSTANCE_IDS: InstanceId[] = ['vijay', 'bheem'];
|
||||
|
||||
const emptySummary = {
|
||||
sessionCount: 0,
|
||||
messageCount: 0,
|
||||
toolCallCount: 0,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
totalTokens: 0,
|
||||
estimatedCostUsd: 0,
|
||||
actualCostUsd: 0,
|
||||
firstStartedAt: null as string | null,
|
||||
lastActivityAt: null as string | null,
|
||||
};
|
||||
|
||||
function formatInt(value: number) {
|
||||
return new Intl.NumberFormat('en-US', { maximumFractionDigits: 0 }).format(value);
|
||||
}
|
||||
|
||||
function formatCompact(value: number) {
|
||||
return new Intl.NumberFormat('en-US', { notation: 'compact', maximumFractionDigits: 1 }).format(value);
|
||||
}
|
||||
|
||||
function formatUsd(value: number) {
|
||||
return new Intl.NumberFormat('en-US', { style: 'currency', currency: 'USD', maximumFractionDigits: 4 }).format(value);
|
||||
}
|
||||
|
||||
function formatDate(value: string | null) {
|
||||
if (!value) return '—';
|
||||
return new Intl.DateTimeFormat('en', { month: 'short', day: 'numeric', hour: 'numeric', minute: '2-digit' }).format(new Date(value));
|
||||
}
|
||||
|
||||
function tokenTotal(session: HermesTokenSession) {
|
||||
return session.totalTokens;
|
||||
}
|
||||
|
||||
export default function HermesTokenAnalyticsPage() {
|
||||
const { selectedInstance } = useHermesInstance();
|
||||
const [snapshots, setSnapshots] = useState<Partial<Record<InstanceId, HermesTokenUsageSnapshot>>>({});
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [query, setQuery] = useState('');
|
||||
const [minimumTokens, setMinimumTokens] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
const load = async () => {
|
||||
try {
|
||||
const ids = selectedInstance === 'all' ? INSTANCE_IDS : [selectedInstance as InstanceId];
|
||||
const pairs = await Promise.all(ids.map(async (id) => [id, await api.getHermesTokenUsage(id)] as const));
|
||||
if (!active) return;
|
||||
setSnapshots((current) => ({ ...current, ...Object.fromEntries(pairs) }));
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
if (!active) return;
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
};
|
||||
void load();
|
||||
const timer = window.setInterval(load, 60_000);
|
||||
return () => {
|
||||
active = false;
|
||||
window.clearInterval(timer);
|
||||
};
|
||||
}, [selectedInstance]);
|
||||
|
||||
const visibleSnapshots = useMemo(() => {
|
||||
const ids = selectedInstance === 'all' ? INSTANCE_IDS : [selectedInstance as InstanceId];
|
||||
return ids.map((id) => snapshots[id]).filter(Boolean) as HermesTokenUsageSnapshot[];
|
||||
}, [selectedInstance, snapshots]);
|
||||
|
||||
const combinedSummary = useMemo(() => visibleSnapshots.reduce((acc, snap) => ({
|
||||
...acc,
|
||||
sessionCount: acc.sessionCount + snap.summary.sessionCount,
|
||||
messageCount: acc.messageCount + snap.summary.messageCount,
|
||||
toolCallCount: acc.toolCallCount + snap.summary.toolCallCount,
|
||||
inputTokens: acc.inputTokens + snap.summary.inputTokens,
|
||||
outputTokens: acc.outputTokens + snap.summary.outputTokens,
|
||||
cacheReadTokens: acc.cacheReadTokens + snap.summary.cacheReadTokens,
|
||||
cacheWriteTokens: acc.cacheWriteTokens + snap.summary.cacheWriteTokens,
|
||||
reasoningTokens: acc.reasoningTokens + snap.summary.reasoningTokens,
|
||||
totalTokens: acc.totalTokens + snap.summary.totalTokens,
|
||||
estimatedCostUsd: acc.estimatedCostUsd + snap.summary.estimatedCostUsd,
|
||||
actualCostUsd: acc.actualCostUsd + snap.summary.actualCostUsd,
|
||||
firstStartedAt: [acc.firstStartedAt, snap.summary.firstStartedAt].filter(Boolean).sort()[0] ?? null,
|
||||
lastActivityAt: [acc.lastActivityAt, snap.summary.lastActivityAt].filter(Boolean).sort().at(-1) ?? null,
|
||||
}), emptySummary), [visibleSnapshots]);
|
||||
|
||||
const byModel = useMemo(() => {
|
||||
const buckets = new Map<string, { model: string; provider: string; instanceIds: Set<InstanceId>; sessionCount: number; totalTokens: number; estimatedCostUsd: number }>();
|
||||
for (const snap of visibleSnapshots) {
|
||||
for (const row of snap.byModel) {
|
||||
const key = `${row.provider}\u0000${row.model}`;
|
||||
const item = buckets.get(key) ?? { model: row.model, provider: row.provider, instanceIds: new Set<InstanceId>(), sessionCount: 0, totalTokens: 0, estimatedCostUsd: 0 };
|
||||
item.instanceIds.add(snap.instanceId);
|
||||
item.sessionCount += row.sessionCount;
|
||||
item.totalTokens += row.totalTokens;
|
||||
item.estimatedCostUsd += row.estimatedCostUsd;
|
||||
buckets.set(key, item);
|
||||
}
|
||||
}
|
||||
return Array.from(buckets.values()).sort((a, b) => b.totalTokens - a.totalTokens).slice(0, 12);
|
||||
}, [visibleSnapshots]);
|
||||
|
||||
const byDay = useMemo(() => {
|
||||
const buckets = new Map<string, number>();
|
||||
for (const snap of visibleSnapshots) {
|
||||
for (const row of snap.byDay) buckets.set(row.day, (buckets.get(row.day) ?? 0) + row.totalTokens);
|
||||
}
|
||||
return Array.from(buckets, ([day, totalTokens]) => ({ day, totalTokens })).sort((a, b) => a.day.localeCompare(b.day)).slice(-30);
|
||||
}, [visibleSnapshots]);
|
||||
|
||||
const recentSessions = useMemo(() => {
|
||||
const q = query.trim().toLowerCase();
|
||||
return visibleSnapshots
|
||||
.flatMap((snap) => snap.recentSessions.map((session) => ({ ...session, instanceId: snap.instanceId })))
|
||||
.filter((session) => tokenTotal(session) >= minimumTokens)
|
||||
.filter((session) => !q || [session.title, session.source, session.model, session.provider].some((value) => value?.toLowerCase().includes(q)))
|
||||
.sort((a, b) => new Date(b.endedAt ?? b.startedAt ?? 0).getTime() - new Date(a.endedAt ?? a.startedAt ?? 0).getTime())
|
||||
.slice(0, 50);
|
||||
}, [minimumTokens, query, visibleSnapshots]);
|
||||
|
||||
const maxDayTokens = Math.max(...byDay.map((row) => row.totalTokens), 1);
|
||||
const maxModelTokens = Math.max(...byModel.map((row) => row.totalTokens), 1);
|
||||
const cachePct = combinedSummary.totalTokens > 0 ? Math.round((combinedSummary.cacheReadTokens / combinedSummary.totalTokens) * 100) : 0;
|
||||
|
||||
return (
|
||||
<HermesShell
|
||||
title="Hermes Token Analytics"
|
||||
description="Historical token usage, model/provider mix, cost fields, and session-level filters for Vijay and Bheem. Message content is not exposed."
|
||||
actions={<Button asChild><Link href="/hermes"><ArrowLeft className="mr-2 h-4 w-4" />Back to mission control</Link></Button>}
|
||||
>
|
||||
<section className="grid gap-4 md:grid-cols-2 xl:grid-cols-4">
|
||||
<MetricCard label="Total tokens" value={formatCompact(combinedSummary.totalTokens)} tone="info" icon={<Sigma className="h-5 w-5" />} helpText={`${formatInt(combinedSummary.totalTokens)} lifetime tokens in selected scope`} />
|
||||
<MetricCard label="Sessions" value={formatInt(combinedSummary.sessionCount)} tone="success" icon={<MessageSquareText className="h-5 w-5" />} helpText={`${formatInt(combinedSummary.messageCount)} messages · ${formatInt(combinedSummary.toolCallCount)} tool calls`} />
|
||||
<MetricCard label="Cache read share" value={`${cachePct}%`} tone="warning" icon={<Gauge className="h-5 w-5" />} helpText={`${formatCompact(combinedSummary.cacheReadTokens)} cached tokens`} />
|
||||
<MetricCard label="Estimated cost" value={formatUsd(combinedSummary.estimatedCostUsd)} tone="default" icon={<Coins className="h-5 w-5" />} helpText={`Actual cost field: ${formatUsd(combinedSummary.actualCostUsd)}`} />
|
||||
</section>
|
||||
|
||||
<SectionCard title="Per-instance token roll-up" subtitle="Vijay and Bheem usage side-by-side from their local Hermes state databases." actions={<Badge variant={error ? 'error' : 'success'}>{error ? 'Load issue' : 'Live state.db'}</Badge>}>
|
||||
{error ? <p className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4 text-sm text-[var(--bl-warning)]">Could not load token analytics: {error}</p> : null}
|
||||
<div className="grid gap-4 md:grid-cols-2">
|
||||
{visibleSnapshots.map((snap) => (
|
||||
<div key={snap.instanceId} className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4">
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div>
|
||||
<p className="text-base font-semibold text-[var(--bl-text-primary)]">{snap.instanceId === 'vijay' ? 'Vijay' : 'Bheem'}</p>
|
||||
<p className="text-xs text-[var(--bl-text-secondary)]">{snap.source ?? 'state.db unavailable'}</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2"><HermesInstanceBadge instanceId={snap.instanceId} /><Badge variant={snap.status === 'up' ? 'success' : 'warning'}>{snap.status}</Badge></div>
|
||||
</div>
|
||||
<dl className="mt-4 grid grid-cols-2 gap-3 text-sm md:grid-cols-4">
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Tokens</dt><dd className="text-lg font-semibold text-[var(--bl-text-primary)]">{formatCompact(snap.summary.totalTokens)}</dd></div>
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Input</dt><dd className="text-lg font-semibold text-[var(--bl-info)]">{formatCompact(snap.summary.inputTokens)}</dd></div>
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Output</dt><dd className="text-lg font-semibold text-[var(--bl-success)]">{formatCompact(snap.summary.outputTokens)}</dd></div>
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Reasoning</dt><dd className="text-lg font-semibold text-[var(--bl-warning)]">{formatCompact(snap.summary.reasoningTokens)}</dd></div>
|
||||
</dl>
|
||||
<p className="mt-3 text-xs text-[var(--bl-text-secondary)]">Window: {formatDate(snap.summary.firstStartedAt)} → {formatDate(snap.summary.lastActivityAt)}</p>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</SectionCard>
|
||||
|
||||
<div className="grid gap-6 xl:grid-cols-[1fr_1fr]">
|
||||
<SectionCard title="30-day token trend" subtitle="Daily buckets across the selected instance scope.">
|
||||
<div className="flex h-64 items-end gap-2 overflow-x-auto rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4">
|
||||
{byDay.map((row) => (
|
||||
<div key={row.day} className="flex min-w-8 flex-1 flex-col items-center gap-2">
|
||||
<div className="w-full rounded-t-md bg-[var(--bl-accent)]" style={{ height: `${Math.max(4, (row.totalTokens / maxDayTokens) * 100)}%` }} title={`${row.day}: ${formatInt(row.totalTokens)} tokens`} />
|
||||
<span className="text-[10px] text-[var(--bl-text-tertiary)]">{row.day.slice(5)}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</SectionCard>
|
||||
|
||||
<SectionCard title="Model/provider mix" subtitle="Top token-consuming model/provider buckets.">
|
||||
<div className="space-y-3">
|
||||
{byModel.map((row) => (
|
||||
<div key={`${row.provider}-${row.model}`} className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 text-sm">
|
||||
<div><p className="font-medium text-[var(--bl-text-primary)]">{row.model}</p><p className="text-xs text-[var(--bl-text-secondary)]">{row.provider} · {row.sessionCount} sessions</p></div>
|
||||
<div className="flex items-center gap-2">{Array.from(row.instanceIds).map((id) => <HermesInstanceBadge key={id} instanceId={id} />)}<span className="font-semibold text-[var(--bl-text-primary)]">{formatCompact(row.totalTokens)}</span></div>
|
||||
</div>
|
||||
<div className="mt-3 h-2 rounded-full bg-[var(--bl-surface-card)]"><div className="h-2 rounded-full bg-[var(--bl-accent)]" style={{ width: `${(row.totalTokens / maxModelTokens) * 100}%` }} /></div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</SectionCard>
|
||||
</div>
|
||||
|
||||
<SectionCard title="Session usage explorer" subtitle="Filter recent sessions by title/source/model/provider and minimum total tokens." actions={<Badge variant="info"><Filter className="mr-1 h-3 w-3" />{recentSessions.length} rows</Badge>}>
|
||||
<div className="mb-4 grid gap-3 md:grid-cols-[1fr_14rem]">
|
||||
<input value={query} onChange={(event) => setQuery(event.target.value)} placeholder="Filter by model, provider, source, or title…" className="rounded-xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] px-3 py-2 text-sm text-[var(--bl-text-primary)] outline-none focus:border-[var(--bl-accent)]" />
|
||||
<input type="number" min={0} step={1000} value={minimumTokens} onChange={(event) => setMinimumTokens(Number(event.target.value) || 0)} aria-label="Minimum tokens" className="rounded-xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] px-3 py-2 text-sm text-[var(--bl-text-primary)] outline-none focus:border-[var(--bl-accent)]" />
|
||||
</div>
|
||||
<div className="space-y-3">
|
||||
{recentSessions.map((session) => (
|
||||
<div key={`${session.instanceId}-${session.id}`} className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<p className="truncate font-medium text-[var(--bl-text-primary)]">{session.title ?? session.id}</p>
|
||||
<p className="text-xs text-[var(--bl-text-secondary)]">{session.model} · {session.provider} · {formatDate(session.endedAt ?? session.startedAt)}</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2"><HermesInstanceBadge instanceId={session.instanceId} /><Badge variant="neutral">{formatCompact(session.totalTokens)} tokens</Badge></div>
|
||||
</div>
|
||||
<dl className="mt-3 grid gap-2 text-sm text-[var(--bl-text-secondary)] md:grid-cols-4">
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Messages</dt><dd>{formatInt(session.messageCount)}</dd></div>
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Tool calls</dt><dd>{formatInt(session.toolCallCount)}</dd></div>
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Cost</dt><dd>{formatUsd(session.estimatedCostUsd)}</dd></div>
|
||||
<div><dt className="text-xs uppercase tracking-[0.18em] text-[var(--bl-text-tertiary)]">Source</dt><dd>{session.source ?? 'unknown'}</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
))}
|
||||
{recentSessions.length === 0 ? <p className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4 text-sm text-[var(--bl-text-secondary)]">No sessions matched the current filters.</p> : null}
|
||||
</div>
|
||||
</SectionCard>
|
||||
|
||||
<SectionCard title="Data contract" subtitle="What is intentionally included and excluded.">
|
||||
<div className="grid gap-3 text-sm text-[var(--bl-text-secondary)] md:grid-cols-3">
|
||||
<div className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4"><Database className="mb-2 h-5 w-5 text-[var(--bl-accent)]" />Source: local Hermes `state.db` sessions table for Vijay and Bheem.</div>
|
||||
<div className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4"><BarChart3 className="mb-2 h-5 w-5 text-[var(--bl-accent)]" />Included: tokens, costs, model/provider, source, titles, dates, messages, and tool-call counts.</div>
|
||||
<div className="rounded-2xl border border-[var(--bl-border)] bg-[var(--bl-surface-muted)] p-4"><Filter className="mb-2 h-5 w-5 text-[var(--bl-accent)]" />Excluded: raw prompt/assistant/tool-result message content.</div>
|
||||
</div>
|
||||
</SectionCard>
|
||||
</HermesShell>
|
||||
);
|
||||
}
|
||||
@ -248,6 +248,64 @@ export interface HermesTelemetrySnapshot {
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export interface HermesTokenSummary {
|
||||
sessionCount: number;
|
||||
messageCount: number;
|
||||
toolCallCount: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
cacheReadTokens: number;
|
||||
cacheWriteTokens: number;
|
||||
reasoningTokens: number;
|
||||
totalTokens: number;
|
||||
estimatedCostUsd: number;
|
||||
actualCostUsd: number;
|
||||
firstStartedAt: string | null;
|
||||
lastActivityAt: string | null;
|
||||
}
|
||||
|
||||
export interface HermesTokenByModel {
|
||||
model: string;
|
||||
provider: string;
|
||||
sessionCount: number;
|
||||
totalTokens: number;
|
||||
estimatedCostUsd: number;
|
||||
}
|
||||
|
||||
export interface HermesTokenByDay {
|
||||
day: string;
|
||||
sessionCount: number;
|
||||
totalTokens: number;
|
||||
estimatedCostUsd: number;
|
||||
}
|
||||
|
||||
export interface HermesTokenSession {
|
||||
id: string;
|
||||
title: string | null;
|
||||
source: string | null;
|
||||
model: string;
|
||||
provider: string;
|
||||
startedAt: string | null;
|
||||
endedAt: string | null;
|
||||
totalTokens: number;
|
||||
messageCount: number;
|
||||
toolCallCount: number;
|
||||
estimatedCostUsd: number;
|
||||
}
|
||||
|
||||
export interface HermesTokenUsageSnapshot {
|
||||
generatedAt: string;
|
||||
cached: boolean;
|
||||
instanceId: 'vijay' | 'bheem';
|
||||
status: HermesProbeStatus;
|
||||
source: string | null;
|
||||
summary: HermesTokenSummary;
|
||||
byModel: HermesTokenByModel[];
|
||||
byDay: HermesTokenByDay[];
|
||||
recentSessions: HermesTokenSession[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export interface HermesOpsSnapshot {
|
||||
generatedAt: string;
|
||||
tailscaleIp: string | null;
|
||||
@ -421,6 +479,8 @@ export const api = {
|
||||
// source isn't readable in the current environment (CI / dev box).
|
||||
getHermesTelemetry: (instance: 'vijay' | 'bheem') =>
|
||||
apiRequest<HermesTelemetrySnapshot>(`/api/hermes/telemetry/${instance}`),
|
||||
getHermesTokenUsage: (instance: 'vijay' | 'bheem') =>
|
||||
apiRequest<HermesTokenUsageSnapshot>(`/api/hermes/telemetry/${instance}/token-usage`),
|
||||
|
||||
// Seed
|
||||
seedServices: () => apiRequest<{ message: string }>('/api/seed', { method: 'POST' }),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user