feat(platform-service): add ai budget governance

This commit is contained in:
root 2026-03-15 09:30:16 +00:00
parent 3f06427038
commit 728d8f2484
7 changed files with 687 additions and 0 deletions

View File

@ -79,6 +79,10 @@ const CONTAINER_DEFS: Record<string, ContainerConfig> = {
agent_evaluation_cases: { partitionKeyPath: '/suiteId' },
agent_evaluation_runs: { partitionKeyPath: '/productId', defaultTtl: 30 * 86400 },
agent_evaluation_results: { partitionKeyPath: '/runId', defaultTtl: 30 * 86400 },
// AI budget and cost governance
ai_budget_policies: { partitionKeyPath: '/productId' },
ai_budget_spend_entries: { partitionKeyPath: '/productId', defaultTtl: 30 * 86400 },
ai_budget_alerts: { partitionKeyPath: '/productId', defaultTtl: 30 * 86400 },
// Telemetry (client diagnostics — see docs/WINDSURF/CLIENT_TELEMETRY_DESIGN.md)
telemetry_events: { partitionKeyPath: '/pk', defaultTtl: 30 * 86400 },
telemetry_error_clusters: { partitionKeyPath: '/pk', defaultTtl: 90 * 86400 },

View File

@ -0,0 +1,73 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { MemoryDatastoreProvider } from '@bytelyst/datastore';
import { _resetDatastoreProvider, setProvider } from '../../lib/datastore.js';
import * as repo from './repository.js';
describe('ai budget repository', () => {
beforeEach(() => {
setProvider(new MemoryDatastoreProvider());
});
afterEach(() => {
_resetDatastoreProvider();
});
it('stores policies, spend entries, and alerts', async () => {
await repo.createPolicy({
id: 'aibudget_1',
productId: 'lysnrai',
scopeType: 'agent',
scopeId: 'agt_1',
name: 'Incident Agent Budget',
status: 'active',
period: 'monthly',
budgetUsd: 25,
softThreshold: 0.8,
hardThreshold: 1,
modelAllowlist: ['gpt-4o-mini'],
createdBy: 'admin_1',
createdAt: '2026-03-15T00:00:00.000Z',
updatedAt: '2026-03-15T00:00:00.000Z',
});
await repo.createSpendEntry({
id: 'aispend_1',
productId: 'lysnrai',
policyId: 'aibudget_1',
scopeType: 'agent',
scopeId: 'agt_1',
agentId: 'agt_1',
agentVersionId: 'agt_1:v2',
model: 'gpt-4o-mini',
tokensUsed: 500,
costUsd: 4.2,
source: 'run',
verdict: 'allow',
recordedAt: '2026-03-15T00:10:00.000Z',
});
await repo.createAlerts([
{
id: 'aialert_1',
productId: 'lysnrai',
policyId: 'aibudget_1',
scopeType: 'agent',
scopeId: 'agt_1',
severity: 'warn',
message: 'Approaching budget',
spendUsd: 20,
budgetUsd: 25,
percentUsed: 0.8,
createdAt: '2026-03-15T00:11:00.000Z',
},
]);
const policies = await repo.listPolicies('lysnrai', { limit: 20 });
const spend = await repo.listSpendEntries('lysnrai', { policyId: 'aibudget_1' });
const alerts = await repo.listAlerts('lysnrai', { limit: 20 });
expect(policies).toHaveLength(1);
expect(spend[0].costUsd).toBe(4.2);
expect(alerts[0].severity).toBe('warn');
});
});

View File

@ -0,0 +1,100 @@
import { NotFoundError } from '../../lib/errors.js';
import { getCollection } from '../../lib/datastore.js';
import type {
BudgetAlertDoc,
BudgetPolicyDoc,
BudgetSpendEntryDoc,
ListBudgetAlertsQuery,
ListBudgetPoliciesQuery,
} from './types.js';
function policyCollection() {
return getCollection<BudgetPolicyDoc>('ai_budget_policies', '/productId');
}
function spendCollection() {
return getCollection<BudgetSpendEntryDoc>('ai_budget_spend_entries', '/productId');
}
function alertCollection() {
return getCollection<BudgetAlertDoc>('ai_budget_alerts', '/productId');
}
export async function createPolicy(doc: BudgetPolicyDoc): Promise<BudgetPolicyDoc> {
return policyCollection().create(doc);
}
export async function listPolicies(
productId: string,
query: ListBudgetPoliciesQuery
): Promise<BudgetPolicyDoc[]> {
return policyCollection().findMany({
filter: {
productId,
...(query.scopeType ? { scopeType: query.scopeType } : {}),
...(query.scopeId ? { scopeId: query.scopeId } : {}),
...(query.status ? { status: query.status } : {}),
},
sort: { createdAt: -1 },
limit: query.limit,
});
}
export async function getPolicy(id: string, productId: string): Promise<BudgetPolicyDoc> {
const policy = await policyCollection().findById(id, productId);
if (!policy) throw new NotFoundError(`Budget policy '${id}' not found`);
return policy;
}
export async function updatePolicy(
id: string,
productId: string,
updates: Partial<BudgetPolicyDoc>
): Promise<BudgetPolicyDoc> {
const updated = await policyCollection().update(id, productId, {
...updates,
updatedAt: new Date().toISOString(),
});
if (!updated) throw new NotFoundError(`Budget policy '${id}' not found`);
return updated;
}
export async function createSpendEntry(doc: BudgetSpendEntryDoc): Promise<BudgetSpendEntryDoc> {
return spendCollection().create(doc);
}
export async function listSpendEntries(
productId: string,
filter: { policyId?: string; scopeType?: string; scopeId?: string; since?: string } = {}
): Promise<BudgetSpendEntryDoc[]> {
return spendCollection().findMany({
filter: {
productId,
...(filter.policyId ? { policyId: filter.policyId } : {}),
...(filter.scopeType ? { scopeType: filter.scopeType } : {}),
...(filter.scopeId ? { scopeId: filter.scopeId } : {}),
...(filter.since ? { recordedAt: { $gte: filter.since } } : {}),
} as import('@bytelyst/datastore').FilterMap,
sort: { recordedAt: -1 },
limit: 1000,
});
}
export async function createAlerts(docs: BudgetAlertDoc[]): Promise<BudgetAlertDoc[]> {
const writes = docs.map(doc => alertCollection().create(doc));
return Promise.all(writes);
}
export async function listAlerts(
productId: string,
query: ListBudgetAlertsQuery
): Promise<BudgetAlertDoc[]> {
return alertCollection().findMany({
filter: {
productId,
...(query.severity ? { severity: query.severity } : {}),
},
sort: { createdAt: -1 },
limit: query.limit,
});
}

View File

@ -0,0 +1,133 @@
import Fastify from 'fastify';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const repoMock = {
listPolicies: vi.fn(),
createPolicy: vi.fn(),
getPolicy: vi.fn(),
updatePolicy: vi.fn(),
createSpendEntry: vi.fn(),
listSpendEntries: vi.fn(),
createAlerts: vi.fn(),
listAlerts: vi.fn(),
};
vi.mock('./repository.js', () => repoMock);
async function buildApp(payload?: { sub: string; productId: string; role?: string }) {
const { aiBudgetRoutes } = await import('./routes.js');
const app = Fastify({ logger: false });
if (payload) {
app.addHook('onRequest', async req => {
req.jwtPayload = payload;
});
}
await app.register(aiBudgetRoutes, { prefix: '/api' });
return app;
}
describe('aiBudgetRoutes', () => {
beforeEach(() => {
vi.clearAllMocks();
});
afterEach(() => {
vi.restoreAllMocks();
});
it('POST /ai-budgets/policies creates a policy', async () => {
repoMock.createPolicy.mockResolvedValue({ id: 'aibudget_1' });
const app = await buildApp({ sub: 'admin_1', productId: 'lysnrai', role: 'admin' });
const res = await app.inject({
method: 'POST',
url: '/api/ai-budgets/policies',
payload: {
scopeType: 'agent',
scopeId: 'agt_1',
name: 'Incident Budget',
budgetUsd: 20,
},
});
expect(res.statusCode).toBe(200);
expect(repoMock.createPolicy).toHaveBeenCalledWith(
expect.objectContaining({
scopeType: 'agent',
scopeId: 'agt_1',
budgetUsd: 20,
})
);
});
it('GET /ai-budgets/policies/:id/status summarizes current spend', async () => {
repoMock.getPolicy.mockResolvedValue({
id: 'aibudget_1',
productId: 'lysnrai',
scopeType: 'agent',
scopeId: 'agt_1',
name: 'Incident Budget',
period: 'monthly',
budgetUsd: 10,
softThreshold: 0.8,
hardThreshold: 1,
});
repoMock.listSpendEntries.mockResolvedValue([
{ costUsd: 3, tokensUsed: 100, recordedAt: '2026-03-15T00:00:00.000Z' },
{ costUsd: 6, tokensUsed: 200, recordedAt: '2026-03-15T01:00:00.000Z' },
]);
const app = await buildApp({ sub: 'admin_1', productId: 'lysnrai', role: 'admin' });
const res = await app.inject({
method: 'GET',
url: '/api/ai-budgets/policies/aibudget_1/status',
});
expect(res.statusCode).toBe(200);
expect(res.json()).toMatchObject({
spendUsd: 9,
tokensUsed: 300,
verdict: 'warn',
});
});
it('POST /ai-budgets/spend blocks disallowed models and persists alerts', async () => {
repoMock.getPolicy.mockResolvedValue({
id: 'aibudget_1',
productId: 'lysnrai',
scopeType: 'agent',
scopeId: 'agt_1',
name: 'Incident Budget',
status: 'active',
period: 'monthly',
budgetUsd: 50,
softThreshold: 0.8,
hardThreshold: 1,
modelAllowlist: ['gpt-4o-mini'],
});
repoMock.createSpendEntry.mockResolvedValue({ id: 'aispend_1', verdict: 'block' });
repoMock.createAlerts.mockResolvedValue([]);
const app = await buildApp({ sub: 'admin_1', productId: 'lysnrai', role: 'admin' });
const res = await app.inject({
method: 'POST',
url: '/api/ai-budgets/spend',
payload: {
policyId: 'aibudget_1',
scopeType: 'agent',
scopeId: 'agt_1',
model: 'gpt-4.1',
costUsd: 2,
},
});
expect(res.statusCode).toBe(200);
expect(repoMock.createAlerts).toHaveBeenCalled();
expect(repoMock.createSpendEntry).toHaveBeenCalledWith(
expect.objectContaining({
verdict: 'block',
model: 'gpt-4.1',
})
);
});
});

View File

@ -0,0 +1,251 @@
import { randomUUID } from 'node:crypto';
import type { FastifyInstance } from 'fastify';
import { BadRequestError, ForbiddenError } from '../../lib/errors.js';
import {
BudgetAlertDoc,
BudgetPolicyDoc,
BudgetSpendEntryDoc,
CreateBudgetPolicySchema,
ListBudgetAlertsQuerySchema,
ListBudgetPoliciesQuerySchema,
RecordBudgetSpendSchema,
UpdateBudgetPolicySchema,
} from './types.js';
import * as repo from './repository.js';
function requireAdmin(req: { jwtPayload?: { sub?: string; role?: string; productId?: string } }): {
userId: string;
productId: string;
} {
const payload = req.jwtPayload;
if (!payload?.sub) throw new ForbiddenError('Authentication required');
if (!payload.role || !['super_admin', 'admin'].includes(payload.role)) {
throw new ForbiddenError('Admin access required');
}
return {
userId: payload.sub,
productId: payload.productId ?? process.env.DEFAULT_PRODUCT_ID ?? 'lysnrai',
};
}
function periodStart(now: Date, period: 'daily' | 'monthly'): string {
if (period === 'daily') {
return now.toISOString().slice(0, 10);
}
return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}-01`;
}
function toIsoDayBoundary(day: string): string {
return `${day}T00:00:00.000Z`;
}
function validationError(message: string): never {
throw new BadRequestError(message);
}
export async function aiBudgetRoutes(app: FastifyInstance) {
app.get('/ai-budgets/policies', async req => {
const access = requireAdmin(req);
const parsed = ListBudgetPoliciesQuerySchema.safeParse(req.query);
if (!parsed.success) {
validationError(parsed.error.issues.map(issue => issue.message).join('; '));
}
return repo.listPolicies(access.productId, parsed.data);
});
app.post('/ai-budgets/policies', async req => {
const access = requireAdmin(req);
const parsed = CreateBudgetPolicySchema.safeParse(req.body);
if (!parsed.success) {
validationError(parsed.error.issues.map(issue => issue.message).join('; '));
}
const now = new Date().toISOString();
const doc: BudgetPolicyDoc = {
id: `aibudget_${randomUUID()}`,
productId: access.productId,
scopeType: parsed.data.scopeType,
scopeId: parsed.data.scopeId,
name: parsed.data.name,
status: 'active',
period: parsed.data.period,
budgetUsd: parsed.data.budgetUsd,
softThreshold: parsed.data.softThreshold,
hardThreshold: parsed.data.hardThreshold,
modelAllowlist: parsed.data.modelAllowlist,
metadata: parsed.data.metadata,
createdBy: access.userId,
createdAt: now,
updatedAt: now,
};
return repo.createPolicy(doc);
});
app.get('/ai-budgets/policies/:id', async req => {
const access = requireAdmin(req);
const { id } = req.params as { id: string };
return repo.getPolicy(id, access.productId);
});
app.patch('/ai-budgets/policies/:id', async req => {
const access = requireAdmin(req);
const { id } = req.params as { id: string };
const parsed = UpdateBudgetPolicySchema.safeParse(req.body);
if (!parsed.success) {
validationError(parsed.error.issues.map(issue => issue.message).join('; '));
}
return repo.updatePolicy(id, access.productId, parsed.data);
});
app.get('/ai-budgets/policies/:id/status', async req => {
const access = requireAdmin(req);
const { id } = req.params as { id: string };
const policy = await repo.getPolicy(id, access.productId);
const since = toIsoDayBoundary(periodStart(new Date(), policy.period));
const entries = await repo.listSpendEntries(access.productId, {
policyId: id,
since,
});
const spendUsd = entries.reduce((sum, entry) => sum + entry.costUsd, 0);
const tokensUsed = entries.reduce((sum, entry) => sum + entry.tokensUsed, 0);
const percentUsed = policy.budgetUsd > 0 ? spendUsd / policy.budgetUsd : 0;
return {
policy,
spendUsd,
tokensUsed,
percentUsed,
verdict:
percentUsed >= policy.hardThreshold
? 'block'
: percentUsed >= policy.softThreshold
? 'warn'
: 'allow',
entries: entries.length,
};
});
app.get('/ai-budgets/alerts', async req => {
const access = requireAdmin(req);
const parsed = ListBudgetAlertsQuerySchema.safeParse(req.query);
if (!parsed.success) {
validationError(parsed.error.issues.map(issue => issue.message).join('; '));
}
return repo.listAlerts(access.productId, parsed.data);
});
app.post('/ai-budgets/spend', async req => {
const access = requireAdmin(req);
const parsed = RecordBudgetSpendSchema.safeParse(req.body);
if (!parsed.success) {
validationError(parsed.error.issues.map(issue => issue.message).join('; '));
}
const candidatePolicies = parsed.data.policyId
? [await repo.getPolicy(parsed.data.policyId, access.productId)]
: await repo.listPolicies(access.productId, {
scopeType: parsed.data.scopeType,
scopeId: parsed.data.scopeId,
status: 'active',
limit: 100,
});
const now = new Date();
const alerts: BudgetAlertDoc[] = [];
let overallVerdict: 'allow' | 'warn' | 'block' = 'allow';
for (const policy of candidatePolicies) {
if (policy.status !== 'active') continue;
if (parsed.data.model && policy.modelAllowlist.length > 0) {
const allowed = policy.modelAllowlist.includes(parsed.data.model);
if (!allowed) {
overallVerdict = 'block';
alerts.push({
id: `aialert_${randomUUID()}`,
productId: access.productId,
policyId: policy.id,
scopeType: policy.scopeType,
scopeId: policy.scopeId,
severity: 'block',
message: `Model '${parsed.data.model}' is not allowed by policy '${policy.name}'`,
spendUsd: parsed.data.costUsd,
budgetUsd: policy.budgetUsd,
percentUsed: 0,
createdAt: now.toISOString(),
});
continue;
}
}
const since = toIsoDayBoundary(periodStart(now, policy.period));
const entries = await repo.listSpendEntries(access.productId, {
policyId: policy.id,
since,
});
const spendUsd = entries.reduce((sum, entry) => sum + entry.costUsd, 0) + parsed.data.costUsd;
const percentUsed = policy.budgetUsd > 0 ? spendUsd / policy.budgetUsd : 0;
if (percentUsed >= policy.hardThreshold) {
overallVerdict = 'block';
alerts.push({
id: `aialert_${randomUUID()}`,
productId: access.productId,
policyId: policy.id,
scopeType: policy.scopeType,
scopeId: policy.scopeId,
severity: 'block',
message: `AI budget hard limit exceeded for policy '${policy.name}'`,
spendUsd,
budgetUsd: policy.budgetUsd,
percentUsed,
createdAt: now.toISOString(),
});
} else if (percentUsed >= policy.softThreshold && overallVerdict !== 'block') {
overallVerdict = 'warn';
alerts.push({
id: `aialert_${randomUUID()}`,
productId: access.productId,
policyId: policy.id,
scopeType: policy.scopeType,
scopeId: policy.scopeId,
severity: 'warn',
message: `AI budget nearing threshold for policy '${policy.name}'`,
spendUsd,
budgetUsd: policy.budgetUsd,
percentUsed,
createdAt: now.toISOString(),
});
}
}
const entry: BudgetSpendEntryDoc = {
id: `aispend_${randomUUID()}`,
productId: access.productId,
policyId: candidatePolicies.length === 1 ? candidatePolicies[0].id : parsed.data.policyId,
scopeType: parsed.data.scopeType,
scopeId: parsed.data.scopeId,
agentId: parsed.data.agentId,
agentVersionId: parsed.data.agentVersionId,
runId: parsed.data.runId,
evaluationRunId: parsed.data.evaluationRunId,
model: parsed.data.model,
tokensUsed: parsed.data.tokensUsed,
costUsd: parsed.data.costUsd,
source: parsed.data.source,
verdict: overallVerdict,
recordedAt: now.toISOString(),
};
const saved = await repo.createSpendEntry(entry);
if (alerts.length > 0) {
await repo.createAlerts(alerts);
}
return {
verdict: overallVerdict,
entry: saved,
alerts,
};
});
}

View File

@ -0,0 +1,124 @@
import { z } from 'zod';
export const BudgetScopeTypeSchema = z.enum(['product', 'agent']);
export const BudgetPolicyStatusSchema = z.enum(['active', 'paused', 'archived']);
export const BudgetPeriodSchema = z.enum(['daily', 'monthly']);
export const BudgetAlertSeveritySchema = z.enum(['warn', 'block']);
export const BudgetVerdictSchema = z.enum(['allow', 'warn', 'block']);
export const BudgetPolicySchema = z.object({
id: z.string().min(1),
productId: z.string().min(1),
scopeType: BudgetScopeTypeSchema,
scopeId: z.string().min(1),
name: z.string().min(1),
status: BudgetPolicyStatusSchema,
period: BudgetPeriodSchema,
budgetUsd: z.number().min(0),
softThreshold: z.number().min(0).max(1),
hardThreshold: z.number().min(0).max(1),
modelAllowlist: z.array(z.string()).default([]),
metadata: z.record(z.unknown()).optional(),
createdBy: z.string().min(1),
createdAt: z.string(),
updatedAt: z.string(),
});
export type BudgetPolicyDoc = z.infer<typeof BudgetPolicySchema> & {
_ts?: number;
_etag?: string;
};
export const BudgetSpendEntrySchema = z.object({
id: z.string().min(1),
productId: z.string().min(1),
policyId: z.string().optional(),
scopeType: BudgetScopeTypeSchema,
scopeId: z.string().min(1),
agentId: z.string().optional(),
agentVersionId: z.string().optional(),
runId: z.string().optional(),
evaluationRunId: z.string().optional(),
model: z.string().optional(),
tokensUsed: z.number().int().min(0).default(0),
costUsd: z.number().min(0),
source: z.string().optional(),
verdict: BudgetVerdictSchema,
recordedAt: z.string(),
});
export type BudgetSpendEntryDoc = z.infer<typeof BudgetSpendEntrySchema> & {
_ts?: number;
_etag?: string;
};
export const BudgetAlertSchema = z.object({
id: z.string().min(1),
productId: z.string().min(1),
policyId: z.string().min(1),
scopeType: BudgetScopeTypeSchema,
scopeId: z.string().min(1),
severity: BudgetAlertSeveritySchema,
message: z.string().min(1),
spendUsd: z.number().min(0),
budgetUsd: z.number().min(0),
percentUsed: z.number().min(0),
createdAt: z.string(),
});
export type BudgetAlertDoc = z.infer<typeof BudgetAlertSchema> & {
_ts?: number;
_etag?: string;
};
export const CreateBudgetPolicySchema = z.object({
scopeType: BudgetScopeTypeSchema,
scopeId: z.string().min(1),
name: z.string().min(1),
period: BudgetPeriodSchema.default('monthly'),
budgetUsd: z.number().min(0.01),
softThreshold: z.number().min(0).max(1).default(0.8),
hardThreshold: z.number().min(0).max(1).default(1),
modelAllowlist: z.array(z.string()).default([]),
metadata: z.record(z.unknown()).optional(),
});
export const UpdateBudgetPolicySchema = z.object({
name: z.string().min(1).optional(),
status: BudgetPolicyStatusSchema.optional(),
period: BudgetPeriodSchema.optional(),
budgetUsd: z.number().min(0.01).optional(),
softThreshold: z.number().min(0).max(1).optional(),
hardThreshold: z.number().min(0).max(1).optional(),
modelAllowlist: z.array(z.string()).optional(),
metadata: z.record(z.unknown()).optional(),
});
export const RecordBudgetSpendSchema = z.object({
scopeType: BudgetScopeTypeSchema,
scopeId: z.string().min(1),
policyId: z.string().min(1).optional(),
agentId: z.string().optional(),
agentVersionId: z.string().optional(),
runId: z.string().optional(),
evaluationRunId: z.string().optional(),
model: z.string().optional(),
tokensUsed: z.number().int().min(0).default(0),
costUsd: z.number().min(0),
source: z.string().optional(),
});
export const ListBudgetPoliciesQuerySchema = z.object({
scopeType: BudgetScopeTypeSchema.optional(),
scopeId: z.string().min(1).optional(),
status: BudgetPolicyStatusSchema.optional(),
limit: z.coerce.number().min(1).max(100).default(20),
});
export const ListBudgetAlertsQuerySchema = z.object({
severity: BudgetAlertSeveritySchema.optional(),
limit: z.coerce.number().min(1).max(100).default(50),
});
export type ListBudgetPoliciesQuery = z.infer<typeof ListBudgetPoliciesQuerySchema>;
export type ListBudgetAlertsQuery = z.infer<typeof ListBudgetAlertsQuerySchema>;

View File

@ -37,6 +37,7 @@ import { magicLinkRoutes } from './modules/auth/magic-link/routes.js';
import { auditRoutes } from './modules/audit/routes.js';
import { agentRoutes } from './modules/agents/routes.js';
import { agentEvalRoutes } from './modules/agent-evals/routes.js';
import { aiBudgetRoutes } from './modules/ai-budgets/routes.js';
import { notificationRoutes } from './modules/notifications/routes.js';
import { flagRoutes } from './modules/flags/routes.js';
import { rateLimitRoutes } from './modules/ratelimit/routes.js';
@ -143,6 +144,7 @@ await app.register(magicLinkRoutes, { prefix: '/api' });
await app.register(auditRoutes, { prefix: '/api' });
await app.register(agentRoutes, { prefix: '/api' });
await app.register(agentEvalRoutes, { prefix: '/api' });
await app.register(aiBudgetRoutes, { prefix: '/api' });
await app.register(notificationRoutes, { prefix: '/api' });
await app.register(flagRoutes, { prefix: '/api' });
await app.register(rateLimitRoutes, { prefix: '/api' });