219 lines
7.7 KiB
TypeScript
219 lines
7.7 KiB
TypeScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
type MonitorOptions = {
|
|
hours: number;
|
|
intervalSec: number;
|
|
outFile: string;
|
|
healthUrl: string;
|
|
stateFile: string;
|
|
};
|
|
|
|
type HealthSnapshot = {
|
|
tradingLoopHealthy?: boolean;
|
|
monitorLoopHealthy?: boolean;
|
|
reconciliationLoopHealthy?: boolean;
|
|
reconciliationMismatchCount?: number;
|
|
reconciliationMissingFromExchange?: number;
|
|
reconciliationMissingInDb?: number;
|
|
reconciliationNoGoTrades?: number;
|
|
reconciliationIntegrityWatchdogTriggered?: boolean;
|
|
tradingControl?: {
|
|
mode?: string;
|
|
lastChangedBy?: string;
|
|
lastChangedAt?: number;
|
|
reason?: string;
|
|
};
|
|
};
|
|
|
|
const toNumber = (value: unknown, fallback: number): number => {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
};
|
|
|
|
const parseArgs = (argv: string[]): MonitorOptions => {
|
|
let hours = 12;
|
|
let intervalSec = 60;
|
|
let outFile = '';
|
|
let healthUrl = 'http://127.0.0.1:5000/internal/health';
|
|
let stateFile = path.resolve(process.cwd(), 'bot_state.json');
|
|
|
|
for (const arg of argv) {
|
|
if (arg.startsWith('--hours=')) {
|
|
hours = Math.max(1, Math.floor(toNumber(arg.slice('--hours='.length), 12)));
|
|
continue;
|
|
}
|
|
if (arg.startsWith('--interval-sec=')) {
|
|
intervalSec = Math.max(15, Math.floor(toNumber(arg.slice('--interval-sec='.length), 60)));
|
|
continue;
|
|
}
|
|
if (arg.startsWith('--out=')) {
|
|
outFile = String(arg.slice('--out='.length) || '').trim();
|
|
continue;
|
|
}
|
|
if (arg.startsWith('--health-url=')) {
|
|
healthUrl = String(arg.slice('--health-url='.length) || '').trim() || healthUrl;
|
|
continue;
|
|
}
|
|
if (arg.startsWith('--state-file=')) {
|
|
const candidate = String(arg.slice('--state-file='.length) || '').trim();
|
|
if (candidate) stateFile = path.resolve(process.cwd(), candidate);
|
|
}
|
|
}
|
|
|
|
if (!outFile) {
|
|
const stamp = new Date().toISOString().replace(/[:]/g, '-');
|
|
outFile = path.resolve(process.cwd(), `logs/fresh-window-monitor-${stamp}.jsonl`);
|
|
} else {
|
|
outFile = path.resolve(process.cwd(), outFile);
|
|
}
|
|
|
|
return {
|
|
hours,
|
|
intervalSec,
|
|
outFile,
|
|
healthUrl,
|
|
stateFile
|
|
};
|
|
};
|
|
|
|
const wait = async (ms: number): Promise<void> => {
|
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
};
|
|
|
|
const readStateEventSummary = (stateFile: string, fromTimestampMs: number): {
|
|
operationalEventsCount: number;
|
|
latestEventType: string;
|
|
latestEventSeverity: string;
|
|
latestEventMessage: string;
|
|
latestEventAt: number;
|
|
} => {
|
|
try {
|
|
if (!fs.existsSync(stateFile)) {
|
|
return {
|
|
operationalEventsCount: 0,
|
|
latestEventType: '',
|
|
latestEventSeverity: '',
|
|
latestEventMessage: '',
|
|
latestEventAt: 0
|
|
};
|
|
}
|
|
const raw = fs.readFileSync(stateFile, 'utf8');
|
|
const parsed = JSON.parse(raw);
|
|
const allEvents = Array.isArray(parsed?.operationalEvents) ? parsed.operationalEvents : [];
|
|
const events = allEvents.filter((row: any) => {
|
|
const ts = Number(row?.timestamp || 0);
|
|
return ts >= fromTimestampMs;
|
|
});
|
|
const latest = events.length > 0 ? events[events.length - 1] : null;
|
|
return {
|
|
operationalEventsCount: events.length,
|
|
latestEventType: String(latest?.type || '').trim(),
|
|
latestEventSeverity: String(latest?.severity || '').trim(),
|
|
latestEventMessage: String(latest?.message || '').trim(),
|
|
latestEventAt: Number(latest?.timestamp || 0) || 0
|
|
};
|
|
} catch {
|
|
return {
|
|
operationalEventsCount: 0,
|
|
latestEventType: 'STATE_READ_ERROR',
|
|
latestEventSeverity: 'WARN',
|
|
latestEventMessage: 'Failed to parse bot_state.json',
|
|
latestEventAt: Date.now()
|
|
};
|
|
}
|
|
};
|
|
|
|
const appendJsonLine = (file: string, row: Record<string, unknown>): void => {
|
|
const dir = path.dirname(file);
|
|
if (!fs.existsSync(dir)) {
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
fs.appendFileSync(file, `${JSON.stringify(row)}\n`, 'utf8');
|
|
};
|
|
|
|
const fetchHealth = async (healthUrl: string): Promise<HealthSnapshot | null> => {
|
|
try {
|
|
const response = await fetch(healthUrl, { method: 'GET' });
|
|
if (!response.ok) return null;
|
|
const payload = await response.json();
|
|
return payload as HealthSnapshot;
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
|
|
const run = async (): Promise<void> => {
|
|
const options = parseArgs(process.argv.slice(2));
|
|
const startedAtMs = Date.now();
|
|
const endAtMs = startedAtMs + (options.hours * 60 * 60 * 1000);
|
|
|
|
appendJsonLine(options.outFile, {
|
|
type: 'monitor_started',
|
|
at: startedAtMs,
|
|
iso: new Date(startedAtMs).toISOString(),
|
|
options
|
|
});
|
|
|
|
while (Date.now() < endAtMs) {
|
|
const now = Date.now();
|
|
const health = await fetchHealth(options.healthUrl);
|
|
const stateSummary = readStateEventSummary(options.stateFile, startedAtMs);
|
|
const mode = String(health?.tradingControl?.mode || 'UNKNOWN').toUpperCase();
|
|
const mismatch = Number(health?.reconciliationMismatchCount || 0);
|
|
const missingDb = Number(health?.reconciliationMissingInDb || 0);
|
|
const noGo = Number(health?.reconciliationNoGoTrades || 0);
|
|
const watchdog = Boolean(health?.reconciliationIntegrityWatchdogTriggered);
|
|
const healthyLoops = Boolean(health?.tradingLoopHealthy) && Boolean(health?.monitorLoopHealthy) && Boolean(health?.reconciliationLoopHealthy);
|
|
const severity = (mode !== 'RUNNING' || mismatch > 0 || missingDb > 0 || noGo > 0 || watchdog || !healthyLoops) ? 'WARN' : 'INFO';
|
|
|
|
appendJsonLine(options.outFile, {
|
|
type: 'monitor_tick',
|
|
severity,
|
|
at: now,
|
|
iso: new Date(now).toISOString(),
|
|
mode,
|
|
tradingLoopHealthy: Boolean(health?.tradingLoopHealthy),
|
|
monitorLoopHealthy: Boolean(health?.monitorLoopHealthy),
|
|
reconciliationLoopHealthy: Boolean(health?.reconciliationLoopHealthy),
|
|
reconciliationMismatchCount: mismatch,
|
|
reconciliationMissingFromExchange: Number(health?.reconciliationMissingFromExchange || 0),
|
|
reconciliationMissingInDb: missingDb,
|
|
reconciliationNoGoTrades: noGo,
|
|
reconciliationIntegrityWatchdogTriggered: watchdog,
|
|
tradingControlChangedBy: String(health?.tradingControl?.lastChangedBy || ''),
|
|
tradingControlReason: String(health?.tradingControl?.reason || ''),
|
|
operationalEventsCount: stateSummary.operationalEventsCount,
|
|
latestEventType: stateSummary.latestEventType,
|
|
latestEventSeverity: stateSummary.latestEventSeverity,
|
|
latestEventMessage: stateSummary.latestEventMessage,
|
|
latestEventAt: stateSummary.latestEventAt
|
|
});
|
|
|
|
await wait(options.intervalSec * 1000);
|
|
}
|
|
|
|
const endedAtMs = Date.now();
|
|
appendJsonLine(options.outFile, {
|
|
type: 'monitor_finished',
|
|
at: endedAtMs,
|
|
iso: new Date(endedAtMs).toISOString(),
|
|
elapsedSec: Math.floor((endedAtMs - startedAtMs) / 1000)
|
|
});
|
|
|
|
console.log(JSON.stringify({
|
|
success: true,
|
|
outFile: options.outFile,
|
|
startedAt: new Date(startedAtMs).toISOString(),
|
|
endedAt: new Date(endedAtMs).toISOString()
|
|
}, null, 2));
|
|
};
|
|
|
|
run().catch((error) => {
|
|
console.error(JSON.stringify({
|
|
success: false,
|
|
error: error instanceof Error ? error.message : String(error)
|
|
}, null, 2));
|
|
process.exit(1);
|
|
});
|