import { config } from '../config/index.js'; import logger from '../utils/logger.js'; import { healthTracker } from './healthTracker.js'; import { observabilityService } from './observabilityService.js'; export interface WatchdogAutoResumeCycleMetrics { success: boolean; mismatchCount: number; missingFromExchange: number; missingInDb: number; noGoTrades: number; parityMismatchTrades: number; parityQuarantinedTrades: number; failedProfiles: number; integrityWatchdogTriggered: boolean; } export interface WatchdogAutoResumeDecision { attempted: boolean; resumed: boolean; reason?: string; cleanStreak: number; } const toPositiveInt = (value: unknown, fallback: number): number => { const parsed = Number(value); if (!Number.isFinite(parsed)) return fallback; return Math.max(0, Math.floor(parsed)); }; export class ReconciliationWatchdogAutoResumeService { private cleanCycleStreak = 0; private lastAutoResumeAt = 0; private isCleanCycle(metrics: WatchdogAutoResumeCycleMetrics): boolean { return metrics.success && metrics.failedProfiles === 0 && metrics.mismatchCount === 0 && metrics.missingFromExchange === 0 && metrics.missingInDb === 0 && metrics.noGoTrades === 0 && metrics.parityMismatchTrades === 0 && metrics.parityQuarantinedTrades === 0 && !metrics.integrityWatchdogTriggered; } public evaluateCycle(metrics: WatchdogAutoResumeCycleMetrics): WatchdogAutoResumeDecision { if (!config.ENABLE_RECON_WATCHDOG_AUTO_RESUME) { this.cleanCycleStreak = 0; return { attempted: false, resumed: false, reason: 'disabled', cleanStreak: 0 }; } const snapshot = healthTracker.getSnapshot(); const control = snapshot.tradingControl; if (control.mode !== 'PAUSED') { this.cleanCycleStreak = 0; return { attempted: false, resumed: false, reason: 'not_paused', cleanStreak: 0 }; } const changedBy = String(control.lastChangedBy || '').trim(); if (!changedBy.startsWith('system:recon_parity_watchdog')) { this.cleanCycleStreak = 0; return { attempted: false, resumed: false, reason: 'pause_not_from_parity_watchdog', cleanStreak: 0 }; } const now = Date.now(); const minPauseMs = toPositiveInt(config.RECON_WATCHDOG_AUTO_RESUME_MIN_PAUSE_MS, 900_000); const pauseAgeMs = Math.max(0, now - Number(control.lastChangedAt || 0)); if (pauseAgeMs < minPauseMs) { this.cleanCycleStreak = 0; return { attempted: true, resumed: false, reason: 'min_pause_window', cleanStreak: 0 }; } if (!this.isCleanCycle(metrics)) { this.cleanCycleStreak = 0; return { attempted: true, resumed: false, reason: 'cycle_not_clean', cleanStreak: 0 }; } const requiredCleanCycles = Math.max(1, toPositiveInt(config.RECON_WATCHDOG_AUTO_RESUME_CLEAN_CYCLES, 2)); this.cleanCycleStreak += 1; if (this.cleanCycleStreak < requiredCleanCycles) { return { attempted: true, resumed: false, reason: `awaiting_clean_streak_${this.cleanCycleStreak}_of_${requiredCleanCycles}`, cleanStreak: this.cleanCycleStreak }; } const cooldownMs = toPositiveInt(config.RECON_WATCHDOG_AUTO_RESUME_COOLDOWN_MS, 1_800_000); const sinceLastResumeMs = Math.max(0, now - this.lastAutoResumeAt); if (cooldownMs > 0 && this.lastAutoResumeAt > 0 && sinceLastResumeMs < cooldownMs) { return { attempted: true, resumed: false, reason: 'resume_cooldown', cleanStreak: this.cleanCycleStreak }; } const reason = `Auto-resume after ${requiredCleanCycles} clean reconciliation cycles (pause_age_ms=${pauseAgeMs}, source=${changedBy}).`; healthTracker.recordTradingControl({ mode: 'RUNNING', lastChangedBy: 'system:recon_parity_auto_resume', lastChangedAt: now, reason }); logger.info(`[Reconcile] ${reason}`); observabilityService.emitEvent({ type: 'PARITY_WARNING', severity: 'INFO', message: reason }); this.lastAutoResumeAt = now; this.cleanCycleStreak = 0; return { attempted: true, resumed: true, reason: 'resumed', cleanStreak: 0 }; } } export const reconciliationWatchdogAutoResumeService = new ReconciliationWatchdogAutoResumeService();