118 lines
4.5 KiB
TypeScript
118 lines
4.5 KiB
TypeScript
import { config } from '../config/index.js';
|
|
import logger from '../utils/logger.js';
|
|
import { healthTracker } from './healthTracker.js';
|
|
import { observabilityService } from './observabilityService.js';
|
|
|
|
export interface WatchdogAutoResumeCycleMetrics {
|
|
success: boolean;
|
|
mismatchCount: number;
|
|
missingFromExchange: number;
|
|
missingInDb: number;
|
|
noGoTrades: number;
|
|
parityMismatchTrades: number;
|
|
parityQuarantinedTrades: number;
|
|
failedProfiles: number;
|
|
integrityWatchdogTriggered: boolean;
|
|
}
|
|
|
|
export interface WatchdogAutoResumeDecision {
|
|
attempted: boolean;
|
|
resumed: boolean;
|
|
reason?: string;
|
|
cleanStreak: number;
|
|
}
|
|
|
|
const toPositiveInt = (value: unknown, fallback: number): number => {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed)) return fallback;
|
|
return Math.max(0, Math.floor(parsed));
|
|
};
|
|
|
|
export class ReconciliationWatchdogAutoResumeService {
|
|
private cleanCycleStreak = 0;
|
|
private lastAutoResumeAt = 0;
|
|
|
|
private isCleanCycle(metrics: WatchdogAutoResumeCycleMetrics): boolean {
|
|
return metrics.success
|
|
&& metrics.failedProfiles === 0
|
|
&& metrics.mismatchCount === 0
|
|
&& metrics.missingFromExchange === 0
|
|
&& metrics.missingInDb === 0
|
|
&& metrics.noGoTrades === 0
|
|
&& metrics.parityMismatchTrades === 0
|
|
&& metrics.parityQuarantinedTrades === 0
|
|
&& !metrics.integrityWatchdogTriggered;
|
|
}
|
|
|
|
public evaluateCycle(metrics: WatchdogAutoResumeCycleMetrics): WatchdogAutoResumeDecision {
|
|
if (!config.ENABLE_RECON_WATCHDOG_AUTO_RESUME) {
|
|
this.cleanCycleStreak = 0;
|
|
return { attempted: false, resumed: false, reason: 'disabled', cleanStreak: 0 };
|
|
}
|
|
|
|
const snapshot = healthTracker.getSnapshot();
|
|
const control = snapshot.tradingControl;
|
|
if (control.mode !== 'PAUSED') {
|
|
this.cleanCycleStreak = 0;
|
|
return { attempted: false, resumed: false, reason: 'not_paused', cleanStreak: 0 };
|
|
}
|
|
|
|
const changedBy = String(control.lastChangedBy || '').trim();
|
|
if (!changedBy.startsWith('system:recon_parity_watchdog')) {
|
|
this.cleanCycleStreak = 0;
|
|
return { attempted: false, resumed: false, reason: 'pause_not_from_parity_watchdog', cleanStreak: 0 };
|
|
}
|
|
|
|
const now = Date.now();
|
|
const minPauseMs = toPositiveInt(config.RECON_WATCHDOG_AUTO_RESUME_MIN_PAUSE_MS, 900_000);
|
|
const pauseAgeMs = Math.max(0, now - Number(control.lastChangedAt || 0));
|
|
if (pauseAgeMs < minPauseMs) {
|
|
this.cleanCycleStreak = 0;
|
|
return { attempted: true, resumed: false, reason: 'min_pause_window', cleanStreak: 0 };
|
|
}
|
|
|
|
if (!this.isCleanCycle(metrics)) {
|
|
this.cleanCycleStreak = 0;
|
|
return { attempted: true, resumed: false, reason: 'cycle_not_clean', cleanStreak: 0 };
|
|
}
|
|
|
|
const requiredCleanCycles = Math.max(1, toPositiveInt(config.RECON_WATCHDOG_AUTO_RESUME_CLEAN_CYCLES, 2));
|
|
this.cleanCycleStreak += 1;
|
|
if (this.cleanCycleStreak < requiredCleanCycles) {
|
|
return {
|
|
attempted: true,
|
|
resumed: false,
|
|
reason: `awaiting_clean_streak_${this.cleanCycleStreak}_of_${requiredCleanCycles}`,
|
|
cleanStreak: this.cleanCycleStreak
|
|
};
|
|
}
|
|
|
|
const cooldownMs = toPositiveInt(config.RECON_WATCHDOG_AUTO_RESUME_COOLDOWN_MS, 1_800_000);
|
|
const sinceLastResumeMs = Math.max(0, now - this.lastAutoResumeAt);
|
|
if (cooldownMs > 0 && this.lastAutoResumeAt > 0 && sinceLastResumeMs < cooldownMs) {
|
|
return { attempted: true, resumed: false, reason: 'resume_cooldown', cleanStreak: this.cleanCycleStreak };
|
|
}
|
|
|
|
const reason = `Auto-resume after ${requiredCleanCycles} clean reconciliation cycles (pause_age_ms=${pauseAgeMs}, source=${changedBy}).`;
|
|
healthTracker.recordTradingControl({
|
|
mode: 'RUNNING',
|
|
lastChangedBy: 'system:recon_parity_auto_resume',
|
|
lastChangedAt: now,
|
|
reason
|
|
});
|
|
|
|
logger.info(`[Reconcile] ${reason}`);
|
|
observabilityService.emitEvent({
|
|
type: 'PARITY_WARNING',
|
|
severity: 'INFO',
|
|
message: reason
|
|
});
|
|
|
|
this.lastAutoResumeAt = now;
|
|
this.cleanCycleStreak = 0;
|
|
return { attempted: true, resumed: true, reason: 'resumed', cleanStreak: 0 };
|
|
}
|
|
}
|
|
|
|
export const reconciliationWatchdogAutoResumeService = new ReconciliationWatchdogAutoResumeService();
|