learning_ai_invt_trdg/backend/reconcileAttributionRepair.ts

762 lines
28 KiB
TypeScript

import 'dotenv/config';
import { randomUUID } from 'crypto';
import { createClient } from '@supabase/supabase-js';
import { normalizeOrderAction, normalizeTradeSide } from '../src/domain/tradingEnums.js';
type CliOptions = {
apply: boolean;
tradeIds: string[];
restoreBatchId?: string;
allowCrossTrade: boolean;
allowPartial: boolean;
};
type OrderRow = {
id?: string;
order_id?: string | null;
user_id?: string | null;
profile_id?: string | null;
trade_id?: string | null;
symbol?: string | null;
action?: string | null;
side?: string | null;
qty?: number | string | null;
quantity?: number | string | null;
price?: number | string | null;
status?: string | null;
source?: string | null;
created_at?: string | null;
updated_at?: string | null;
};
type AuditRow = {
id: number;
batch_id?: string | null;
profile_id?: string | null;
symbol?: string | null;
trade_id?: string | null;
exchange_order_id?: string | null;
backfill_order_id?: string | null;
decision?: string | null;
reason?: string | null;
metadata?: Record<string, any> | null;
created_at?: string | null;
reverted_at?: string | null;
};
type CandidateRow = {
order: OrderRow;
qty: number;
exchangeOrderId: string;
realOrder: OrderRow | null;
safe: boolean;
unsafeReason?: string;
};
type ChosenSubset = {
orderIds: string[];
sumQty: number;
nextOpenQty: number;
withinDust: boolean;
};
type TradeEvaluation = {
tradeId: string;
profileId: string;
symbol: string;
entryQty: number;
exitQty: number;
openQtyBefore: number;
openQtyAfter: number;
dustThreshold: number;
candidates: CandidateRow[];
selectedOrderIds: string[];
decision: 'SKIP' | 'DRY_RUN' | 'APPLIED' | 'NO_GO';
reason: string;
};
const EPSILON = 1e-8;
const DUST_ABS_QTY = 0.001;
const DUST_REL_PCT = 0.002; // 0.2%
const FILLED_STATUSES = new Set(['filled', 'partially_filled', 'partially-filled']);
const APPLIED_DECISIONS = new Set([
'APPLIED',
'MANUAL_OVERRIDE_APPLIED',
'SKIP_EXISTING',
'MANUAL_OVERRIDE_SKIP_EXISTING',
'ATTRIB_REPAIR_APPLIED'
]);
const STATUS_REVERT_TARGET = 'canceled';
const parseArgs = (argv: string[]): CliOptions => {
const tradeIds = new Set<string>();
let apply = false;
let restoreBatchId: string | undefined;
let allowCrossTrade = false;
let allowPartial = false;
for (const raw of argv) {
const arg = String(raw || '').trim();
if (!arg) continue;
if (arg === '--apply') {
apply = true;
continue;
}
if (arg === '--allow-cross-trade') {
allowCrossTrade = true;
continue;
}
if (arg === '--allow-partial') {
allowPartial = true;
continue;
}
if (arg.startsWith('--restore-batch=')) {
const value = String(arg.slice('--restore-batch='.length) || '').trim();
if (value) restoreBatchId = value;
continue;
}
if (arg.startsWith('--trade=')) {
const value = String(arg.slice('--trade='.length) || '').trim();
if (value) tradeIds.add(value);
continue;
}
}
return {
apply,
tradeIds: Array.from(tradeIds),
restoreBatchId,
allowCrossTrade,
allowPartial
};
};
const toNumber = (value: unknown): number => {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : 0;
};
const normalizeStatus = (value: unknown): string => String(value || '').trim().toLowerCase();
const inferAction = (actionRaw?: string | null, sideRaw?: string | null): 'ENTRY' | 'EXIT' | undefined => {
const explicit = normalizeOrderAction(actionRaw || undefined);
if (explicit) return explicit;
const side = normalizeTradeSide(sideRaw || 'BUY');
return side === 'BUY' ? 'ENTRY' : 'EXIT';
};
const parseMetadata = (value: unknown): Record<string, any> => {
if (!value) return {};
if (typeof value === 'object') return value as Record<string, any>;
if (typeof value === 'string') {
try {
const parsed = JSON.parse(value);
if (parsed && typeof parsed === 'object') return parsed as Record<string, any>;
} catch {
return {};
}
}
return {};
};
const pickBestSubset = (
openQtyBefore: number,
candidates: CandidateRow[],
dustThreshold: number,
allowPartial: boolean
): ChosenSubset | null => {
if (candidates.length === 0) return null;
if (!(openQtyBefore < -EPSILON)) return null;
if (candidates.length > 20) return null;
const n = candidates.length;
const totalMasks = 1 << n;
let best: ChosenSubset | null = null;
for (let mask = 1; mask < totalMasks; mask += 1) {
let sumQty = 0;
const orderIds: string[] = [];
for (let idx = 0; idx < n; idx += 1) {
if ((mask & (1 << idx)) === 0) continue;
const candidate = candidates[idx];
sumQty += candidate.qty;
orderIds.push(String(candidate.order.order_id || '').trim());
}
const nextOpenQty = Number((openQtyBefore + sumQty).toFixed(8));
const improved = Math.abs(nextOpenQty) + EPSILON < Math.abs(openQtyBefore);
if (!improved) continue;
const withinDust = Math.abs(nextOpenQty) <= dustThreshold + EPSILON;
if (!withinDust && !allowPartial) continue;
// Never move from over-closed to materially under-closed in strict mode.
if (!allowPartial && nextOpenQty > dustThreshold + EPSILON) continue;
const candidateBest: ChosenSubset = {
orderIds,
sumQty: Number(sumQty.toFixed(8)),
nextOpenQty,
withinDust
};
if (!best) {
best = candidateBest;
continue;
}
const bestAbs = Math.abs(best.nextOpenQty);
const candAbs = Math.abs(candidateBest.nextOpenQty);
if (candAbs + EPSILON < bestAbs) {
best = candidateBest;
continue;
}
if (Math.abs(candAbs - bestAbs) <= EPSILON && candidateBest.orderIds.length < best.orderIds.length) {
best = candidateBest;
}
}
return best;
};
const buildTradeLedger = (rows: OrderRow[]): {
entryQty: number;
exitQty: number;
openQty: number;
profileId: string;
symbol: string;
} => {
let entryQty = 0;
let exitQty = 0;
let profileId = '';
let symbol = '';
for (const row of rows) {
const status = normalizeStatus(row.status);
if (!FILLED_STATUSES.has(status)) continue;
const qty = toNumber(row.qty ?? row.quantity);
if (!(qty > EPSILON)) continue;
const action = inferAction(row.action, row.side);
if (!action) continue;
if (!profileId) profileId = String(row.profile_id || '').trim();
if (!symbol) symbol = String(row.symbol || '').trim();
if (action === 'ENTRY') entryQty += qty;
if (action === 'EXIT') exitQty += qty;
}
return {
entryQty: Number(entryQty.toFixed(8)),
exitQty: Number(exitQty.toFixed(8)),
openQty: Number((entryQty - exitQty).toFixed(8)),
profileId,
symbol
};
};
const runRestoreMode = async (
supabase: ReturnType<typeof createClient>,
options: CliOptions
): Promise<void> => {
const restoreBatchId = String(options.restoreBatchId || '').trim();
if (!restoreBatchId) {
throw new Error('Missing --restore-batch=<BATCH_ID>.');
}
const { data: rows, error } = await supabase
.from('reconciliation_backfill_audit')
.select('id,batch_id,profile_id,symbol,trade_id,backfill_order_id,decision,reason,metadata,created_at,reverted_at')
.eq('batch_id', restoreBatchId)
.in('decision', ['ATTRIB_REPAIR_APPLIED'])
.order('created_at', { ascending: true });
if (error) throw error;
const restoreTargets = ((rows || []) as AuditRow[])
.filter((row) => String(row.backfill_order_id || '').trim().length > 0);
const batchId = `ATTRIB-RESTORE-${randomUUID()}`;
const nowIso = new Date().toISOString();
const updates = restoreTargets.map((row) => {
const metadata = parseMetadata(row.metadata);
const restoreStatus = normalizeStatus(metadata.prevStatus || 'filled') || 'filled';
return {
row,
restoreStatus
};
});
let restoredRows = 0;
if (options.apply && updates.length > 0) {
for (const target of updates) {
const orderId = String(target.row.backfill_order_id || '').trim();
const { error: updateError } = await supabase
.from('orders')
.update({
status: target.restoreStatus,
updated_at: nowIso
})
.eq('order_id', orderId);
if (updateError) throw updateError;
restoredRows += 1;
}
const { error: markRevertedError } = await supabase
.from('reconciliation_backfill_audit')
.update({ reverted_at: nowIso })
.eq('batch_id', restoreBatchId)
.in('decision', ['ATTRIB_REPAIR_APPLIED']);
if (markRevertedError) throw markRevertedError;
const restoreAuditRows = updates.map((target) => ({
batch_id: batchId,
profile_id: String(target.row.profile_id || '').trim(),
symbol: String(target.row.symbol || '').trim(),
trade_id: String(target.row.trade_id || '').trim(),
exchange_order_id: null,
exchange_client_order_id: null,
backfill_order_id: String(target.row.backfill_order_id || '').trim(),
filled_qty: null,
filled_price: null,
filled_at: null,
dry_run: false,
decision: 'ATTRIB_RESTORE_APPLIED',
reason: `status_restored_from_${restoreBatchId}`,
metadata: {
restoredFromBatch: restoreBatchId,
restoredFromAuditId: target.row.id,
restoredToStatus: target.restoreStatus
},
applied_at: nowIso
}));
const { error: restoreAuditError } = await supabase
.from('reconciliation_backfill_audit')
.insert(restoreAuditRows);
if (restoreAuditError) throw restoreAuditError;
}
console.log(JSON.stringify({
mode: options.apply ? 'restore-apply' : 'restore-dry-run',
restoreBatchId,
batchId,
targetRows: updates.length,
restoredRows,
targets: updates.map((target) => ({
backfillOrderId: String(target.row.backfill_order_id || '').trim(),
profileId: String(target.row.profile_id || '').trim(),
tradeId: String(target.row.trade_id || '').trim(),
restoreStatus: target.restoreStatus
}))
}, null, 2));
};
const runRepairMode = async (
supabase: ReturnType<typeof createClient>,
options: CliOptions
): Promise<void> => {
if (options.tradeIds.length === 0) {
throw new Error('Provide at least one --trade=<TRADE_ID> or use --restore-batch=<BATCH_ID>.');
}
const { data: tradeRows, error: tradeError } = await supabase
.from('orders')
.select('id,order_id,user_id,profile_id,trade_id,symbol,action,side,qty,quantity,price,status,source,created_at,updated_at')
.in('trade_id', options.tradeIds)
.order('created_at', { ascending: true });
if (tradeError) throw tradeError;
const rows = (tradeRows || []) as OrderRow[];
const rowsByTrade = new Map<string, OrderRow[]>();
for (const row of rows) {
const tradeId = String(row.trade_id || '').trim();
if (!tradeId) continue;
const list = rowsByTrade.get(tradeId) || [];
list.push(row);
rowsByTrade.set(tradeId, list);
}
const candidateBfillOrderIds = Array.from(new Set(
rows
.map((row) => String(row.order_id || '').trim())
.filter((orderId) => orderId.startsWith('BFILL-'))
));
const preferredAuditByBackfill = new Map<string, AuditRow>();
if (candidateBfillOrderIds.length > 0) {
const { data: auditRows, error: auditError } = await supabase
.from('reconciliation_backfill_audit')
.select('id,batch_id,profile_id,symbol,trade_id,exchange_order_id,backfill_order_id,decision,reason,metadata,created_at,reverted_at')
.in('backfill_order_id', candidateBfillOrderIds)
.order('created_at', { ascending: false });
if (auditError) throw auditError;
const rowsByBackfill = new Map<string, AuditRow[]>();
for (const row of (auditRows || []) as AuditRow[]) {
const backfillOrderId = String(row.backfill_order_id || '').trim();
if (!backfillOrderId) continue;
const list = rowsByBackfill.get(backfillOrderId) || [];
list.push(row);
rowsByBackfill.set(backfillOrderId, list);
}
for (const [backfillOrderId, backfillAuditRows] of rowsByBackfill.entries()) {
const preferred = backfillAuditRows.find((row) => {
const decision = String(row.decision || '').trim();
const exchangeOrderId = String(row.exchange_order_id || '').trim();
return APPLIED_DECISIONS.has(decision) && exchangeOrderId.length > 0;
}) || backfillAuditRows[0];
if (preferred) {
preferredAuditByBackfill.set(backfillOrderId, preferred);
}
}
}
const exchangeOrderIds = Array.from(new Set(
Array.from(preferredAuditByBackfill.values())
.map((row) => String(row.exchange_order_id || '').trim())
.filter(Boolean)
));
const realOrderByOrderId = new Map<string, OrderRow>();
if (exchangeOrderIds.length > 0) {
const { data: realRows, error: realError } = await supabase
.from('orders')
.select('id,order_id,user_id,profile_id,trade_id,symbol,action,side,qty,quantity,price,status,source,created_at,updated_at')
.in('order_id', exchangeOrderIds);
if (realError) throw realError;
for (const row of (realRows || []) as OrderRow[]) {
const orderId = String(row.order_id || '').trim();
if (!orderId) continue;
if (!realOrderByOrderId.has(orderId)) {
realOrderByOrderId.set(orderId, row);
}
}
}
const batchId = `ATTRIB-REPAIR-${randomUUID()}`;
const nowIso = new Date().toISOString();
const evaluations: TradeEvaluation[] = [];
const auditRowsToInsert: Record<string, any>[] = [];
const orderIdsToRevert = new Set<string>();
for (const tradeId of options.tradeIds) {
const tradeRowsForId = rowsByTrade.get(tradeId) || [];
if (tradeRowsForId.length === 0) {
evaluations.push({
tradeId,
profileId: '',
symbol: '',
entryQty: 0,
exitQty: 0,
openQtyBefore: 0,
openQtyAfter: 0,
dustThreshold: DUST_ABS_QTY,
candidates: [],
selectedOrderIds: [],
decision: 'NO_GO',
reason: 'trade_not_found'
});
continue;
}
const ledger = buildTradeLedger(tradeRowsForId);
const dustThreshold = Math.max(DUST_ABS_QTY, ledger.entryQty * DUST_REL_PCT);
const candidates = tradeRowsForId
.filter((row) => String(row.order_id || '').trim().startsWith('BFILL-'))
.filter((row) => FILLED_STATUSES.has(normalizeStatus(row.status)))
.filter((row) => inferAction(row.action, row.side) === 'EXIT')
.map((row): CandidateRow => {
const orderId = String(row.order_id || '').trim();
const latestAudit = preferredAuditByBackfill.get(orderId);
if (!latestAudit) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId: '',
realOrder: null,
safe: false,
unsafeReason: 'missing_audit_link'
};
}
if (!APPLIED_DECISIONS.has(String(latestAudit.decision || '').trim())) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId: String(latestAudit.exchange_order_id || '').trim(),
realOrder: null,
safe: false,
unsafeReason: `audit_not_applied:${String(latestAudit.decision || '').trim() || 'unknown'}`
};
}
const exchangeOrderId = String(latestAudit.exchange_order_id || '').trim();
if (!exchangeOrderId) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder: null,
safe: false,
unsafeReason: 'missing_exchange_order_id'
};
}
const realOrder = realOrderByOrderId.get(exchangeOrderId) || null;
if (!realOrder) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder: null,
safe: false,
unsafeReason: 'exchange_order_not_persisted_in_orders'
};
}
if (!FILLED_STATUSES.has(normalizeStatus(realOrder.status))) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder,
safe: false,
unsafeReason: `exchange_order_not_filled:${normalizeStatus(realOrder.status)}`
};
}
const realTradeId = String(realOrder.trade_id || '').trim();
if (!options.allowCrossTrade && realTradeId !== tradeId) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder,
safe: false,
unsafeReason: `cross_trade_mapping:${realTradeId || 'null'}`
};
}
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder,
safe: true
};
});
if (!(ledger.openQty < -EPSILON)) {
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: ledger.openQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: [],
decision: 'SKIP',
reason: 'trade_not_overclosed'
});
continue;
}
const safeCandidates = candidates.filter((candidate) => candidate.safe && candidate.qty > EPSILON);
if (safeCandidates.length === 0) {
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: ledger.openQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: [],
decision: 'NO_GO',
reason: 'no_safe_candidates'
});
continue;
}
const bestSubset = pickBestSubset(ledger.openQty, safeCandidates, dustThreshold, options.allowPartial);
if (!bestSubset) {
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: ledger.openQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: [],
decision: 'NO_GO',
reason: options.allowPartial ? 'no_improving_subset' : 'no_subset_within_dust'
});
continue;
}
const resolvedDecision: TradeEvaluation['decision'] = options.apply ? 'APPLIED' : 'DRY_RUN';
for (const orderId of bestSubset.orderIds) {
orderIdsToRevert.add(orderId);
}
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: bestSubset.nextOpenQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: bestSubset.orderIds,
decision: resolvedDecision,
reason: bestSubset.withinDust ? 'synthetic_duplicate_subset_selected' : 'synthetic_duplicate_subset_selected_partial'
});
}
if (options.apply && orderIdsToRevert.size > 0) {
for (const orderId of orderIdsToRevert) {
const { error: updateError } = await supabase
.from('orders')
.update({
status: STATUS_REVERT_TARGET,
updated_at: nowIso
})
.eq('order_id', orderId)
.in('status', ['filled', 'partially_filled', 'partially-filled']);
if (updateError) throw updateError;
}
}
for (const evaluation of evaluations) {
if (evaluation.selectedOrderIds.length > 0) {
for (const orderId of evaluation.selectedOrderIds) {
const candidate = evaluation.candidates.find((row) => String(row.order.order_id || '').trim() === orderId);
if (!candidate) continue;
const prevStatus = normalizeStatus(candidate.order.status);
auditRowsToInsert.push({
batch_id: batchId,
profile_id: evaluation.profileId,
symbol: evaluation.symbol,
trade_id: evaluation.tradeId,
exchange_order_id: candidate.exchangeOrderId || null,
exchange_client_order_id: null,
backfill_order_id: orderId,
filled_qty: Number(candidate.qty.toFixed(8)),
filled_price: toNumber(candidate.order.price),
filled_at: null,
dry_run: !options.apply,
decision: options.apply ? 'ATTRIB_REPAIR_APPLIED' : 'ATTRIB_REPAIR_DRY',
reason: 'synthetic_duplicate_status_revert',
metadata: {
prevStatus: prevStatus || 'filled',
nextStatus: options.apply ? STATUS_REVERT_TARGET : `would_${STATUS_REVERT_TARGET}`,
openQtyBefore: evaluation.openQtyBefore,
openQtyAfter: evaluation.openQtyAfter,
dustThreshold: evaluation.dustThreshold,
realOrderTradeId: String(candidate.realOrder?.trade_id || '').trim() || null,
allowCrossTrade: options.allowCrossTrade,
allowPartial: options.allowPartial
},
applied_at: options.apply ? nowIso : null
});
}
} else if (evaluation.decision === 'NO_GO') {
auditRowsToInsert.push({
batch_id: batchId,
profile_id: evaluation.profileId || null,
symbol: evaluation.symbol || null,
trade_id: evaluation.tradeId,
exchange_order_id: null,
exchange_client_order_id: null,
backfill_order_id: null,
filled_qty: null,
filled_price: null,
filled_at: null,
dry_run: !options.apply,
decision: options.apply ? 'ATTRIB_REPAIR_NO_GO' : 'ATTRIB_REPAIR_NO_GO_DRY',
reason: evaluation.reason,
metadata: {
openQtyBefore: evaluation.openQtyBefore,
dustThreshold: evaluation.dustThreshold,
safeCandidates: evaluation.candidates.filter((row) => row.safe).length,
unsafeCandidates: evaluation.candidates.filter((row) => !row.safe).map((row) => ({
orderId: row.order.order_id,
reason: row.unsafeReason
}))
},
applied_at: null
});
}
}
if (auditRowsToInsert.length > 0) {
const { error: auditInsertError } = await supabase
.from('reconciliation_backfill_audit')
.insert(auditRowsToInsert);
if (auditInsertError) throw auditInsertError;
}
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
batchId,
tradeIds: options.tradeIds,
allowCrossTrade: options.allowCrossTrade,
allowPartial: options.allowPartial,
totalTrades: evaluations.length,
appliedOrderStatusReverts: options.apply ? orderIdsToRevert.size : 0,
evaluations: evaluations.map((evaluation) => ({
tradeId: evaluation.tradeId,
profileId: evaluation.profileId,
symbol: evaluation.symbol,
decision: evaluation.decision,
reason: evaluation.reason,
entryQty: evaluation.entryQty,
exitQty: evaluation.exitQty,
openQtyBefore: evaluation.openQtyBefore,
openQtyAfter: evaluation.openQtyAfter,
dustThreshold: evaluation.dustThreshold,
selectedOrderIds: evaluation.selectedOrderIds,
safeCandidateCount: evaluation.candidates.filter((row) => row.safe).length,
unsafeCandidateCount: evaluation.candidates.filter((row) => !row.safe).length
}))
}, null, 2));
};
const run = async (): Promise<void> => {
const options = parseArgs(process.argv.slice(2));
const supabaseUrl = String(process.env.SUPABASE_URL || '').trim();
const supabaseKey = String(
process.env.SUPABASE_KEY
|| process.env.SUPABASE_SERVICE_ROLE_KEY
|| process.env.SUPABASE_ANON_KEY
|| ''
).trim();
if (!supabaseUrl || !supabaseKey) {
throw new Error('Missing Supabase credentials. Expected SUPABASE_URL + SUPABASE_KEY/SUPABASE_SERVICE_ROLE_KEY.');
}
const supabase = createClient(supabaseUrl, supabaseKey);
if (options.restoreBatchId) {
await runRestoreMode(supabase, options);
return;
}
await runRepairMode(supabase, options);
};
run().catch((error) => {
console.error(JSON.stringify({
error: error instanceof Error ? error.message : String(error)
}, null, 2));
process.exit(1);
});