848 lines
33 KiB
TypeScript
848 lines
33 KiB
TypeScript
import 'dotenv/config';
|
|
import Alpaca from '@alpacahq/alpaca-trade-api';
|
|
import { createClient } from '@supabase/supabase-js';
|
|
|
|
type OrderRow = {
|
|
id?: string;
|
|
order_id?: string;
|
|
profile_id?: string | null;
|
|
symbol?: string;
|
|
side?: string;
|
|
type?: string;
|
|
qty?: number | string | null;
|
|
quantity?: number | string | null;
|
|
price?: number | string | null;
|
|
status?: string;
|
|
timestamp?: number | string | null;
|
|
created_at?: string | null;
|
|
trade_id?: string | null;
|
|
action?: string | null;
|
|
source?: string | null;
|
|
};
|
|
|
|
type HistoryRow = {
|
|
id?: string;
|
|
profile_id?: string | null;
|
|
trade_id?: string | null;
|
|
symbol?: string;
|
|
side?: string;
|
|
size?: number | string | null;
|
|
entry_price?: number | string | null;
|
|
exit_price?: number | string | null;
|
|
pnl?: number | string | null;
|
|
pnl_percent?: number | string | null;
|
|
reason?: string | null;
|
|
source?: string | null;
|
|
timestamp?: number | string | null;
|
|
created_at?: string | null;
|
|
};
|
|
|
|
type AlpacaOrder = {
|
|
id?: string;
|
|
client_order_id?: string;
|
|
symbol?: string;
|
|
side?: string;
|
|
qty?: number | string;
|
|
filled_qty?: number | string;
|
|
type?: string;
|
|
limit_price?: number | string | null;
|
|
filled_avg_price?: number | string | null;
|
|
status?: string;
|
|
submitted_at?: string;
|
|
filled_at?: string | null;
|
|
};
|
|
|
|
type ReconciliationSample = Record<string, any>;
|
|
|
|
const cliArgs = process.argv.slice(2);
|
|
const positionalArgs = cliArgs.filter((arg) => !arg.startsWith('--'));
|
|
const LOOKBACK_DAYS = Number(positionalArgs[0] || '7');
|
|
const MAX_SAMPLES = Number(positionalArgs[1] || '12');
|
|
const carryLookbackArg = cliArgs.find((arg) => arg.startsWith('--carry-lookback-days='));
|
|
const CARRY_IN_LOOKBACK_DAYS = Number(
|
|
(carryLookbackArg ? carryLookbackArg.split('=')[1] : '')
|
|
|| process.env.RECONCILE_CARRY_IN_LOOKBACK_DAYS
|
|
|| '90'
|
|
);
|
|
const PAGE_SIZE = 1000;
|
|
const ALPACA_PAGE_LIMIT = 500;
|
|
const MAX_ALPACA_PAGES = 60;
|
|
|
|
const supabaseUrl = String(process.env.SUPABASE_URL || '').trim();
|
|
const supabaseKey = String(
|
|
process.env.SUPABASE_KEY ||
|
|
process.env.SUPABASE_SERVICE_ROLE_KEY ||
|
|
process.env.SUPABASE_ANON_KEY ||
|
|
''
|
|
).trim();
|
|
const alpacaApiKey = String(process.env.ALPACA_API_KEY || '').trim();
|
|
const alpacaApiSecret = String(process.env.ALPACA_API_SECRET || '').trim();
|
|
const paperTrading = String(process.env.PAPER_TRADING || 'true').toLowerCase() === 'true';
|
|
|
|
if (!supabaseUrl || !supabaseKey) {
|
|
throw new Error('Missing Supabase credentials. Expected SUPABASE_URL + SUPABASE_KEY/SUPABASE_SERVICE_ROLE_KEY.');
|
|
}
|
|
if (!alpacaApiKey || !alpacaApiSecret) {
|
|
throw new Error('Missing Alpaca credentials. Expected ALPACA_API_KEY + ALPACA_API_SECRET.');
|
|
}
|
|
if (!Number.isFinite(LOOKBACK_DAYS) || LOOKBACK_DAYS <= 0) {
|
|
throw new Error(`Invalid lookback days: ${positionalArgs[0] || process.argv[2]}`);
|
|
}
|
|
if (!Number.isFinite(CARRY_IN_LOOKBACK_DAYS) || CARRY_IN_LOOKBACK_DAYS <= 0) {
|
|
throw new Error(`Invalid carry lookback days: ${carryLookbackArg || process.env.RECONCILE_CARRY_IN_LOOKBACK_DAYS}`);
|
|
}
|
|
|
|
const supabase = createClient(supabaseUrl, supabaseKey);
|
|
const alpaca = new (Alpaca as any)({
|
|
keyId: alpacaApiKey,
|
|
secretKey: alpacaApiSecret,
|
|
paper: paperTrading
|
|
});
|
|
|
|
const now = new Date();
|
|
const startDate = new Date(now.getTime() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000);
|
|
const carryStartDate = new Date(startDate.getTime() - CARRY_IN_LOOKBACK_DAYS * 24 * 60 * 60 * 1000);
|
|
const startIso = startDate.toISOString();
|
|
const carryStartIso = carryStartDate.toISOString();
|
|
const nowIso = now.toISOString();
|
|
|
|
const toNumber = (value: unknown): number => {
|
|
const num = Number(value);
|
|
return Number.isFinite(num) ? num : 0;
|
|
};
|
|
|
|
const toTimestampMs = (value: unknown, fallback: number = 0): number => {
|
|
if (typeof value === 'number') {
|
|
return value > 1_000_000_000_000 ? value : value * 1000;
|
|
}
|
|
if (typeof value === 'string') {
|
|
if (/^\d+(\.\d+)?$/.test(value.trim())) {
|
|
return toTimestampMs(Number(value.trim()), fallback);
|
|
}
|
|
const parsed = Date.parse(value);
|
|
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
|
}
|
|
return fallback;
|
|
};
|
|
|
|
const normalizeStatus = (status: string | undefined | null): string => {
|
|
const s = String(status || '').trim().toLowerCase();
|
|
if (s === 'filled') return 'filled';
|
|
if (s === 'partially_filled' || s === 'partiallyfilled' || s === 'partial_fill') return 'partially_filled';
|
|
if (s === 'canceled' || s === 'cancelled') return 'canceled';
|
|
if (s === 'expired') return 'expired';
|
|
if (s === 'rejected') return 'rejected';
|
|
if (s === 'unknown') return 'unknown';
|
|
return 'pending_new';
|
|
};
|
|
|
|
const normalizeSide = (side: string | undefined | null): 'BUY' | 'SELL' => {
|
|
const s = String(side || '').trim().toUpperCase();
|
|
return s === 'SELL' || s === 'SHORT' ? 'SELL' : 'BUY';
|
|
};
|
|
|
|
const normalizeSymbol = (symbol: string | undefined | null): string => {
|
|
const raw = String(symbol || '').trim().toUpperCase().replace(/[\/\-_]/g, '');
|
|
if (raw.endsWith('USDT')) {
|
|
return `${raw.slice(0, -4)}USD`;
|
|
}
|
|
return raw;
|
|
};
|
|
|
|
const getOrderQty = (row: OrderRow): number => {
|
|
const qty = toNumber(row.qty);
|
|
if (qty > 0) return qty;
|
|
return toNumber(row.quantity);
|
|
};
|
|
|
|
const pctDiff = (left: number, right: number): number => {
|
|
const denom = Math.max(Math.abs(left), Math.abs(right), 1e-9);
|
|
return Math.abs(left - right) / denom;
|
|
};
|
|
|
|
const pushSample = (bucket: ReconciliationSample[], sample: ReconciliationSample) => {
|
|
if (bucket.length < MAX_SAMPLES) bucket.push(sample);
|
|
};
|
|
|
|
const isQuarantinedHistoryReason = (reason: string | undefined | null): boolean => {
|
|
const normalized = String(reason || '').trim().toUpperCase();
|
|
return normalized.startsWith('[INVALID_')
|
|
|| normalized.startsWith('[DUPLICATE_')
|
|
|| normalized.startsWith('[RECONCILED_TO_');
|
|
};
|
|
|
|
const getSubmittedTsMs = (order: AlpacaOrder): number => toTimestampMs(order.submitted_at || 0, 0);
|
|
const getFillTsMs = (order: AlpacaOrder): number => toTimestampMs(order.filled_at || order.submitted_at || 0, 0);
|
|
|
|
const isFilledExecutionOrder = (order: AlpacaOrder): boolean => {
|
|
const filledQty = toNumber(order.filled_qty);
|
|
const filledPrice = toNumber(order.filled_avg_price);
|
|
if (filledQty <= 0 || filledPrice <= 0) return false;
|
|
const status = normalizeStatus(order.status);
|
|
return status === 'filled' || status === 'partially_filled';
|
|
};
|
|
|
|
const statusBreakdown = (rows: Array<{ status?: string | null }>): Record<string, number> => {
|
|
const out: Record<string, number> = {};
|
|
for (const row of rows) {
|
|
const status = normalizeStatus(row.status || undefined);
|
|
out[status] = (out[status] || 0) + 1;
|
|
}
|
|
return out;
|
|
};
|
|
|
|
const fetchPaged = async <T>(
|
|
table: 'orders' | 'trade_history',
|
|
columns: string,
|
|
startIsoFilter: string
|
|
): Promise<T[]> => {
|
|
const rows: T[] = [];
|
|
let offset = 0;
|
|
|
|
while (true) {
|
|
const { data, error } = await supabase
|
|
.from(table)
|
|
.select(columns)
|
|
.gte('created_at', startIsoFilter)
|
|
.order('created_at', { ascending: false })
|
|
.range(offset, offset + PAGE_SIZE - 1);
|
|
|
|
if (error) {
|
|
throw error;
|
|
}
|
|
|
|
const chunk = (data || []) as T[];
|
|
if (!chunk.length) break;
|
|
rows.push(...chunk);
|
|
if (chunk.length < PAGE_SIZE) break;
|
|
offset += PAGE_SIZE;
|
|
}
|
|
|
|
return rows;
|
|
};
|
|
|
|
const fetchSupabaseOrders = async (startIsoFilter: string = startIso): Promise<OrderRow[]> => {
|
|
const v2 = 'id,order_id,profile_id,symbol,type,side,qty,quantity,price,status,timestamp,created_at,trade_id,action,source';
|
|
const legacy = 'id,order_id,profile_id,symbol,type,side,qty,price,status,timestamp,created_at,trade_id,action,source';
|
|
try {
|
|
return await fetchPaged<OrderRow>('orders', v2, startIsoFilter);
|
|
} catch (error: any) {
|
|
const msg = String(error?.message || '').toLowerCase();
|
|
if (msg.includes('column') && msg.includes('quantity')) {
|
|
return await fetchPaged<OrderRow>('orders', legacy, startIsoFilter);
|
|
}
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
const fetchTradeHistory = async (): Promise<HistoryRow[]> => {
|
|
const v2 = 'id,profile_id,trade_id,symbol,side,size,entry_price,exit_price,pnl,pnl_percent,reason,source,timestamp,created_at';
|
|
const v1 = 'id,profile_id,symbol,side,size,entry_price,exit_price,pnl,pnl_percent,reason,timestamp,created_at';
|
|
try {
|
|
return await fetchPaged<HistoryRow>('trade_history', v2, startIso);
|
|
} catch (error: any) {
|
|
const msg = String(error?.message || '').toLowerCase();
|
|
if (msg.includes('column') && (msg.includes('trade_id') || msg.includes('source'))) {
|
|
return await fetchPaged<HistoryRow>('trade_history', v1, startIso);
|
|
}
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
type PositionState = { qty: number; avg: number };
|
|
type PositionSnapshot = Record<string, { qty: number; avg: number; notional: number }>;
|
|
type AlpacaRealizedPnlResult = {
|
|
totalRealized: number;
|
|
perSymbol: Record<string, number>;
|
|
openingPositions: PositionSnapshot;
|
|
endingPositions: PositionSnapshot;
|
|
openingExposureNotional: number;
|
|
endingExposureNotional: number;
|
|
preWindowFillCount: number;
|
|
inWindowFillCount: number;
|
|
};
|
|
|
|
const cloneStateMap = (source: Map<string, PositionState>): Map<string, PositionState> => {
|
|
const cloned = new Map<string, PositionState>();
|
|
for (const [symbol, state] of source.entries()) {
|
|
cloned.set(symbol, { qty: state.qty, avg: state.avg });
|
|
}
|
|
return cloned;
|
|
};
|
|
|
|
const snapshotStateMap = (stateBySymbol: Map<string, PositionState>): {
|
|
positions: PositionSnapshot;
|
|
exposureNotional: number;
|
|
} => {
|
|
const positions: PositionSnapshot = {};
|
|
let exposureNotional = 0;
|
|
|
|
for (const [symbol, state] of stateBySymbol.entries()) {
|
|
const qty = Number(state.qty || 0);
|
|
const avg = Number(state.avg || 0);
|
|
if (!Number.isFinite(qty) || !Number.isFinite(avg) || Math.abs(qty) <= 1e-12) continue;
|
|
const notional = qty * avg;
|
|
exposureNotional += Math.abs(notional);
|
|
positions[symbol] = {
|
|
qty: Number(qty.toFixed(8)),
|
|
avg: Number(avg.toFixed(8)),
|
|
notional: Number(notional.toFixed(8))
|
|
};
|
|
}
|
|
|
|
return {
|
|
positions,
|
|
exposureNotional: Number(exposureNotional.toFixed(8))
|
|
};
|
|
};
|
|
|
|
const applyFilledOrderToState = (
|
|
stateBySymbol: Map<string, PositionState>,
|
|
order: AlpacaOrder,
|
|
onRealized?: (symbol: string, pnl: number) => void
|
|
): void => {
|
|
const symbol = normalizeSymbol(order.symbol);
|
|
const side = normalizeSide(order.side);
|
|
const qty = toNumber(order.filled_qty);
|
|
const price = toNumber(order.filled_avg_price);
|
|
if (!(qty > 0 && price > 0)) return;
|
|
|
|
const current = stateBySymbol.get(symbol) || { qty: 0, avg: 0 };
|
|
let positionQty = current.qty;
|
|
let avg = current.avg;
|
|
|
|
if (side === 'BUY') {
|
|
if (positionQty >= 0) {
|
|
const newQty = positionQty + qty;
|
|
avg = newQty > 0 ? ((avg * positionQty) + (price * qty)) / newQty : 0;
|
|
positionQty = newQty;
|
|
} else {
|
|
const closeQty = Math.min(qty, Math.abs(positionQty));
|
|
if (closeQty > 0) {
|
|
onRealized?.(symbol, (avg - price) * closeQty);
|
|
positionQty += closeQty;
|
|
}
|
|
const remainQty = qty - closeQty;
|
|
if (remainQty > 0) {
|
|
positionQty = remainQty;
|
|
avg = price;
|
|
} else if (Math.abs(positionQty) < 1e-12) {
|
|
positionQty = 0;
|
|
avg = 0;
|
|
}
|
|
}
|
|
} else {
|
|
if (positionQty <= 0) {
|
|
const baseQty = Math.abs(positionQty);
|
|
const newBaseQty = baseQty + qty;
|
|
avg = newBaseQty > 0 ? ((avg * baseQty) + (price * qty)) / newBaseQty : 0;
|
|
positionQty = -newBaseQty;
|
|
} else {
|
|
const closeQty = Math.min(qty, positionQty);
|
|
if (closeQty > 0) {
|
|
onRealized?.(symbol, (price - avg) * closeQty);
|
|
positionQty -= closeQty;
|
|
}
|
|
const remainQty = qty - closeQty;
|
|
if (remainQty > 0) {
|
|
positionQty = -remainQty;
|
|
avg = price;
|
|
} else if (Math.abs(positionQty) < 1e-12) {
|
|
positionQty = 0;
|
|
avg = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
stateBySymbol.set(symbol, { qty: positionQty, avg });
|
|
};
|
|
|
|
const calculateAlpacaRealizedPnl = (
|
|
allOrders: AlpacaOrder[],
|
|
windowStartMs: number,
|
|
windowEndMs: number
|
|
): AlpacaRealizedPnlResult => {
|
|
const sortedFills = [...allOrders]
|
|
.filter((order) => isFilledExecutionOrder(order))
|
|
.sort((a, b) => getFillTsMs(a) - getFillTsMs(b));
|
|
|
|
const openingState = new Map<string, PositionState>();
|
|
let preWindowFillCount = 0;
|
|
for (const order of sortedFills) {
|
|
const fillTs = getFillTsMs(order);
|
|
if (fillTs <= 0 || fillTs >= windowStartMs) continue;
|
|
applyFilledOrderToState(openingState, order);
|
|
preWindowFillCount += 1;
|
|
}
|
|
|
|
const openingSnapshot = snapshotStateMap(openingState);
|
|
const currentState = cloneStateMap(openingState);
|
|
let totalRealized = 0;
|
|
const perSymbol: Record<string, number> = {};
|
|
let inWindowFillCount = 0;
|
|
|
|
for (const order of sortedFills) {
|
|
const fillTs = getFillTsMs(order);
|
|
if (fillTs < windowStartMs || fillTs > windowEndMs) continue;
|
|
inWindowFillCount += 1;
|
|
applyFilledOrderToState(currentState, order, (symbol, pnl) => {
|
|
totalRealized += pnl;
|
|
perSymbol[symbol] = (perSymbol[symbol] || 0) + pnl;
|
|
});
|
|
}
|
|
|
|
const endingSnapshot = snapshotStateMap(currentState);
|
|
|
|
return {
|
|
totalRealized,
|
|
perSymbol,
|
|
openingPositions: openingSnapshot.positions,
|
|
endingPositions: endingSnapshot.positions,
|
|
openingExposureNotional: openingSnapshot.exposureNotional,
|
|
endingExposureNotional: endingSnapshot.exposureNotional,
|
|
preWindowFillCount,
|
|
inWindowFillCount
|
|
};
|
|
};
|
|
|
|
const fetchAlpacaOrdersPaged = async (after: Date, until: Date): Promise<AlpacaOrder[]> => {
|
|
const out: AlpacaOrder[] = [];
|
|
const seen = new Set<string>();
|
|
let cursorUntil = new Date(until);
|
|
const afterMs = after.getTime();
|
|
|
|
for (let page = 0; page < MAX_ALPACA_PAGES; page++) {
|
|
const raw = await (alpaca as any).getOrders({
|
|
status: 'all',
|
|
after,
|
|
until: cursorUntil,
|
|
direction: 'desc',
|
|
limit: ALPACA_PAGE_LIMIT
|
|
});
|
|
const batch: AlpacaOrder[] = Array.isArray(raw) ? raw : [];
|
|
if (!batch.length) break;
|
|
|
|
let oldestTs = Number.POSITIVE_INFINITY;
|
|
for (const order of batch) {
|
|
const id = String(order.id || '').trim();
|
|
if (id && seen.has(id)) continue;
|
|
if (id) seen.add(id);
|
|
out.push(order);
|
|
|
|
const ts = getSubmittedTsMs(order) || getFillTsMs(order);
|
|
if (ts > 0 && ts < oldestTs) oldestTs = ts;
|
|
}
|
|
|
|
if (batch.length < ALPACA_PAGE_LIMIT) break;
|
|
if (!Number.isFinite(oldestTs) || oldestTs <= 0) break;
|
|
|
|
const nextUntilMs = oldestTs - 1;
|
|
if (nextUntilMs <= afterMs) break;
|
|
if (nextUntilMs >= cursorUntil.getTime()) break;
|
|
cursorUntil = new Date(nextUntilMs);
|
|
}
|
|
|
|
return out;
|
|
};
|
|
|
|
const calculateSupabaseRealizedPnl = (historyRows: HistoryRow[]) => {
|
|
let totalPnl = 0;
|
|
const perSymbol: Record<string, number> = {};
|
|
const formulaMismatches: ReconciliationSample[] = [];
|
|
const tradeIdCounts = new Map<string, number>();
|
|
const rankedRows: Array<HistoryRow & { __absPnl: number }> = [];
|
|
|
|
for (const row of historyRows) {
|
|
const symbol = normalizeSymbol(row.symbol);
|
|
const side = normalizeSide(row.side);
|
|
const size = toNumber(row.size);
|
|
const entry = toNumber(row.entry_price);
|
|
const exit = toNumber(row.exit_price);
|
|
const pnl = toNumber(row.pnl);
|
|
const reason = String(row.reason || '');
|
|
totalPnl += pnl;
|
|
perSymbol[symbol] = (perSymbol[symbol] || 0) + pnl;
|
|
rankedRows.push({ ...row, __absPnl: Math.abs(pnl) });
|
|
|
|
const tradeId = String(row.trade_id || '').trim();
|
|
if (tradeId) {
|
|
tradeIdCounts.set(tradeId, (tradeIdCounts.get(tradeId) || 0) + 1);
|
|
}
|
|
|
|
if (!isQuarantinedHistoryReason(reason) && size > 0 && entry > 0 && exit > 0) {
|
|
const expected = side === 'BUY'
|
|
? (exit - entry) * size
|
|
: (entry - exit) * size;
|
|
const absDiff = Math.abs(expected - pnl);
|
|
if (absDiff > 0.02) {
|
|
pushSample(formulaMismatches, {
|
|
id: row.id,
|
|
trade_id: row.trade_id,
|
|
symbol: row.symbol,
|
|
side: row.side,
|
|
size,
|
|
entry_price: entry,
|
|
exit_price: exit,
|
|
pnl_recorded: pnl,
|
|
pnl_expected: Number(expected.toFixed(8)),
|
|
pnl_abs_diff: Number(absDiff.toFixed(8))
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
const duplicateTradeIds = Array.from(tradeIdCounts.entries())
|
|
.filter(([, count]) => count > 1)
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, MAX_SAMPLES)
|
|
.map(([tradeId, count]) => ({ trade_id: tradeId, count }));
|
|
|
|
const topRows = rankedRows
|
|
.sort((a, b) => b.__absPnl - a.__absPnl)
|
|
.slice(0, MAX_SAMPLES)
|
|
.map((row) => ({
|
|
id: row.id,
|
|
trade_id: row.trade_id,
|
|
profile_id: row.profile_id,
|
|
symbol: row.symbol,
|
|
side: row.side,
|
|
size: toNumber(row.size),
|
|
entry_price: toNumber(row.entry_price),
|
|
exit_price: toNumber(row.exit_price),
|
|
pnl: toNumber(row.pnl),
|
|
reason: row.reason,
|
|
created_at: row.created_at
|
|
}));
|
|
|
|
return {
|
|
totalPnl,
|
|
perSymbol,
|
|
formulaMismatches,
|
|
duplicateTradeIds,
|
|
topRows
|
|
};
|
|
};
|
|
|
|
const sortRecordByAbsValueDesc = (record: Record<string, number>) => (
|
|
Object.entries(record)
|
|
.sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
|
|
.map(([key, value]) => ({ key, value: Number(value.toFixed(8)) }))
|
|
);
|
|
|
|
const run = async () => {
|
|
const windowStartMs = startDate.getTime();
|
|
const windowEndMs = now.getTime();
|
|
const alpacaAllOrders = await fetchAlpacaOrdersPaged(carryStartDate, now);
|
|
const alpacaOrders = alpacaAllOrders.filter((order) => {
|
|
const submittedTs = getSubmittedTsMs(order);
|
|
if (submittedTs > 0) {
|
|
return submittedTs >= windowStartMs && submittedTs <= windowEndMs;
|
|
}
|
|
const fillTs = getFillTsMs(order);
|
|
return fillTs >= windowStartMs && fillTs <= windowEndMs;
|
|
});
|
|
|
|
const [dbOrders, dbOrdersCarryScope, tradeHistory] = await Promise.all([
|
|
fetchSupabaseOrders(startIso),
|
|
fetchSupabaseOrders(carryStartIso),
|
|
fetchTradeHistory()
|
|
]);
|
|
const dbPreWindowFilledOrderCount = dbOrdersCarryScope.filter((order) => {
|
|
const createdAtTs = toTimestampMs(order.created_at || 0, 0);
|
|
if (createdAtTs <= 0 || createdAtTs >= windowStartMs) return false;
|
|
const status = normalizeStatus(order.status);
|
|
return status === 'filled' || status === 'partially_filled';
|
|
}).length;
|
|
const comparableSymbols = new Set(
|
|
dbOrders
|
|
.map((order) => normalizeSymbol(order.symbol))
|
|
.filter((symbol) => !!symbol)
|
|
);
|
|
const useComparableScope = comparableSymbols.size > 0;
|
|
const alpacaOrdersForPnlAll = useComparableScope
|
|
? alpacaAllOrders.filter((order) => comparableSymbols.has(normalizeSymbol(order.symbol)))
|
|
: alpacaAllOrders;
|
|
const alpacaOrdersForPnlWindow = useComparableScope
|
|
? alpacaOrders.filter((order) => comparableSymbols.has(normalizeSymbol(order.symbol)))
|
|
: alpacaOrders;
|
|
|
|
let alpacaAccountSummary: Record<string, number> | null = null;
|
|
try {
|
|
const account = await (alpaca as any).getAccount();
|
|
alpacaAccountSummary = {
|
|
equity: toNumber((account as any)?.equity),
|
|
cash: toNumber((account as any)?.cash),
|
|
portfolio_value: toNumber((account as any)?.portfolio_value),
|
|
buying_power: toNumber((account as any)?.buying_power),
|
|
last_equity: toNumber((account as any)?.last_equity)
|
|
};
|
|
} catch {
|
|
alpacaAccountSummary = null;
|
|
}
|
|
|
|
let alpacaPortfolioSummary: Record<string, any> | null = null;
|
|
try {
|
|
const portfolioHistory = await (alpaca as any).getPortfolioHistory({
|
|
date_start: startIso.slice(0, 10),
|
|
date_end: nowIso.slice(0, 10),
|
|
timeframe: '1D',
|
|
extended_hours: true
|
|
});
|
|
|
|
const timestamps = Array.isArray((portfolioHistory as any)?.timestamp)
|
|
? (portfolioHistory as any).timestamp
|
|
: [];
|
|
const equity = Array.isArray((portfolioHistory as any)?.equity)
|
|
? (portfolioHistory as any).equity
|
|
: [];
|
|
const profitLoss = Array.isArray((portfolioHistory as any)?.profit_loss)
|
|
? (portfolioHistory as any).profit_loss
|
|
: [];
|
|
const profitLossPct = Array.isArray((portfolioHistory as any)?.profit_loss_pct)
|
|
? (portfolioHistory as any).profit_loss_pct
|
|
: [];
|
|
|
|
const firstEquity = equity.length ? toNumber(equity[0]) : 0;
|
|
const lastEquity = equity.length ? toNumber(equity[equity.length - 1]) : 0;
|
|
const lastProfitLoss = profitLoss.length ? toNumber(profitLoss[profitLoss.length - 1]) : 0;
|
|
const lastProfitLossPct = profitLossPct.length ? toNumber(profitLossPct[profitLossPct.length - 1]) : 0;
|
|
|
|
const firstTsRaw = timestamps.length ? Number(timestamps[0]) : 0;
|
|
const lastTsRaw = timestamps.length ? Number(timestamps[timestamps.length - 1]) : 0;
|
|
const firstTsMs = firstTsRaw > 1_000_000_000_000 ? firstTsRaw : firstTsRaw * 1000;
|
|
const lastTsMs = lastTsRaw > 1_000_000_000_000 ? lastTsRaw : lastTsRaw * 1000;
|
|
|
|
alpacaPortfolioSummary = {
|
|
points: equity.length,
|
|
first_timestamp_utc: firstTsMs > 0 ? new Date(firstTsMs).toISOString() : null,
|
|
last_timestamp_utc: lastTsMs > 0 ? new Date(lastTsMs).toISOString() : null,
|
|
first_equity: Number(firstEquity.toFixed(8)),
|
|
last_equity: Number(lastEquity.toFixed(8)),
|
|
equity_change: Number((lastEquity - firstEquity).toFixed(8)),
|
|
latest_profit_loss: Number(lastProfitLoss.toFixed(8)),
|
|
latest_profit_loss_pct: Number(lastProfitLossPct.toFixed(8))
|
|
};
|
|
} catch {
|
|
alpacaPortfolioSummary = null;
|
|
}
|
|
|
|
const alpacaById = new Map<string, AlpacaOrder>();
|
|
for (const order of alpacaOrders) {
|
|
const id = String(order.id || '').trim();
|
|
if (id) alpacaById.set(id, order);
|
|
}
|
|
|
|
const dbByOrderId = new Map<string, OrderRow[]>();
|
|
for (const order of dbOrders) {
|
|
const orderId = String(order.order_id || '').trim();
|
|
if (!orderId) continue;
|
|
if (!dbByOrderId.has(orderId)) dbByOrderId.set(orderId, []);
|
|
dbByOrderId.get(orderId)!.push(order);
|
|
}
|
|
|
|
const missingInDb: ReconciliationSample[] = [];
|
|
const missingInAlpaca: ReconciliationSample[] = [];
|
|
const statusMismatches: ReconciliationSample[] = [];
|
|
const qtyMismatches: ReconciliationSample[] = [];
|
|
const sideMismatches: ReconciliationSample[] = [];
|
|
const symbolMismatches: ReconciliationSample[] = [];
|
|
const priceMismatches: ReconciliationSample[] = [];
|
|
|
|
for (const alpacaOrder of alpacaOrders) {
|
|
const alpacaId = String(alpacaOrder.id || '').trim();
|
|
if (!alpacaId) continue;
|
|
|
|
const matches = dbByOrderId.get(alpacaId) || [];
|
|
if (!matches.length) {
|
|
pushSample(missingInDb, {
|
|
alpaca_order_id: alpacaId,
|
|
symbol: alpacaOrder.symbol,
|
|
side: alpacaOrder.side,
|
|
status: alpacaOrder.status,
|
|
submitted_at: alpacaOrder.submitted_at
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const dbOrder = matches[0];
|
|
const alpacaStatus = normalizeStatus(alpacaOrder.status);
|
|
const dbStatus = normalizeStatus(dbOrder.status);
|
|
if (alpacaStatus !== dbStatus) {
|
|
pushSample(statusMismatches, {
|
|
order_id: alpacaId,
|
|
alpaca_status: alpacaOrder.status,
|
|
db_status: dbOrder.status,
|
|
alpaca_status_normalized: alpacaStatus,
|
|
db_status_normalized: dbStatus
|
|
});
|
|
}
|
|
|
|
const alpacaQty = toNumber(alpacaOrder.qty);
|
|
const dbQty = getOrderQty(dbOrder);
|
|
if (alpacaQty > 0 && dbQty > 0 && pctDiff(alpacaQty, dbQty) > 0.000001) {
|
|
pushSample(qtyMismatches, {
|
|
order_id: alpacaId,
|
|
symbol: alpacaOrder.symbol,
|
|
alpaca_qty: alpacaQty,
|
|
db_qty: dbQty
|
|
});
|
|
}
|
|
|
|
const alpacaSide = normalizeSide(alpacaOrder.side);
|
|
const dbSide = normalizeSide(dbOrder.side);
|
|
if (alpacaSide !== dbSide) {
|
|
pushSample(sideMismatches, {
|
|
order_id: alpacaId,
|
|
symbol: alpacaOrder.symbol,
|
|
alpaca_side: alpacaOrder.side,
|
|
db_side: dbOrder.side
|
|
});
|
|
}
|
|
|
|
const alpacaSymbol = normalizeSymbol(alpacaOrder.symbol);
|
|
const dbSymbol = normalizeSymbol(dbOrder.symbol);
|
|
if (alpacaSymbol !== dbSymbol) {
|
|
pushSample(symbolMismatches, {
|
|
order_id: alpacaId,
|
|
alpaca_symbol: alpacaOrder.symbol,
|
|
db_symbol: dbOrder.symbol,
|
|
alpaca_symbol_normalized: alpacaSymbol,
|
|
db_symbol_normalized: dbSymbol
|
|
});
|
|
}
|
|
|
|
const alpacaFilledPrice = toNumber(alpacaOrder.filled_avg_price);
|
|
const dbPrice = toNumber(dbOrder.price);
|
|
if (alpacaFilledPrice > 0 && dbPrice > 0 && pctDiff(alpacaFilledPrice, dbPrice) > 0.005) {
|
|
pushSample(priceMismatches, {
|
|
order_id: alpacaId,
|
|
symbol: alpacaOrder.symbol,
|
|
alpaca_filled_avg_price: alpacaFilledPrice,
|
|
db_price: dbPrice,
|
|
diff_percent: Number((pctDiff(alpacaFilledPrice, dbPrice) * 100).toFixed(4))
|
|
});
|
|
}
|
|
}
|
|
|
|
for (const dbOrder of dbOrders) {
|
|
const orderId = String(dbOrder.order_id || '').trim();
|
|
if (!orderId) continue;
|
|
if (!alpacaById.has(orderId)) {
|
|
pushSample(missingInAlpaca, {
|
|
db_order_id: orderId,
|
|
symbol: dbOrder.symbol,
|
|
side: dbOrder.side,
|
|
status: dbOrder.status,
|
|
profile_id: dbOrder.profile_id,
|
|
created_at: dbOrder.created_at
|
|
});
|
|
}
|
|
}
|
|
|
|
const alpacaRealizedWithCarry = calculateAlpacaRealizedPnl(alpacaOrdersForPnlAll, windowStartMs, windowEndMs);
|
|
const alpacaRealizedFlatStart = calculateAlpacaRealizedPnl(alpacaOrdersForPnlWindow, windowStartMs, windowEndMs);
|
|
const carryCoverageGap = Math.max(0, alpacaRealizedWithCarry.preWindowFillCount - dbPreWindowFilledOrderCount);
|
|
const carryCoverageThreshold = Math.max(20, dbPreWindowFilledOrderCount * 3);
|
|
const useCarryAsPrimary = dbPreWindowFilledOrderCount > 0
|
|
&& carryCoverageGap <= carryCoverageThreshold;
|
|
const alpacaRealizedPrimary = useCarryAsPrimary
|
|
? alpacaRealizedWithCarry
|
|
: alpacaRealizedFlatStart;
|
|
const dbRealized = calculateSupabaseRealizedPnl(tradeHistory);
|
|
const pnlDiff = dbRealized.totalPnl - alpacaRealizedPrimary.totalRealized;
|
|
const alpacaFillCashFlow = alpacaOrdersForPnlWindow.reduce((sum, order) => {
|
|
const status = normalizeStatus(order.status);
|
|
if (status !== 'filled' && status !== 'partially_filled') return sum;
|
|
const qty = toNumber(order.filled_qty);
|
|
const price = toNumber(order.filled_avg_price);
|
|
if (!(qty > 0 && price > 0)) return sum;
|
|
const side = normalizeSide(order.side);
|
|
const signedNotional = side === 'BUY' ? -qty * price : qty * price;
|
|
return sum + signedNotional;
|
|
}, 0);
|
|
|
|
const output = {
|
|
meta: {
|
|
window_start_utc: startIso,
|
|
window_end_utc: nowIso,
|
|
lookback_days: LOOKBACK_DAYS,
|
|
carry_in_lookback_days: CARRY_IN_LOOKBACK_DAYS,
|
|
carry_in_start_utc: carryStartIso,
|
|
pnl_symbol_scope: useComparableScope ? 'db-order-symbols' : 'all-symbols',
|
|
paper_trading: paperTrading
|
|
},
|
|
counts: {
|
|
alpaca_orders_total: alpacaOrders.length,
|
|
alpaca_orders_total_with_carry_window: alpacaAllOrders.length,
|
|
alpaca_orders_total_in_pnl_scope: alpacaOrdersForPnlWindow.length,
|
|
alpaca_orders_total_with_carry_in_pnl_scope: alpacaOrdersForPnlAll.length,
|
|
supabase_orders_total: dbOrders.length,
|
|
supabase_trade_history_total: tradeHistory.length
|
|
},
|
|
alpaca_account: alpacaAccountSummary,
|
|
alpaca_portfolio_history: alpacaPortfolioSummary,
|
|
status_breakdown: {
|
|
alpaca_orders: statusBreakdown(alpacaOrders.map((o) => ({ status: o.status }))),
|
|
supabase_orders: statusBreakdown(dbOrders.map((o) => ({ status: o.status })))
|
|
},
|
|
order_conflicts: {
|
|
missing_in_supabase_count: missingInDb.length,
|
|
missing_in_alpaca_count: missingInAlpaca.length,
|
|
status_mismatch_count: statusMismatches.length,
|
|
qty_mismatch_count: qtyMismatches.length,
|
|
side_mismatch_count: sideMismatches.length,
|
|
symbol_mismatch_count: symbolMismatches.length,
|
|
price_mismatch_count: priceMismatches.length,
|
|
samples: {
|
|
missing_in_supabase: missingInDb,
|
|
missing_in_alpaca: missingInAlpaca,
|
|
status_mismatches: statusMismatches,
|
|
qty_mismatches: qtyMismatches,
|
|
side_mismatches: sideMismatches,
|
|
symbol_mismatches: symbolMismatches,
|
|
price_mismatches: priceMismatches
|
|
}
|
|
},
|
|
pnl_comparison: {
|
|
caveat: 'Alpaca fill-derived realized PnL publishes both flat_start and with_carry_in. Primary metric auto-selects carry only when pre-window Alpaca fill volume is consistent with Supabase pre-window coverage.',
|
|
supabase_trade_history_realized_pnl: Number(dbRealized.totalPnl.toFixed(8)),
|
|
alpaca_fill_derived_realized_pnl: Number(alpacaRealizedPrimary.totalRealized.toFixed(8)),
|
|
alpaca_fill_derived_realized_pnl_with_carry_in: Number(alpacaRealizedWithCarry.totalRealized.toFixed(8)),
|
|
alpaca_fill_derived_realized_pnl_flat_start: Number(alpacaRealizedFlatStart.totalRealized.toFixed(8)),
|
|
alpaca_fill_realized_carry_minus_flat: Number((alpacaRealizedWithCarry.totalRealized - alpacaRealizedFlatStart.totalRealized).toFixed(8)),
|
|
alpaca_fill_cash_flow: Number(alpacaFillCashFlow.toFixed(8)),
|
|
pnl_diff_supabase_minus_alpaca: Number(pnlDiff.toFixed(8)),
|
|
supabase_per_symbol_realized_pnl: sortRecordByAbsValueDesc(dbRealized.perSymbol),
|
|
alpaca_per_symbol_realized_pnl: sortRecordByAbsValueDesc(alpacaRealizedPrimary.perSymbol),
|
|
alpaca_per_symbol_realized_pnl_flat_start: sortRecordByAbsValueDesc(alpacaRealizedFlatStart.perSymbol),
|
|
alpaca_per_symbol_realized_pnl_with_carry_in: sortRecordByAbsValueDesc(alpacaRealizedWithCarry.perSymbol),
|
|
alpaca_opening_positions_at_window_start: alpacaRealizedWithCarry.openingPositions,
|
|
alpaca_opening_exposure_notional: Number(alpacaRealizedWithCarry.openingExposureNotional.toFixed(8)),
|
|
alpaca_ending_positions_after_window: alpacaRealizedWithCarry.endingPositions,
|
|
alpaca_ending_exposure_notional: Number(alpacaRealizedWithCarry.endingExposureNotional.toFixed(8)),
|
|
alpaca_carry_in_bootstrap: {
|
|
pre_window_fill_count: alpacaRealizedWithCarry.preWindowFillCount,
|
|
in_window_fill_count: alpacaRealizedWithCarry.inWindowFillCount,
|
|
has_pre_window_fills: alpacaRealizedWithCarry.preWindowFillCount > 0,
|
|
symbol_scope: useComparableScope ? 'db-order-symbols' : 'all-symbols',
|
|
supabase_pre_window_filled_order_count: dbPreWindowFilledOrderCount,
|
|
carry_coverage_gap: carryCoverageGap,
|
|
carry_coverage_threshold: carryCoverageThreshold,
|
|
primary_mode: useCarryAsPrimary ? 'with_carry_in' : 'flat_start'
|
|
},
|
|
trade_history_formula_mismatch_count: dbRealized.formulaMismatches.length,
|
|
trade_history_formula_mismatch_samples: dbRealized.formulaMismatches,
|
|
duplicate_trade_id_samples: dbRealized.duplicateTradeIds,
|
|
top_trade_history_pnl_rows: dbRealized.topRows
|
|
}
|
|
};
|
|
|
|
console.log(JSON.stringify(output, null, 2));
|
|
};
|
|
|
|
run().catch((error) => {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
console.error(JSON.stringify({ error: message }, null, 2));
|
|
process.exit(1);
|
|
});
|