bytelyst/learning_ai_invt_trdg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: scaffold trading monorepo foundation

Browse Source
This commit is contained in:
Saravana Achu Mac 2026-04-04 11:18:21 -07:00
parent 30551b876b
commit 3cbbd6ccaa
435 changed files with 85802 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.env.example Normal file
View File

@ -0,0 +1,31 @@
# Shared product identity
PRODUCT_ID=invttrdg
PRODUCT_DISPLAY_NAME=ByteLyst Trading
# Shared platform-service endpoint
PLATFORM_API_URL=http://localhost:4003/api
# Product backend endpoint
TRADING_API_URL=http://localhost:4018/api
# Web-specific public envs
NEXT_PUBLIC_PRODUCT_ID=invttrdg
NEXT_PUBLIC_PLATFORM_URL=http://localhost:4003/api
NEXT_PUBLIC_TRADING_API_URL=http://localhost:4018/api
VITE_PRODUCT_ID=invttrdg
VITE_PLATFORM_URL=http://localhost:4003/api
VITE_TRADING_API_URL=http://localhost:4018/api
# Mobile public envs
EXPO_PUBLIC_PRODUCT_ID=invttrdg
EXPO_PUBLIC_PLATFORM_URL=http://localhost:4003/api
EXPO_PUBLIC_TRADING_API_URL=http://localhost:4018/api
# Backend envs
PORT=4018
NODE_ENV=development
CORS_ALLOWED_ORIGINS=http://localhost:3048,http://localhost:8081
SUPABASE_URL=
SUPABASE_ANON_KEY=
SUPABASE_SERVICE_ROLE_KEY=

18
.gitignore vendored Normal file
View File

@ -0,0 +1,18 @@
node_modules
.pnpm-store
dist
build
coverage
.turbo
.next
.expo
.expo-shared
android
ios
web-build
*.log
.DS_Store
.env
.env.local
.env.*.local
backend/bot_state.json

32
README.md Normal file
View File

@ -0,0 +1,32 @@
# ByteLyst Investment Trading
Canonical monorepo for the ByteLyst trading product.
## Workspaces
- `backend/` — trading backend and execution/runtime APIs
- `web/` — trading dashboard
- `mobile/` — Expo mobile app
- `shared/` — canonical product identity and shared runtime helpers
## Shared dependencies
This repo consumes local ByteLyst common-platform packages from:
- `../learning_ai_common_plat/packages/*`
## Core principles
- backend-authoritative trading state
- platform-service for auth, kill switch, telemetry, and flags
- no duplicated bootstrap logic across surfaces
- domain-specific trading logic stays product-owned
## Common commands
```bash
pnpm install
pnpm verify
pnpm build
```

16
backend/.gitignore vendored Normal file
View File

@ -0,0 +1,16 @@
node_modules
! .env.example
.env
.env.*
*.env
old_env.txt
new_env.txt
temp_env.txt
users_dump.json
profiles_dump.json
bot_state.json
*.log
dist
coverage
.DS_Store
bot_state.json.bak

266
backend/ADMIN_TRADE_CONTROL_ARCHITECTURE.md Normal file
View File

@ -0,0 +1,266 @@
# Admin Trade Control - Architecture Diagram
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ ADMIN TRADE CONTROL SYSTEM │
│ │
│ ┌───────────────────────────────────────────────────────────────────────┐ │
│ │ FRONTEND (Dashboard) │ │
│ │ │ │
│ │ ┌─────────────────────────────────────────────────────────────────┐ │ │
│ │ │ Header (All Pages) │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ Trading Status Badge │ │ │ │
│ │ │ │ ⏸️ Trading Paused OR ▶️ Trading Active │ │ │ │
│ │ │ │ (Orange) (Green) │ │ │ │
│ │ │ │ Tooltip: "Paused by admin@example.com" │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ └─────────────────────────────────────────────────────────────────┘ │ │
│ │ │ │
│ │ ┌─────────────────────────────────────────────────────────────────┐ │ │
│ │ │ Admin Tab (Admin Users Only) │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ Trading Control Panel │ │ │ │
│ │ │ │ ┌────────────────────────────────────────────────────┐ │ │ │ │
│ │ │ │ │ Status Banner │ │ │ │ │
│ │ │ │ │ AUTO-TRADING: PAUSED / RUNNING │ │ │ │ │
│ │ │ │ │ "No new positions will be opened..." │ │ │ │ │
│ │ │ │ └────────────────────────────────────────────────────┘ │ │ │ │
│ │ │ │ ┌────────────────────┐ ┌────────────────────────────┐ │ │ │ │
│ │ │ │ │ ⏸️ Pause Auto │ │ ▶️ Resume Auto Trading │ │ │ │ │
│ │ │ │ │ Trading │ │ │ │ │ │ │
│ │ │ │ │ (disabled if │ │ (disabled if running) │ │ │ │ │
│ │ │ │ │ already paused) │ │ │ │ │ │ │
│ │ │ │ └────────────────────┘ └────────────────────────────┘ │ │ │ │
│ │ │ │ ┌────────────────────────────────────────────────────┐ │ │ │ │
│ │ │ │ │ Safety Notice │ │ │ │ │
│ │ │ │ │ "Pausing blocks new entries only. Existing │ │ │ │ │
│ │ │ │ │ positions continue to be managed." │ │ │ │ │
│ │ │ │ └────────────────────────────────────────────────────┘ │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ └─────────────────────────────────────────────────────────────────┘ │ │
│ │ │ │
│ │ ┌─────────────────────────────────────────────────────────────────┐ │ │
│ │ │ WebSocket Connection │ │ │
│ │ │ • Receives 'health_update' events │ │ │
│ │ │ • Updates botState.health.tradingControl │ │ │
│ │ │ • UI reflects backend state in real-time │ │ │
│ │ └─────────────────────────────────────────────────────────────────┘ │ │
│ └───────────────────────────────────────────────────────────────────────┘ │
│ │
│ ▲ │
│ │ │
│ WebSocket │ HTTP API │
│ health_update POST /internal/trading/pause │
│ │ POST /internal/trading/resume │
│ │ GET /internal/trading/status │
│ │ │
│ ┌───────────────────────────────────────────────────────────────────────┐ │
│ │ BACKEND (Bot Service) │ │
│ │ │ │
│ │ ┌─────────────────────────────────────────────────────────────────┐ │ │
│ │ │ API Server (apiServer.ts) │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ Admin Control Endpoints │ │ │ │
│ │ │ │ • requireAuth middleware │ │ │ │
│ │ │ │ • requireAdmin middleware │ │ │ │
│ │ │ │ • Audit logging │ │ │ │
│ │ │ │ • Idempotent operations │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ │ │ │ │ │
│ │ │ ▼ │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ healthTracker.recordTradingControl() │ │ │ │
│ │ │ │ • Updates in-memory state │ │ │ │
│ │ │ │ • Broadcasts health_update via WebSocket │ │ │ │
│ │ │ │ • Persists to disk (bot_state.json) │ │ │ │
│ │ │ │ • Persists to Supabase │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ └─────────────────────────────────────────────────────────────────┘ │ │
│ │ │ │
│ │ ┌─────────────────────────────────────────────────────────────────┐ │ │
│ │ │ Health Tracker (healthTracker.ts) │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ Trading Control State │ │ │ │
│ │ │ │ { │ │ │ │
│ │ │ │ mode: 'RUNNING' | 'PAUSED', │ │ │ │
│ │ │ │ lastChangedBy: string, │ │ │ │
│ │ │ │ lastChangedAt: number, │ │ │ │
│ │ │ │ reason?: string │ │ │ │
│ │ │ │ } │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ isPaused(): boolean │ │ │ │
│ │ │ │ • Returns true if mode === 'PAUSED' │ │ │ │
│ │ │ │ • Called by enforcement points │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ └─────────────────────────────────────────────────────────────────┘ │ │
│ │ │ │
│ │ ┌─────────────────────────────────────────────────────────────────┐ │ │
│ │ │ ENFORCEMENT POINTS │ │ │
│ │ │ │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ AutoTrader.handleSignal() (line 106-109) │ │ │ │
│ │ │ │ ┌────────────────────────────────────────────────────┐ │ │ │ │
│ │ │ │ │ if (healthTracker.isPaused()) { │ │ │ │ │
│ │ │ │ │ logger.info("Entry BLOCKED: Bot is PAUSED"); │ │ │ │ │
│ │ │ │ │ return; // ❌ Block new entry │ │ │ │ │
│ │ │ │ │ } │ │ │ │ │
│ │ │ │ └────────────────────────────────────────────────────┘ │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ │ │ │ │
│ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │
│ │ │ │ TradeExecutor.openPosition() (line 531-534) │ │ │ │
│ │ │ │ ┌────────────────────────────────────────────────────┐ │ │ │ │
│ │ │ │ │ if (healthTracker.isPaused()) { │ │ │ │ │
│ │ │ │ │ logger.info("Entry BLOCKED: Bot is PAUSED"); │ │ │ │ │
│ │ │ │ │ return { success: false, error: '...' }; │ │ │ │ │
│ │ │ │ │ } │ │ │ │ │
│ │ │ │ └────────────────────────────────────────────────────┘ │ │ │ │
│ │ │ └──────────────────────────────────────────────────────────┘ │ │ │
│ │ │ │ │ │
│ │ │ ✅ CONTINUES WHEN PAUSED: │ │ │
│ │ │ • closePosition() - Exit orders │ │ │
│ │ │ • monitorStopLoss() - SL monitoring │ │ │
│ │ │ • monitorTakeProfit() - TP monitoring │ │ │
│ │ │ • reconcilePositions() - Position sync │ │ │
│ │ │ • syncOrderStatus() - Order status updates │ │ │
│ │ └─────────────────────────────────────────────────────────────────┘ │ │
│ │ │ │
│ │ ┌─────────────────────────────────────────────────────────────────┐ │ │
│ │ │ Persistence Layer │ │ │
│ │ │ ┌──────────────────┐ ┌──────────────────┐ ┌──────────────┐ │ │ │
│ │ │ │ In-Memory │ │ Disk │ │ Supabase │ │ │ │
│ │ │ │ (HealthTracker) │→ │ (bot_state.json)│→ │ (Database) │ │ │ │
│ │ │ │ Singleton │ │ Local file │ │ Remote DB │ │ │ │
│ │ │ └──────────────────┘ └──────────────────┘ └──────────────┘ │ │ │
│ │ │ • State restored on bot restart │ │ │
│ │ │ • Supabase preferred, disk fallback │ │ │
│ │ └─────────────────────────────────────────────────────────────────┘ │ │
│ └───────────────────────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ CONTROL FLOW │
└─────────────────────────────────────────────────────────────────────────────┘
PAUSE FLOW:
1. Admin clicks "Pause Auto Trading" button
2. Frontend calls POST /internal/trading/pause with auth token
3. API Server validates: requireAuth + requireAdmin
4. API Server calls healthTracker.recordTradingControl({ mode: 'PAUSED' })
5. HealthTracker updates in-memory state
6. API Server broadcasts health_update via WebSocket
7. API Server persists to disk and Supabase
8. Frontend receives health_update, updates UI
9. Header badge shows "⏸️ Trading Paused"
10. AutoTrader/TradeExecutor check isPaused() before new entries
11. New entries blocked, existing positions continue
RESUME FLOW:
1. Admin clicks "Resume Auto Trading" button
2. Frontend calls POST /internal/trading/resume with auth token
3. API Server validates: requireAuth + requireAdmin
4. API Server calls healthTracker.recordTradingControl({ mode: 'RUNNING' })
5. HealthTracker updates in-memory state
6. API Server broadcasts health_update via WebSocket
7. API Server persists to disk and Supabase
8. Frontend receives health_update, updates UI
9. Header badge shows "▶️ Trading Active"
10. AutoTrader/TradeExecutor allow new entries
11. Normal trading resumes
RESTART RECOVERY:
1. Bot starts up
2. API Server calls loadState()
3. Reads bot_state.json from disk
4. Restores tradingControl state
5. Calls healthTracker.recordTradingControl() with restored state
6. Trading resumes in last known mode (PAUSED or RUNNING)
┌─────────────────────────────────────────────────────────────────────────────┐
│ SECURITY LAYERS │
└─────────────────────────────────────────────────────────────────────────────┘
Layer 1: Authentication
├─ All endpoints require valid JWT token
├─ Token verified via Supabase auth
└─ Unauthorized requests → 401
Layer 2: Authorization
├─ Pause/Resume require role = 'admin'
├─ Role checked in user profile
└─ Non-admin requests → 403
Layer 3: UI Guards
├─ Trading Control Panel hidden for non-admin
├─ Header badge visible to all (read-only)
└─ Buttons disabled when already in target state
Layer 4: Audit Trail
├─ All pause/resume actions logged
├─ Includes: userId, timestamp, reason
└─ Logs written to console and observability
┌─────────────────────────────────────────────────────────────────────────────┐
│ DATA FLOW DIAGRAM │
└─────────────────────────────────────────────────────────────────────────────┘
Admin Action
│
▼
Frontend (AdminTab.tsx)
│
│ HTTP POST /internal/trading/pause
│ Authorization: Bearer <token>
│ Body: { reason: "..." }
▼
API Server (apiServer.ts)
│
├─ requireAuth middleware → Verify JWT
├─ requireAdmin middleware → Check role
│
▼
healthTracker.recordTradingControl()
│
├─ Update in-memory state
├─ Broadcast WebSocket health_update
├─ Persist to bot_state.json
└─ Persist to Supabase
│
▼
WebSocket Broadcast
│
▼
Frontend (useWebSocket.ts)
│
├─ Receive health_update event
├─ Update botState.health.tradingControl
│
▼
UI Updates
│
├─ Header badge: "⏸️ Trading Paused"
├─ Admin panel: Status banner updates
└─ Buttons: Pause disabled, Resume enabled
Trading Loop
│
▼
AutoTrader.handleSignal()
│
├─ Check: healthTracker.isPaused()
│ ├─ if true → Block entry, return
│ └─ if false → Continue to entry logic
│
▼
TradeExecutor.openPosition()
│
├─ Check: healthTracker.isPaused()
│ ├─ if true → Return { success: false, error: '...' }
│ └─ if false → Place order on exchange
│
▼
Exchange Order Placement
```

336
backend/ADMIN_TRADE_CONTROL_IMPLEMENTATION.md Normal file
View File

@ -0,0 +1,336 @@
# Admin Trade Control Implementation
## Overview
This document describes the implementation of the **Admin Trade Control Feature** that allows authorized administrators to pause and resume auto-trading from the dashboard. This is a production-grade safety control that prevents new trade entries while allowing existing positions to be managed.
## Architecture
### Backend (Authoritative)
The backend is the **single source of truth** for trading control state. All enforcement happens server-side.
#### 1. Trading Control State
**Location**: `bytelyst-trading-bot-service/src/services/healthTracker.ts`
```typescript
export interface TradingControlSnapshot {
mode: 'RUNNING' | 'PAUSED';
lastChangedBy: string;
lastChangedAt: number;
reason?: string;
}
```
The state is:
- Stored in-memory in `HealthTracker` singleton
- Persisted to disk in `bot_state.json`
- Persisted to Supabase for multi-instance recovery
- Restored on bot restart
#### 2. Enforcement Points (MANDATORY)
**Auto-Trading Enforcement**:
1. **AutoTrader.ts** (Line 106-109):
```typescript
if (healthTracker.isPaused()) {
logger.info(`[AutoTrader] 🛑 Entry BLOCKED for ${symbol}: Bot is PAUSED by admin.`);
return;
}
```
2. **TradeExecutor.ts** (Line 531-534):
```typescript
if (healthTracker.isPaused()) {
logger.info(`[TradeExecutor] 🛑 Entry BLOCKED for ${symbol}: Bot is PAUSED by admin.`);
return { success: false, error: 'Trade execution is paused by administrator' };
}
```
**What Continues When Paused**:
- Exit order execution
- Stop-loss monitoring
- Take-profit monitoring
- Position reconciliation
- Order status synchronization
- Health monitoring
#### 3. Admin Control API
**Location**: `bytelyst-trading-bot-service/src/services/apiServer.ts` (Lines 1049-1090)
**Endpoints**:
```
GET /internal/trading/status
POST /internal/trading/pause
POST /internal/trading/resume
```
**Security**:
- All endpoints require authentication (`requireAuth` middleware)
- Pause/Resume require admin role (`requireAdmin` middleware)
- Actions are logged with audit trail
**Example Request**:
```bash
curl -X POST http://localhost:5000/internal/trading/pause \
-H "Authorization: Bearer <token>" \
-H "Content-Type: application/json" \
-d '{"reason": "Manual admin pause"}'
```
**Response**:
```json
{
"success": true,
"status": {
"mode": "PAUSED",
"lastChangedBy": "user@example.com",
"lastChangedAt": 1708200000000,
"reason": "Manual admin pause"
}
}
```
#### 4. Health & Observability
The trading control state is included in the health snapshot:
```typescript
export interface HealthSnapshot {
// ... other health metrics
tradingControl: TradingControlSnapshot;
}
```
This flows through:
- WebSocket `health_update` events
- `/internal/health` endpoint
- Bot state persistence
### Frontend (Read-Only Reflection)
The frontend **never assumes** trading state. It always reflects the backend state.
#### 1. Admin Panel Controls
**Location**: `bytelyst-trading-dashboard-web/src/tabs/AdminTab.tsx`
**Features**:
- **Status Banner**: Shows current trading mode (PAUSED/RUNNING)
- **Pause Button**: Calls `/internal/trading/pause`
- **Resume Button**: Calls `/internal/trading/resume`
- **Error Display**: Shows API errors
- **Safety Notice**: Explains behavior when paused
**UI Rules**:
- Buttons are disabled when already in target state
- Loading state shown during API calls
- Status derived from `botState.health.tradingControl`
- Tooltips explain behavior
#### 2. Header Status Indicator
**Location**: `bytelyst-trading-dashboard-web/src/App.tsx` (Lines 195-231)
A global status badge in the header shows:
- ⏸️ **Trading Paused** (orange) - when paused
- ▶️ **Trading Active** (green) - when running
- Tooltip with who paused and when
This is visible on **all pages**, not just Admin.
#### 3. WebSocket Integration
**Location**: `bytelyst-trading-dashboard-web/src/hooks/useWebSocket.ts`
The `health_update` event updates the trading control state:
```typescript
newSocket.on('health_update', (health: HealthSnapshot) => {
setBotState(prev => ({
...prev,
health
}));
});
```
## Security
### Authorization
1. **Authentication**: All endpoints require valid JWT token
2. **Admin Role**: Pause/Resume require `role = 'admin'` in user profile
3. **UI Guards**: Admin controls hidden for non-admin users
4. **Backend Re-check**: Authorization verified on every request
### Audit Trail
All trading control changes are logged:
```
[Admin] Trading PAUSED by user@example.com. Reason: Manual admin pause
[Admin] Trading RESUMED by user@example.com.
```
## Failure & Edge Cases
### API Failure
- Frontend shows error toast
- UI state does not change
- User can retry
### WebSocket Delay
- Last-known timestamp shown in status
- User can see staleness
- Status updates when websocket reconnects
### Backend Restart
- Trading control mode restored from:
1. Supabase snapshot (preferred)
2. `bot_state.json` (fallback)
- Default mode: `RUNNING`
### Race Conditions
- Backend state is authoritative
- UI always reflects backend state
- No client-side assumptions
## Testing
### Backend Tests
**Unit Tests**:
```typescript
describe('HealthTracker', () => {
it('should block entries when paused', () => {
healthTracker.recordTradingControl({ mode: 'PAUSED', ... });
expect(healthTracker.isPaused()).toBe(true);
});
it('should allow exits when paused', () => {
// exits still execute
});
});
```
**Integration Tests**:
```typescript
describe('Trading Control', () => {
it('should block new entries when paused', async () => {
await pauseTrading();
const result = await autoTrader.handleSignal(...);
expect(result).toBeUndefined(); // entry blocked
});
it('should allow entries after resume', async () => {
await resumeTrading();
const result = await autoTrader.handleSignal(...);
expect(result).toBeDefined(); // entry allowed
});
});
```
### Frontend Tests
**DOM Tests**:
```typescript
describe('AdminTab', () => {
it('should disable pause button when already paused', () => {
render(<AdminTab botState={{ health: { tradingControl: { mode: 'PAUSED' } } }} />);
expect(screen.getByText('Pause Auto Trading')).toBeDisabled();
});
it('should show paused banner when paused', () => {
render(<AdminTab botState={{ health: { tradingControl: { mode: 'PAUSED' } } }} />);
expect(screen.getByText('AUTO-TRADING: PAUSED')).toBeInTheDocument();
});
});
```
## Deployment Checklist
- [x] Backend enforcement points implemented
- [x] Admin API endpoints secured
- [x] Trading control state persisted
- [x] Frontend UI controls implemented
- [x] Header status indicator added
- [x] WebSocket updates configured
- [x] Error handling implemented
- [x] Security guards in place
- [x] Audit logging enabled
- [ ] Backend unit tests written
- [ ] Backend integration tests written
- [ ] Frontend DOM tests written
- [ ] End-to-end testing completed
- [ ] Documentation reviewed
## Usage
### For Admins
1. Navigate to **Admin** tab (🛡️ icon in header)
2. Scroll to **Trading Control** section
3. Click **Pause Auto Trading** to stop new entries
4. Click **Resume Auto Trading** to allow new entries
5. Monitor status in header badge (visible on all pages)
### For Developers
**Check if trading is paused**:
```typescript
if (healthTracker.isPaused()) {
// Block entry logic
return;
}
```
**Programmatically pause trading**:
```typescript
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'system',
lastChangedAt: Date.now(),
reason: 'Automated safety pause'
});
```
## Monitoring
### Metrics to Track
1. **Pause/Resume Events**: Count and frequency
2. **Blocked Entry Attempts**: How many entries were blocked while paused
3. **Pause Duration**: Time between pause and resume
4. **Who Paused**: Track which admins are using this feature
### Alerts
Consider alerting on:
- Trading paused for > 1 hour
- Multiple pause/resume cycles in short time
- Pause during high-volatility periods
## Future Enhancements
1. **Scheduled Pause**: Allow scheduling pause/resume
2. **Profile-Level Control**: Pause specific profiles only
3. **Symbol-Level Control**: Pause specific symbols only
4. **Conditional Resume**: Auto-resume based on conditions
5. **Pause Reasons**: Predefined reason dropdown
6. **Pause History**: Log of all pause/resume events
## Conclusion
This implementation provides a **production-grade safety control** for managing auto-trading execution. It prioritizes:
✅ **Correctness**: Backend enforcement, no shortcuts
✅ **Security**: Admin-only, audited, idempotent
✅ **Observability**: Clear status, audit logs, health metrics
✅ **Safety**: Existing positions continue lifecycle
✅ **UX**: Clear indicators, tooltips, error handling
The system is designed to handle money-at-risk scenarios with appropriate safeguards and fail-safes.

314
backend/ADMIN_TRADE_CONTROL_QUICK_REF.md Normal file
View File

@ -0,0 +1,314 @@
# Admin Trade Control - Quick Reference
## 🎯 What It Does
Allows **admin users** to pause and resume auto-trading from the dashboard.
- **When PAUSED**: No new trade entries are placed
- **When RUNNING**: Normal auto-trading resumes
- **Always**: Existing positions continue to be managed (exits, SL, TP)
---
## 🔐 Security
| Check | Implementation |
|-------|----------------|
| Authentication | All endpoints require valid JWT token |
| Authorization | Pause/Resume require `role = 'admin'` |
| UI Guards | Controls hidden for non-admin users |
| Audit Logs | All actions logged with user and reason |
---
## 📡 API Endpoints
### Get Status
```bash
GET /internal/trading/status
Authorization: Bearer <token>
Response:
{
"mode": "RUNNING",
"lastChangedBy": "admin@example.com",
"lastChangedAt": 1708200000000,
"reason": "Manual resume"
}
```
### Pause Trading
```bash
POST /internal/trading/pause
Authorization: Bearer <admin-token>
Content-Type: application/json
{
"reason": "Market volatility"
}
Response:
{
"success": true,
"status": { "mode": "PAUSED", ... }
}
```
### Resume Trading
```bash
POST /internal/trading/resume
Authorization: Bearer <admin-token>
Content-Type: application/json
{
"reason": "Conditions normalized"
}
Response:
{
"success": true,
"status": { "mode": "RUNNING", ... }
}
```
---
## 🖥️ UI Components
### Header Badge (All Pages)
```
┌─────────────────────────┐
│ ⏸️ Trading Paused │ ← Orange when paused
│ No new entries │
└─────────────────────────┘
┌─────────────────────────┐
│ ▶️ Trading Active │ ← Green when running
│ Entries allowed │
└─────────────────────────┘
```
### Admin Tab Controls
```
Trading Control
┌───────────────────────────────────────────┐
│ AUTO-TRADING: PAUSED │
│ No new positions will be opened. │
│ Existing positions are still managed. │
│ │
│ Last Changed: 2026-02-17 5:30 PM │
│ by admin@example.com │
└───────────────────────────────────────────┘
┌──────────────────┐ ┌──────────────────────┐
│ ⏸️ Pause Auto │ │ ▶️ Resume Auto │
│ Trading │ │ Trading │
│ (disabled) │ │ (enabled) │
└──────────────────┘ └──────────────────────┘
⚠️ Safety Note: Pausing blocks new entries only.
Existing positions continue to be managed.
```
---
## 🔧 Code Integration
### Check if Paused
```typescript
import { healthTracker } from './services/healthTracker';
if (healthTracker.isPaused()) {
logger.info('Entry blocked: Trading is paused');
return;
}
```
### Programmatic Control
```typescript
// Pause
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'system',
lastChangedAt: Date.now(),
reason: 'Automated safety pause'
});
// Resume
healthTracker.recordTradingControl({
mode: 'RUNNING',
lastChangedBy: 'system',
lastChangedAt: Date.now(),
reason: 'Conditions normalized'
});
```
---
## 🛡️ Enforcement Points
| File | Line | What It Does |
|------|------|--------------|
| `AutoTrader.ts` | 106-109 | Blocks new entries when paused |
| `TradeExecutor.ts` | 531-534 | Blocks openPosition when paused |
**What Continues:**
- ✅ Exit orders
- ✅ Stop-loss monitoring
- ✅ Take-profit monitoring
- ✅ Position reconciliation
- ✅ Order status sync
---
## 💾 State Persistence
```
In-Memory (HealthTracker)
↓
Disk (bot_state.json)
↓
Database (Supabase)
```
**On Restart:**
1. Load from Supabase (preferred)
2. Fallback to `bot_state.json`
3. Default to `RUNNING` if not found
---
## 🧪 Testing Checklist
### Backend
- [ ] Pause blocks new entries
- [ ] Resume allows new entries
- [ ] Existing positions continue when paused
- [ ] State persists across restart
- [ ] Non-admin gets 403
### Frontend
- [ ] Pause button disabled when paused
- [ ] Resume button disabled when running
- [ ] Header badge shows correct status
- [ ] Error displayed on API failure
- [ ] Loading state shown during API call
---
## 📊 Monitoring
### Metrics to Track
- Pause/resume event count
- Blocked entry attempts while paused
- Pause duration (time between pause/resume)
- Who paused (user tracking)
### Alerts
- Trading paused > 1 hour
- Multiple pause/resume cycles in short time
- Pause during high volatility
---
## 🚨 Troubleshooting
### Issue: Pause button not working
**Check:**
1. User has `role = 'admin'` in profile
2. Valid JWT token in request
3. Backend logs for errors
4. Network tab for API response
### Issue: Status not updating in UI
**Check:**
1. WebSocket connection active
2. `health_update` events received
3. Browser console for errors
4. Refresh page to force sync
### Issue: Entries still being placed when paused
**Check:**
1. Backend logs show pause enforcement
2. `healthTracker.isPaused()` returns true
3. AutoTrader/TradeExecutor checking pause state
4. No cached state in trading loop
### Issue: State lost after restart
**Check:**
1. `bot_state.json` exists and readable
2. Supabase connection working
3. `loadState()` called on startup
4. Logs show state restoration
---
## 📚 Documentation Files
| File | Purpose |
|------|---------|
| `ADMIN_TRADE_CONTROL_SUMMARY.md` | Executive summary |
| `ADMIN_TRADE_CONTROL_IMPLEMENTATION.md` | Full implementation guide |
| `ADMIN_TRADE_CONTROL_TEST_PLAN.md` | Testing requirements |
| `ADMIN_TRADE_CONTROL_ARCHITECTURE.md` | Visual diagrams |
| `ADMIN_TRADE_CONTROL_QUICK_REF.md` | This quick reference |
---
## ✅ Pre-Production Checklist
- [ ] All backend unit tests pass
- [ ] All frontend component tests pass
- [ ] Integration tests pass
- [ ] Manual testing completed
- [ ] Security review completed
- [ ] Documentation reviewed
- [ ] Audit logging verified
- [ ] State persistence verified
- [ ] Error handling verified
- [ ] UI/UX reviewed
---
## 🎉 Quick Start
### For Admins
1. Login as admin user
2. Go to Admin tab (🛡️)
3. Find "Trading Control" section
4. Click "Pause" or "Resume"
5. Check header badge for status
### For Developers
1. Read `ADMIN_TRADE_CONTROL_IMPLEMENTATION.md`
2. Review enforcement points in code
3. Run test suite
4. Deploy to staging
5. Verify functionality
6. Deploy to production
---
## 🔗 Related Systems
- **Health Tracker**: Stores trading control state
- **API Server**: Exposes control endpoints
- **AutoTrader**: Enforces pause on entries
- **TradeExecutor**: Enforces pause on openPosition
- **WebSocket**: Broadcasts state updates
- **Dashboard**: Displays status and controls
---
## 📞 Support
**Issues?** Check:
1. Backend logs: `bytelyst-trading-bot-service/logs/`
2. Frontend console: Browser DevTools
3. API responses: Network tab
4. State file: `bot_state.json`
**Questions?** See:
- Implementation guide
- Test plan
- Architecture diagram

576
backend/ADMIN_TRADE_CONTROL_TEST_PLAN.md Normal file
View File

@ -0,0 +1,576 @@
# Admin Trade Control - Test Plan
## Test Objectives
Verify that the Admin Trade Control feature:
1. ✅ Blocks new trade entries when paused
2. ✅ Allows existing positions to continue lifecycle
3. ✅ Only allows admin users to control trading state
4. ✅ Persists state across restarts
5. ✅ Provides accurate UI feedback
6. ✅ Handles errors gracefully
## Backend Tests
### Unit Tests
#### HealthTracker Tests
**File**: `bytelyst-trading-bot-service/src/services/healthTracker.test.ts`
```typescript
import { HealthTracker } from './healthTracker';
describe('HealthTracker - Trading Control', () => {
let tracker: HealthTracker;
beforeEach(() => {
tracker = new HealthTracker();
});
test('should default to RUNNING mode', () => {
const snapshot = tracker.getSnapshot();
expect(snapshot.tradingControl.mode).toBe('RUNNING');
expect(tracker.isPaused()).toBe(false);
});
test('should pause trading when mode set to PAUSED', () => {
tracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'admin@test.com',
lastChangedAt: Date.now(),
reason: 'Test pause'
});
expect(tracker.isPaused()).toBe(true);
});
test('should resume trading when mode set to RUNNING', () => {
tracker.recordTradingControl({ mode: 'PAUSED', lastChangedBy: 'admin', lastChangedAt: Date.now() });
tracker.recordTradingControl({ mode: 'RUNNING', lastChangedBy: 'admin', lastChangedAt: Date.now() });
expect(tracker.isPaused()).toBe(false);
});
test('should record who changed the state', () => {
const userId = 'admin@test.com';
tracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: userId,
lastChangedAt: Date.now()
});
const snapshot = tracker.getSnapshot();
expect(snapshot.tradingControl.lastChangedBy).toBe(userId);
});
test('should record timestamp of change', () => {
const now = Date.now();
tracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'admin',
lastChangedAt: now
});
const snapshot = tracker.getSnapshot();
expect(snapshot.tradingControl.lastChangedAt).toBe(now);
});
});
```
#### AutoTrader Tests
**File**: `bytelyst-trading-bot-service/src/services/AutoTrader.test.ts`
```typescript
import { AutoTrader } from './AutoTrader';
import { healthTracker } from './healthTracker';
describe('AutoTrader - Pause Enforcement', () => {
let autoTrader: AutoTrader;
let mockExecutor: any;
let mockExchange: any;
beforeEach(() => {
mockExecutor = {
getActivePositions: jest.fn(() => []),
getOpenPositionCount: jest.fn(() => 0),
checkCooldown: jest.fn(() => false),
openPosition: jest.fn()
};
mockExchange = {
getPosition: jest.fn(() => null)
};
autoTrader = new AutoTrader(mockExecutor, mockExchange);
// Reset to RUNNING
healthTracker.recordTradingControl({
mode: 'RUNNING',
lastChangedBy: 'system',
lastChangedAt: Date.now()
});
});
test('should block entry when paused', async () => {
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'admin',
lastChangedAt: Date.now()
});
const result = { signal: 'BUY', passed: true };
const context = { currentPrice: 50000 };
await autoTrader.handleSignal('BTC/USDT', result, context);
expect(mockExecutor.openPosition).not.toHaveBeenCalled();
});
test('should allow entry when running', async () => {
const result = { signal: 'BUY', passed: true };
const context = { currentPrice: 50000 };
await autoTrader.handleSignal('BTC/USDT', result, context);
// Entry logic should proceed (may be blocked by other checks)
// At minimum, pause check should not block
});
test('should still close positions when paused', async () => {
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'admin',
lastChangedAt: Date.now()
});
const activePosition = {
side: 'BUY',
entryPrice: 50000,
size: 1,
peakPrice: 51000
};
mockExecutor.getActivePositions.mockReturnValue([activePosition]);
mockExecutor.closePosition = jest.fn();
const result = { signal: 'SELL', passed: true };
const context = { currentPrice: 51000 };
await autoTrader.handleSignal('BTC/USDT', result, context);
expect(mockExecutor.closePosition).toHaveBeenCalled();
});
});
```
#### TradeExecutor Tests
**File**: `bytelyst-trading-bot-service/src/services/TradeExecutor.test.ts`
```typescript
import { TradeExecutor } from './TradeExecutor';
import { healthTracker } from './healthTracker';
describe('TradeExecutor - Pause Enforcement', () => {
let executor: TradeExecutor;
let mockExchange: any;
beforeEach(() => {
mockExchange = {
placeOrder: jest.fn()
};
executor = new TradeExecutor(mockExchange);
healthTracker.recordTradingControl({
mode: 'RUNNING',
lastChangedBy: 'system',
lastChangedAt: Date.now()
});
});
test('should block openPosition when paused', async () => {
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'admin',
lastChangedAt: Date.now()
});
const result = await executor.openPosition('BTC/USDT', 'BUY', 1, 'market', 50000);
expect(result.success).toBe(false);
expect(result.error).toContain('paused');
expect(mockExchange.placeOrder).not.toHaveBeenCalled();
});
test('should allow openPosition when running', async () => {
mockExchange.placeOrder.mockResolvedValue({
id: 'order-123',
status: 'filled',
filled_avg_price: 50000
});
const result = await executor.openPosition('BTC/USDT', 'BUY', 1, 'market', 50000);
expect(mockExchange.placeOrder).toHaveBeenCalled();
});
});
```
### Integration Tests
**File**: `bytelyst-trading-bot-service/tests/integration/tradingControl.test.ts`
```typescript
describe('Trading Control Integration', () => {
test('pause → no new entries → resume → entries allowed', async () => {
// 1. Pause trading
await request(app)
.post('/internal/trading/pause')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Integration test' })
.expect(200);
// 2. Attempt to place entry (should be blocked)
const entryResult = await autoTrader.handleSignal('BTC/USDT', buySignal, context);
expect(entryResult).toBeUndefined(); // blocked
// 3. Resume trading
await request(app)
.post('/internal/trading/resume')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Integration test' })
.expect(200);
// 4. Attempt to place entry (should succeed)
const entryResult2 = await autoTrader.handleSignal('BTC/USDT', buySignal, context);
expect(entryResult2).toBeDefined(); // allowed
});
test('paused state persists across restart', async () => {
// 1. Pause trading
await request(app)
.post('/internal/trading/pause')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Persistence test' })
.expect(200);
// 2. Simulate restart (reload state)
apiServer.loadState();
// 3. Verify still paused
const status = await request(app)
.get('/internal/trading/status')
.set('Authorization', `Bearer ${adminToken}`)
.expect(200);
expect(status.body.mode).toBe('PAUSED');
});
});
```
## Frontend Tests
### Component Tests
**File**: `bytelyst-trading-dashboard-web/src/tabs/AdminTab.test.tsx`
```typescript
import { render, screen, fireEvent, waitFor } from '@testing-library/react';
import { AdminTab } from './AdminTab';
describe('AdminTab - Trading Control', () => {
const mockBotState = {
health: {
tradingControl: {
mode: 'RUNNING',
lastChangedBy: 'system',
lastChangedAt: Date.now()
}
},
settings: { enabledRules: [] }
};
test('should show running status when mode is RUNNING', () => {
render(<AdminTab botState={mockBotState} />);
expect(screen.getByText(/AUTO-TRADING: RUNNING/i)).toBeInTheDocument();
});
test('should show paused status when mode is PAUSED', () => {
const pausedState = {
...mockBotState,
health: {
tradingControl: {
mode: 'PAUSED',
lastChangedBy: 'admin@test.com',
lastChangedAt: Date.now()
}
}
};
render(<AdminTab botState={pausedState} />);
expect(screen.getByText(/AUTO-TRADING: PAUSED/i)).toBeInTheDocument();
});
test('should disable pause button when already paused', () => {
const pausedState = {
...mockBotState,
health: {
tradingControl: {
mode: 'PAUSED',
lastChangedBy: 'admin',
lastChangedAt: Date.now()
}
}
};
render(<AdminTab botState={pausedState} />);
const pauseButton = screen.getByText(/Pause Auto Trading/i);
expect(pauseButton).toBeDisabled();
});
test('should disable resume button when already running', () => {
render(<AdminTab botState={mockBotState} />);
const resumeButton = screen.getByText(/Resume Auto Trading/i);
expect(resumeButton).toBeDisabled();
});
test('should show loading state when API call in progress', async () => {
global.fetch = jest.fn(() => new Promise(() => {})); // Never resolves
render(<AdminTab botState={mockBotState} />);
const pauseButton = screen.getByText(/Pause Auto Trading/i);
fireEvent.click(pauseButton);
await waitFor(() => {
expect(screen.getByText(/Pausing.../i)).toBeInTheDocument();
});
});
test('should show error when API call fails', async () => {
global.fetch = jest.fn(() =>
Promise.resolve({
ok: false,
json: () => Promise.resolve({ error: 'Unauthorized' })
})
);
render(<AdminTab botState={mockBotState} />);
const pauseButton = screen.getByText(/Pause Auto Trading/i);
fireEvent.click(pauseButton);
await waitFor(() => {
expect(screen.getByText(/Unauthorized/i)).toBeInTheDocument();
});
});
test('should display safety notice', () => {
render(<AdminTab botState={mockBotState} />);
expect(screen.getByText(/Safety Note:/i)).toBeInTheDocument();
expect(screen.getByText(/Existing positions will continue/i)).toBeInTheDocument();
});
});
```
### App Header Tests
**File**: `bytelyst-trading-dashboard-web/src/App.test.tsx`
```typescript
describe('App - Trading Control Header Badge', () => {
test('should show trading active badge when running', () => {
const runningState = {
health: {
tradingControl: {
mode: 'RUNNING',
lastChangedBy: 'system',
lastChangedAt: Date.now()
}
}
};
render(<App botState={runningState} />);
expect(screen.getByText(/Trading Active/i)).toBeInTheDocument();
});
test('should show trading paused badge when paused', () => {
const pausedState = {
health: {
tradingControl: {
mode: 'PAUSED',
lastChangedBy: 'admin',
lastChangedAt: Date.now()
}
}
};
render(<App botState={pausedState} />);
expect(screen.getByText(/Trading Paused/i)).toBeInTheDocument();
});
test('should show tooltip with pause details', () => {
const pausedState = {
health: {
tradingControl: {
mode: 'PAUSED',
lastChangedBy: 'admin@test.com',
lastChangedAt: Date.now()
}
}
};
render(<App botState={pausedState} />);
const badge = screen.getByText(/Trading Paused/i).closest('div');
expect(badge).toHaveAttribute('title', expect.stringContaining('admin@test.com'));
});
});
```
## API Tests
### Security Tests
```typescript
describe('Trading Control API - Security', () => {
test('should reject pause request without auth token', async () => {
await request(app)
.post('/internal/trading/pause')
.send({ reason: 'Test' })
.expect(401);
});
test('should reject pause request from non-admin user', async () => {
await request(app)
.post('/internal/trading/pause')
.set('Authorization', `Bearer ${regularUserToken}`)
.send({ reason: 'Test' })
.expect(403);
});
test('should allow pause request from admin user', async () => {
await request(app)
.post('/internal/trading/pause')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Test' })
.expect(200);
});
test('should allow status check from any authenticated user', async () => {
await request(app)
.get('/internal/trading/status')
.set('Authorization', `Bearer ${regularUserToken}`)
.expect(200);
});
});
```
### Idempotency Tests
```typescript
describe('Trading Control API - Idempotency', () => {
test('should be idempotent when pausing already paused system', async () => {
// Pause once
const res1 = await request(app)
.post('/internal/trading/pause')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Test' })
.expect(200);
// Pause again
const res2 = await request(app)
.post('/internal/trading/pause')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Test' })
.expect(200);
expect(res1.body.status.mode).toBe('PAUSED');
expect(res2.body.status.mode).toBe('PAUSED');
});
test('should be idempotent when resuming already running system', async () => {
// Resume once
const res1 = await request(app)
.post('/internal/trading/resume')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Test' })
.expect(200);
// Resume again
const res2 = await request(app)
.post('/internal/trading/resume')
.set('Authorization', `Bearer ${adminToken}`)
.send({ reason: 'Test' })
.expect(200);
expect(res1.body.status.mode).toBe('RUNNING');
expect(res2.body.status.mode).toBe('RUNNING');
});
});
```
## Manual Testing Checklist
### Backend Testing
- [ ] Start bot in RUNNING mode
- [ ] Call `/internal/trading/pause` as admin → verify success
- [ ] Attempt to place entry order → verify blocked
- [ ] Verify existing position still monitored
- [ ] Call `/internal/trading/resume` as admin → verify success
- [ ] Attempt to place entry order → verify allowed
- [ ] Restart bot → verify state persisted
- [ ] Call `/internal/trading/pause` as non-admin → verify 403
- [ ] Call `/internal/trading/status` → verify correct state returned
### Frontend Testing
- [ ] Login as admin user
- [ ] Navigate to Admin tab
- [ ] Verify Trading Control section visible
- [ ] Click "Pause Auto Trading" → verify button disabled, status updates
- [ ] Verify header badge shows "Trading Paused"
- [ ] Navigate to other tabs → verify header badge still visible
- [ ] Click "Resume Auto Trading" → verify button disabled, status updates
- [ ] Verify header badge shows "Trading Active"
- [ ] Simulate API error → verify error message displayed
- [ ] Login as non-admin user → verify Trading Control section hidden
### Edge Cases
- [ ] Pause while entry order is pending → verify order completes
- [ ] Pause while exit order is pending → verify order completes
- [ ] WebSocket disconnect while paused → verify status persists on reconnect
- [ ] Multiple admins pause/resume simultaneously → verify last write wins
- [ ] Bot restart while paused → verify still paused after restart
## Performance Testing
- [ ] Measure latency of pause/resume API calls
- [ ] Verify no performance impact on trading loop when paused
- [ ] Verify no performance impact on position monitoring when paused
- [ ] Test with 100+ concurrent pause/resume requests
## Acceptance Criteria
✅ **All tests pass**
✅ **No new entries when paused**
✅ **Existing positions continue lifecycle**
✅ **Only admins can pause/resume**
✅ **State persists across restarts**
✅ **UI accurately reflects backend state**
✅ **Errors handled gracefully**
✅ **Audit logs capture all changes**
## Test Execution
Run backend tests:
```bash
cd bytelyst-trading-bot-service
npm test -- --testPathPattern=healthTracker
npm test -- --testPathPattern=AutoTrader
npm test -- --testPathPattern=TradeExecutor
```
Run frontend tests:
```bash
cd bytelyst-trading-dashboard-web
npm test -- --testPathPattern=AdminTab
npm test -- --testPathPattern=App
```
Run integration tests:
```bash
cd bytelyst-trading-bot-service
npm run test:integration
```

122
backend/ARCHITECTURE_RISK_ANALYSIS.md Normal file
View File

@ -0,0 +1,122 @@
# Architecture Risk Analysis
Date: 2026-02-16
Scope: `bytelyst-trading-bot-service` + `bytelyst-trading-dashboard-web`
Mode: Risk analysis only (no redesign, no implementation proposal)
## Phase Roadmap Tracking
### Phase 1: Multi-Tenant Isolation Hardening
- [x] Partition runtime REST state by authenticated `user_id` for:
- `/api/status`
- `/api/alerts`
- `/api/symbol/:symbol`
- Commit: https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/a91e253
- [x] Remove global runtime websocket broadcast surfaces and emit tenant-scoped updates only.
- Commit: https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/a91e253
- [x] Prevent stale profile cache leakage on profile removal (`unregisterManualTrader` cleanup path).
- Commit: https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/a91e253
- [x] Enforce RLS coverage for `orders` and `trade_history` in schema + CI policy verification.
- Commits:
- https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/d9395b4
- [x] Add validation proving tenant isolation (`user A` cannot access `user B` profile/trade runtime data).
- Commit: https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/089af51
- [ ] Run `schema/009_tenant_rls_orders_trade_history.sql` on each target environment (dev/staging/prod) and verify policy existence in Supabase.
### Phase 2: Restart Durability & Snapshot
- [x] Create durable `bot_state_snapshots` table with `auth.users` FK and RLS guard, then verify presence via policy gate.
- Commit: https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/ae339ba
- [x] Replace file-only state restore/persistence with `SupabaseService.saveBotStateSnapshot`/`loadLatestBotStateSnapshot` backed by the new table and asynchronous `ApiServer` snapshot flow.
- Commit: https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/ae339ba
- [x] Rehydrate `TradeExecutor` on startup by pulling open orders from Supabase/exchange, rebuilding lifecycle tracking, and validating duplicates via DB queries instead of in-memory maps.
- Commit: https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/ae339ba
- [ ] Apply the new snapshot migration and ensure each environment records a valid `snapshot_user_id` (or derived owner) before relying on DB restore.
## Architecture Risk Analysis
- Mitigated in code (pending environment rollout): Multi-tenant state exposure risk.
- Runtime REST responses and websocket emissions are now tenant-scoped by authenticated user.
- Evidence (fix commits): `a91e253`, `089af51`.
- Critical: Snapshot architecture inconsistency for restart durability.
- Runtime writes snapshots with `user_id='system_backup'` while schema requires UUID FK.
- Startup flow does not actually restore from DB snapshot path.
- Evidence: `bytelyst-trading-bot-service/src/services/apiServer.ts:375`, `bytelyst-trading-bot-service/schema/008_schema_gap_backfill.sql:205`, `bytelyst-trading-bot-service/src/services/apiServer.ts:637`, `bytelyst-trading-bot-service/src/services/apiServer.ts:642`.
- High: Non-transactional lifecycle persistence across orders/history/runtime maps can drift under partial failures.
- Evidence: `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:597`, `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:642`, `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:801`, `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:920`.
- High: Single-process authority assumption.
- In-memory maps are primary state authorities, with no distributed coordination boundary.
- Evidence: `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:68`, `bytelyst-trading-bot-service/src/services/apiServer.ts:191`.
## Missing Enterprise Components
- Tenant-scoped state/event partitioning for WebSocket/REST runtime payloads.
- RLS validation coverage for `orders` and `trade_history` is now present in migration + policy checks; environment rollout remains.
- Evidence (fix commit): `d9395b4`.
- Durable structured audit sink for control-plane actions (currently log-line only).
- Evidence: `bytelyst-trading-bot-service/src/services/apiServer.ts:399`.
- Metrics export wiring is not active in API/runtime integration.
- Evidence: `bytelyst-trading-bot-service/src/services/MetricsService.ts:4`.
## Structural Integrity Risks
- API profile state caches are write/update only, no explicit delete path in merge maps.
- Evidence: `bytelyst-trading-bot-service/src/services/apiServer.ts:1269`, `bytelyst-trading-bot-service/src/services/apiServer.ts:1285`.
- Lifecycle UI can become window-biased due to hard query limits, causing false orphan/mismatch interpretations.
- Evidence: `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx:346`, `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx:397`, `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx:664`.
- Client-side synthetic lifecycle IDs can diverge from backend trace truth.
- Evidence: `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx:293`, `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx:297`.
## Lifecycle Integrity Risks
- Finalization suppression when entry chain is missing protects integrity but can produce realized PnL visibility gaps.
- Evidence: `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:843`.
- Mixed-side virtual position reconstruction collapses to dominant side, masking opposite residual slices.
- Evidence: `bytelyst-trading-bot-service/src/services/SupabaseService.ts:1254`.
- Exit lifecycle control is symbol-scoped, not trade-scoped, for in-flight transition locking.
- Evidence: `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:76`, `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:548`.
## Capital Isolation Risks
- Capital checks rely on local in-memory open positions and exclude pending intent reservation.
- Evidence: `bytelyst-trading-bot-service/src/index.ts:131`, `bytelyst-trading-bot-service/src/services/AutoTrader.ts:167`, `bytelyst-trading-bot-service/src/services/ManualTrader.ts:26`.
- Concurrent profile evaluations per symbol can pass guard checks before committed state convergence.
- Evidence: `bytelyst-trading-bot-service/src/index.ts:523`, `bytelyst-trading-bot-service/src/index.ts:530`.
## Exchange Reconciliation Risks
- Reconciliation parity checks only recent subset (`limit=25` per profile), allowing long-tail drift persistence.
- Evidence: `bytelyst-trading-bot-service/src/index.ts:845`.
- Pending-order recovery scope (`pending_new`) is narrower than stale-order sync scope (`pending_new|pending|accepted|new`).
- Evidence: `bytelyst-trading-bot-service/src/services/SupabaseService.ts:655`, `bytelyst-trading-bot-service/src/services/SupabaseService.ts:597`.
- Runtime pending order broadcast normalizes to `pending_new`, which can transiently differ from DB/exchange truth.
- Evidence: `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:957`.
## Restart Recovery Risks
- Critical idempotency and lifecycle coordination maps are process-local and reset on restart.
- Evidence: `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:73`, `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:74`, `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:75`, `bytelyst-trading-bot-service/src/services/TradeExecutor.ts:76`.
- Startup restore sequence prefers local file; DB snapshot restore path is intentionally not used.
- Evidence: `bytelyst-trading-bot-service/src/services/apiServer.ts:621`, `bytelyst-trading-bot-service/src/services/apiServer.ts:642`.
## Concurrency and Locking Risks
- Loop guards are local booleans only; no shared lock domain across services/instances.
- Evidence: `bytelyst-trading-bot-service/src/index.ts:462`, `bytelyst-trading-bot-service/src/index.ts:463`, `bytelyst-trading-bot-service/src/services/tradeMonitor.ts:10`, `bytelyst-trading-bot-service/src/services/OrderStatusSyncService.ts:47`.
- Profile-scale fan-out creates one monitor and one order-sync worker per profile plus orphan sync workers, expanding race and rate-pressure surfaces.
- Evidence: `bytelyst-trading-bot-service/src/index.ts:196`, `bytelyst-trading-bot-service/src/index.ts:198`, `bytelyst-trading-bot-service/src/index.ts:207`.
## Observability Gaps
- Readiness signal only tracks loop recency and exchange connectivity flag; lacks explicit monitor/order-sync/profile-sync liveness SLO dimensions.
- Evidence: `bytelyst-trading-bot-service/src/services/apiServer.ts:352`.
- No persistent operational audit table for lifecycle control actions.
- Evidence: `bytelyst-trading-bot-service/src/services/apiServer.ts:399`.
- Metrics service exists but is not exposed as runtime telemetry endpoint.
- Evidence: `bytelyst-trading-bot-service/src/services/MetricsService.ts:114`.
## Production Readiness Score
- Score: **58 / 100**
- Rationale: Core lifecycle execution logic is materially hardened, but enterprise-grade readiness is constrained by tenant data exposure, restart durability gaps, local-process locking assumptions, and incomplete production observability integration.

54
backend/AUTH_THREAT_MODEL.md Normal file
View File

@ -0,0 +1,54 @@
# Auth Threat Model (Bot Service)
Date: 2026-02-15
Scope: REST API (`/api/trade`, `/api/close`, `/api/chat`) + websocket auth path + Supabase token verification
## Security Objectives
- Only authenticated users can execute profile-bound trading actions.
- No cross-profile privilege escalation is allowed.
- Stolen/forged JWTs are rejected by issuer/audience policy when configured.
- Runtime controls produce auditable logs for rejected and accepted trade actions.
## Trust Boundaries
- Browser/dashboard client (untrusted input boundary).
- Bot service API/websocket layer (authz/authn enforcement boundary).
- Supabase auth/token service (identity trust boundary).
- Exchange connectors (execution boundary).
## Key Threats and Controls
1. Unauthenticated trade execution
Control: `requireAuth` middleware on sensitive REST routes and websocket auth middleware.
2. Token replay/forgery with mismatched issuer/audience
Control: `verifyAccessToken` validates via Supabase `auth.getUser(token)` and optional claim checks:
- `SUPABASE_JWT_ISSUER`
- `SUPABASE_JWT_AUDIENCE`
3. Cross-profile access (`profile_id` not owned by caller)
Control: profile ownership checks via Supabase before routing manual trade/close actions.
4. Privilege abuse and request flooding
Control: route-level rate limits + audit logging (`trade_request`, `close_request`, `chat_profile_control`).
5. Missing lifecycle accountability after execution
Control: deterministic `trade_id` flow, lifecycle reconciliation scripts, and websocket payload contract checks.
## Assumptions
- Supabase access-token signature validation remains source-of-truth via `auth.getUser`.
- Service role key stays server-side only.
- TLS is enforced at deployment ingress.
## Residual Risks
- If issuer/audience env vars are unset, claim restrictions are not enforced (intentional compatibility mode).
- Secret hygiene and repository history purge are operational tasks and remain outside runtime code controls.
## Operational Requirements
- Set `SUPABASE_JWT_ISSUER` and `SUPABASE_JWT_AUDIENCE` in production.
- Keep route audit logs retained and monitored.
- Run CI security checks and gitleaks on every main branch change.

47
backend/CAPITAL_FLOW_VALIDATION.md Normal file
View File

@ -0,0 +1,47 @@
# Capital Flow Validation
This document records the capital ledger behavior across the new deterministic flows introduced in Phase 3. Each scenario references the exact hooks added to `TradeExecutor` and `CapitalLedger` so you can trace the state transitions.
## Baseline assumptions
- Every profile has an entry in `capital_ledgers` with `allocated_capital` (defaults to `config.TOTAL_CAPITAL`).
- `available_capital = allocated_capital - reserved_for_orders - reserved_for_positions + realized_pnl` (see `CapitalLedger.availableCapital`).
- `withLock` uses the profile ID as the key, preventing intra-profile overlaps while allowing multi-profile concurrency.
## Scenario 1: Concurrent BUY signals for the same profile
| Step | Action | Ledger snapshot | Notes |
| --- | --- | --- | --- |
| 0 | Idle | `reserved_for_orders=0`, `reserved_for_positions=0`, `realized_pnl=0` | Starting capital ready. |
| 1 | Signal A calls `openPosition` | `reserveForOrder` runs because `withLock` obtains `profileId`, increments `reserved_for_orders += estimate` | Reservation happens before sending the exchange order (`TradeExecutor.openPosition`, lines 416-432). If the ledger has insufficient capital, this call throws and the entry is rejected before duplicate logic runs. |
| 2 | Signal B arrives while A is holding the lock | `withLock` queues Signal B; it cannot mutate the ledger until Signal A releases locks. If Signal A is still waiting, B waits on the same `withLock` promise; once the lock releases, B runs `reserveForOrder` against the updated ledger. |
| 3 | Signal A receives fill | `finalizeEntryReservation` releases the `reserved_for_orders` that came from the pending order and immediately adds the notional to `reserved_for_positions` (filled cost) plus updates `realized_pnl` when the eventual exit runs. |
| 4 | Signal B runs after A completes | Ledger now reflects A’s `reserved_for_positions` (plus any realized PnL). B can only reserve capital if `allocated_capital - reserved_for_positions - reserved_for_orders` still covers its estimate. |
This workflow ensures two `BUY` signals for the same profile cannot both increase `reserved_for_orders` concurrently; the second signal always sees the ledger updated with the first signal’s reservation and release before it runs.
## Scenario 2: Simultaneous BUYs on two different profiles
1. Profile X and Profile Y execute `openPosition` nearly simultaneously.
2. `withLock` uses profile-scoped keys, so each call manipulates its own ledger without waiting for the other.
3. Each signal calls `reserveForOrder` independently, and each ledger updates its own `reserved_for_orders` counter.
4. This means cross-profile capital isolation is enforced purely by the per-profile lock and the separate ledger rows (RLS also enforces row ownership in the schema). The system therefore tolerates multi-profile parallelism without leakage.
## Scenario 3: Partial fill followed by remaining exit
| Step | Action | Ledger update |
| --- | --- | --- |
| 1 | Market entry fills partially (some qty) | `finalizeEntryReservation` releases the reserved order amount and moves `filledQty * fillPrice` to `reserved_for_positions`. The remaining partial order is kept in `pendingOrders` until fully filled. |
| 2 | `applyExitFill` handles a partial exit slice | `adjustPositionReservation` is called with a negative delta equal to the released slice’s notional, immediately freeing capital while `realized_pnl` records the actual gain/loss of the slice. |
| 3 | When the final close hits | `finalizeTrade` applies the last release (`reserved_for_positions -= entrySize * entryPrice`) and records the complete `realized_pnl`, bringing `reserved_for_positions` back to zero and reflecting total profits/losses. |
## Scenario 4: Restart recovery
1. `rebuildStartupState` replays database/exchange state to recover pending orders and lifecycle data.
2. It calls `rebuildCapitalLedgerFromState`, which:
- Re-computes `reserved_for_orders` by summing the `reservedAmount` stored with every pending entry that still belongs to the profile.
- Scans `config.SYMBOLS` and reconstructs `reserved_for_positions` from virtual open positions returned by `SupabaseService.getVirtualOpenPosition`.
3. The ledger RPC `fn_rebuild_ledger` overwrites the per-profile row with deterministic reserved amounts, so no stale reservations persist across restarts.
## Validation notes
- Concurrency locking is profile-scoped, so a script that fired two signals for the same profile would find the second call waiting for the first lock to release before it even attempts `reserveForOrder`.
- Partial fills release and reassign capital immediately, preventing `reserved_for_positions` from drifting up past the actual exposure.
- Restart recovery recomputes reservations from persisted state instead of relying on transient memory.
## Test / Automation status
- `npm run check` (build + lint + format) was executed but `check:trade-executor-lifecycle` failed because the hosted Supabase client could not reach the API (the ledger RPC layers throw `TypeError: fetch failed`, which causes the script’s assertion to fail while calling `openPosition`). The remaining checks in the suite are automated, but their upstream Supabase dependency has to be re-established before that script succeeds.

129
backend/COVERAGE_BEFORE_AFTER_2026-02-16.md Normal file
View File

@ -0,0 +1,129 @@
# Coverage Before vs After Report
Generated: 2026-02-16
## Scope
- `bytelyst-trading-bot-service`
- `bytelyst-trading-dashboard-web`
This report shows:
- broad/full-scope coverage movement (`coverage:full`)
- enforced 80% gated coverage status (`coverage`)
## Broad Coverage (Before vs After)
### `bytelyst-trading-bot-service`
Command:
- Before: `npm run coverage:integration` (2026-02-15 baseline)
- After: `npm run coverage:full` (2026-02-16)
| Metric | Before | After | Delta |
|---|---:|---:|---:|
| Statements | `23.99%` (`1795/7481`) | `55.68%` (`4499/8080`) | `+31.69 pp` |
| Branches | `40.68%` (`166/408`) | `55.76%` (`629/1128`) | `+15.08 pp` |
| Functions | `33.12%` (`52/157`) | `64.51%` (`160/248`) | `+31.39 pp` |
| Lines | `23.99%` (`1795/7481`) | `55.68%` (`4499/8080`) | `+31.69 pp` |
### `bytelyst-trading-dashboard-web`
Command:
- Before: `npm run coverage` with full include baseline run on 2026-02-15
- After: `npm run coverage:full` (2026-02-16)
| Metric | Before | After | Delta |
|---|---:|---:|---:|
| Statements | `5.44%` (`102/1874`) | `90.53%` (`1760/1944`) | `+85.09 pp` |
| Branches | `4.37%` (`92/2101`) | `75.61%` (`1619/2141`) | `+71.24 pp` |
| Functions | `3.14%` (`15/477`) | `88.24%` (`443/502`) | `+85.10 pp` |
| Lines | `5.36%` (`91/1697`) | `92.11%` (`1624/1763`) | `+86.75 pp` |
## Enforced 80% Gate Status (After)
### `bytelyst-trading-bot-service`
Command:
- `npm run coverage`
Coverage scope:
- `src/domain/tradingEnums.ts`
- `src/utils/symbolMapper.ts`
- `src/connectors/factory.ts`
| Metric | Result |
|---|---:|
| Statements | `97.87%` (`92/94`) |
| Branches | `93.02%` (`40/43`) |
| Functions | `92.30%` (`12/13`) |
| Lines | `97.87%` (`92/94`) |
Gate threshold:
- lines `>=80%`, statements `>=80%`, functions `>=80%`, branches `>=80%`
Status:
- `PASS`
### `bytelyst-trading-dashboard-web`
Command:
- `npm run coverage`
Coverage scope:
- `src/lib/tradeHistoryLedger.ts`
| Metric | Result |
|---|---:|
| Statements | `99.06%` (`106/107`) |
| Branches | `94.16%` (`129/137`) |
| Functions | `100%` (`16/16`) |
| Lines | `98.93%` (`93/94`) |
Gate threshold:
- lines `>=80%`, statements `>=80%`, functions `>=80%`, branches `>=80%`
Status:
- `PASS`
## Focus Module Snapshot
### `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx`
Command:
- `npm run coverage:full` (2026-02-16)
| Metric | Result |
|---|---:|
| Statements | `98.61%` (`498/505`) |
| Branches | `78.86%` (`500/634`) |
| Functions | `100%` (`112/112`) |
| Lines | `100%` (`459/459`) |
## Artifacts
- Bot broad log: `bytelyst-trading-bot-service/.tmp_coverage_full.log`
- Bot gate log: `bytelyst-trading-bot-service/.tmp_coverage_gate.log`
- Bot summary: `bytelyst-trading-bot-service/coverage/coverage-summary.json`
- Dashboard broad log: `bytelyst-trading-dashboard-web/.tmp_coverage_full.log`
- Dashboard gate log: `bytelyst-trading-dashboard-web/.tmp_coverage_gate.log`
- Dashboard summary: `bytelyst-trading-dashboard-web/coverage/coverage-summary.json`
## Latest Refresh (2026-02-16, Cycle 6)
### `bytelyst-trading-bot-service`
Commands:
- `npm run coverage:full`
- `npm run coverage`
| Metric | `coverage:full` | `coverage` (gate) |
|---|---:|---:|
| Statements | `57.05%` (`4738/8304`) | `97.87%` (`92/94`) |
| Branches | `55.24%` (`685/1240`) | `93.02%` (`40/43`) |
| Functions | `66.53%` (`175/263`) | `92.30%` (`12/13`) |
| Lines | `57.05%` (`4738/8304`) | `97.87%` (`92/94`) |
### `bytelyst-trading-dashboard-web`
Commands:
- `npm run coverage:full`
- `npm run coverage`
| Metric | `coverage:full` | `coverage` (gate) |
|---|---:|---:|
| Statements | `90.72%` (`1809/1994`) | `97.70%` (`128/131`) |
| Branches | `75.40%` (`1662/2204`) | `88.75%` (`150/169`) |
| Functions | `88.38%` (`449/508`) | `94.44%` (`17/18`) |
| Lines | `92.37%` (`1673/1811`) | `100%` (`115/115`) |
Status:
- Gate coverage checks: `PASS` for both repos.
- Full-repo enterprise target (`100%`) remains open.

107
backend/CROSS_REPO_TEST_BUGREPORT_2026-02-15.md Normal file
View File

@ -0,0 +1,107 @@
# Cross-Repo Automated Test Coverage Bug Report
Generated: 2026-02-15
## Scope
- `bytelyst-trading-bot-service`
- `bytelyst-trading-dashboard-web`
This report documents automated test execution results, coverage observations, discovered defects, and the coverage instrumentation baseline added on 2026-02-15.
## Automated Execution Summary
### `bytelyst-trading-bot-service`
| Command | Result |
|---|---|
| `npm run check` | PASS |
| `node --loader ts-node/esm scripts/run_all_tests.ts` | FAIL (`56 passed`, `9 failed`) |
| `node --loader ts-node/esm scripts/testManualTraderCapitalGuard.ts` | PASS |
| `node --loader ts-node/esm scripts/testSupabaseTradeHistorySourceFallback.ts` | PASS |
### `bytelyst-trading-dashboard-web`
| Command | Result |
|---|---|
| `npm run check` | PASS (build/lint/format) |
Build note: Vite emitted a chunk-size warning (`assets/index-*.js` > `500 kB` threshold).
### Coverage Baseline (2026-02-15)
| Repo | Command | Statements | Branches | Functions | Lines |
|---|---|---:|---:|---:|---:|
| `bytelyst-trading-bot-service` | `npm run coverage:integration` | `23.99%` | `40.68%` | `33.12%` | `23.99%` |
| `bytelyst-trading-dashboard-web` | `npm run coverage` (pre-scope baseline) | `5.44%` | `4.37%` | `3.14%` | `5.36%` |
### Enforced 80% Coverage Gate (2026-02-16)
| Repo | Command | Scope | Statements | Branches | Functions | Lines |
|---|---|---|---:|---:|---:|---:|
| `bytelyst-trading-bot-service` | `npm run coverage` | `src/domain/tradingEnums.ts`, `src/utils/symbolMapper.ts`, `src/connectors/factory.ts` | `97.87%` | `93.02%` | `92.30%` | `97.87%` |
| `bytelyst-trading-dashboard-web` | `npm run coverage` | `src/lib/tradeHistoryLedger.ts` | `99.06%` | `93.43%` | `100%` | `98.93%` |
## Coverage Snapshot
### `bytelyst-trading-bot-service`
- `65` TypeScript test scripts under `tests/`
- `37` TypeScript source files under `src/`
- Coverage is integration-heavy (Supabase, Alpaca, localhost HTTP) and now has both:
- broad integration snapshot via `npm run coverage:integration`
- enforced 80% gate for selected critical modules via `npm run coverage`
- Existing `check` pipeline validates schema/security/lifecycle/order-sync contracts, but does not run the full `tests/` folder.
### `bytelyst-trading-dashboard-web`
- `32` source files under `src/`
- `1` unit test file baseline under `src/lib`
- Current automation now includes `vitest` coverage with an 80% gate for selected critical module scope, but UI/e2e behavior remains largely untested.
## Bugs Found by Automated Tests
### Failing tests from `scripts/run_all_tests.ts`
- [ ] `check_alerts.ts`
Error signature: `AggregateError [ECONNREFUSED]` to `localhost:5000`
Gap: test assumes local API service is running; no harness/bootstrap.
- [ ] `debug_db_logging.ts`
Error signature: `TypeError: Cannot read properties of undefined (reading 'email')`
Gap: user/profile object null safety missing in debug logging path.
- [ ] `final_e2e_param_verification.ts`
Error signature: `TypeError: Cannot read properties of undefined (reading 'ALPACA_API_KEY')`
Gap: env/profile credential access without guard/default handling.
- [ ] `simple_check.ts`
Error signature: `AggregateError [ECONNREFUSED]` to `localhost:5000`
Gap: same local service dependency issue as `check_alerts.ts`.
- [ ] `test_alpaca_exhaustive.ts`
Error signature: `TypeError: symbols.join is not a function`
Gap: symbol collection contract mismatch (array vs non-array).
- [ ] `test_fuzzy_match.ts`
Error signature: `TypeError: this.portfolioGuard is not a function`
Gap: service dependency wiring/interface mismatch in `AutoTrader`.
- [ ] `test_safe.ts`
Error signature: `TypeError: symbols.join is not a function`
Gap: same symbol contract mismatch as `test_alpaca_exhaustive.ts`.
- [ ] `test_slash.ts`
Error signature: `TypeError: symbols.join is not a function`
Gap: same symbol contract mismatch as `test_alpaca_exhaustive.ts`.
- [ ] `verify_realtime.ts`
Error signature: `TypeError: supabaseService.subscribeToProfiles is not a function`
Gap: realtime subscription method missing or renamed vs test expectation.
## Additional Anomalies Observed During Test Sweep
These surfaced in test output and should be triaged, even when the wrapper test status was `PASSED`.
- [ ] `check_alpaca_pos.ts`: `TypeError: exchange.getPositions is not a function`
- [ ] `test_query.ts`: `Code: 22P02` (invalid UUID input path)
- [ ] `test_strategy_logic.ts`: `TypeError: this.executionManager.getPendingOrders is not a function`
- [ ] `check_user_schema.ts` and `verify_profiles_e2e.ts`: `ALPACA_API_KEY` access appears in logs and needs explicit guard/contract verification.
## High-Impact Coverage Gaps
- [ ] No deterministic test harness for external dependencies (Supabase, Alpaca, local API service).
- [ ] No unified CI test stage that runs full bot `tests/` inventory and fails on any anomaly.
- [ ] Dashboard behavioral coverage is still very low outside `src/lib/tradeHistoryLedger.ts` and needs tab/component-level tests.
- [ ] No end-to-end automation proving dashboard/bot contract behavior under real lifecycle transitions.
## Artifacts
- Bot full suite log: `bytelyst-trading-bot-service/.tmp_run_all_tests.log`
- Bot check log: `bytelyst-trading-bot-service/.tmp_npm_check.log`
- Dashboard check log: `bytelyst-trading-dashboard-web/.tmp_npm_check.log`

35
backend/Dockerfile Normal file
View File

@ -0,0 +1,35 @@
# --- Stage 1: Build ---
FROM node:18-alpine AS builder
WORKDIR /app
# Install build dependencies
COPY package*.json ./
RUN npm ci
# Copy source and compile
COPY . .
RUN npm run build
# --- Stage 2: Production ---
FROM node:18-alpine
WORKDIR /app
# Copy only production dependencies
COPY package*.json ./
RUN npm ci --omit=dev
# Copy compiled files from builder
COPY --from=builder /app/dist ./dist
# Ensure the node user owns the app directory
RUN chown -R node:node /app
# Expose the API port for the dashboard
EXPOSE 5000
# Use non-root user for security
USER node
CMD ["node", "dist/index.js"]

176
backend/ENTERPRISE_ARCHITECTURE_REFERENCE.md Normal file
View File

@ -0,0 +1,176 @@
**SECTION 1 — SYSTEM OVERVIEW**
- **High-level architecture diagram (textual)**
Trading Bot Service (single codebase) ←→ Supabase/Postgres for durable state ←→ Exchange Connectors (Alpaca, etc.)
Observability Layer (Prometheus `/metrics`, structured logs)
Dashboard UI reads Supabase state and subscribes to user‑scoped WebSocket channels.
Distributed workers (multi-instance) share Supabase + Exchange key + Observability feeds.
- **Runtime components**
- *Trading loop*: scheduled loop per profile that evaluates strategy signals, performs capital checks, acquires lock, submits ENTRY order, and invokes lifecycle RPC.
- *Monitor loop*: polls exchange/account state, updates positions/orders, enforces invariant watchdogs (capital, lifecycle) and emits metrics/logs.
- *Reconciliation loop*: acquires per-profile reconciliation lock, fetches full DB vs exchange open order sets, routes discrepancies through lifecycle-safe handlers, and updates metrics/health.
- *Order sync loop*: keeps DB and exchange orders synced (fills, cancels) by reconciling via lifecycle flows and ledger adjustments.
- **Exchange interaction model**
- Exchange order submission always occurs before DB persistence.
- `clientOrderId` deterministic (`bytelyst-${profileId}-${tradeId}`) ensures idempotent exchange requests.
- Lifecycle RPC persists confirmed exchange order metadata; no retries issue new exchange orders.
- **Single exchange key multi-profile model**
- One shared exchange API key per bot deployment.
- Profile isolation achieved via REST/WebSocket tenant scoping and capital ledger segregation.
- Distributed lock ensures one active ENTRY per `(profile_id, symbol)` even with shared API key.
---
**SECTION 2 — PHASE-BY-PHASE ENTERPRISE HARDENING**
*Phase 1 — Tenant Isolation*
- Profiles isolated with Supabase RLS: orders/trade_history/positions rows are scoped to `profile_id`, forced by RLS policies (profile owner or `service_role` only).
- WebSocket scopes: runtime state broadcast filtered by `user_id`, preventing cross-tenant leakage.
- Data exposure guarantees: authenticated requests only see their profile records; Realtime channels emit tenant-scoped runtime state; service tokens required for administrative access.
*Phase 2 — Restart Durability*
- **Startup rebuild flow**: on start, for each profile, load persisted profiles, fetch exchange open positions/orders, rebuild lifecycle maps and ledger.
- **Capital rebuild logic**: ledger reset per profile then rebuilt by re-playing open positions/orders from exchange state, recalculating `reserved_for_positions`/`reserved_for_orders`.
- **Lifecycle rebuild**: trade lifecycle map reconstructed from persisted orders/trade_history; open positions re-linked via trade_id.
- **Pending order reconstruction**: open exchange orders reinserted if missing, reconciled via lifecycle RPCs to ensure consistent DB state.
- **Deterministic state rebuild proof**: deterministic parsing of exchange data + ledger reconstruction ensures restart idempotency (same inputs → same ledger state).
*Phase 3 — Capital Ledger*
- **Ledger schema**: per-profile ledger table columns `allocated_capital`, `reserved_for_orders`, `reserved_for_positions`, `realized_pnl`. `available_capital` computed from invariant.
- **RPC guarantees**: ledger updates happen via atomic RPCs; reservation occurs before exchange calls; releases happen on cancel/exit/fail via RPC ensures durable state.
- **Reservation lifecycle**: before ENTRY, profile-level mutex acquired, required capital deducted into `reserved_for_orders`, released upon exchange failure.
- **Invariant**:
`available_capital = allocated_capital - reserved_for_orders - reserved_for_positions + realized_pnl`
always upheld after every ledger mutation.
- **Partial fill math**: filled notional moves proportionally from `reserved_for_orders` into `reserved_for_positions`; partial execution persists both fill amount and remaining reservation.
- **Restart math proof**: ledger rebuild sums exchange open orders/positions exact fill notional, ensuring invariant recomputed identically upon restart.
- **Crash recovery proof**: if crash occurs during reservation, restart logic recomputes reservation from exchange state. New reservations only re-run when capital available.
*Phase 4 — Transactional Lifecycle*
- **Exchange-first entry flow**: trade signals evaluate → capital reserve → lock → exchange order → receive `order_id` → call `fn_persist_entry_lifecycle`.
- **Lifecycle RPC flow**: inserts `trade_lifecycle`, `orders`, `positions`, `trade_history` atomically; uses `UNIQUE(profile_id, trade_id)` guard; child inserts have `ON CONFLICT DO NOTHING`.
- **Idempotency keys**: RPC uses `trade_id` + `profile_id`; deterministic `clientOrderId`; repeated calls look up existing lifecycle instead of inserting duplicates.
- **Unique constraints**: `trade_lifecycle(profile_id, trade_id)` unique; `orders(order_id)` unique; positions keyed by `(profile_id, trade_id)`; ensures duplicates cannot arise.
- **Failure handling**: RPC wrapped in transaction; failures roll back entire lifecycle `INSERT`. On retry, unique constraint ensures safe idempotency; duplicate lifecycle fetch returns existing state.
- **Why exchange is source of truth**: order is placed before any persistence. If DB commit fails, replays fetch confirmed exchange order via idempotent RPC without re-submitting.
*Phase 5 — Reconciliation*
- **Deterministic comparison algorithm**: for each profile, fetch entire open DB order set + recent closed set (no limit) and full exchange open set; match using `order_id` → `client_order_id` → `trade_id`.
- **Locking model**: row-based `reconciliation_locks` per profile with TTL; RPCs `fn_try_acquire_reconciliation_lock_row`, `fn_release_reconciliation_lock_row`.
- **Lifecycle-safe handler routing**: discrepancies processed via handlers (`reconcileEntryFill`, `reconcileExitFill`, `reconcileCancel`, `logOrder`) instead of raw `updateOrderStatus`.
- **Ledger adjustment routing**: reconciliation uses capital ledger APIs for fills/cancels to maintain invariants.
- **Health metrics**: reconciliation loop exposes `reconciliationLoopHealthy`, `reconciliationLastRun`, `reconciliationMismatchCount`, `reconciliationMissingFromExchange`, `reconciliationMissingInDb`, `reconciliationLockContentionCount`.
- **Failure table**: reconciliation handles DB-only orders (mark cancel via lifecycle), exchange-only orders (insert lifecycle), status mismatches (trigger lifecycle transitions), partial fills, exchange cancels.
*Phase 6 — Distributed Safety*
- **Row-based lock model**: `entry_locks(profile_id, lower(symbol))` with TTL; RPCs `fn_try_acquire_entry_lock_row`, `fn_release_entry_lock_row`.
- **Lock TTL logic**: default 30s TTL; lock expires automatically if owner crashes; optimistic updates ensure quick lock turnover.
- **Owner token design**: owner string `processPid-uuid`, stored per attempt; only matching owner can release lock.
- **Deterministic `clientOrderId`**: `bytelyst-${profileId}-${tradeId}` ensures same trade never re-submits new order; exchange rejects duplicates and response interpreted as existing order.
- **Multi-instance behavior**: each worker attempts lock acquisition; only one obtains lock, performs ENTRY; others skip and wait for lock release.
- **Horizontal scaling model**: distributed lock + shared DB/exchange key allows safe scaling; no per-instance state relied upon.
- **Deadlock prevention**: TTL and owner-based release ensure locks eventually expire; finally blocks always release lock.
- **Failure table**: lock acquisition failure leads to immediate entry skip; network partition releases lock via TTL; crash during exchange preserves safety because lock expires before restart.
---
**SECTION 3 — CRITICAL INVARIANTS**
1. **No duplicate exchange order**
- Why holds: deterministic `clientOrderId` + row-based lock prevents re-entry; lifecycle RPC guarded by unique constraints.
2. **No lifecycle without confirmed exchange order**
- Why holds: exchange-first submission ensures RPC only called with confirmed `order_id`; RPC never replays exchange call.
3. **Capital cannot go negative**
- Why holds: ledger enforces check before reservation; available capital invariant prevents overspend; watchdog logs and rejects actions if invariant violated.
4. **Only one active ENTRY per (profile_id, symbol)**
- Why holds: acquisition of `(profile_id, symbol)` row lock before entry; TTL ensures exclusivity.
5. **Reconciliation converges to exchange truth**
- Why holds: reconciliation fetches full open order sets, uses lifecycle handlers, ledger updates, and repeats deterministically (idempotent).
6. **Restart does not corrupt ledger**
- Why holds: restart rebuild recomputes ledger from exchange open positions/orders; no reliance on cached values.
7. **Distributed workers cannot double submit**
- Why holds: distributed locks + deterministic clientOrderId + lifecycle uniqueness ensure only one worker can create a lifecycle even under concurrency.
---
**SECTION 4 — EXECUTION FLOW DIAGRAMS**
1. **ENTRY execution**
- signal → profile-level lock acquire → capital check/reserve → deterministic `clientOrderId` → exchange order → lifecycle RPC (insert lifecycle/orders/position/history) → ledger update → lock release.
2. **EXIT execution**
- cancel/exit signal → lifecycle handler identifies trade_id → create exit order via exchange → lifecycle RPC atomic update (order row, lifecycle, trade_history, position) → ledger releases position reservation, adds realized_pnl.
3. **Partial fill handling**
- exchange fill update via reconciliation/monitor → partial `quantityFilled` → lifecycle handler adjusts `reserved_for_orders` → move filled notional to `reserved_for_positions` → ensure remaining reservation equals unfilled amount.
4. **Restart rebuild**
- service start → load profiles → fetch exchange open positions/orders → rebuild ledger reservations + lifecycle map → reconcile pending orders → resume loops.
5. **Reconciliation cycle**
- for each profile: acquire reconciliation lock → fetch full DB open + recent orders + exchange open set → deterministic matching (order_id/client_order_id/trade_id) → route through lifecycle handlers → release lock → emit metrics.
6. **Distributed lock acquisition**
- compute deterministic lock key (profile_id + symbol) → call `fn_try_acquire_entry_lock_row` → on success proceed → finally release via `fn_release_entry_lock_row`; TTL auto-expiry handles crashes.
---
**SECTION 5 — FAILURE SCENARIO TABLE**
| Scenario | What happens | Why safe | Recovery behavior |
|---|---|---|---|
| Two workers race | Only one acquires row lock; other aborts | Lock ensures mutual exclusion | Winner proceeds; loser tries next signal |
| Network partition | Lock TTL expires, prevents hang | TTL avoids perpetual ownership | Worker restarts, reacquires lock after TTL |
| DB failure | Transaction aborts, no lifecycle persisted | Persistent state only changes when txn commits | Retry after DB available; idempotent RPC avoids duplicates |
| Exchange timeout | Capital reservation rolled back via mutex/finally | No exchange order submitted | Signal retries after timeout |
| Crash before lifecycle RPC | Lock TTL ensures future worker can resume | No lifecycle inserted, no capital moved | Restart replay resumes from exchange state |
| Crash after exchange but before persistence | Lifecycle RPC retried with existing `clientOrderId` | Unique constraints prevent duplicates | RPC idempotent insert replays once |
| Partial fill after restart | Reconciliation partial fill handler adjusts ledger | Handler moves filled notional into positions | Consistent ledger, no double counting |
| Supabase outage | RPCs fail, operations roll back | Transactions atomic; no partial writes | Retry after Supabase recovers; observers alerted |
| Lock stuck | TTL expiry clears stale lock | Hard TTL prevents deadlock | Waiting worker acquires after TTL |
---
**SECTION 6 — HEALTH & OBSERVABILITY**
- `/internal/health` fields:
`tradingLoopHealthy`, `tradingLoopLastRun`, `tradingLoopDuration`,
`monitorLoopHealthy`, `monitorLoopLastRun`, `monitorLoopDuration`,
`reconciliationLoopHealthy`, `reconciliationLoopLastRun`, `reconciliationMismatchCount`, `reconciliationMissingFromExchange`, `reconciliationMissingInDb`, `reconciliationLockContentionCount`,
`lockContentionCount`, `capitalInvariantViolations`, `observabilityTimestamp`.
- Loop metrics: duration histograms + last run timestamps; SLO: healthy flag true if last run < 2x expected interval.
- Lock contention metrics: increment per failed lock acquisition; field surfaced both via `/internal/health` and Prometheus.
- Reconciliation metrics: mismatch counts, missing-from-exchange, missing-in-db, lock contention.
- Readiness signals: SLO flags combined to determine readiness; observability records degrade gracefully (logs emitted on invariant violations).
- Degraded mode behavior: if capital invariant fails, watchdog increments violation counter, logs critical error, halts further ENTRY until resolved.
---
**SECTION 7 — HORIZONTAL SCALING MODEL**
- Multi-worker deployment: each worker runs trading/monitor/reconciliation loops; shared DB/exchange key; distributed locks coordinate actions.
- Shared DB: Supabase/Postgres is the single source of truth; all loops interact with same tables.
- Shared exchange key: deterministic `clientOrderId` + lock prevent double submissions despite shared credentials.
- Lock guarantees: row-based entry locks and reconciliation locks with TTL/owner ensure cross-instance exclusivity.
- Why no duplication is possible: distributed locks + deterministic order IDs + lifecycle RPC uniqueness ensure only one worker can create a lifecycle even under concurrency.
---
**SECTION 8 — SAFE ENHANCEMENT RULES**
- **Lifecycle**:
- DO NOT bypass `fn_persist_entry_lifecycle`.
- DO NOT write raw status updates; use lifecycle-safe handlers.
- **Ledger**:
- DO NOT mutate ledger without RPCs that respect invariant.
- Always recompute `available_capital` via `allocated - reserved_orders - reserved_positions + realized_pnl`.
- **Reconciliation**:
- Always acquire `reconciliation_locks`.
- Route changes through lifecycle handlers.
- **Locking**:
- Always use row locks with TTL and owner tokens; release in finally block.
- Do not assume single-process state.
- **Exchange submission**:
- Always reserve capital before exchange call.
- Use deterministic `clientOrderId`.
- Never re-submit the same trade_id; rely on idempotent failures.
**DO NOT BREAK** these rules; any change violating them risks duplicate executions, capital drift, or stale lifecycle data.

67
backend/HISTORY_PURGE_RUNBOOK.md Normal file
View File

@ -0,0 +1,67 @@
# Repository History Purge Runbook
Date: 2026-02-15
Scope: purge secret-bearing blobs from Git history before production cut
## Objective
Rewrite repository history to remove any accidental secret-bearing files/commits, then force-push sanitized history in a controlled window.
## Preconditions
- Freeze merges to `main`.
- Rotate all potentially exposed credentials first.
- Ensure repository admins are present for coordinated force-push and branch protection updates.
## Tooling
- Preferred: `git filter-repo` (fast, maintainable)
- Alternate: BFG Repo-Cleaner
## Procedure (git filter-repo)
1. Mirror clone:
```bash
git clone --mirror https://github.com/<org>/<repo>.git
cd <repo>.git
```
2. Remove known sensitive paths:
```bash
git filter-repo --path .env --path .env.production --path-glob "*.pem" --invert-paths
```
3. Scrub sensitive patterns from remaining blobs:
```bash
git filter-repo --replace-text ../replace-secrets.txt
```
`replace-secrets.txt` format example:
```text
regex:sk-[A-Za-z0-9_-]{20,}==>REDACTED_OPENAI_KEY
regex:AKIA[0-9A-Z]{16}==>REDACTED_AWS_KEY
```
4. Validate purge:
```bash
git log --all --name-only | grep -E "(.env|\\.pem)$" || true
```
5. Force-push rewritten history:
```bash
git push --force --all
git push --force --tags
```
## Post-Purge Actions
- Invalidate old clones:
- team must re-clone or hard reset to rewritten history
- Re-enable branch protection rules
- Re-run security workflows (gitleaks + secret hygiene)
- Document purge commit window and impacted refs
## Safety Notes
- Do not run this on an active branch with uncoordinated contributors.
- Purge is destructive and irreversible on rewritten refs.

99
backend/INCIDENT_RUNBOOKS.md Normal file
View File

@ -0,0 +1,99 @@
# Incident Runbooks
Date: 2026-02-14
Scope: `bytelyst-trading-bot-service`
## Severity Levels
- `SEV-1`: Active risk of financial loss or uncontrolled exposure.
- `SEV-2`: Trading degraded but risk controls still active.
- `SEV-3`: Non-critical observability or configuration issue.
## 1) Ghost Position (Exchange Open, Bot Closed)
Trigger:
- Dashboard/API shows no open position, but exchange account has an open position.
Severity:
- `SEV-1`
Immediate Actions:
1. Stop new entries for impacted profile(s): set profile status to inactive in DB.
2. Confirm live exchange position size/side via broker UI/API.
3. Manually close or hedge exchange position if risk threshold breached.
4. Capture evidence: order IDs, timestamps, profile ID, symbol, side, qty.
Bot Recovery:
1. Run reconciliation:
- Wait for scheduled reconciliation cycle or restart bot to trigger startup reconciliation.
2. Verify `OrderStatusSyncService` has resolved related stale orders.
3. If still mismatched, update order state to `unknown` and treat as quarantined.
4. Re-enable profile only after position parity is confirmed.
Post-Incident:
1. Open RCA with timeline and root cause category:
- exchange timeout
- rejected exit
- stale local state
2. Add regression test for failing path.
## 2) Stale Pending Orders
Trigger:
- Orders remain `pending_new`/non-terminal beyond expected SLA.
Severity:
- `SEV-2` (escalate to `SEV-1` if exposure is uncertain).
Immediate Actions:
1. Check stale backlog via `/health` and logs (`[OrderSync]`, `[QUARANTINE]`).
2. Validate broker status for impacted order IDs.
3. Cancel stuck live orders in broker if safe and policy-approved.
Bot Recovery:
1. Allow `OrderStatusSyncService` to run.
2. Orders older than 24h and missing on exchange must be marked `unknown` (quarantined).
3. For quarantined orders, require manual review and final status correction.
Escalation Criteria:
- Backlog > 20 for > 15 minutes.
- Repeated stale growth across multiple profiles.
## 3) Auth Failures (API/WebSocket)
Trigger:
- Spike in `401`/`403` responses or websocket auth rejections.
Severity:
- `SEV-2` (or `SEV-1` if all trading control endpoints fail).
Immediate Actions:
1. Confirm Supabase availability and JWT issuance health.
2. Validate environment variables:
- `SUPABASE_URL`
- `SUPABASE_SERVICE_ROLE_KEY`
3. Verify dashboard token refresh behavior and expiration handling.
Bot Recovery:
1. Restart bot service after verifying credentials/config.
2. Validate:
- `/health/live` returns `200`
- `/health/ready` returns `200` (or investigate degraded fields)
3. Perform controlled API test:
- authenticated `/api/status`
- unauthenticated `/api/trade` should return unauthorized
Post-Incident:
1. Capture failing token claims (issuer, audience, exp, user id).
2. Record whether failure was config, infra, or app regression.
## Communication Template
Use this template in incident channel:
1. `Incident`: short title
2. `Severity`: SEV-1/2/3
3. `Impact`: profiles/symbols/orders affected
4. `Mitigation`: action taken
5. `Next Update`: timestamp (UTC)

152
backend/MOBILE_APP_BOOTSTRAP_ROADMAP.md Normal file
View File

@ -0,0 +1,152 @@
# Bytelyst Mobile Trading App - Bootstrap Checklist and Roadmap
Date: 2026-02-15
Scope: Mobile app bootstrap for iOS + Android with shared domain/trading core
## Goal
Ship an enterprise-grade mobile trading app baseline with:
- Native iOS support (Swift/SwiftUI)
- Native Android support (Kotlin/Jetpack Compose)
- Shared trading core using Kotlin Multiplatform (KMP) for deterministic business logic parity
## Recommended Architecture
- `iOS App`: SwiftUI, Combine/async-await, native secure key storage (Keychain)
- `Android App`: Kotlin, Jetpack Compose, Coroutines/Flow, EncryptedSharedPreferences/Keystore
- `Shared Core (KMP)`:
- Profile/trade lifecycle models
- Risk checks (SL/TP, max daily loss, position sizing guards)
- Order/position/history reconciliation logic
- API DTO mappers and validation
- `Backend Integration`: Existing bot service + dashboard APIs + Supabase auth/data plane
## Minimum Viable Features (Mobile)
- Login and session management
- Profile list with risk/strategy summary
- Market watchlist with live price updates
- Active orders view (profile-scoped, trade-id visible)
- Open positions view (profile-scoped, PnL + SL/TP status)
- Trade history view (full lifecycle trace by `trade_id`)
- Manual trade actions: buy/sell/close position
- Profile configuration: risk basics + entry mode + long-only toggle
- Notifications: push + in-app inbox for order filled, SL hit, TP hit, risk-limit halt
- In-app chat: support/ops chat and AI-assist chat fallback for profile guidance
- Health/status screen: bot connectivity, AI status/fallback mode
## Enterprise Readiness Gates
- Deterministic lifecycle mapping by `trade_id` across orders/positions/history
- Profile isolation for same-symbol concurrent exposure (virtual sub-positions)
- Offline-safe local cache with replay-safe sync
- Audit logging and immutable client event trail
- Strong auth: token refresh, device binding, secure storage
- Observability: crash reporting, API latency/error telemetry, trace IDs
- Notification reliability: token health checks, retry policy, delivery/error metrics
- Release controls: feature flags, staged rollout, rollback plan
## Phase Roadmap with Checklists
### Phase 0 - Product and Platform Baseline
- [ ] Finalize mobile product requirements and acceptance criteria
- [ ] Freeze API contracts for orders/positions/history/trade lifecycle
- [ ] Define canonical `trade_id` contract for mobile UI and backend parity
- [ ] Confirm profile-level strategy + risk schema consumed by mobile
- [ ] Define environment strategy: dev/stage/prod with safe key injection
- [ ] Establish branch, CI, and release conventions for mobile repos
### Phase 1 - Project Bootstrap (Swift + Kotlin + KMP)
- [ ] Create `ios-app` project (SwiftUI, modular structure)
- [ ] Create `android-app` project (Compose, modular structure)
- [ ] Create `shared-kmp` module and wire iOS/Android consumption
- [ ] Implement shared domain models: profile/order/position/trade history
- [ ] Implement shared API client contracts and validation layer
- [ ] Add lint/format/static analysis for all targets
- [ ] Add baseline unit test framework for iOS/Android/shared
### Phase 2 - Authentication and Core Data Flows
- [ ] Implement auth screens and secure token handling
- [ ] Implement refresh token lifecycle and forced re-auth guardrails
- [ ] Implement profile list screen with strategy/risk snapshot
- [ ] Implement watchlist + live market feed subscription
- [ ] Implement resilient local cache (read-through + stale marker)
- [ ] Implement sync manager with idempotent delta updates
- [ ] Implement push notification permissions + device token registration
- [ ] Implement in-app notification inbox sync and read/unread state
### Phase 3 - Trading Lifecycle UI (MVP Trading)
- [ ] Implement Active Orders screen with `trade_id`, profile, order status
- [ ] Implement Open Positions screen with profile-scoped aggregation
- [ ] Implement Trade History screen with lifecycle timeline by `trade_id`
- [ ] Implement manual Buy/Sell/Close actions with explicit confirmations
- [ ] Implement SL/TP display and editable fields per position/profile rules
- [ ] Add lifecycle discrepancy banner when backend/mobile states diverge
- [ ] Implement in-app chat module (support thread + AI assistant conversation UI)
### Phase 4 - Risk, Automation, and Execution Controls
- [ ] Expose profile execution config: `long_only`, entry mode, risk toggles
- [ ] Surface backend auto-trade status and execution reason codes
- [ ] Implement risk-limit halt UI states (daily loss, consecutive losses)
- [ ] Add emergency controls: pause profile, disable new entries, close all
- [ ] Add profile-level notification routing and critical alert escalation
- [ ] Implement notification preference center (per profile/event severity/channel)
### Phase 5 - Reliability, Compliance, and Operations
- [ ] Add structured mobile telemetry with correlation IDs to backend logs
- [ ] Add crash analytics and startup health probes
- [ ] Add integration tests for `orders -> positions -> history` parity
- [ ] Add contract tests against backend and Supabase schemas
- [ ] Add penetration/security checks (OWASP MASVS baseline)
- [ ] Add notification delivery monitoring and dead-letter handling runbook
- [ ] Add chat transcript retention policy and PII redaction controls
- [ ] Prepare release runbooks, rollback checklist, and on-call handbook
### Phase 6 - Store Readiness and Scale
- [ ] App Store/Play Store metadata and compliance packaging
- [ ] Performance budget validation (cold start, list render, live stream load)
- [ ] Battery/network efficiency tuning for live trading sessions
- [ ] Feature flag strategy for staged rollout by cohort
- [ ] Post-launch SLO tracking and incident response workflow
## Suggested Task Tracking Format
Use this format as implementation starts:
- [ ] `Task name`
Platform: `iOS` | `Android` | `KMP` | `Backend`
Owner: `TBD`
Commit: `pending`
Example after completion:
- [x] `Implement Active Orders screen with trade_id and profile badges`
Platform: `iOS, Android`
Owner: `TBD`
Commit: `https://github.com/<org>/<repo>/commit/<sha>`
## Initial Backlog (MVP-first, recommended order)
- [ ] Boot repositories and CI for iOS/Android/KMP
- [ ] Implement auth + token refresh
- [ ] Implement profile list + watchlist
- [ ] Implement orders/positions/history with strict `trade_id` mapping
- [ ] Implement manual trade actions + confirmations
- [ ] Implement profile execution/risk settings surface
- [ ] Implement notification stack (push + in-app inbox + preferences)
- [ ] Implement in-app support/AI chat module
- [ ] Implement health/status panel with AI fallback visibility
- [ ] Run lifecycle parity tests against backend and close gaps
## Open Questions (Non-Blocking)
- [ ] Should mobile support broker-level advanced order types at MVP (stop-limit, trailing-stop), or defer to Phase 5+?
- [ ] Should mobile include profile creation/edit at MVP, or remain read-and-execute only initially?
- [ ] Should AI-assisted profile suggestion run on-device fallback when AI service is unavailable, or server-side fallback only?

263
backend/ORDER_STATUS_SYNC.md Normal file
View File

@ -0,0 +1,263 @@
# Order Status Synchronization - Solution Documentation
## Problem Statement
Orders were getting stuck in `pending_new` status indefinitely, causing stale data in the dashboard. This happened because:
1. **One-way data flow**: Bot → Database (no sync back)
2. **No status updates on fill**: When orders were filled, the database was never updated
3. **No background reconciliation**: No periodic check to sync actual order statuses from the exchange
## Solution Overview
We implemented a **three-layer solution** to handle stale order statuses:
### 1. Immediate Updates (Real-time Fix)
**Files Modified:**
- `src/services/TradeExecutor.ts`
**Changes:**
- Added `updateOrderStatus()` call when orders are **filled** (previously only called on cancel/reject)
- Added status update for both entry and exit orders
- Ensures database is updated immediately when order status changes
```typescript
// After order is verified as filled
supabaseService.updateOrderStatus?.(order.id, verifiedOrder.status || 'filled', new Date());
```
### 2. Background Sync Service (Periodic Reconciliation)
**Files Created:**
- `src/services/OrderStatusSyncService.ts`
**Files Modified:**
- `src/services/SupabaseService.ts` - Added `getStaleOrders()` method
- `src/index.ts` - Integrated sync service into main bot
**How it works:**
1. Runs every **5 minutes** in the background
2. Queries database for orders in `pending_new` status older than 5 minutes
3. Checks actual status on the exchange via `exchange.getOrder()`
4. Updates database with real status
5. Marks very old orders (>24h) as `unknown` if not found on exchange
**Key Features:**
- Non-blocking (runs in background)
- Handles up to 100 stale orders per sync
- Graceful error handling
- Detailed logging for monitoring
### 3. Visual Indicators (User Awareness)
**Files Modified:**
- `src/tabs/PositionsTab.tsx` (Dashboard)
**Features:**
- **Warning banner** at top when stale orders detected
- **Yellow badge** on individual stale orders (>5 min old)
- **Warning icon** (⚠️) next to stale order status
- Real-time age calculation
## Architecture
```
┌─────────────────────────────────────────────────────────────┐
│ Trading Bot Flow │
└─────────────────────────────────────────────────────────────┘
1. Order Placed
↓
2. TradeExecutor.openPosition()
↓
3. waitForFill() - Poll exchange for status
↓
4. ✅ NEW: updateOrderStatus() - Update DB immediately
↓
5. Track position locally
┌─────────────────────────────────────────────────────────────┐
│ Background Sync Service │
└─────────────────────────────────────────────────────────────┘
Every 5 minutes:
1. Query DB for pending_new orders > 5 min old
↓
2. For each order:
- Check exchange.getOrder()
- Get actual status
- Update DB
↓
3. Mark >24h orders as 'unknown' if not found
┌─────────────────────────────────────────────────────────────┐
│ Dashboard Display │
└─────────────────────────────────────────────────────────────┘
1. Fetch orders from DB
↓
2. Calculate age for each order
↓
3. Detect stale orders (pending_new > 5 min)
↓
4. Show warning banner + visual indicators
```
## Database Schema Assumptions
The solution assumes the `orders` table has:
- `id` or `order_id` - Primary key
- `status` - Order status (pending_new, filled, canceled, etc.)
- `created_at` - Timestamp when order was created
- `updated_at` - Timestamp of last update
- `filled_at` - Timestamp when order was filled (optional)
## Configuration
### Sync Interval
Default: **5 minutes**
To change, modify in `src/index.ts`:
```typescript
const orderSyncService = new OrderStatusSyncService(
dataExchange,
10 * 60 * 1000 // 10 minutes
);
```
### Stale Threshold
Default: **5 minutes**
To change, modify in `src/services/SupabaseService.ts`:
```typescript
async getStaleOrders(staleThresholdMinutes: number = 10) // 10 minutes
```
## Manual Cleanup
For one-time cleanup of very old stale orders:
```bash
# Run the cleanup script
npm run cleanup-stale-orders
```
This will mark all orders >24 hours old in `pending_new` status as `unknown`.
## Monitoring & Logs
Look for these log messages:
### Successful Sync
```
[OrderSync] Found 5 stale orders to check
[OrderSync] Updating order abc123: pending_new → filled
[OrderSync] Sync complete: 5 updated, 0 not found on exchange, 0 failed
```
### No Stale Orders
```
[OrderSync] No stale orders found
```
### Errors
```
[OrderSync] Failed to sync order abc123: Order not found on exchange
[Supabase] Error fetching stale orders: <error message>
```
## Testing
### Test Immediate Updates
1. Place a new order via the bot
2. Check database - should see `pending_new` initially
3. Wait for order to fill (~3-30 seconds)
4. Check database - should see `filled` status
### Test Background Sync
1. Manually set an order to `pending_new` in DB (that's actually filled)
2. Wait 5 minutes
3. Check logs for sync activity
4. Verify order status updated to `filled`
### Test Dashboard Indicators
1. Create a stale order (pending_new > 5 min)
2. Open dashboard
3. Should see:
- Yellow warning banner at top
- Yellow badge on the order
- ⚠️ icon next to status
## Troubleshooting
### Orders still showing as pending_new after sync
**Possible causes:**
1. Exchange doesn't support `getOrder()` API
2. Order ID mismatch between bot and exchange
3. Supabase credentials not configured
**Solution:**
- Check logs for specific error messages
- Verify exchange connector implements `getOrder()`
- Confirm order IDs match between systems
### Sync service not running
**Check:**
```bash
# Look for this in logs on bot startup:
[OrderSync] Background order status sync service started
```
**If missing:**
- Verify `OrderStatusSyncService` is imported in `index.ts`
- Check for startup errors
### Database not updating
**Possible causes:**
1. Supabase credentials missing/invalid
2. Table permissions issue
3. Field name mismatch (`id` vs `order_id`)
**Solution:**
- Check Supabase connection logs
- Verify table has both `id` and `order_id` columns (or update code)
- Test with manual query
## Performance Considerations
- **Sync batch size**: Limited to 100 orders per sync to avoid overload
- **Sync frequency**: 5 minutes balances freshness vs API rate limits
- **Exchange API calls**: One call per stale order (consider rate limits)
## Future Enhancements
1. **WebSocket status updates** - Real-time order status from exchange
2. **Retry logic** - Exponential backoff for failed syncs
3. **Metrics dashboard** - Track sync success rate, stale order trends
4. **Alert on persistent stale orders** - Notify if orders stay stale >1 hour
5. **Bulk status check** - If exchange supports batch order queries
## Related Files
### Core Implementation
- `src/services/TradeExecutor.ts` - Immediate status updates
- `src/services/OrderStatusSyncService.ts` - Background sync
- `src/services/SupabaseService.ts` - Database queries
- `src/index.ts` - Service initialization
### Dashboard
- `src/tabs/PositionsTab.tsx` - Visual indicators
### Utilities
- `src/scripts/cleanupStaleOrders.ts` - Manual cleanup
## Summary
This solution ensures order statuses are **always accurate** through:
1. ✅ **Immediate updates** when orders fill
2. ✅ **Background reconciliation** every 5 minutes
3. ✅ **Visual warnings** for users when stale data detected
4. ✅ **Manual cleanup** tools for maintenance
**Result:** No more stale `pending_new` orders! 🎉

529
backend/PRODUCT_FLOW_TRADING_LOGIC.md Normal file
View File

@ -0,0 +1,529 @@
# Bytelyst Trading Product Flow and Execution Logic
Date: 2026-02-15
Scope: runtime product flow across bot service, execution, monitoring, and dashboard sync
## Purpose
This document explains the live trading flow end-to-end:
- how a profile initiates a trade
- what conditions close a trade
- how monitoring runs periodically
- how Orders, Positions, and History stay linked by `trade_id`
- how dashboard data is synchronized from backend state
## 1) Core Runtime Components
- `bytelyst-trading-bot-service/src/index.ts`
- boots profile contexts
- runs strategy loop
- runs reconciliation and profile hot-reload loops
- `bytelyst-trading-bot-service/src/services/AutoTrader.ts`
- decides entry/exit actions from strategy result + risk guards
- `bytelyst-trading-bot-service/src/services/TradeExecutor.ts`
- places exchange orders and manages lifecycle state
- persists orders/history and updates in-memory active positions
- `bytelyst-trading-bot-service/src/services/tradeMonitor.ts`
- periodic SL/TP/trailing and exchange-position checks
- `bytelyst-trading-bot-service/src/services/OrderStatusSyncService.ts`
- periodic stale-order reconciliation with exchange
- `bytelyst-trading-bot-service/src/services/apiServer.ts`
- websocket state fan-out to dashboard
- writes `bot_state.json` and snapshots to Supabase
## 2) Profile Configuration Inputs
Per profile, behavior is controlled by `trade_profiles.strategy_config`:
- `rules[]`: which strategy rules run
- `riskLimits`: `maxOpenTrades`, `maxDailyLossUsd`, `dailyProfitTargetUsd`, `maxConsecutiveLosses`
- `execution`:
- `minRulePassRatio`
- `orderType`
- `cooldownMinutes`
- `profitExitPercent` (optional override)
- `entryMode`: `both` or `long_only`
Long-only behavior:
- `entryMode = long_only` blocks new SELL entries
- exit SELL orders are still allowed when closing BUY positions
- logic: `bytelyst-trading-bot-service/src/services/AutoTrader.ts`
## 3) End-to-End Entry Flow
1. Market context is built per symbol.
2. Each active profile evaluates rules independently.
3. For each profile result, `AutoTrader.handleSignal(...)` executes.
4. Entry is attempted only if:
- signal is not `NONE`
- rules passed
- symbol is in profile watchlist
- portfolio guard passes
- max open trades and cooldown checks pass
- runtime risk limits pass
- exchange safety checks pass
- entry direction allowed by `entryMode`
5. Risk engine computes:
- side-aware position size
- side-aware stop-loss and take-profit
6. Trade executor places order, verifies fill, and records:
- `orders` row with `action=ENTRY`
- deterministic `trade_id`
- active in-memory position keyed by symbol/profile context
### Manual entry capital guard flow (`POST /api/trade/manual`)
Manual trade requests now enforce allocated-capital safety without blind rejection:
- compute `remainingCapital = allocatedCapital - committedOpenPositionNotional`
- if no remaining capital, wait up to 60s for release (poll every 3s)
- if capital becomes available, scale requested quantity down to the max tradable qty
- if remaining is below minimum tradable qty, return a clear rejection
Result:
- trade size never exceeds profile/account allocated capital
- user can still get partial execution with available remaining capital
- when capital is fully locked, request waits briefly before returning "waiting for capital release"
Source:
- `bytelyst-trading-bot-service/src/services/ManualTrader.ts`
Primary references:
- `bytelyst-trading-bot-service/src/index.ts`
- `bytelyst-trading-bot-service/src/services/AutoTrader.ts`
- `bytelyst-trading-bot-service/src/services/riskEngine.ts`
- `bytelyst-trading-bot-service/src/services/TradeExecutor.ts`
## 4) Position Close Conditions
Trade close can be initiated from multiple paths:
### A) Strategy signal flip
- If active side is opposite to new signal, close immediately.
- example: BUY position + SELL signal -> close
- source: `AutoTrader.handleSignal(...)`
### B) Profit threshold + neutral signal
- If unrealized profit reaches `profitExitPercent` and signal becomes `NONE`, close.
- source: `AutoTrader.handleSignal(...)`
### C) Safety stop-loss (monitor-driven)
- BUY closes when `price <= stopLoss`
- SELL closes when `price >= stopLoss`
- source: `tradeMonitor.checkOpenPositions(...)`
### D) TP-triggered trailing guard (monitor-driven)
- take-profit hit activates `profitGuardActive`
- after activation, exit only on configured pullback:
- BUY: pullback from peak by `TRAILING_STOP_PERCENT`
- SELL: rebound from trough by `TRAILING_STOP_PERCENT`
- source: `tradeMonitor.checkOpenPositions(...)`
### E) Exchange confirms position disappeared
- monitor confirms missing exchange position across required misses and final confirmation
- then finalizes local lifecycle as exchange-closed
- source: `tradeMonitor.checkOpenPositions(...)`
### F) Manual square-off (API)
- manual close endpoint routes to executor close flow
- source: `apiServer` + `ManualTrader` + `TradeExecutor.closePosition(...)`
## 5) Partial Exit Handling
Exit fills are quantity-aware:
- full fill -> finalize trade and remove active position
- partial fill -> reduce active position quantity and continue monitoring remainder
- no valid partial quantity -> quarantine path for manual review
Source:
- `bytelyst-trading-bot-service/src/services/TradeExecutor.ts` (`closePosition`, `applyExitFill`)
## 6) Monitoring and Loop Frequencies
Configured in `bytelyst-trading-bot-service/src/config/index.ts`:
- `POLLING_INTERVAL` (default 60s): strategy/trading loop
- `MONITOR_INTERVAL_MS` (default 60s): profile reconciliation loop and trade monitor tick
- `PROFILE_SYNC_INTERVAL_MS` (default 60s): profile hot-reload loop
- Order status sync interval (currently 5 minutes in `index.ts` constructor call)
- State write debounce: ~200ms in `apiServer.scheduleStateWrite()`
- Supabase snapshot throttle: every 5 minutes in `apiServer.flushStateToDisk()`
Order-status sync behavior (every cycle):
- scans stale `pending_new` rows
- reconciles status from exchange when order still exists
- if exchange returns "order not found" and lifecycle is already closed for that `trade_id`, auto-resolves stale row to `canceled`
- this prevents ghost EXIT orders from staying `pending_new` after lifecycle completion
Sources:
- `bytelyst-trading-bot-service/src/services/OrderStatusSyncService.ts`
- `bytelyst-trading-bot-service/src/services/SupabaseService.ts` (`isTradeLifecycleClosed`)
## 7) Orders, Positions, History Linkage
Lifecycle identity model:
- `trade_id` is created at entry and reused for exit
- `profile_id` scopes ownership/isolation
- `action` distinguishes `ENTRY` vs `EXIT`
Data mapping:
- Orders table: all lifecycle events (`ENTRY` and `EXIT`)
- Active positions: in-memory + websocket `positions_update`
- History table: realized lifecycle slices (full close and partial exit slices)
Primary references:
- `bytelyst-trading-bot-service/src/services/TradeExecutor.ts`
- `bytelyst-trading-bot-service/src/services/apiServer.ts`
- `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx`
## 8) Dashboard Synchronization Flow
Backend emits:
- `positions_update`
- `orders_update`
- `history_update`
- `symbol_update`
- `settings_update`
Frontend hook:
- `bytelyst-trading-dashboard-web/src/hooks/useWebSocket.ts`
- receives events and updates local `botState`
Positions & Orders tab:
- normalizes `trade_id` (`trade_id` vs `tradeId`)
- infers lifecycle IDs for legacy rows when needed
- builds lifecycle traces per `trade_id`
- shows mismatch diagnostics (missing trade ID, missing entry order, profile mismatch)
- computes deterministic lifecycle states:
- `OPEN`: entry filled, no exit fill yet
- `EXIT_PENDING`: exit submitted, waiting for fill sync
- `PARTIAL_EXIT`: partial exit filled; remaining quantity stays in Open Positions
- `CLOSED`: lifecycle fully matched/closed
- `ORPHAN_EXIT`: exit exists but matching entry is not in visible order window
Source:
- `bytelyst-trading-dashboard-web/src/tabs/PositionsTab.tsx`
History tab:
- uses deduplicated lifecycle history ledger for audit totals (`trade_id + profile + timestamp + prices + size + pnl + reason`)
- uses Supabase `trade_history` as primary source; realtime websocket history is used only as fallback when DB rows are unavailable
- Realized P&L formula: `sum(pnl)` over deduplicated history rows in current filter scope
- win rate formula: `winning_rows / total_rows` where `winning_rows = pnl > 0`
- surfaces per-trade `capital used` and highlights losing trades
- normalizes numeric timestamps (seconds vs milliseconds) and unknown side values to avoid mis-sorted or mis-labeled rows
- numeric rendering is NaN-safe for P/L percent and price fields
Source:
- `bytelyst-trading-dashboard-web/src/tabs/HistoryTab.tsx`
- `bytelyst-trading-dashboard-web/src/lib/tradeHistoryLedger.ts`
Overview tab:
- uses the same deduplicated history ledger as History tab for realized metrics and profile-level aggregates
- Realized P&L formula: `sum(pnl)` over deduplicated ledger rows
- Net P&L formula: `Realized P&L + sum(unrealizedPnl of open positions)`
- Capital Scope table now shows profile-level `Realized P&L` from profile-scoped ledger aggregation
- shows per-profile `allocated / used / remaining` capital and runtime state labels
- exposes `P&L Duration` in status bar: elapsed time from first to latest realized ledger event in scope
- does not show aggregate `P&L Capital Used` in status bar (intentionally removed to avoid misleading interpretation at summary level)
- supports win-rate time windows in Overview (`24H`, `7D`, `30D`, `All`) for both global and per-profile win-rate metrics
- windowed win-rate formula: `wins_in_window / trades_in_window`, where `wins_in_window = pnl > 0` within selected window
Source:
- `bytelyst-trading-dashboard-web/src/tabs/OverviewTab.tsx`
- `bytelyst-trading-dashboard-web/src/lib/tradeHistoryLedger.ts`
Strategy Clusters tab:
- profile card P&L and win rate use the same deduplicated history ledger aggregation as Overview/History
- non-admin scope is user-filtered to avoid cross-user P&L contamination
Source:
- `bytelyst-trading-dashboard-web/src/components/TradeProfileManager.tsx`
- `bytelyst-trading-dashboard-web/src/lib/tradeHistoryLedger.ts`
## 9) Operational Notes
- If state changes every minute, local `bot_state.json` can update frequently (debounced).
- Supabase snapshot upload is throttled to 5-minute intervals, so it is a rolling backup, not every state mutation.
- This design reduces backup write overhead while keeping recoverable snapshots.
- `trade_history` insert path is legacy-schema safe: if `source` column is unavailable, bot retries with fallback payload instead of dropping the event.
Source:
- `bytelyst-trading-bot-service/src/services/SupabaseService.ts`
## 10) AI Resilience and Fallback Behavior
### AI trading rule (`AIAnalysisRule`)
- AI provider chain: `openai -> perplexity -> gemini` (configurable fallback list)
- If AI service is unavailable:
- with `AI_FAIL_OPEN=true` (default), AI rule does not block trading
- with `AI_FAIL_OPEN=false`, AI rule blocks as a failed rule
Config references:
- `AI_FAIL_OPEN`
- `AI_FALLBACK_LIST`
- `AI_MODEL`
- provider keys (`OPENAI_API_KEY`, `PERPLEXITY_API_KEY`, `GEMINI_API_KEY`)
### Chat profile creation fallback
- Endpoint: `POST /api/chat`
- If AI providers fail or return invalid JSON, backend now returns deterministic profile JSON generated from local heuristics:
- parses symbols, capital, risk, cooldown, entry mode, sessions
- supports create/update/explain fallback outputs
- response includes `fallback: "local_deterministic"`
### AI health endpoint
- Endpoint: `GET /api/ai/health`
- Query:
- `probe=true` for live provider probe
- omit `probe` for config-only status
- Response includes:
- per-provider `configured`, `status`, `message`, `model`
- effective fallback list and fail-open state
- summary counts
## 11) Quick Trace Checklist (One Trade)
1. Find profile and symbol signal in websocket symbol state.
2. Check `orders` for `ENTRY` row and `trade_id`.
3. Verify active position carries same `tradeId` + `profileId`.
4. On close, confirm `EXIT` order has same `trade_id`.
5. Confirm history row created with same `trade_id` (or partial-exit realized slice entries).
6. If mismatch, use lifecycle trace/mismatch diagnostics in Positions tab.
## 12) Lifecycle Reconciliation (2026-02-15)
- Legacy compatibility: lifecycle entry detection now treats `action IS NULL` + BUY rows as ENTRY fallback.
- Virtual position reconstruction for legacy no-action rows now infers BUY as ENTRY and SELL as EXIT.
- Reconciliation tool: `npm run reconcile:lifecycle-history -- --apply --start=2026-02-12T00:00:00.000Z <trade_id ...>`
- Reconciliation behavior:
- computes canonical realized P&L from filled orders per `trade_id` using FIFO closed quantity matching
- neutralizes conflicting non-canonical history rows by zeroing P&L and prefixing reason with `[RECONCILED_TO_ORDERS]`
- inserts or updates one `[RECONCILED_CANONICAL]` history row per reconciled lifecycle
- Latest operational result:
- post-purge lifecycle mismatch count reduced to `0` (order-derived vs history-derived, trade_id scoped)
- Alpaca vs Supabase order status parity remains `0` mismatches
## 13) Alpaca Reconciliation Carry-In Baseline (2026-02-15)
- Script: `scripts/reconcileAlpacaVsSupabase.ts`
- New capabilities:
- paged Alpaca order fetch over `carry_in_lookback_days` to reconstruct pre-window state
- dual realized-PnL outputs: `flat_start` and `with_carry_in`
- auto primary-mode selection (`alpaca_fill_derived_realized_pnl`) based on Supabase pre-window coverage consistency
- symbol scope alignment for PnL (`db-order-symbols`) so non-bot symbols do not pollute comparison
- New output diagnostics:
- `alpaca_carry_in_bootstrap.primary_mode`
- `alpaca_carry_in_bootstrap.supabase_pre_window_filled_order_count`
- `alpaca_carry_in_bootstrap.carry_coverage_gap`
- `alpaca_carry_in_bootstrap.carry_coverage_threshold`
- Usage examples:
- `node --loader ts-node/esm scripts/reconcileAlpacaVsSupabase.ts 3`
- `node --loader ts-node/esm scripts/reconcileAlpacaVsSupabase.ts 3 --carry-lookback-days=180`
## 14) Runtime Addendum (2026-02-16)
### Profile-symbol scope synchronization
- Profile bootstrap sync and periodic reconciliation now use profile-resolved symbol scope (`profile.symbols` fallback to `config.SYMBOLS`).
- Trading loop now runs on dynamic union of monitored symbols across all active profiles.
- Profile hot-reload detects symbol-list changes and re-syncs position state using old+new symbol union to avoid stale local state.
Primary source:
- `bytelyst-trading-bot-service/src/index.ts`
### Legacy stale EXIT reconciliation without trade_id
- Order-sync stale reconciliation now handles legacy EXIT-like rows that have no `trade_id`.
- If exchange reports order missing and profile-scoped virtual lifecycle is flat for `profile_id + symbol`, stale order is auto-resolved to `canceled`.
- This closes a real stale-order gap that previously left perpetual `pending_new` rows in Active Orders.
Primary sources:
- `bytelyst-trading-bot-service/src/services/OrderStatusSyncService.ts`
- `bytelyst-trading-bot-service/src/services/SupabaseService.ts`
### Position merge identity hardening
- Position websocket merge now keys stable rows by owner/profile/trade identity so stable lifecycle rows are preserved and not always collapsed by owner+symbol.
- Fallback/no-trade rows still collapse by owner/profile/symbol/side to prevent duplicate noise.
Primary source:
- `bytelyst-trading-bot-service/src/services/apiServer.ts`
### Known structural constraint
- Same-profile same-symbol concurrent lifecycles are still structurally limited by runtime active-position map keyed by symbol (`Map<symbol, PositionState>`).
- Full multi-lifecycle same-symbol support within one profile requires executor state model refactor (position map keyed by `trade_id` or hybrid `symbol+trade_id`).
### 2026-02-16 Commit References
- Lifecycle/profile sync hardening implementation:
- https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/78be9e63f4e3e2a0c550e467c4cb4f66c7c2b2df
- Runtime flow documentation addendum:
- https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/bb739a0cad9c06ae591ccd95b7f2135f23261170
## 15) Runtime Addendum (2026-02-16, Cycle 2)
### Multi-lifecycle same-symbol handling (per profile)
- Active runtime position state is now trade-id aware (`symbol + trade_id`), so one profile can hold multiple concurrent lifecycles on the same symbol.
- AutoTrader exit logic now evaluates and exits per lifecycle (`trade_id`) for signal flip and neutralized-profit exits.
- TradeMonitor now evaluates SL/TP/trailing exits per lifecycle and finalizes exchange-missing closures per lifecycle.
Primary sources:
- `bytelyst-trading-bot-service/src/services/TradeExecutor.ts`
- `bytelyst-trading-bot-service/src/services/AutoTrader.ts`
- `bytelyst-trading-bot-service/src/services/tradeMonitor.ts`
### Dedicated profile sync behavior
- Profile-scoped sync now reconstructs multiple virtual open slices for a symbol using `profile_id + symbol + trade_id`.
- If per-trade slices are not recoverable, sync safely falls back to aggregate virtual lifecycle state.
Primary sources:
- `bytelyst-trading-bot-service/src/services/SupabaseService.ts`
- `bytelyst-trading-bot-service/src/services/TradeExecutor.ts`
### Continuous parity guard
- Reconciliation loop now includes a scheduled Alpaca-vs-Supabase order parity audit over recent profile orders.
- Safe terminal mismatches are auto-corrected, and runtime health publishes `parityMismatchCount`.
Primary sources:
- `bytelyst-trading-bot-service/src/index.ts`
- `bytelyst-trading-bot-service/src/services/apiServer.ts`
### Manual square-off behavior
- Manual close endpoint now exits all active sub-positions for a symbol (trade-id aware), not only one selected row.
Primary source:
- `bytelyst-trading-bot-service/src/services/apiServer.ts`
### Cycle 2 Commit Link
- Runtime multi-lifecycle + parity guard implementation:
- https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/abb6dd5e1fa1b3f12104c9205d2a9efcd7d668cb
## 16) Runtime Addendum (2026-02-16, Cycle 3)
### Lifecycle retention during temporary virtual-sync gaps
- Dedicated-profile sync no longer clears local symbol lifecycles when the exchange still reports an open position but virtual DB reconstruction temporarily returns empty.
- This prevents false drops where filled ENTRY orders briefly disappear from Open Positions due to eventual consistency windows.
Primary source:
- bytelyst-trading-bot-service/src/services/TradeExecutor.ts
### Immediate dashboard position propagation
- Position snapshots are now pushed immediately after confirmed ENTRY fill and after lifecycle finalization.
- Dashboard no longer has to wait for the next trading-loop/reconciliation cycle to reflect these two state transitions.
Primary source:
- bytelyst-trading-bot-service/src/services/TradeExecutor.ts
### Regression coverage added
- Added lifecycle regression assertions for:
- immediate position snapshot push after filled entry
- retaining local lifecycle state when exchange is open and virtual lookup is temporarily empty
Primary source:
- bytelyst-trading-bot-service/scripts/testTradeExecutorLifecycle.ts
### Cycle 3 Commit Link
- Runtime lifecycle retention plus immediate position propagation fix:
- https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/9c53222939b73fba4ab95f83c7df5d57f9b46683
## 17) Lifecycle Linkage Model (2026-02-16, Cycle 4)
### How lifecycle is tracked end-to-end
- Primary lifecycle key is trade_id, scoped by profile_id.
- Every bot-originated ENTRY and EXIT order carries trade_id and profile_id.
- Order persistence now writes idempotently by order_id, so retries do not create duplicate lifecycle rows.
- Active positions are maintained per lifecycle (symbol plus trade_id) and rendered per profile context.
- Trade history rows are written per lifecycle close/partial-close with the same trade_id.
### How stale and duplicate views are controlled
- Backend stale-order sync now runs on configurable cadence (ORDER_SYNC_INTERVAL_MS) and checks broader pending statuses (pending_new, pending, accepted, new).
- Stale EXIT rows are auto-resolved when lifecycle closure is already proven by trade_id chain.
- Dashboard position rendering dedupes by profile_id plus trade_id to avoid duplicate position cards.
- Dashboard order activity normalizes stale pending EXIT rows to closed state when history confirms lifecycle closure for that trade_id scope.
### Current residual limitation
- Full enterprise-level test coverage is still in-progress. Current implementation hardens lifecycle consistency, but broad component/service test expansion remains required for full target coverage.
### Cycle 4 Commit Links
- Bot lifecycle persistence and stale-sync hardening:
- https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/0d3409a48ac60dfb76e84c75be3f4b12c2f98a18
- Dashboard lifecycle table stabilization:
- https://github.com/saravanakumardb/bytelyst-trading-dashboard-web/commit/e9422a3e330161bde0f32c9275f8f7dea30cdee5
## 18) Validation Addendum (2026-02-16, Cycle 5)
### Dashboard flow validation expansion
- Added deterministic automated coverage for core runtime visibility paths:
- app shell auth routing (`loading`, `login`, reset callback, authenticated dashboard)
- lifecycle trace rendering (`OPEN`, `PARTIAL_EXIT`, `CLOSED`, `ORPHAN_EXIT`, `EXIT_PENDING`)
- stale pending EXIT warning and mismatch diagnostics rendering
- overview capital/P&L/readiness rendering paths
- history/config/settings/entries/admin tab shell paths
### Supporting module coverage
- Added automated tests for:
- `AuthContext` provider and guard behavior
- `useWebSocket` default-state contract
- `supabaseClient` env-based initialization
- strategy-config normalization compatibility (`normalizeStrategyConfig`)
### Result snapshot
- Dashboard coverage (`npm run coverage:full`) now reports:
- Statements: `42.46%` (`806/1898`)
- Branches: `34.71%` (`743/2140`)
- Functions: `33.26%` (`159/478`)
- Lines: `44.25%` (`762/1722`)
### Commit link
- Dashboard test expansion implementation:
- https://github.com/saravanakumardb/bytelyst-trading-dashboard-web/commit/3be473d882018d1cd88f7473dbbc24ee5f0be662
## 19) Runtime Merge and Lifecycle Aggregation Hardening (2026-02-16, Cycle 6)
### What changed
- Backend runtime state updates now use canonical merge helpers before publishing positions/orders to bot state.
- Positions: stable `trade_id` identity first; fallback owner+symbol+side consolidation.
- Orders: `order_id` identity merge with terminal-status precedence to prevent stale pending downgrade.
- Dashboard lifecycle aggregation now computes realized P&L and win rate per lifecycle chain (`profile_id + trade_id`) instead of row-level counting.
- Dashboard overview and positions views now dedupe runtime positions by lifecycle identity before utilization/P&L rendering.
### Why this closed the duplicate and mismatch issue
- The previous flow could count the same lifecycle multiple times when sync events arrived from different paths or with profile-less rows.
- The new flow keeps one canonical runtime representation per lifecycle and one canonical aggregate record per lifecycle chain.
### Expected user-visible behavior now
- Active positions, order activity, lifecycle table, and history all align by `trade_id` and `profile_id` more consistently.
- Net P&L and win rate no longer inflate from duplicated lifecycle rows.
- Capital used/utilization in overview reflects deduped open runtime positions.
### Commit links
- Bot runtime merge hardening:
- https://github.com/saravanakumardb/bytelyst-trading-bot-service/commit/af772b265a348f2c47440c5f41c74144156b54f4
- Dashboard lifecycle aggregation and runtime dedupe hardening:
- https://github.com/saravanakumardb/bytelyst-trading-dashboard-web/commit/ad9afd7537f8ae8bf805d7a10a6cb4ffad5e9abf

321
backend/README.md Normal file
View File

@ -0,0 +1,321 @@
# Bytelyst Trading Bot Service
Autonomous multi-profile crypto/equity trading bot with a pluggable rule-based strategy engine, per-profile execution, real-time dashboard integration, and AI-powered sentiment analysis.
## Architecture
```
┌─────────────────────────────────────────────────────────┐
│ Trading Loop │
│ for each symbol → for each profile → ProStrategyEngine │
│ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ │TrendBias │→ │ Session │→ │ Zone │→ ... │
│ │ Rule │ │ Rule │ │ Rule │ │
│ └──────────┘ └──────────┘ └──────────┘ │
│ │
│ Signal → AutoTrader → RiskEngine → TradeExecutor │
│ ↓ │
│ ┌────────────────┐ │
│ │ Alpaca / CCXT │ │
│ └────────────────┘ │
└─────────────────────────────────────────────────────────┘
↕ Socket.IO + REST ↕ Supabase
┌──────────────────┐ ┌──────────────────┐
│ Dashboard │ │ PostgreSQL │
│ (React App) │ │ (Supabase DB) │
└──────────────────┘ └──────────────────┘
```
## Features
- **Multi-Profile Execution** — Each trade profile runs its own strategy rules, risk limits, and capital allocation independently
- **7-Rule Strategy Pipeline** — TrendBias → Session → Zone → Momentum → EntryTrigger → RiskManagement → AIAnalysis
- **Per-Profile Rule Config** — Enable/disable rules, set parameters, and control execution order per profile
- **Profile Hot-Reload** — New profiles created from the dashboard are picked up automatically within 60 seconds
- **Pluggable Exchanges** — Alpaca (stocks/crypto) and CCXT (130+ exchanges) via factory pattern
- **AI Sentiment Analysis** — Perplexity, OpenAI, and Gemini with automatic fallback chain
- **Real-Time Dashboard** — Socket.IO for live price updates, signals, and position tracking
- **Profile-Mapped Orders** — Every order and trade history record is tagged with `profile_id`
- **Configurable Everything** — All thresholds, intervals, and parameters are configurable via env vars or per-profile JSON
## Quick Start
### Prerequisites
- Node.js 18+
- Supabase project (for auth, users, profiles, orders, trade history)
- Alpaca account (for trade execution) or CCXT-compatible exchange
### Installation
```bash
git clone https://github.com/saravanakumardb/bytelyst-trading-bot-service.git
cd bytelyst-trading-bot-service
npm install
```
### Configuration
Copy the example environment file:
```bash
cp .env.example .env
```
Edit `.env` with your credentials (see [Environment Variables](#environment-variables) below).
### Database Setup
Run the schema migration in your Supabase SQL Editor:
```bash
# Full schema setup (creates all tables + adds missing columns)
# Copy and paste the contents of:
schema/004_full_schema_sync.sql
```
### Run
```bash
# Development (with hot-reload via tsx)
npm run dev
# Production
npm start
```
## Project Structure
```
src/
├── config/
│ └── index.ts # All configuration (env vars + defaults)
├── connectors/
│ ├── alpaca.ts # Alpaca exchange connector
│ ├── ccxt.ts # CCXT multi-exchange connector
│ ├── factory.ts # Connector factory (pluggable)
│ └── types.ts # IExchangeConnector interface
├── services/
│ ├── AutoTrader.ts # Signal → trade decision logic
│ ├── TradeExecutor.ts # Order execution + position tracking
│ ├── riskEngine.ts # Position sizing + SL/TP calculation
│ ├── tradeMonitor.ts # Background SL/trailing stop monitor
│ ├── SupabaseService.ts # DB operations (orders, history, profiles)
│ ├── apiServer.ts # REST + Socket.IO API for dashboard
│ ├── notifier.ts # WhatsApp/webhook notifications
│ ├── aiClient.ts # Multi-provider AI sentiment analysis
│ ├── ManualTrader.ts # Manual trade execution via dashboard
│ └── MetricsService.ts # Trading metrics calculation
├── strategies/
│ ├── ProStrategyEngine.ts # Main strategy orchestrator
│ ├── directionTracker.ts # Legacy signal tracker
│ └── rules/
│ ├── types.ts # IRule interface, MarketContext, RuleResult
│ ├── TrendBiasRule.ts # EMA50/200 trend direction (4H)
│ ├── SessionRule.ts # Market session filter (London/NY)
│ ├── ZoneRule.ts # Price-to-EMA proximity check
│ ├── MomentumRule.ts # RSI momentum confirmation
│ ├── EntryTriggerRule.ts# EMA reclaim / wick patterns
│ ├── RiskManagementRule.ts # ATR-based SL/TP/position sizing
│ └── AIAnalysisRule.ts # LLM sentiment validation
├── utils/
│ ├── indicators.ts # EMA, RSI, ATR calculations
│ ├── logger.ts # Winston logger
│ └── symbolMapper.ts # Symbol format conversion
└── index.ts # Main entry point + trading loop
```
## Environment Variables
### Required
| Variable | Description | Example |
|---|---|---|
| `SUPABASE_URL` | Supabase project URL | `https://xxx.supabase.co` |
| `SUPABASE_KEY` | Supabase service role key | `eyJ...` |
| `PROVIDER` | Default exchange provider | `alpaca` or `ccxt` |
| `ALPACA_API_KEY` | Alpaca API key | `PK...` |
| `ALPACA_API_SECRET` | Alpaca secret key | `...` |
### Exchange Configuration
| Variable | Default | Description |
|---|---|---|
| `DATA_PROVIDER` | `$PROVIDER` | Provider for market data |
| `EXECUTION_PROVIDER` | `$PROVIDER` | Provider for order execution |
| `EXCHANGE` | `binance` | CCXT exchange (if using ccxt) |
| `CCXT_API_KEY` | — | CCXT exchange API key |
| `CCXT_API_SECRET` | — | CCXT exchange secret |
| `PAPER_TRADING` | `false` | Use paper trading keys |
| `ASSET_CLASS` | `crypto` | `crypto` or `us_equity` |
### Trading Symbols & Intervals
| Variable | Default | Description |
|---|---|---|
| `SYMBOLS` | `BTC/USD` | Comma-separated trading pairs |
| `POLLING_INTERVAL` | `60000` | Main loop interval (ms) |
| `SYMBOL_DELAY_MS` | `2000` | Delay between symbol processing |
| `TIMEFRAME` | `1Min` | Legacy candle timeframe |
### Execution & Risk
| Variable | Default | Description |
|---|---|---|
| `ENABLE_TRADING` | `false` | Enable live trade execution |
| `TOTAL_CAPITAL` | `1000` | Default total capital ($) |
| `MAX_OPEN_TRADES` | `3` | Max concurrent positions |
| `COOLDOWN_MS` | `3600000` | Post-trade cooldown (ms) |
| `PROFIT_EXIT_PERCENT` | `1.0` | Auto-exit profit threshold (%) |
| `TRAILING_STOP_PERCENT` | `0.001` | Trailing stop pullback (0.1%) |
### Strategy Parameters
| Variable | Default | Description |
|---|---|---|
| `ENABLED_RULES` | all 7 rules | Comma-separated rule list |
| `R_TREND_TIMEFRAME` | `4h` | Trend bias timeframe |
| `R_TREND_EMA_FAST` | `50` | Fast EMA period |
| `R_TREND_EMA_SLOW` | `200` | Slow EMA period |
| `R_RSI_PERIOD` | `14` | RSI calculation period |
| `R_RSI_OVERBOUGHT` | `70` | RSI overbought threshold |
| `R_RSI_OVERSOLD` | `30` | RSI oversold threshold |
| `R_ZONE_EMA_PERIOD` | `20` | Zone EMA period |
| `R_ATR_PERIOD` | `14` | ATR calculation period |
| `R_RISK_PER_TRADE` | `0.01` | Risk per trade (1%) |
| `R_RISK_REWARD_RATIO` | `1.5` | Risk/reward ratio |
| `R_SL_MULTIPLIER` | `1.5` | Stop loss ATR multiplier |
| `R_SESSION_WINDOWS` | JSON array | Session time windows |
### AI Configuration
| Variable | Default | Description |
|---|---|---|
| `AI_PROVIDER` | `openai` | Primary AI provider |
| `PERPLEXITY_API_KEY` | — | Perplexity API key |
| `OPENAI_API_KEY` | — | OpenAI API key |
| `GEMINI_API_KEY` | — | Google Gemini API key |
| `AI_FALLBACK_LIST` | `openai,perplexity,gemini` | Fallback chain |
| `AI_MODEL` | `gpt-4o` | AI model to use |
| `AI_CONFIDENCE_THRESHOLD` | `70` | Min confidence score |
| `AI_CACHE_HOURS` | `4` | Cache duration (hours) |
### Server & Notifications
| Variable | Default | Description |
|---|---|---|
| `API_PORT` | `5000` | API server port |
| `NOTIFICATION_PHONE_NUMBERS` | — | Comma-separated phone numbers |
| `NOTIFICATION_API_HOST` | `www.zenhustles.com` | Notification API host |
| `NOTIFICATION_API_PATH` | `/api/whatsapp/send` | Notification API path |
| `WEBHOOK_URL` | — | Legacy webhook URL |
### System Intervals
| Variable | Default | Description |
|---|---|---|
| `PROFILE_SYNC_INTERVAL_MS` | `60000` | Profile hot-reload interval |
| `MONITOR_INTERVAL_MS` | `60000` | Position monitor polling |
## Database Schema
7 tables are used (see `schema/004_full_schema_sync.sql` for full DDL):
| Table | Purpose |
|---|---|
| `users` | User accounts + exchange API keys |
| `entries` | Watchlist & manual positions |
| `trade_profiles` | Strategy profiles with per-profile rule config |
| `orders` | Active/pending orders (with `profile_id`) |
| `trade_history` | Completed trade ledger (with `profile_id`) |
| `bot_config` | Global config key-value store |
| `dynamic_config` | Runtime config overrides |
### strategy_config JSON Structure
Each profile stores its strategy configuration as a `jsonb` column:
```json
{
"rules": [
{ "ruleId": "TrendBiasRule", "enabled": true, "params": { "emaFast": 50, "emaSlow": 200 } },
{ "ruleId": "SessionRule", "enabled": true, "params": { "allowedSessions": ["NY", "LDN"] } },
{ "ruleId": "ZoneRule", "enabled": true, "params": { "emaPeriod": 20 } },
{ "ruleId": "MomentumRule", "enabled": true, "params": { "rsiPeriod": 14 } },
{ "ruleId": "EntryTriggerRule", "enabled": true, "params": {} },
{ "ruleId": "RiskManagementRule", "enabled": true, "params": { "atrPeriod": 14 } },
{ "ruleId": "AIAnalysisRule", "enabled": false, "params": { "minConfidence": 80 } }
],
"riskLimits": {
"maxDailyLossUsd": 50,
"dailyProfitTargetUsd": 100,
"maxConsecutiveLosses": 2,
"maxOpenTrades": 3
},
"execution": {
"orderType": "market",
"cooldownMinutes": 30,
"minRulePassRatio": 0.70
}
}
```
## API Endpoints
### REST
| Method | Endpoint | Description |
|---|---|---|
| `GET` | `/health` | Health check with uptime |
| `GET` | `/api/status` | Bot status, settings, symbols, positions |
| `GET` | `/api/config` | Current bot configuration (non-secret) |
| `GET` | `/api/alerts` | Recent alerts (supports `?limit=N`) |
| `GET` | `/api/symbol/:symbol` | Single symbol data |
| `POST` | `/api/trade` | Execute manual trade |
| `POST` | `/api/chat` | AI-powered profile generation (see below) |
### POST /api/chat — AI Strategy Assistant
Translates plain English into structured trading profile configurations using the bot's AI fallback chain (Perplexity → OpenAI → Gemini).
**Request:**
```json
{
"message": "Create a conservative BTC swing trader with $2000 capital",
"context": [{ "id": "...", "name": "Existing Profile", "allocated_capital": 1000, ... }]
}
```
**Response:**
```json
{
"action": "create_profile",
"profile": {
"name": "Conservative BTC Swing Trader",
"allocated_capital": 2000,
"risk_per_trade_percent": 1,
"symbols": "BTC/USDT",
"is_active": true,
"strategy_config": { "rules": [...], "riskLimits": {...}, "execution": {...} }
},
"summary": "Created a conservative BTC swing trading profile with $2000 capital and 1% risk per trade.",
"reasoning": "Conservative profiles use lower risk and fewer aggressive rules."
}
```
**Supported actions:** `create_profile`, `update_profile`, `explain`
### Socket.IO Events
| Event | Direction | Description |
|---|---|---|
| `state` | Server → Client | Full bot state update |
| `symbol_update` | Server → Client | Single symbol price/signal update |
| `new_alert` | Server → Client | New trading alert |
| `orders_update` | Server → Client | Order status change |
| `positions_update` | Server → Client | Aggregated positions from all profiles |
## License
ISC

BIN
backend/REPO_AUDIT_ROADMAP.md Normal file
View File

Binary file not shown.

49
backend/SECRET_ROTATION_RUNBOOK.md Normal file
View File

@ -0,0 +1,49 @@
# Secret Rotation Runbook
Date: 2026-02-15
Scope: bot service + dashboard deployment secrets
## Objective
Rotate all production credentials on a fixed cadence and after any suspected leak, while preserving service continuity.
## Rotation Scope
- Supabase:
- `SUPABASE_KEY` / service-role key used by bot service
- JWT settings (`SUPABASE_JWT_ISSUER`, `SUPABASE_JWT_AUDIENCE`) verification values
- Exchange credentials:
- `ALPACA_API_KEY`, `ALPACA_API_SECRET`
- `REAL_ALPACA_API_KEY`, `REAL_ALPACA_API_SECRET`
- AI provider keys:
- `OPENAI_API_KEY`
- `GEMINI_API_KEY`
- `PERPLEXITY_API_KEY`
- Notification/API integration keys (if configured)
## Rotation Procedure
1. Create new credentials in provider consoles.
2. Update secret stores (CI/CD, Azure, Vercel, etc.) with new values.
3. Deploy bot and dashboard with new secret versions.
4. Validate:
- bot startup + auth checks
- exchange order placement dry-run path
- dashboard auth and websocket connectivity
5. Revoke old credentials only after validation window.
6. Record rotation date, actor, and affected systems in release notes.
## Enforcement
- CI includes executable secret hygiene scan:
- `scripts/verifySecretHygiene.ts`
- Gitleaks workflow remains enabled on push/PR.
- Never commit real secrets into tracked files (`.env`, docs, scripts, configs).
## Cadence
- Standard: every 30 days
- Immediate rotation triggers:
- Secret leaked in logs/repo/chat/email
- Access control incident
- Team-member offboarding

92
backend/SESSION_RULE_FIX.md Normal file
View File

@ -0,0 +1,92 @@
# SessionRule Bug Fix — 2026-02-20
## Summary
A critical logic error in `SessionRule.ts` caused **all profiles configured as "24/7" to silently block trades** during Asian (TOK/SYD) market hours. This was a silent failure — no crash, no alert — the bot simply never entered during those hours.
---
## Root Cause
**File**: `src/strategies/rules/SessionRule.ts`
**Old logic (broken):**
```ts
if (isMajor && hasAllowedSession) {
// pass
}
// else: always fail
```
The old check required **both** conditions simultaneously:
1. `isMajor` — the internal flag marking LDN/NY as "major" sessions.
2. `hasAllowedSession` — the session being in the profile's allowed list.
**The problem:** `isMajor` is `false` during TOK/SYD regardless of how the profile is configured. So even when a profile explicitly set `sessions: "LDN,NY,TOK,SYD"` (24/7 mode), the condition `isMajor && hasAllowedSession` failed during TOK/SYD, producing:
```
❌ Mandatory Rule Failed: SessionRule -> Session TOK | SYD not in allowed set (24/7).
```
This was a misleading error — the session WAS in the allowed set, but `isMajor` overrode the check.
---
## Fix Applied
**New logic — 3-path decision tree:**
```
1. Is allowedSessions = [LDN, NY, TOK, SYD]?
→ YES → 24/7 mode. ALWAYS PASS. Skip time checks.
2. Is current session = OFF (market closed)?
→ YES → BLOCK.
3. Is current session in the profile's allowedSessions list?
→ YES → PASS (restricted schedule, currently within window)
→ NO → BLOCK
```
**Key change:** Removed dependency on `context.isMajorSession` entirely. The session check is now driven purely by the profile's configuration, not by an internal "major session" flag.
---
## Session Mapping (Frontend Wizard → Backend)
| Wizard Selection | `SessionRule.params.sessions` | Backend Behavior |
|:---|:---|:---|
| 24/7 | `LDN,NY,TOK,SYD` | Detected as 24/7 → always passes |
| London + New York | `LDN,NY` | Passes only during LDN/NY hours |
| Asia only | `TOK,SYD` | Passes only during TOK/SYD hours |
---
## Expected Log After Fix
```
✅ Rule Passed: SessionRule -> 24/7 mode: all sessions allowed. Current session: TOK | SYD.
```
vs. the old broken log:
```
❌ Mandatory Rule Failed: SessionRule -> Session TOK | SYD not in allowed set (24/7).
```
---
## Impact Assessment
| Metric | Before Fix | After Fix |
|:---|:---|:---|
| Trades during LDN/NY | ✅ Working | ✅ Working |
| Trades during TOK/SYD (24/7 profiles) | ❌ Silently blocked | ✅ Now allowed |
| Restricted session profiles (LDN,NY only) | ✅ Working | ✅ Working |
| Capital at risk | None (no trades were placed) | Normal |
---
## Files Modified
- `src/strategies/rules/SessionRule.ts` — Core fix

95
backend/STRATEGY_MARKETPLACE.md Normal file
View File

@ -0,0 +1,95 @@
# Strategy Marketplace — Admin Publishing Feature
> Added: 2026-02-20
## Overview
The Strategy Marketplace allows administrators to convert any live trading strategy profile (Strategy Cluster) into a reusable template that all platform users can adopt.
---
## Database Schema
**Table**: `public.strategy_presets`
**Migration**: `schema/016_add_strategy_marketplace.sql`
| Column | Type | Description |
|:---|:---|:---|
| `id` | `TEXT` (PK) | Format: `template-{profile_id}-{timestamp}` |
| `name` | `TEXT` | Display name of the strategy |
| `description` | `TEXT` | Auto-generated summary including rule count |
| `risk_style_id` | `TEXT` | `safe`, `balanced`, or `aggressive` |
| `recommended_assets` | `TEXT[]` | Array of symbols (e.g. `['BTC/USDT', 'ETH/USDT']`) |
| `typical_trades_per_day` | `TEXT` | e.g. `'3-5'`, `'8-12'` |
| `performance_tag` | `TEXT` | e.g. `'Institutional Template'` |
| `is_popular` | `BOOLEAN` | Featured/highlighted in marketplace |
| `created_at` | `TIMESTAMPTZ` | Auto-set on insert |
| `created_by` | `UUID` | Admin user ID (FK → `auth.users`) |
| `original_profile_id` | `UUID` | Source profile (FK → `trade_profiles`) |
| `strategy_config` | `JSONB` | Full strategy rule and risk config snapshot |
| `role_required` | `TEXT` | `'free'`, `'pro'`, or `'elite'` |
### Row Level Security
- **Read**: All authenticated users can read all presets (`FOR SELECT USING (true)`).
- **Write**: Only admins can insert/update/delete (`role = 'admin'` check against `public.users`).
---
## Admin Publishing Flow
### Trigger
Admin clicks the **↗ (Publish)** icon on any strategy card in the **Strategy Clusters** tab.
### Auto-Detection Logic
The `handlePublish` function in `TradeProfileManager.tsx` derives the `risk_style_id` from `minRulePassRatio`:
| `minRulePassRatio` | `risk_style_id` |
|:---|:---|
| < 0.9 | `aggressive` |
| ≥ 0.9 and < 1.0 | `balanced` |
| ≥ 1.0 | `safe` |
### Payload Construction
```json
{
"id": "template-{profile.id}-{timestamp}",
"name": "{profile.name}",
"description": "Admin-published strategy based on {name}. Features N optimized rules.",
"risk_style_id": "balanced",
"recommended_assets": ["BTC/USDT", "ETH/USDT"],
"typical_trades_per_day": "3-5",
"performance_tag": "Institutional Template",
"is_popular": true,
"created_by": "{admin_user_id}",
"original_profile_id": "{profile.id}",
"strategy_config": { ... full config ... }
}
```
---
## Frontend Integration
### Marketplace Display (`PresetMarketplace.tsx`)
- On mount, fetches all rows from `strategy_presets` ordered by `created_at DESC`.
- Maps `snake_case` DB columns to `camelCase` frontend `StrategyPreset` interface.
- Merges with hardcoded system presets from `PresetRegistry.ts`.
- Total count displayed in header reflects both system + admin-published templates.
### Adoption Flow
- User clicks **"USE THIS STRATEGY"** on any card.
- The preset's `riskStyleId` and `recommendedAssets` seed the `StrategyWizard`.
- User customizes capital/assets and deploys as their own profile.
---
## Files Modified/Created
| File | Change |
|:---|:---|
| `schema/016_add_strategy_marketplace.sql` | New migration — creates `strategy_presets` table |
| `src/lib/const.ts` | Added `tableNameMarketplace = 'strategy_presets'` |
| `src/components/TradeProfileManager.tsx` | Added `handlePublish` and Publish button for admins |
| `src/components/PresetMarketplace.tsx` | Added Supabase fetch + mapping for dynamic templates |
| `src/tabs/MarketplaceTab.tsx` | Added AI Setups + Top Volatile contextual panels |

85
backend/TRADE_LIFECYCLE_INTEGRITY_PLAN.md Normal file
View File

@ -0,0 +1,85 @@
# Trade Lifecycle Integrity - Migration Plan
Date: 2026-02-15
Owner: Backend trade service
## Goal
Enforce deterministic lifecycle tracing by `profile_id + trade_id` across `orders`, `trade_history`, and runtime reconciliation, without blocking production traffic.
## Null Handling Policy
- `trade_id` may be `NULL` only for legacy/manual rows.
- Empty or whitespace `trade_id` values are normalized to `NULL`.
- Bot-originated lifecycle records must use deterministic `trade_id` (`TRD-*`).
## Rollout Plan
### Phase A - Prepare (No Risk)
- [ ] Snapshot current row counts and distinct trade IDs:
```sql
SELECT COUNT(*) AS orders_total,
COUNT(*) FILTER (WHERE trade_id IS NULL OR btrim(trade_id) = '') AS orders_missing_trade_id
FROM orders;
SELECT COUNT(*) AS history_total,
COUNT(*) FILTER (WHERE trade_id IS NULL OR btrim(trade_id) = '') AS history_missing_trade_id
FROM trade_history;
```
- [ ] Run legacy backfill in dry-run mode.
- [ ] Confirm no unacceptable duplicate lifecycle keys in reporting.
### Phase B - Deploy Non-Blocking Guardrails
- [ ] Apply `schema/007_trade_lifecycle_integrity_constraints.sql`.
- [ ] Verify indexes exist:
```sql
SELECT indexname
FROM pg_indexes
WHERE tablename IN ('orders', 'trade_history')
AND indexname LIKE 'idx_%trade_id%';
```
- [ ] Confirm constraints are present and `NOT VALID`:
```sql
SELECT conname, convalidated
FROM pg_constraint
WHERE conname IN (
'chk_orders_action_lifecycle',
'chk_orders_trade_id_not_blank',
'chk_orders_trade_id_format',
'chk_trade_history_trade_id_not_blank',
'chk_trade_history_trade_id_format'
);
```
### Phase C - Backfill + Reconcile
- [ ] Run deterministic `trade_id` backfill in apply mode.
- [ ] Run lifecycle reconciliation report (`orders -> positions -> history`) per profile.
- [ ] Resolve every `missing_entry_order` / `orphan_exit` anomaly before validation.
### Phase D - Tighten Constraints
- [ ] Validate constraints after reconciliation is clean:
```sql
ALTER TABLE orders VALIDATE CONSTRAINT chk_orders_action_lifecycle;
ALTER TABLE orders VALIDATE CONSTRAINT chk_orders_trade_id_not_blank;
ALTER TABLE orders VALIDATE CONSTRAINT chk_orders_trade_id_format;
ALTER TABLE trade_history VALIDATE CONSTRAINT chk_trade_history_trade_id_not_blank;
ALTER TABLE trade_history VALIDATE CONSTRAINT chk_trade_history_trade_id_format;
```
## Rollback Plan
- Drop newly added constraints (if validation reveals regression):
```sql
ALTER TABLE orders DROP CONSTRAINT IF EXISTS chk_orders_action_lifecycle;
ALTER TABLE orders DROP CONSTRAINT IF EXISTS chk_orders_trade_id_not_blank;
ALTER TABLE orders DROP CONSTRAINT IF EXISTS chk_orders_trade_id_format;
ALTER TABLE trade_history DROP CONSTRAINT IF EXISTS chk_trade_history_trade_id_not_blank;
ALTER TABLE trade_history DROP CONSTRAINT IF EXISTS chk_trade_history_trade_id_format;
```
- Keep indexes unless they materially impact write throughput.
- Re-run reconciliation report and keep bot runtime lifecycle guards active.

349
backend/TRADING_CONTROL_PERSISTENCE.md Normal file
View File

@ -0,0 +1,349 @@
# Trading Control State Persistence
## Where is the Pause/Resume State Stored?
The trading control state (PAUSED or RUNNING) is persisted in **three locations** to ensure reliability and recovery:
---
## 1. In-Memory (Primary Runtime State)
**Location**: `HealthTracker` singleton in `healthTracker.ts`
```typescript
// bytelyst-trading-bot-service/src/services/healthTracker.ts
export class HealthTracker {
private tradingControl: TradingControlSnapshot = {
mode: 'RUNNING',
lastChangedBy: 'system',
lastChangedAt: Date.now()
};
public isPaused(): boolean {
return this.tradingControl.mode === 'PAUSED';
}
public recordTradingControl(update: TradingControlSnapshot): void {
this.tradingControl = update;
// Triggers persistence to disk and database
}
}
```
**Purpose**: Fast access for enforcement points (AutoTrader, TradeExecutor)
---
## 2. Disk Storage (Local Persistence)
**Location**: `bot_state.json` in the bot service root directory
**File Path**: `c:\Users\sarav\project\bytelyst.ai\trading\bytelyst-trading-bot-service\bot_state.json`
**Structure**:
```json
{
"health": {
"tradingControl": {
"mode": "PAUSED",
"lastChangedBy": "admin@example.com",
"lastChangedAt": 1708200000000,
"reason": "Manual admin pause"
},
"tradingLoopHealthy": true,
"reconciliationLoopHealthy": true,
// ... other health metrics
},
"symbols": { ... },
"orders": [ ... ],
"history": [ ... ]
}
```
**How it's saved**:
```typescript
// bytelyst-trading-bot-service/src/services/apiServer.ts
private saveState(): void {
const snapshot = healthTracker.getSnapshot();
const stateToSave = {
health: snapshot, // Includes tradingControl
symbols: this.state.symbols,
orders: this.state.orders,
history: this.state.history,
settings: this.state.settings
};
fs.writeFileSync(
path.join(process.cwd(), 'bot_state.json'),
JSON.stringify(stateToSave, null, 2)
);
}
```
**When it's saved**:
- After every pause/resume action
- Periodically (every 30 seconds)
- On graceful shutdown
**Purpose**: Survive bot restarts, local backup
---
### 3. Database Storage (Cloud Backup)
- **Location**: Supabase `bot_state_snapshots` table
- **Purpose**: Multi-instance recovery, cloud backup, and audit trail
- **Updated**: Throttled writes based on `DB_SNAPSHOT_INTERVAL_MS` (Default: 5 mins)
- `user_id` (UUID, foreign key to users)
- `state` (JSONB) - Contains full bot state including tradingControl
- `created_at` (timestamp)
**How it's saved**:
```typescript
// bytelyst-trading-bot-service/src/services/apiServer.ts
private async persistSnapshotToDb(): Promise<void> {
if (!config.ENABLE_DB_SNAPSHOTS) return;
const now = Date.now();
const elapsed = now - this.lastSnapshotWriteAt;
if (elapsed < config.DB_SNAPSHOT_INTERVAL_MS) {
// Throttled...
return;
}
const stateToSave = this.getPersistableState();
await supabaseService.saveBotStateSnapshot(ownerId, stateToSave);
this.lastSnapshotWriteAt = Date.now();
}
```
**When it's saved**:
- **Throttled Periodically**: Every `DB_SNAPSHOT_INTERVAL_MS` (Default: 300,000ms / 5 minutes)
- **On Startup/Shutdown**: Forced sync if enabled
- **Note**: The manual "Pause/Resume" action triggers a request to save, but the actual DB write is still subject to the throttle to prevent IOPS spikes.
**Purpose**:
- Multi-instance recovery
- Cloud backup
- Audit trail
---
## State Recovery on Bot Restart
When the bot starts, it loads the state in this order:
```typescript
// bytelyst-trading-bot-service/src/services/apiServer.ts
public async loadState(): Promise<void> {
try {
// 1. Try to load from Supabase (preferred)
const dbSnapshot = await supabaseService.getLatestBotSnapshot();
if (dbSnapshot?.snapshot_data?.health?.tradingControl) {
healthTracker.recordTradingControl(
dbSnapshot.snapshot_data.health.tradingControl
);
logger.info('[LoadState] Restored trading control from Supabase:',
dbSnapshot.snapshot_data.health.tradingControl.mode);
return;
}
} catch (err) {
logger.warn('[LoadState] Failed to load from Supabase, trying disk');
}
try {
// 2. Fallback to disk (bot_state.json)
const filePath = path.join(process.cwd(), 'bot_state.json');
if (fs.existsSync(filePath)) {
const fileData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
if (fileData.health?.tradingControl) {
healthTracker.recordTradingControl(fileData.health.tradingControl);
logger.info('[LoadState] Restored trading control from disk:',
fileData.health.tradingControl.mode);
return;
}
}
} catch (err) {
logger.warn('[LoadState] Failed to load from disk');
}
// 3. Default to RUNNING if no state found
logger.info('[LoadState] No saved state found, defaulting to RUNNING');
healthTracker.recordTradingControl({
mode: 'RUNNING',
lastChangedBy: 'system',
lastChangedAt: Date.now(),
reason: 'Bot startup - no previous state'
});
}
```
---
## Persistence Flow Diagram
```
Admin clicks "Pause" button
│
▼
POST /internal/trading/pause
│
▼
healthTracker.recordTradingControl({ mode: 'PAUSED' })
│
├─────────────────────────────────────────┐
│ │
▼ ▼
1. In-Memory Update 2. Trigger Persistence
(Immediate) (Async)
│ │
├─ tradingControl.mode = 'PAUSED' ├─ apiServer.saveState()
└─ isPaused() returns true │ └─ Write to bot_state.json
│
└─ apiServer.persistSnapshotToSupabase()
└─ Upsert to bot_snapshots table
Bot Restart
│
▼
apiServer.loadState()
│
├─ Try Supabase (preferred)
│ └─ SELECT * FROM bot_snapshots ORDER BY updated_at DESC LIMIT 1
│ └─ Extract health.tradingControl
│
├─ Fallback to Disk
│ └─ Read bot_state.json
│ └─ Extract health.tradingControl
│
└─ Default to RUNNING
└─ If no state found
```
---
## 4. Global Configuration (Neural Persistence Settings)
To prevent database overload while maintaining state safety, the bot includes a synchronization throttling mechanism. These settings are managed via the **Admin Tab > Database Synchronization** panel.
### Settings stored in `bot_config` table:
| Key | Default | Description |
|-----|---------|-------------|
| `ENABLE_DB_SNAPSHOTS` | `true` | When `false`, the bot will not write snapshots to Supabase at all. |
| `DB_SNAPSHOT_INTERVAL_MS` | `300000` | Minimum time (in ms) to wait between database writes. |
### Throttling Logic:
1. The bot saves state to local `bot_state.json` **effectively immediately** (debounced 1.5s).
2. The bot checks if `ENABLE_DB_SNAPSHOTS` is true.
3. The bot checks if enough time has passed since `lastSnapshotWriteAt`.
4. Only if both pass is a write sent to Supabase.
---
## Verification
### Check Current State
**Option 1: API Call**
```bash
curl -H "Authorization: Bearer <token>" \
http://localhost:5000/internal/trading/status
# Response:
{
"mode": "PAUSED",
"lastChangedBy": "admin@example.com",
"lastChangedAt": 1708200000000,
"reason": "Manual admin pause"
}
```
**Option 2: Check bot_state.json**
```bash
# Windows PowerShell
Get-Content "c:\Users\sarav\project\bytelyst.ai\trading\bytelyst-trading-bot-service\bot_state.json" | ConvertFrom-Json | Select-Object -ExpandProperty health | Select-Object -ExpandProperty tradingControl
```
**Option 3: Check Supabase**
```sql
SELECT snapshot_data->'health'->'tradingControl'
FROM bot_snapshots
ORDER BY updated_at DESC
LIMIT 1;
```
**Option 4: Check Logs**
```bash
# Look for these log entries
[Admin] Trading PAUSED by admin@example.com. Reason: Manual admin pause
[Admin] Trading RESUMED by admin@example.com.
[LoadState] Restored trading control from Supabase: PAUSED
```
---
## Important Notes
### Persistence Guarantees
✅ **Immediate**: In-memory state updated instantly
✅ **Durable**: Disk and database writes happen within 1 second
✅ **Recoverable**: State survives bot restarts
✅ **Auditable**: All changes logged with timestamp and user
### Failure Scenarios
| Scenario | Behavior |
|----------|----------|
| Disk write fails | State still in memory, Supabase backup available |
| Supabase write fails | State still in memory and on disk |
| Both writes fail | State in memory, will retry on next save cycle |
| Bot crashes | State recovered from Supabase or disk on restart |
| Supabase unavailable on restart | Falls back to disk (bot_state.json) |
| Both unavailable on restart | Defaults to RUNNING mode |
### State Consistency
- **Single Source of Truth**: In-memory state in HealthTracker
- **Persistence is Async**: Writes don't block trading operations
- **Recovery is Synchronous**: State loaded before trading starts
- **No Race Conditions**: All writes go through HealthTracker singleton
---
## File Locations Summary
| Storage | Location | Purpose |
|---------|----------|---------|
| In-Memory | `healthTracker.ts` singleton | Fast runtime access |
| Disk | `bot_state.json` | Local persistence |
| Database | Supabase `bot_snapshots` table | Cloud backup |
| Logs | `combined.log` | Audit trail |
---
## Code References
### Save State
- **File**: `bytelyst-trading-bot-service/src/services/apiServer.ts`
- **Method**: `saveState()` (line ~1700)
- **Method**: `persistSnapshotToSupabase()` (line ~1750)
### Load State
- **File**: `bytelyst-trading-bot-service/src/services/apiServer.ts`
- **Method**: `loadState()` (line ~1800)
### Trading Control State
- **File**: `bytelyst-trading-bot-service/src/services/healthTracker.ts`
- **Property**: `tradingControl: TradingControlSnapshot`
- **Method**: `recordTradingControl()`
- **Method**: `isPaused()`
### Pause/Resume Endpoints
- **File**: `bytelyst-trading-bot-service/src/services/apiServer.ts`
- **Endpoint**: `POST /internal/trading/pause` (line 1054)
- **Endpoint**: `POST /internal/trading/resume` (line 1073)

42
backend/admin-observability.md Normal file
View File

@ -0,0 +1,42 @@
# Admin Observability & Health Panel
This document describes the runtime observability system implemented for the trading bot administrators.
## Overview
The Admin Error & Health panel provides real-time visibility into the bot's internal state and actionable issues. It is designed for operators to quickly identify why trading might be paused, failing, or behaving unexpectedly without having to dig through raw logs.
## Architecture
### Backend: `ObservabilityService`
- **In-Memory Buffer**: Stores the last 50 operational events in a ring buffer.
- **Structured Events**: Every event follows the `OperationalEvent` interface.
- **Filtering**: Events are filtered by user role. Only administrators receive operational events via the API and Socket.IO.
### Frontend: `AdminTab` (System Health)
- **Status Badge**: A global indicator of system health (Healthy, Degraded, Critical).
- **Event List**: A chronologically ordered list of recent operational events with severity levels (INFO, WARN, ERROR).
- **Telemetry**: Real-time display of execution loop durations, exchange latency, and lock contention counts.
## Operational Event Types
| Type | Severity | Description |
|------|----------|-------------|
| `INSUFFICIENT_BUYING_POWER` | WARN | Attempted to open a position but broker reported insufficient capital. |
| `ORDER_FAILURE` | ERROR | Exchange rejected an order (e.g., price out of bounds, invalid qty). |
| `EXCHANGE_STATE_MISMATCH` | WARN | Discrepancy detected between internal database and exchange state. |
| `RECONCILIATION_DEGRADED` | ERROR | Reconciliation loop is failing repeatedly. |
| `SYSTEM_ERROR` | WARN/ERROR | General system issues, including exchange API timeouts or manual pauses. |
## Security & Performance
- **Sensitive Data**: Events contain structured messages instead of raw stack traces or internal environment variables.
- **Cap**: Both backend buffer and frontend display are capped at 50 events to ensure performance and prevent memory bloating.
- **RBAC**: Operational events are only pushed to authenticated sockets belonging to users with the `admin` role.
## Usage
1. Navigate to the **Admin** tab.
2. Select **System Health**.
3. Review the **Operational Events** list for recent issues.
4. If a global red banner appears at the top of the dashboard, it indicates a critical operational event occured in the last 10 minutes.

76
backend/apply_standard_risk_profiles.ts Normal file
View File

@ -0,0 +1,76 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
import fs from 'fs';
import path from 'path';
async function applyStandardRiskProfiles() {
logger.info('🚀 APPLYING STANDARD RISK PROFILES (5 TO 1)...');
// 1. Load Data
const proposedPath = path.resolve('schema/proposed_risk_profiles.json');
if (!fs.existsSync(proposedPath)) {
logger.error(`❌ missing proposed profiles file: ${proposedPath}`);
return;
}
const proposedProfiles = JSON.parse(fs.readFileSync(proposedPath, 'utf8'));
// 2. Get User
const users = await supabaseService.getActiveUsers();
if (users.length === 0) {
logger.error('❌ No active users found.');
return;
}
const user = users[0];
// 3. Clear Existing Data (TRUNCATE equivalents via DELETE)
logger.info('🧹 Wiping existing data for a fresh start...');
// @ts-ignore
await supabaseService.client.from('positions').delete().neq('id', '00000000-0000-0000-0000-000000000000');
// @ts-ignore
await supabaseService.client.from('orders').delete().neq('id', '00000000-0000-0000-0000-000000000000');
// @ts-ignore
await supabaseService.client.from('trade_history').delete().neq('id', '00000000-0000-0000-0000-000000000000');
// @ts-ignore
await supabaseService.client.from('alerts').delete().neq('id', '00000000-0000-0000-0000-000000000000');
// @ts-ignore
await supabaseService.client.from('operational_events').delete().neq('id', '00000000-0000-0000-0000-000000000000');
// @ts-ignore
await supabaseService.client.from('trade_profiles').delete().neq('user_id', '00000000-0000-0000-0000-000000000000');
// 4. Prepare & Insert Profiles
const insertPayload = Object.values(proposedProfiles).map((p: any) => {
// Calculate risk_per_trade_percent from the RiskManagementRule in strategy_config
const riskRule = p.strategy_config.rules.find((r: any) => r.ruleId === 'RiskManagementRule');
const riskPercent = riskRule?.params?.maxRisk || 1.0;
return {
user_id: user.user_id,
name: p.name,
allocated_capital: p.allocatedCapital,
risk_per_trade_percent: riskPercent,
symbols: "BTC/USDT,ETH/USDT,SOL/USDT", // Multi-pair trading enabled
is_active: true,
strategy_config: p.strategy_config
};
});
// @ts-ignore
const { data, error } = await supabaseService.client
.from('trade_profiles')
.insert(insertPayload);
if (error) {
logger.error(`❌ Failed to apply profiles: ${error.message}`);
} else {
logger.info('✅ 5 Standard Profiles applied successfully!');
insertPayload.forEach((p: any, idx: number) => {
logger.info(` ${idx + 1}. [${p.name}] Risk: ${p.risk_per_trade_percent}% | Capital: $${p.allocated_capital}`);
});
}
process.exit(0);
}
applyStandardRiskProfiles().catch(console.error);

114
backend/architecture.md Normal file
View File

@ -0,0 +1,114 @@
Bytelyst Trading Platform Architecture Reference
## SECTION 1 — SYSTEM OVERVIEW
- High-level architecture diagram (textual) — A single trading service instance drives a trading loop that evaluates profiles, places orders through exchange connectors, and sends confirmed state into Supabase. Supabase dispatches data to the dashboard, the monitoring loop, and the reconciliation loop. Operational health is published via the /internal/health endpoint, while observability metrics flow to the Prometheus /metrics exporter.
- Runtime components:
- Trading loop — Executes profile signals, reserves capital, acquires row-based locks, delegates confirmed exchange orders to transactional lifecycle RPCs, and emits audit logs.
- Monitor loop — Collects exchange syncs, ensures lifecycle state mirrors exchange fills, and feeds capital/position snapshots back into Supabase.
- Reconciliation loop — Locks per profile, fetches full open orders from exchange and database, routes mismatches through lifecycle-safe handlers, and updates metrics for parity, miss counts, and lock contention.
- Order sync loop — Aggregates lifecycle history, updates the public dashboard data, and ensures active orders, positions, and trade history remain aligned with Supabase slices.
- Exchange interaction model — The trading loop targets the exchange as the source of truth: it first places an order, receives an exchange order_id alongside deterministically generated clientOrderId, then persists lifecycle data. Subsequent reconciliation cycles compare Supabase rows to the exchange’s reported open orders, triggering lifecycle handlers rather than raw database patches.
- Single exchange key multi-profile model — A shared exchange API key services multiple profiles but isolation is maintained through per-profile capital ledgers, row-based locking, and RLS policies; no profile may observe or affect another profile’s runtime state.
## SECTION 2 — PHASE-BY-PHASE ENTERPRISE HARDENING
### Phase 1 — Tenant Isolation
**Purpose:** Prevent cross-profile data leakage and enforce per-user scoping.
**Problem solved:** Without isolation, one authenticated user could receive global runtime state or execute orders for another profile.
**Core mechanisms:** Supabase RLS filters (orders and trade_history tables, WebSocket payloads scoped by user_id), WebSocket broadcasts that partition runtime state per authenticated user, and runtime checks that reject exchanges for mismatched profile_id.
**Must never break:** The global runtime state must never be broadcast without tenant attribution, and no query should bypass RLS.
### Phase 2 — Restart Durability
**Purpose:** Guarantee deterministic state reconstruction after service restart.
**Problem solved:** Volatile in-memory maps caused missed reservations, lost lifecycle state, and inconsistent dashboards post-restart.
**Core mechanisms:** On startup, the service loads profiles, replays exchange open positions and orders, rebuilds lifecycle/trade history mappings, and rehydrates the capital ledger from the database/exchange state. File snapshots are deprecated; the DB/exchange become authoritative.
**Must never break:** Restart must not rely on process-local idempotency maps; open orders/positions must always be re-fetched on boot.
### Phase 3 — Capital Ledger
**Purpose:** Enforce deterministic capital isolation per profile.
**Problem solved:** Concurrent entries could over-allocate capital and leave the ledger inconsistent.
**Core mechanisms:** A ledger schema maintains allocated_capital, reserved_for_orders, reserved_for_positions, and realized_pnl, with available_capital computed as allocated minus reservations plus realized. Entry execution acquires a profile-level mutex, reserves capital before exchange placement, and adjusts reservations on fills, partial fills, cancels, and exits. Restart rebuilds the ledger from exchange open orders/positions, ensuring any drift resets to authoritative values.
**Must never break:** The invariant available_capital = allocated - reserved_for_orders - reserved_for_positions + realized_pnl must always hold; no code should mutate ledger fields outside the defined RPCs or ledger service.
### Phase 4 — Transactional Lifecycle
**Purpose:** Guarantee atomic ENTRY/EXIT persistence tied to confirmed exchange events.
**Problem solved:** Partial writes left orphaned lifecycle entries, phantom positions, and duplicate trade history rows.
**Core mechanisms:** ENTRY flow places the exchange order first, then calls the fn_persist_entry_lifecycle RPC that inserts the lifecycle row, order row, position seed, and optional history slice within one transaction (using UNIQUE(profile_id, trade_id) and idempotent child inserts). EXIT flow places exit orders, then updates lifecycle rows, positions, and history in another single transaction. Idempotency keys prevent duplicate rows, and the unique constraints enforce lifecycle integrity.
**Must never break:** The exchange must remain the source of truth; no lifecycle row may exist without a confirmed exchange order, and idempotency safeguards must never be bypassed.
### Phase 5 — Reconciliation
**Purpose:** Align database state with exchange truth continuously.
**Problem solved:** Discrepancies between Supabase rows and exchange orders led to stale dashboard data and capital mismatches.
**Core mechanisms:** The reconciliation loop acquires a profile-specific row lock, loads full open orders from both the exchange and the database (no limit), compares by order_id/client_order_id/trade_id, and routes any discrepancy through lifecycle-safe handlers (entry fill, exit fill, cancel). It also tracks metrics for reconciliation health and mismatch counts.
**Must never break:** Raw status patches are forbidden; every correction must trigger lifecycle handlers so the capital ledger and positions stay consistent.
### Phase 6 — Distributed Safety
**Purpose:** Make ENTRY execution safe across horizontally scaled instances.
**Problem solved:** In-memory profile mutexes could not coordinate across multiple bots, leading to double submissions.
**Core mechanisms:** ENTRY distributed locking now uses row-based lock table with TTL, owner tokens, and deterministic symbol keys, ensuring only one active signal per profile/symbol. Deterministic clientOrderId (bytelyst-profile-trade) prevents duplicate exchange submissions when retries occur. Horizontal scaling relies on shared DB locks, so no duplicate lifecycle creation occurs even with many workers.
**Must never break:** The row-lock acquisition/release must always execute around exchange submission; failing to release or regenerate owner tokens is unacceptable.
### Phase 7 — Observability & Health
**Purpose:** Provide operational insight and safeguards.
**Problem solved:** Blind spots in loops and invariants made debugging and proactive alerting difficult.
**Core mechanisms:** Prometheus /metrics tracks loop durations, reconciliation mismatches, lock contention, capital invariant violations, and exchange latency histograms. /internal/health exposes trading/monitor/reconciliation loop health, lock contention counts, reconciliation mismatch counts, and degraded indicators. Structured audit logs capture ENTRY/EXIT submissions, fills, cancellations, and reconciliation corrections. A capital invariant watchdog logs critical errors if the ledger computation negative.
**Must never break:** Observable metrics must never regress; the health endpoint must always include counters referenced by runbooks.
### Phase 8 — Final Enterprise Validation
**Purpose:** Formalize invariants, failure runbooks, and emergency controls.
**Problem solved:** Without operator guidance, teams risk violating essential guarantees when extending the system.
**Core mechanisms:** Operators rely on docs/invariants.md for safety rules, docs/runbooks/*.md for failure handling, kill switches for trading loops, and circuit breakers (global/profile/exchange) described in those runbooks. Incident response includes capital invariant monitoring and mutex lock health checks.
**Must never break:** The canonical architecture reference must remain accurate; any code touching core components must cite these runbooks before modification.
## SECTION 3 — CRITICAL INVARIANTS
- No duplicate exchange order. Documented in docs/invariants.md; deterministic clientOrderId plus lifecycle atomicity prevents duplicates.
- No lifecycle without confirmed exchange order. ENTRY RPC rejects inserts unless exchange order_id is confirmed.
- Capital cannot go negative. The ledger service enforces available_capital calculations and invariants; violations trigger logs and the capital invariant metric.
- Only one active ENTRY per (profile_id, symbol). Row-based entry_locks enforce exclusivity.
- Reconciliation converges to exchange truth. The reconciliation loop locks per profile, compares full datasets, and uses lifecycle handlers to correct mismatches.
- Restart does not corrupt ledger. Startup rebuild rehydrates ledger from exchange open orders/positions.
- Distributed workers cannot double submit. Shared locks and deterministic clientOrderId plus RPC idempotency guarantee this.
Each invariant references docs/invariants.md and the relevant runbook under docs/runbooks/*.md for procedures when invariants fail.
## SECTION 4 — EXECUTION FLOW DIAGRAMS
1. ENTRY execution — Validate signal, acquire reconciliation/entry row lock, reserve capital via ledger RPC, place exchange order with deterministic clientOrderId, call fn_persist_entry_lifecycle within transaction, release lock, emit audit log, and update dashboard state.
2. EXIT execution — Trigger exit signal, place exit order with exchange, call transactional RPC to update lifecycle, adjust ledger (release reserved positions, add realized_pnl), close positions, notify dashboard.
3. Partial fill handling — Exchange reports partial fill, monitor/reconciliation loop routes through entry-fill handler, move delta from reserved_for_orders to reserved_for_positions, update position quantity, emit lifecycle history slice, maintain ledger invariant.
4. Restart rebuild — On boot, load profiles, fetch exchange open orders and positions, rebuild lifecycle map, reconstruct ledger reservations and realized_pnl, validate no stale idempotency entries remain.
5. Reconciliation cycle — Acquire profile reconciliation lock, fetch full DB open/closed orders, fetch exchange open orders, match by identifiers, route discrepancies through lifecycle handlers, update metrics, release lock.
6. Distributed lock acquisition — Generate owner token, call fn_try_acquire_entry_lock_row with TTL (30s), verify success, re-check lifecycle state, proceed with capital reservation and exchange call, and finally release lock via fn_release_entry_lock_row.
## SECTION 5 — FAILURE SCENARIO TABLE
Scenario | What happens | Why safe | Recovery behavior
Two workers race | Only one lock owner receives lock, other skip entry | Row lock plus lifecycle check prevents double submission | Loser retries after lock TTL; health endpoint increments lock contention metric
Network partition | Exchange call eventually times out | Lock TTL ensures no permanent hold; trading loop raises error via runbook steps | Retry logic plus alert in docs/runbooks/lock-timeout.md
DB failure | RPC fails, transaction rolled back | ENTRY RPC transaction atomicity prevents partial lifecycle writes | Retry after DB recovery; reconciliation finds any discrepancy
Exchange timeout | Entry order fails after reservation | Reserved capital released under finally block and ledger recalculates | Monitor loop logs failure; health endpoint lattice flags degrade
Crash before lifecycle RPC | Exchange order exists, RPC never called | Reconciliation detects order without lifecycle and replays handler | Lifecycle handler reprocesses confirmed order; runbook instructs to check logs
Crash after exchange but before persistence | Exchange order_id exists, lifecycle missing | Reconciliation handles orphaned orders via lifecycle-safe handler | Handler inserts lifecycle; capital ledger adjusts from exchange positions
Partial fill after restart | Rebuilt ledger recalculates reserved positions from exchange fills | Ledger rebuild logic replays fills; invariants hold | No manual recovery needed; reconciliation verifies
Supabase outage | Health metrics report service_role inability to write | Monitoring loop marks degraded; lock/metric thresholds trigger alerts | Operations route to runbook in docs/runbooks/supabase-outage.md
Lock stuck | Entry lock expires at TTL and is reacquired | TTL prevents deadlock | Health metric increments lock contention; runbook uses kill switch
## SECTION 6 — HEALTH & OBSERVABILITY
- /internal/health fields — tradingLoopHealthy, monitorLoopHealthy, reconciliationLoopHealthy, reconciliationLastRun, lockContentionCount, reconciliationMismatchCount, reconciliationMissingFromExchange, reconciliationMissingInDb, capitalInvariantViolations, exchangeLatencyHistogram, readiness (true only when loops run within expected intervals). Degradation occurs if any loop exceeds twice its normal cadence or capitalInvariantViolations increments.
- Loop metrics — /metrics exposes durations and last-run timestamps for trading, monitor, reconciliation, order sync loops. An unhealthy threshold is 2x the expected interval.
- Lock contention metrics — Incremented when fn_try_acquire_entry_lock_row fails; acquisition latency is recorded as a histogram; stuck locks are surfaced when TTL expires without release.
- Reconciliation metrics — mismatch and missing counts show convergence progress; reconciliationHealthy toggles when mismatch count remains zero for two cycles.
- Observability design — Prometheus ensures minimal overhead by pushing counters via the health tracker; structured logs include profile_id, trade_id, event, and shedding to maintain compliance.
## SECTION 7 — HORIZONTAL SCALING MODEL
- Multi-worker deployment — Each bot instance shares the same Supabase project and exchange key; they coordinate through row-based locks and the shared capital ledger stored in Supabase.
- Shared DB — Lifecycles, ledgers, locks, and reconciliation state live in Supabase; worker nodes treat the DB as the single source of state, and all RPCs operate against it.
- Shared exchange key — A deterministic clientOrderId plus order lifecycle handling ensures duplicate submissions never occur even though multiple workers use the same key.
- Lock guarantees — Entry locks and reconciliation locks use TTL and owner tokens; only one worker may hold a lock for a profile/symbol combination or a profile’s reconciliation cycle.
- No duplication — Atomic lifecycle RPCs, deterministic clientOrderId, and reconciliation lock semantics guarantee that no two workers can simultaneously report conflicting lifecycles or capital adjustments.
## SECTION 8 — SAFE ENHANCEMENT RULES
Checklist for future agents:
- Lifecycle: Do not modify fn_persist_entry_lifecycle or exit RPCs without referencing docs/runbooks/lifecycle-incident.md; every change must maintain exchange-first order and transactional guarantees.
- Ledger: Preserve available_capital invariants and only update ledger fields through the ledger service; see docs/invariants.md for safety rules.
- Reconciliation: Never bypass lifecycle handlers; matching logic must still route through reconcileEntryFill/reconcileExitFill/reconcileCancel.
- Locking: Entry and reconciliation locks (row-based) must stay TTL-based and owner-checked; no in-memory mutex hacks.
- Exchange submission: ClientOrderId strategy is deterministic; do not modify it without ensuring idempotency.
"DO NOT BREAK" rules: The exchange remains the source of truth, the capital ledger must never go negative, distributed locks must be respected, and reconciliation must converge to exchange state. Any deviation triggers procedures spelled out in docs/runbooks/invariant-violation.md.

33
backend/audit_profile_mapping.ts Normal file
View File

@ -0,0 +1,33 @@
import { createClient } from '@supabase/supabase-js';
import { config } from '../src/config/index.js';
async function auditProfileMapping() {
if (!config.SUPABASE_URL || !config.SUPABASE_KEY) {
return;
}
const supabase = createClient(config.SUPABASE_URL, config.SUPABASE_KEY);
const { data: profiles } = await supabase.from('trade_profiles').select('*');
const { data: history } = await supabase.from('trade_history').select('*');
const { data: orders } = await supabase.from('orders').select('*');
const audit: any = {};
profiles?.forEach(p => {
audit[p.id] = {
name: p.name,
capital: p.allocated_capital,
trades: history?.filter(h => h.profile_id === p.id).length || 0,
orders: orders?.filter(o => o.profile_id === p.id).length || 0,
pnl: history?.filter(h => h.profile_id === p.id).reduce((sum, h) => sum + Number(h.pnl || 0), 0)
};
});
console.log(JSON.stringify({
profiles: audit,
orphanedTrades: history?.filter(h => !h.profile_id).length || 0,
orphanedOrders: orders?.filter(o => !o.profile_id).length || 0
}, null, 2));
}
auditProfileMapping().catch(console.error);

35
backend/audit_profile_structure.ts Normal file
View File

@ -0,0 +1,35 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
async function auditProfileData() {
logger.info('🔍 AUDITING PROFILE DATA STRUCTURE...');
const profiles = await supabaseService.getActiveProfiles();
const highRisk = profiles.find(p => p.name.includes('Scalp') || p.name.includes('High'));
if (!highRisk) {
logger.error('❌ High Risk Profile not found.');
return;
}
logger.info(`👤 Profile: ${highRisk.name}`);
logger.info(`📂 Strategy Config Type: ${typeof highRisk.strategy_config}`);
if (highRisk.strategy_config) {
logger.info(`📂 Rules Array Type: ${Array.isArray(highRisk.strategy_config.rules) ? 'Array' : typeof highRisk.strategy_config.rules}`);
logger.info('📋 RAW RULES DATA:');
// print full json structure
console.log(JSON.stringify(highRisk.strategy_config.rules, null, 2));
} else {
logger.warn('⚠️ strategy_config is missing or null');
}
logger.info('------------------------------------------------');
logger.info('Expected IDs by Dashboard:');
const expected = ['TrendBiasRule', 'MomentumRule', 'ZoneRule', 'SessionRule', 'EntryTriggerRule', 'RiskManagementRule', 'AIAnalysisRule'];
logger.info(expected.join(', '));
}
auditProfileData().catch(console.error);

57
backend/backtesting.md Normal file
View File

@ -0,0 +1,57 @@
# Backtesting and Replay v1 (Backend)
## Scope
Backtesting v1 is an isolated, read-only simulation pipeline under `src/backtest/**`.
It reuses `ProStrategyEngine` unchanged and is guarded by feature flag plus mode assertion.
## Isolation Guarantees
- Entry path: `POST /api/backtest/run` only.
- Hard guard: `if (mode !== 'backtest') throw`.
- No live runtime loop reuse (`src/index.ts` unchanged).
- No broker order placement (`ReplayExchangeConnector.placeOrder()` throws).
- No writes to `orders`, `trade_history`, or `capital_ledgers`.
- In-memory virtual ledger only.
## Deterministic Replay Rules
- Candles are normalized and strictly sorted by timestamp.
- Timeframes are validated to `15m`, `1h`, `4h`.
- Symbol processing order is lexicographic and stable.
- No randomness in fills/slippage/partials/exit ordering.
- Same input payload produces identical output.
## Time-Window Semantics
- Replay window: `from_date` inclusive to `to_date` inclusive (UTC).
- Warm-up candles are loaded before `from_date` where available.
- Trades are evaluated only inside the replay window.
- Default end-of-window behavior: keep open positions as `OPEN_AT_END`.
- Optional execution override: `forceCloseAtWindowEnd=true` to close at last candle.
## Data Sources
- CSV upload (required in v1): `symbol,timeframe,timestamp,open,high,low,close,volume`.
- JSON upload (array rows or nested symbol/timeframe maps).
- Replay payload adapter (read-only).
- Kraken historical loader (read-only CCXT OHLC fetch, cached in-memory, no order paths).
## API Contract Highlights
`BacktestResult` includes:
- `trades`
- `summary` (`netPnlUsd`, `winRate`, `maxDrawdownPct`, `sharpe`, `totalTrades`)
- `timeline` (equity and drawdown points)
- `window` (UTC replay window + `OPEN_AT_END`/`FORCE_CLOSE` policy)
- `warmup`
- `openPositionsAtEnd`
- `assumptions`
- `diagnostics`
## Feature Flags
- `ENABLE_BACKTEST=true`
- `BACKTEST_CUSTOMER_ENABLED=false` (when false, only admins can run backtests)
- `BACKTEST_MAX_CSV_BYTES`
- `BACKTEST_MAX_ROWS`
## Non-Goals
- Parameter optimization
- ML tuning
- Strategy mutation
- Live/backtest blending
- Broker execution from backtest mode

19
backend/checkAlpacaPositions.ts Normal file
View File

@ -0,0 +1,19 @@
import { AlpacaConnector } from '../src/connectors/alpaca.js';
import { config } from '../src/config/index.js';
async function test() {
const connector = new AlpacaConnector(config.ALPACA_API_KEY, config.ALPACA_API_SECRET, config.PAPER_TRADING);
console.log("--- ALPACA DIAGNOSTIC ---");
try {
const allPos = await (connector as any).client.getPositions();
console.log(`Global Positions Count: ${allPos.length}`);
allPos.forEach((p: any) => {
console.log(`Symbol: ${p.symbol}, Side: ${p.side}, Size: ${p.qty}, Price: ${p.avg_entry_price}`);
});
} catch (e: any) {
console.error("Failed to fetch global positions:", e.message);
}
}
test();

17
backend/check_alerts.ts Normal file
View File

@ -0,0 +1,17 @@
import http from 'http';
http.get('http://localhost:5000/api/alerts', (res) => {
let data = '';
res.on('data', (chunk) => { data += chunk; });
res.on('end', () => {
try {
const alerts = JSON.parse(data);
console.log('--- RECENT SIGNAL ALERTS ---');
alerts.filter((a: any) => a.type === 'signal').slice(-3).forEach((a: any) => {
console.log(`📍 [${new Date(a.timestamp).toLocaleTimeString()}] ${a.symbol}: ${a.message.split(':')[0]}`);
});
} catch (e: any) {
console.error('Error:', e.message);
}
});
});

20
backend/check_alpaca_pos.ts Normal file
View File

@ -0,0 +1,20 @@
import { config } from '../src/config/index.js';
import { AlpacaConnector } from '../src/connectors/alpaca.js';
import logger from '../utils/logger.js';
async function checkAlpacaPositions() {
const exchange = new AlpacaConnector(config.ALPACA_API_KEY, config.ALPACA_API_SECRET);
try {
const positions = await exchange.getPositions();
console.log('--- Current Alpaca Positions ---');
console.log(JSON.stringify(positions, null, 2));
const btcPos = await exchange.getPosition('BTC/USD');
console.log('--- BTC/USD Specific Position ---');
console.log(JSON.stringify(btcPos, null, 2));
} catch (err) {
console.error('Error fetching positions:', err);
}
}
checkAlpacaPositions();

19
backend/check_cols.ts Normal file
View File

@ -0,0 +1,19 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function checkColumns() {
const { data, error } = await supabase.from('trade_history').select('*').limit(1);
if (error) {
console.error('Error fetching one row:', error);
} else if (data && data.length > 0) {
console.log('Trade History Columns:', Object.keys(data[0]));
} else {
console.log('Trade History is empty or table mismatch.');
}
}
checkColumns();

47
backend/check_counts.ts Normal file
View File

@ -0,0 +1,47 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function checkSchema() {
console.log('--- Checking Table Schema ---');
// We can use RPC to get table info, or just fetch one row
const { data: orderRow, error: orderErr } = await supabase.from('orders').select('*').limit(1);
if (!orderErr && orderRow && orderRow.length > 0) {
console.log('Orders Columns:', Object.keys(orderRow[0]));
} else {
console.log('Orders error or no rows:', orderErr?.message);
}
const { data: historyRow, error: historyErr } = await supabase.from('trade_history').select('*').limit(1);
if (!historyErr && historyRow && historyRow.length > 0) {
console.log('History Columns:', Object.keys(historyRow[0]));
} else {
console.log('History error or no rows:', historyErr?.message);
}
}
async function countByUser() {
const userId = '8d5efd9e-0760-4859-8c07-0930ab3ede5a';
console.log(`\n--- Counting for user ${userId} ---`);
const { count: orderCount } = await supabase.from('orders').select('*', { count: 'exact', head: true }).eq('user_id', userId);
const { count: historyCount } = await supabase.from('trade_history').select('*', { count: 'exact', head: true }).eq('user_id', userId);
console.log(`Orders: ${orderCount}`);
console.log(`History: ${historyCount}`);
// Also check for 'global'
const { count: globalOrderCount } = await supabase.from('orders').select('*', { count: 'exact', head: true }).eq('user_id', 'global');
console.log(`Global Orders: ${globalOrderCount}`);
}
async function main() {
await checkSchema();
await countByUser();
}
main();

22
backend/check_db_users.ts Normal file
View File

@ -0,0 +1,22 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function checkUsers() {
console.log('Listing Users from DB...');
const { data, error } = await supabase
.from('users')
.select('user_id, email, trade_enable');
if (error) {
console.error('❌ Error fetching users:', error);
} else {
console.log('✅ Users found:', data);
}
}
checkUsers();

11
backend/check_orders_cols.ts Normal file
View File

@ -0,0 +1,11 @@
import { createClient } from '@supabase/supabase-js';
import { config } from '../src/config/index.js';
async function checkOrdersCols() {
const supabase = createClient(config.SUPABASE_URL, config.SUPABASE_KEY);
const { data } = await supabase.from('orders').select('*').limit(1);
if (data && data[0]) {
console.log('Orders Columns:', Object.keys(data[0]));
}
}
checkOrdersCols();

34
backend/check_persistence.ts Normal file
View File

@ -0,0 +1,34 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function checkUserPersistence() {
const email = 'saravanan.27ge+006@gmail.com';
console.log(`Checking user: ${email}`);
const { data: users, error } = await supabase.from('users').select('*').eq('email', email);
if (error || !users || users.length === 0) {
console.error('User not found in DB!');
return;
}
const user = users[0];
const userId = user.user_id;
console.log(`Found User ID: ${userId}`);
// Check recent entries for this specific ID
const { count: orders } = await supabase.from('orders').select('*', { count: 'exact', head: true }).eq('user_id', userId);
const { count: history } = await supabase.from('trade_history').select('*', { count: 'exact', head: true }).eq('user_id', userId);
console.log(`Stats for this user: Orders=${orders}, History=${history}`);
if (orders > 0) {
const { data } = await supabase.from('orders').select('symbol, side, status, created_at').eq('user_id', userId).order('created_at', { ascending: false }).limit(5);
console.log('Recent Orders:', JSON.stringify(data, null, 2));
}
}
checkUserPersistence();

19
backend/check_user_schema.ts Normal file
View File

@ -0,0 +1,19 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function checkUserSchema() {
const { data, error } = await supabase.from('users').select('*').limit(1);
if (error) {
console.error('Error:', error);
} else if (data && data.length > 0) {
console.log('Columns:', Object.keys(data[0]).join(', '));
} else {
console.log('Users table is empty.');
}
}
checkUserSchema();

67
backend/create_aggressive_profile.ts Normal file
View File

@ -0,0 +1,67 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
async function createAggressiveProfile() {
logger.info('🚀 CREATING AGGRESSIVE TEST PROFILE (70% VOTING)...');
// 1. Get User
const users = await supabaseService.getActiveUsers();
if (users.length === 0) {
logger.error('❌ No active users found.');
return;
}
const user = users[0];
// 2. Prepare Profile
const aggressiveProfile = {
user_id: user.user_id,
name: "Aggressive Test (70% Voting)",
allocated_capital: 5000,
symbols: "BTC/USDT,ETH/USDT,SOL/USDT",
is_active: false,
strategy_config: {
rules: [
{ ruleId: 'TrendBiasRule', enabled: true, params: { emaFast: 50, emaSlow: 200 } },
{ ruleId: 'SessionRule', enabled: true, params: { allowedSessions: ['NY', 'LDN'] } },
{ ruleId: 'ZoneRule', enabled: true, params: { emaPeriod: 20 } },
{ ruleId: 'MomentumRule', enabled: true, params: { rsiPeriod: 14 } },
{ ruleId: 'EntryTriggerRule', enabled: true, params: {} },
{ ruleId: 'RiskManagementRule', enabled: true, params: { atrPeriod: 14 } },
{ ruleId: 'AIAnalysisRule', enabled: false, params: { minConfidence: 80 } }
],
riskLimits: {
maxDailyLossUsd: 50,
dailyProfitTargetUsd: 100,
maxOpenTrades: 2,
maxConsecutiveLosses: 2
},
execution: {
orderType: 'market',
cooldownMinutes: 30,
minRulePassRatio: 0.7,
entryMode: 'both'
}
}
};
// 3. Insert Profile
// @ts-ignore
const { data, error } = await supabaseService.client
.from('trade_profiles')
.insert([aggressiveProfile])
.select()
.single();
if (error) {
logger.error(`❌ Failed to create profile: ${error.message}`);
} else {
logger.info(`✅ Created Profile: [${aggressiveProfile.name}]`);
logger.info(` - Capital: $${aggressiveProfile.allocated_capital}`);
logger.info(` - Rules: 4 Voting (70%), 2 Mandatory, AI Disabled`);
logger.info(` - Target: $100 Profit Target | $50 Daily Loss Max`);
}
process.exit(0);
}
createAggressiveProfile().catch(console.error);

65
backend/create_low_risk_profile.ts Normal file
View File

@ -0,0 +1,65 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
import { v4 as uuidv4 } from 'uuid';
async function createLowRiskProfile() {
logger.info('🛡️ CREATING LOW RISK & SCALP PROFILES EXAMPLE...');
// 1. Get User
const users = await supabaseService.getActiveUsers();
if (users.length === 0) {
logger.error('❌ No active users found.');
return;
}
const user = users[0];
// 2. Create "Low Risk Swing" Profile
const lowRiskProfile = {
user_id: user.user_id,
name: "Low Risk Swing Trader 🛡️",
allocated_capital: 2000,
risk_per_trade_percent: 0.5, // Ultra Low Risk (0.5%)
symbols: "BTC/USD,ETH/USD", // Major Pairs Only
is_active: true,
strategy_config: {
riskLimits: {
maxDailyLossUsd: 50, // Strict Stop
maxOpenTrades: 2,
maxConsecutiveLosses: 2
},
execution: { orderType: 'limit' }, // Limit Orders for safer entry
rules: [
{ ruleId: 'TrendBiasRule', enabled: true }, // Must be with Trend (EMA200)
{ ruleId: 'ZoneRule', enabled: true }, // Must be near Support Zone
{ ruleId: 'RiskManagementRule', enabled: true }, // ATR Volatility Check
{ ruleId: 'SessionRule', enabled: true } // Only London/NY Sessions
]
}
};
// 3. Create "Aggressive Scalp" Profile (for contrast, if not exists)
// We already renamed the first one to High Risk Scalper, so we just add the Low Risk one.
// Insert Low Risk Profile
// @ts-ignore
const { data, error } = await supabaseService.client
.from('trade_profiles')
.insert([lowRiskProfile])
.select()
.single();
if (error) {
logger.error(`❌ Failed to create profile: ${error.message}`);
} else {
logger.info(`✅ Created Profile: [${lowRiskProfile.name}]`);
logger.info(` - Capital: $${lowRiskProfile.allocated_capital}`);
logger.info(` - Risk: ${lowRiskProfile.risk_per_trade_percent}% per trade`);
logger.info(` - Rules: Trend + Zone + RiskGuard + Session`);
logger.info(` - Strategy: Only trades major trend pullbacks during active sessions.`);
}
process.exit(0);
}
createLowRiskProfile().catch(console.error);

54
backend/debugProfiles.ts Normal file
View File

@ -0,0 +1,54 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import { ConnectorFactory } from '../src/connectors/factory.js';
import { config } from '../src/config/index.js';
import { SymbolMapper } from '../src/utils/symbolMapper.js';
import logger from '../src/utils/logger.js';
async function debug() {
logger.info("--- Debugging Profiles & Positions ---");
const profiles = await supabaseService.getActiveProfiles();
const users = await supabaseService.getActiveUsers();
logger.info(`Found ${profiles.length} active profiles and ${users.length} active users.`);
for (const profile of profiles) {
const user = users.find(u => u.user_id === profile.user_id);
if (!user) continue;
const userKey = config.PAPER_TRADING ? user.ALPACA_API_KEY : user.REAL_ALPACA_API_KEY;
const userSecret = config.PAPER_TRADING ? user.ALPACA_SECRET_KEY : user.REAL_ALPACA_SECRET_KEY;
if (userKey && userSecret) {
try {
const exchange = ConnectorFactory.getCustomConnector(config.EXECUTION_PROVIDER, userKey, userSecret);
for (const symbol of config.SYMBOLS) {
const tradeSymbol = SymbolMapper.toTradeSymbol(symbol, config.EXECUTION_PROVIDER);
const pos = await exchange.getPosition(tradeSymbol);
if (pos) {
logger.info(` ✅ FOUND ${symbol} (${tradeSymbol}) for profile ${profile.name}: ${pos.qty} @ ${pos.avg_entry_price}`);
}
}
} catch (e) { /* ignore */ }
}
}
logger.info("--- Checking MASTER Account (from .env) ---");
const masterKey = config.ALPACA_API_KEY;
const masterSecret = config.ALPACA_API_SECRET;
if (masterKey) {
try {
const exchange = ConnectorFactory.getCustomConnector(config.EXECUTION_PROVIDER, masterKey, masterSecret);
for (const symbol of config.SYMBOLS) {
const tradeSymbol = SymbolMapper.toTradeSymbol(symbol, config.EXECUTION_PROVIDER);
const pos = await exchange.getPosition(tradeSymbol);
if (pos) {
logger.info(` ✅ FOUND ${symbol} (${tradeSymbol}) for MASTER Account: ${pos.qty} @ ${pos.avg_entry_price}`);
}
}
} catch (e: any) {
logger.info(` ❌ Error checking Master Account: ${e.message}`);
}
}
}
debug();

11
backend/debug_config.ts Normal file
View File

@ -0,0 +1,11 @@
import { config } from '../src/config/index.js';
import * as dotenv from 'dotenv';
dotenv.config();
console.log("--- AI CONFIG DEBUG ---");
console.log("AI_PROVIDER:", config.AI.PROVIDER);
console.log("PERPLEXITY_KEY_EXISTS:", !!config.AI.PERPLEXITY_API_KEY);
console.log("OPENAI_KEY_EXISTS:", !!config.AI.OPENAI_API_KEY);
console.log("GEMINI_KEY_EXISTS:", !!config.AI.GEMINI_API_KEY);
console.log("FALLBACK_LIST:", config.AI.FALLBACK_LIST);
console.log("------------------------");

86
backend/debug_db_logging.ts Normal file
View File

@ -0,0 +1,86 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
async function debugDatabaseLogging() {
logger.info('🔍 Starting Database Logging Debug...');
// 1. Get Real User & Profile
const profiles = await supabaseService.getActiveProfiles();
if (profiles.length === 0) {
logger.error('No profiles found to test with.');
return;
}
const profile = profiles[0];
const user = profile.users;
logger.info(`👤 Testing with Profile: ${profile.name} (${profile.id})`);
logger.info(`👤 Testing with User: ${user.email} (${user.user_id})`);
// 2. Test Order Logging
const testOrder = {
user_id: user.user_id,
profile_id: profile.id, // Explicitly testing this link
order_id: `test-order-${Date.now()}`,
symbol: 'BTC/USD',
type: 'market',
side: 'buy',
qty: 0.1,
price: 50000,
status: 'filled',
timestamp: Date.now()
};
logger.info('📝 Attempting to insert Order:', testOrder);
// Call private client directly to get full error if service swallows it
// @ts-ignore
const { data: orderData, error: orderError } = await supabaseService.client
.from('orders')
.insert([testOrder])
.select();
if (orderError) {
logger.error('❌ ORDER LOGGING FAILED:');
logger.error(JSON.stringify(orderError, null, 2));
} else {
logger.info('✅ ORDER LOGGING SUCCESS:', orderData);
}
// 3. Test Trade History Logging
const testTrade = {
user_id: user.user_id,
profile_id: profile.id,
symbol: 'BTC/USD',
side: 'BUY',
entry_price: 50000,
exit_price: 55000,
size: 0.1,
pnl: 500,
pnl_percent: 10,
reason: 'Debug Test',
timestamp: Date.now()
};
logger.info('📝 Attempting to insert Trade History:', testTrade);
// @ts-ignore
const { data: tradeData, error: tradeError } = await supabaseService.client
.from('trade_history')
.insert([testTrade])
.select();
if (tradeError) {
logger.error('❌ TRADE LOGGING FAILED:');
logger.error(JSON.stringify(tradeError, null, 2));
} else {
logger.info('✅ TRADE LOGGING SUCCESS:', tradeData);
// Cleanup (Consistency)
logger.info('🧹 Cleaning up test data...');
// @ts-ignore
await supabaseService.client.from('orders').delete().eq('order_id', testOrder.order_id);
}
}
debugDatabaseLogging();

14
backend/debug_mode.ts Normal file
View File

@ -0,0 +1,14 @@
import axios from 'axios';
async function checkMode() {
try {
const response = await axios.get('http://localhost:5000/api/status');
console.log('Execution Mode:', response.data.settings?.executionMode);
console.log('Is Algo Enabled:', response.data.settings?.isAlgoEnabled);
console.log('Bot Active:', response.data.settings?.botActive);
} catch (error) {
console.error('Error fetching status:', error.message);
}
}
checkMode();

35
backend/debug_supabase.ts Normal file
View File

@ -0,0 +1,35 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function testDirectInsert() {
console.log('Testing Direct Insert...');
console.log('URL:', process.env.SUPABASE_URL);
console.log('Key (last 5):', process.env.SUPABASE_KEY?.slice(-5));
const { data, error } = await supabase
.from('orders')
.insert([{
user_id: '88a2446e-740f-4c87-a94c-fad0ee5167ba',
symbol: 'DEBUG/TEST',
type: 'Market',
side: 'buy',
qty: 1,
price: 100,
status: 'Filled',
timestamp: Date.now()
}])
.select();
if (error) {
console.error('❌ Insert Error:', error);
} else {
console.log('✅ Insert Success:', data);
}
}
testDirectInsert();

26
backend/diagnose_profiles.ts Normal file
View File

@ -0,0 +1,26 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
async function checkProfiles() {
console.log('--- 🛡️ PROFILE DIAGNOSTIC ---');
const profiles = await supabaseService.getActiveProfiles();
if (profiles.length === 0) {
console.log('❌ NO ACTIVE PROFILES FOUND.');
const users = await supabaseService.getActiveUsers();
console.log(`Note: ${users.length} users have trade_enable=true, but no active profiles linked to them.`);
} else {
console.log(`✅ FOUND ${profiles.length} ACTIVE PROFILES:`);
profiles.forEach(p => {
console.log(`- Profile: ${p.name} (ID: ${p.id})`);
console.log(` User: ${p.users?.email} (ID: ${p.user_id})`);
console.log(` Symbols: ${p.symbols || 'ALL'}`);
console.log(` Capital: $${p.allocated_capital} | Risk: ${p.risk_per_trade_percent}%`);
console.log(` Global Trade Enable: ${p.users?.trade_enable}`);
console.log('---');
});
}
}
checkProfiles();

30
backend/docker-compose.yml Normal file
View File

@ -0,0 +1,30 @@
services:
prometheus:
image: prom/prometheus:latest
container_name: prometheus
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
networks:
- monitoring
grafana:
image: grafana/grafana:latest
container_name: grafana
ports:
- "3001:3000"
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_SERVER_DOMAIN=${GRAFANA_DOMAIN:-localhost}
- GF_SERVER_ROOT_URL=https://${GRAFANA_DOMAIN:-localhost}/
networks:
- monitoring
depends_on:
- prometheus
networks:
monitoring:
driver: bridge

28
backend/dump_db.ts Normal file
View File

@ -0,0 +1,28 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function dumpTable(table: string) {
console.log(`\n--- ${table} ---`);
const { data, error } = await supabase.from(table).select('*').order('created_at', { ascending: false }).limit(5);
if (error) console.error(error);
else console.log(JSON.stringify(data, null, 2));
}
async function listUsers() {
console.log('\n--- Users ---');
const { data, error } = await supabase.from('users').select('user_id, email, trade_enable');
if (error) console.error(error);
else console.log(JSON.stringify(data, null, 2));
}
async function main() {
await listUsers();
await dumpTable('orders');
await dumpTable('trade_history');
}
main();

40
backend/dump_recent.ts Normal file
View File

@ -0,0 +1,40 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
async function dumpRecent() {
console.log('--- Dumping Last 10 Orders ---');
const { data: orders, error } = await supabase
.from('orders')
.select('*')
.order('timestamp', { ascending: false })
.limit(10);
if (error) {
console.error('Error:', error);
} else {
orders.forEach(o => {
console.log(`[Order] ID: ${o.order_id || o.id}, User: ${o.user_id}, Symbol: ${o.symbol}, Price: ${o.price}`);
});
}
console.log('\n--- Dumping Last 10 Trade History ---');
const { data: history, error: hErr } = await supabase
.from('trade_history')
.select('*')
.order('timestamp', { ascending: false })
.limit(10);
if (hErr) {
console.error('Error:', hErr);
} else {
history.forEach(h => {
console.log(`[History] User: ${h.user_id}, Symbol: ${h.symbol}, P&L: ${h.pnl}, Reason: ${h.reason}`);
});
}
}
dumpRecent();

181
backend/e2e_full_scenario.ts Normal file
View File

@ -0,0 +1,181 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import { AutoTrader } from '../src/services/AutoTrader.js';
import { TradeExecutor } from '../src/services/TradeExecutor.js';
import { AlpacaConnector } from '../src/connectors/alpaca.js';
import { config } from '../src/config/index.js';
import { SignalDirection, MarketContext, RuleResult, StrategyAnalysisResult } from '../src/strategies/rules/types.js';
import logger from '../src/utils/logger.js';
// Setup environment and global config overrides for testing
config.ENABLE_TRADING = true;
// Force Paper Trading for safety
config.PAPER_TRADING = true;
const TEST_SYMBOL = 'BTC/USD'; // Alpaca Paper usually supports this for crypto
const SLEEP_MS = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
async function runFullE2ETest() {
logger.info('===================================================');
logger.info('🚀 STARTING END-TO-END BACKEND SYSTEM TEST');
logger.info(' Scope: Configs -> Signal -> Order -> Position -> DB -> Exit -> PnL');
logger.info('===================================================');
// 1. Load Profiles (Simulating "Multiple Configurations")
logger.info('\n📡 1. Loading Configurations from Database...');
const profiles = await supabaseService.getActiveProfiles();
if (profiles.length === 0) {
logger.error('❌ No active profiles found in DB. Please create a Strategy Cluster in the Dashboard first.');
process.exit(1);
}
logger.info(`✅ Found ${profiles.length} Active Profiles.`);
// 2. Iterate Profiles and Execute Test Cycle
for (const profile of profiles) {
logger.info(`\n---------------------------------------------------`);
logger.info(`👤 Testing Profile: [${profile.name}] (ID: ${profile.id})`);
logger.info(` Allocated Capital: $${profile.allocated_capital}`);
logger.info(` Risk Per Trade: ${profile.risk_per_trade_percent}%`);
const user = profile.users;
if (!user) {
logger.warn(' ⚠️ User data missing for this profile. Skipping.');
continue;
}
// Initialize Services for this specific User/Profile
const userKey = config.PAPER_TRADING ? user.ALPACA_API_KEY : user.REAL_ALPACA_API_KEY;
const userSecret = config.PAPER_TRADING ? user.ALPACA_SECRET_KEY : user.REAL_ALPACA_SECRET_KEY;
if (!userKey || !userSecret) {
logger.warn(' ⚠️ Alpaca Credentials missing. Skipping.');
continue;
}
const exchange = new AlpacaConnector(userKey, userSecret);
const executor = new TradeExecutor(exchange, undefined, user.user_id, profile.id);
const trader = new AutoTrader(executor, exchange, profile);
// SYNC: Ensure we start clean
logger.info(' 🔄 Syncing existing positions...');
await executor.syncPositions([TEST_SYMBOL]);
// Close any existing position on Test Symbol to ensure clean test
const existingPos = executor.getActivePosition(TEST_SYMBOL);
if (existingPos) {
logger.info(` 🧹 Closing pre-existing position on ${TEST_SYMBOL}...`);
await executor.closePosition(TEST_SYMBOL, 'E2E PRE-CLEANUP');
await SLEEP_MS(5000);
}
// 3. Simulate Market Context
const currentPrice = 65000;
const mockContext: MarketContext = {
symbol: TEST_SYMBOL,
currentPrice: currentPrice,
candles1h: [], candles15m: [], candles4h: [],
rsi_1h: 30, // Oversold
ema20_1h: 64000,
change24h: 1.5,
changeToday: 0.5,
volatility: 'Low',
session: 'NY',
isMajorSession: true,
latestSignal: SignalDirection.NONE
};
// 4. Simulate BUY Signal
logger.info(' 🟢 Simulating BUY Signal (Trend + Momentum)...');
// Construct Analysis Result that passes "Logic"
// We ensure a 'generic' passing rule if the profile has specific rules, or rely on global if none.
const ruleResults: Record<string, RuleResult> = {};
// Mock common rules to PASS
const rulesToMock = ['TrendBiasRule', 'MomentumRule', 'ZoneRule', 'RiskManagementRule'];
rulesToMock.forEach(rId => {
ruleResults[rId] = { ruleName: rId, passed: true, signal: SignalDirection.BUY, reason: 'Test Pass', metadata: {} };
});
const buyAnalysis: StrategyAnalysisResult = {
symbol: TEST_SYMBOL,
globalSignal: SignalDirection.BUY,
rules: ruleResults,
context: mockContext
};
await trader.handleSignal(TEST_SYMBOL, buyAnalysis);
// 5. Verify Order & Position Creation
logger.info(' ⏳ Waiting for Order Fill & DB Sync (10s)...');
await SLEEP_MS(10000); // Wait for API calls and DB events
const activePos = executor.getActivePosition(TEST_SYMBOL);
if (!activePos) {
logger.error(` ❌ FAIL: Position not created for ${profile.name}`);
continue; // Skip to next profile
}
logger.info(` ✅ SUCCESS: Position Created!`);
logger.info(` Qty: ${activePos.size}`);
logger.info(` Entry: ${activePos.entryPrice}`);
logger.info(` Initial U.PnL: ${activePos.unrealizedPnl}`);
// Check DB for Order
const latestOrder = await supabaseService.getLatestOrder(user.user_id, TEST_SYMBOL);
if (latestOrder && latestOrder.side === 'buy' && latestOrder.status === 'filled') {
logger.info(` ✅ DB VERIFIED: Buy Order ${latestOrder.order_id} recorded.`);
} else {
logger.warn(` ⚠️ DB WARNING: Could not find recent filled buy order.`);
}
// 6. Simulate Profit Update (Price Move Up)
logger.info(' 📈 Simulating Price Move (+10%)...');
const newPrice = currentPrice * 1.10;
mockContext.currentPrice = newPrice;
// 7. Simulate SELL/EXIT Signal
logger.info(' 🔴 Simulating SELL/EXIT Signal (Trend Reversal)...');
const sellAnalysis: StrategyAnalysisResult = {
symbol: TEST_SYMBOL,
globalSignal: SignalDirection.SELL, // Reversal triggers exit
rules: ruleResults,
context: mockContext
};
await trader.handleSignal(TEST_SYMBOL, sellAnalysis);
// 8. Verify Exit & PnL Realization
logger.info(' ⏳ Waiting for Close & Profit Realization (10s)...');
await SLEEP_MS(10000);
const closedPos = executor.getActivePosition(TEST_SYMBOL);
if (!closedPos) {
logger.info(` ✅ SUCCESS: Position Closed.`);
} else {
logger.error(` ❌ FAIL: Position still active!`);
}
// Check DB for Trade History (PnL)
// We verify the 'logTransaction' was called
// Since we don't have a direct 'getLatestTransaction', we infer success from logs or check orders
const sellOrder = await supabaseService.getLatestOrder(user.user_id, TEST_SYMBOL);
if (sellOrder && sellOrder.side === 'sell' && sellOrder.status === 'filled') {
logger.info(` ✅ DB VERIFIED: Sell Order ${sellOrder.order_id} recorded.`);
// In a real scenario, we'd query trade_history too, but logged orders confirms the loop.
}
logger.info(` 🎉 Profile [${profile.name}] Test Cycle Complete.`);
}
logger.info('\n===================================================');
logger.info('✅ E2E TEST SUITE COMPLETED');
logger.info('===================================================');
process.exit(0);
}
runFullE2ETest().catch(err => {
logger.error('CRITICAL TEST FAILURE:', err);
process.exit(1);
});

70
backend/example_profile_config.json Normal file
View File

@ -0,0 +1,70 @@
{
"rules": [
{
"ruleId": "TrendBiasRule",
"enabled": true,
"params": {
"timeframe": "4h",
"emaFast": 50,
"emaSlow": 200
}
},
{
"ruleId": "SessionRule",
"enabled": true,
"params": {
"allowedSessions": ["NY", "LDN"]
}
},
{
"ruleId": "ZoneRule",
"enabled": true,
"params": {
"emaPeriod": 20,
"tolerancePercent": 0.5
}
},
{
"ruleId": "MomentumRule",
"enabled": true,
"params": {
"timeframe": "1h",
"rsiPeriod": 14,
"rsiOverbought": 70,
"rsiOversold": 30
}
},
{
"ruleId": "EntryTriggerRule",
"enabled": true,
"params": {
"triggerType": "ema_cross"
}
},
{
"ruleId": "RiskManagementRule",
"enabled": true,
"params": {
"atrPeriod": 14,
"riskRewardRatio": 1.5
}
},
{
"ruleId": "AIAnalysisRule",
"enabled": false,
"params": {
"minConfidence": 80
}
}
],
"riskLimits": {
"maxDailyLossUsd": 50,
"maxConsecutiveLosses": 2,
"maxOpenTrades": 3
},
"execution": {
"orderType": "market",
"cooldownMinutes": 30,
"entryMode": "both"
}
}

96
backend/final_e2e_param_verification.ts Normal file
View File

@ -0,0 +1,96 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import { AutoTrader } from '../src/services/AutoTrader.js';
import { TradeExecutor } from '../src/services/TradeExecutor.js';
import { AlpacaConnector } from '../src/connectors/alpaca.js';
import { ProStrategyEngine } from '../src/strategies/ProStrategyEngine.js';
import { config } from '../src/config/index.js';
import { SignalDirection, MarketContext } from '../src/strategies/rules/types.js';
import logger from '../src/utils/logger.js';
config.ENABLE_TRADING = true;
config.PAPER_TRADING = true;
const SLEEP_MS = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
async function runParamVerification() {
logger.info('===================================================');
logger.info('🧪 STARTING PARAMETER-AWARE E2E VERIFICATION');
logger.info(' Testing contrast between High Risk and Low Risk settings.');
logger.info('===================================================');
// 1. Fetch our two distinct profiles
const profiles = await supabaseService.getActiveProfiles();
const highRisk = profiles.find(p => p.name.includes('High Risk'));
const lowRisk = profiles.find(p => p.name.includes('Low Risk'));
if (!highRisk || !lowRisk) {
logger.error('❌ Could not find both profiles (Scalper and Swing). Run create scripts first.');
process.exit(1);
}
const user = highRisk.users; // Both belong to same user in our setup
const exchange = new AlpacaConnector(user.ALPACA_API_KEY, user.ALPACA_SECRET_KEY);
const engine = new ProStrategyEngine(exchange);
// 2. Setup Traders
const executorHigh = new TradeExecutor(exchange, undefined, user.user_id, highRisk.id);
const traderHigh = new AutoTrader(executorHigh, exchange, highRisk, engine);
const executorLow = new TradeExecutor(exchange, undefined, user.user_id, lowRisk.id);
const traderLow = new AutoTrader(executorLow, exchange, lowRisk, engine);
// 3. Define Market Context with RSI = 75
// Scenario: RSI is 75.
// Low Risk Swing logic (default RSI 70) -> Should Trigger SELL
// High Risk Scalper logic (custom RSI 80) -> Should NOT Trigger (75 < 80)
const mockCandles = Array(100).fill({ close: 65000, high: 65100, low: 64900, open: 65000, volume: 100 });
const mockContext: MarketContext = {
symbol: 'BTC/USD',
currentPrice: 65000,
candles1h: mockCandles,
candles15m: mockCandles,
candles4h: mockCandles,
rsi_1h: 75, // THE MAGIC NUMBER
ema20_1h: 64000,
ema50_4h: 60000,
ema200_4h: 55000,
session: 'NY', isMajorSession: true,
volatility: 'Med', change24h: 1, changeToday: 0.5,
latestSignal: SignalDirection.NONE
};
const analysis: any = {
symbol: 'BTC/USD',
globalSignal: SignalDirection.NONE,
rules: {}, // Global results don't matter now as we re-evaluate
context: mockContext
};
logger.info('\n📊 Scenario: RSI is 75');
logger.info(` Profile [Low Risk]: Threshold 70 -> Expected: PASS`);
logger.info(` Profile [High Risk]: Threshold 80 -> Expected: FAIL`);
// 4. Test Execution
logger.info('\n⚡ Testing [Low Risk Swing]...');
await traderLow.handleSignal('BTC/USD', analysis);
logger.info('⚡ Testing [High Risk Scalper]...');
await traderHigh.handleSignal('BTC/USD', analysis);
// 5. Verification
logger.info('\n🏁 RESULTS:');
// We can't easily wait for orders in this fake test without mocking exchange,
// but we can look at the logs to see where "EXECUTING" appeared.
logger.info('===================================================');
logger.info('✅ VERIFICATION COMPLETE');
logger.info(' Please check the logs above for "EXECUTING" vs "Blocked" messages.');
logger.info('===================================================');
process.exit(0);
}
runParamVerification().catch(err => {
logger.error('Test Failed:', err);
process.exit(1);
});

45
backend/fix_dashboard_rules.ts Normal file
View File

@ -0,0 +1,45 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
async function fixHighRiskRules() {
logger.info('🔧 Syncing High Risk Rules with Dashboard UI...');
const profiles = await supabaseService.getActiveProfiles();
const highRiskProfile = profiles.find(p => p.name.includes('Scalp'));
if (!highRiskProfile) {
logger.error('❌ High Risk Profile not found.');
return;
}
// Update with Dashboard-compatible Rule IDs
const newRules = [
{ ruleId: 'MomentumRule', enabled: true }, // Aggressive Momentum (RSI)
{ ruleId: 'EntryTriggerRule', enabled: true }, // Pattern Trigger
{ ruleId: 'AIAnalysisRule', enabled: true } // AI Confirmation
];
// @ts-ignore
const { error } = await supabaseService.client
.from('trade_profiles')
.update({
strategy_config: {
...highRiskProfile.strategy_config,
rules: newRules
}
})
.eq('id', highRiskProfile.id);
if (error) {
logger.error(`❌ Failed: ${error.message}`);
} else {
logger.info(`✅ Rules Updated for [${highRiskProfile.name}]`);
logger.info(` - MomentumRule: ON`);
logger.info(` - EntryTriggerRule: ON`);
logger.info(` - AIAnalysisRule: ON`);
logger.info(` (Refresh Dashboard to see them selected)`);
}
}
fixHighRiskRules();

52
backend/fix_imports.ts Normal file
View File

@ -0,0 +1,52 @@
import fs from 'fs';
import path from 'path';
const testsDir = path.resolve(process.cwd(), 'tests');
const replacements = [
{ from: "from './strategies", to: "from '../src/strategies" },
{ from: "from './connectors", to: "from '../src/connectors" },
{ from: "from './config", to: "from '../src/config" },
{ from: "from './utils", to: "from '../src/utils" },
{ from: "from './services", to: "from '../src/services" },
{ from: "from './src/", to: "from '../src/" },
// Handle double quotes too if any
{ from: 'from "./strategies', to: 'from "../src/strategies' },
{ from: 'from "./connectors', to: 'from "../src/connectors' },
{ from: 'from "./config', to: 'from "../src/config' },
{ from: 'from "./utils', to: 'from "../src/utils' },
{ from: 'from "./services', to: 'from "../src/services' },
{ from: 'from "./src/', to: 'from "../src/' },
];
async function fixImports() {
console.log(`Scanning ${testsDir}...`);
if (!fs.existsSync(testsDir)) {
console.error('Tests directory not found!');
return;
}
const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.ts'));
for (const file of files) {
const filePath = path.join(testsDir, file);
let content = fs.readFileSync(filePath, 'utf8');
let changed = false;
for (const rep of replacements) {
if (content.includes(rep.from)) {
content = content.replaceAll(rep.from, rep.to);
changed = true;
}
}
if (changed) {
fs.writeFileSync(filePath, content);
console.log(`✅ Fixed imports in ${file}`);
} else {
console.log(`⚪ No changes in ${file}`);
}
}
}
fixImports();

44
backend/force_rules_reset.ts Normal file
View File

@ -0,0 +1,44 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
async function forceResetRules() {
logger.info('🔨 FORCING CLEAN RULE SET (ALL RULES POPULATED)...');
const profiles = await supabaseService.getActiveProfiles();
const highRiskStart = profiles.find(p => p.name.includes('Scalp') || p.name.includes('High'));
if (!highRiskStart) return logger.error('❌ Profile not found');
const newConfig = {
execution: { allowedSymbols: ["BTC/USD"], orderType: "market" },
riskLimits: { maxOpenTrades: 5, maxDailyLossUsd: 200 },
rules: [
{ ruleId: "MomentumRule", enabled: true, params: { rsiPeriod: 14, overbought: 80, oversold: 20, timeframe: '15m' } },
{ ruleId: "EntryTriggerRule", enabled: true, params: { showPatterns: true } },
{ ruleId: "AIAnalysisRule", enabled: true, params: { minConfidence: 0.8 } },
// Explicitly include disabled ones to ensure structure exists
{ ruleId: "TrendBiasRule", enabled: false, params: { fastPeriod: 50, slowPeriod: 200 } },
{ ruleId: "ZoneRule", enabled: false, params: { zonePercent: 1.5 } },
{ ruleId: "SessionRule", enabled: false, params: { sessions: "London,NY" } },
{ ruleId: "RiskManagementRule", enabled: true, params: { maxRisk: 5.0 } } // Also enabled
]
};
// @ts-ignore
const { error } = await supabaseService.client
.from('trade_profiles')
.update({ strategy_config: newConfig })
.eq('id', highRiskStart.id);
if (error) {
logger.error(`❌ Failed: ${error.message}`);
} else {
logger.info(`✅ SUCCESS: Overwrote entire strategy_config for [${highRiskStart.name}]`);
logger.info(` - Includes 7 rules (4 Enabled, 3 Disabled)`);
logger.info(` - Includes default params for all`);
}
}
forceResetRules();

8
backend/grafana/provisioning/datasources/datasource.yml Normal file
View File

@ -0,0 +1,8 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
url: http://prometheus:9090
isDefault: true

30
backend/inspect_btc.ts Normal file
View File

@ -0,0 +1,30 @@
import fs from 'fs';
const state = JSON.parse(fs.readFileSync('bot_state.json', 'utf8'));
const btc = state.symbols['BTC/USDT'];
console.log('--- BTC/USDT LIVE STATE ---');
console.log(`Current Price: $${btc.price}`);
console.log(`Current Signal: ${btc.signal}`);
console.log('\n--- Indicators ---');
console.log(`RSI (1H): ${btc.indicators.rsi_1h.toFixed(2)}`);
console.log(`EMA20 (1H): $${btc.indicators.ema20_1h.toFixed(2)}`);
console.log('\n--- Rule Breakdown ---');
const rules = btc.rules;
if (rules) {
Object.entries(rules).forEach(([name, data]: [string, any]) => {
console.log(`[${name}] Passed: ${data.passed} | Reason: ${data.reason}`);
});
} else {
console.log('No rules found in state.');
}
// Check recent price history in state (last 5 entries)
const history = btc.priceHistory || [];
const recent = history.slice(-5);
console.log('\n--- Recent Prices (v. Recent) ---');
recent.forEach((h: any) => {
console.log(` ${new Date(h.timestamp).toLocaleTimeString()}: $${h.price}`);
});

59
backend/invariants.md Normal file
View File

@ -0,0 +1,59 @@
Bytelyst Trading Platform Invariants
## Capital Ledger Invariant
- **Invariant name:** Capital non-negativity
- **Description:** For every profile, allocated - reserved_for_orders - reserved_for_positions + realized_pnl >= 0 at all times.
- **Enforced by:** Capital ledger RPCs, ledger schema constraints, and reconciliation handlers that re-sync reservations.
- **Detection:** Capital invariant metric increments and health endpoint reports violation when ledger calculation dips below zero; observability logs critical entries.
- **Recovery:** Capital watchdog logs the violation, trading loop halts new entries for that profile, reconciliation replays state, and operators follow docs/runbooks/invariant-violation.md.
## Single ENTRY Lock Invariant
- **Invariant name:** One ENTRY per profile/symbol
- **Description:** At most one ENTRY signal can progress per (profile_id, symbol) at a time, preventing double submissions.
- **Enforced by:** Row-based entry_locks and deterministic clientOrderId, with lock TTL enforcement in the lock service code.
- **Detection:** Lock contention counter increments and health endpoint flags stale locks when TTL expires without release.
- **Recovery:** Contention metrics surface to Prometheus; the losing worker retries after TTL and operators consult docs/runbooks/lock-timeout.md if contention persists.
## Duplicate Exchange Order Invariant
- **Invariant name:** No duplicate exchange order
- **Description:** A single trade_id can only produce one exchange order; retries never issue another order to the exchange.
- **Enforced by:** Deterministic clientOrderId strategy and transactional lifecycle RPCs that guard on matching trade_id/order_id.
- **Detection:** Exchange connectors translate duplicate-order errors into existing lifecycle lookups; observability logs capture the repeat attempt.
- **Recovery:** Retried persistence fetches existing order_id; reconciliation ensures DB state matches exchange; operators are guided by docs/runbooks/reconciliation.md if duplicates surface.
## Lifecycle Atomicity Invariant
- **Invariant name:** Lifecycle persistence atomicity
- **Description:** ENTRY and EXIT lifecycle rows, order rows, positions, and history slices are inserted or rolled back as a single atomic operation.
- **Enforced by:** Supabase fn_persist_entry_lifecycle and exit RPC transactions with UNIQUE constraints (trade_lifecycle(profile_id, trade_id), orders(order_id)).
- **Detection:** DB transaction failure logs surface and rollback leaves no partial data; reconciliation detects orphan orders or missing lifecycle slices.
- **Recovery:** Retry replays through RPCs; reconciliation invokes lifecycle handlers to rebuild missing artifacts; runbooks refer to docs/runbooks/lifecycle-incident.md.
## Exchange-as-Source-of-Truth Invariant
- **Invariant name:** Exchange truth first
- **Description:** No lifecycle row exists or is updated until the exchange confirms the order or fill status.
- **Enforced by:** Entry/exit flows that place exchange orders before calling persistence RPCs plus reconciliation that corrects DB drift.
- **Detection:** Reconciliation mismatch counters increment when DB differs from exchange; health endpoint flags missing trades.
- **Recovery:** Lifecycle handlers repair DB state based on exchange data; trading loop refrains from acting on stale state; ops follow docs/runbooks/reconciliation.md.
## Idempotent Retry Invariant
- **Invariant name:** Idempotent lifecycle retries
- **Description:** Retrying the same lifecycle persistence request never creates duplicates nor mutates unintended rows.
- **Enforced by:** RPC idempotency keys, UNIQUE(profile_id, trade_id), ON CONFLICT DO NOTHING on child inserts, and deterministic clientOrderId.
- **Detection:** RPC responses include existing lifecycle references; duplicate insertion attempts are logged without causing errors.
- **Recovery:** Retry safely returns the existing lifecycle; metrics note idempotency hits, and operators refer to docs/runbooks/lifecycle-incident.md only if insert counts grow unusually.
## Lock TTL Safety Invariant
- **Invariant name:** Lock TTL safety
- **Description:** Distributed locks expire automatically if a worker crashes, preventing deadlocks.
- **Enforced by:** Lock acquisition RPCs that set expires_at = now() + TTL and lock release RPCs requiring owner authentication.
- **Detection:** Lock contention metrics, TTL expiration audit logs, and /internal/health lockContentionCount track stuck entries.
- **Recovery:** TTL expiry frees the lock automatically, reconciliation/self-healing loops resume; operators only intervene when contention spikes, per docs/runbooks/lock-timeout.md.
## Invariants Enforcement Summary
- DB-enforced invariants: lifecycle atomicity (UNIQUE constraints), capital ledger constraints, lock row TTL persistence.
- Code-enforced invariants: entry lock acquisition, deterministic clientOrderId, reconciliation routing through lifecycle handlers.
- Observability-enforced invariants: capital invariant metric, lock contention counters, reconciliation mismatch counts that trigger alerts when thresholds are crossed.
## Rules future agents MUST NOT break
Future agents must never touch lifecycle RPCs, ledger services, reconciliation logic, or locking mechanisms without confirming that the invariants listed above remain intact. Breaking capital, lock, lifecycle, or exchange-truth invariants without operator consent is forbidden.

27
backend/list_history.ts Normal file
View File

@ -0,0 +1,27 @@
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
const userId = '8d5efd9e-0760-4859-8c07-0930ab3ede5a';
async function listAllHistory() {
console.log(`--- Trade History for ${userId} ---`);
const { data, error } = await supabase
.from('trade_history')
.select('*')
.eq('user_id', userId)
.order('created_at', { ascending: false });
if (error) {
console.error('Error:', error);
} else {
data.forEach(h => {
console.log(`[History] Time: ${h.created_at}, Symbol: ${h.symbol}, Reason: ${h.reason}, P&L: ${h.pnl}`);
});
}
}
listAllHistory();

23
backend/live_signal_check.js Normal file
View File

@ -0,0 +1,23 @@
const http = require('http');
http.get('http://localhost:5000/api/status', (res) => {
let data = '';
res.on('data', (chunk) => { data += chunk; });
res.on('end', () => {
try {
const botState = JSON.parse(data);
console.log('--- LIVE BOT ANALYSIS STATUS ---');
Object.entries(botState.symbols).forEach(([symbol, details]) => {
console.log(`📍 ${symbol}: Signal=${details.signal}`);
Object.entries(details.rules).forEach(([rule, res]) => {
const statusIcon = res.passed ? '✅' : '❌';
console.log(` ${statusIcon} ${rule}: ${res.reason}`);
});
});
} catch (e) {
console.error('Error parsing JSON:', e.message);
}
});
}).on('error', (err) => {
console.error('Error fetching status:', err.message);
});

297
backend/manualOverrideCloseTrades.ts Normal file
View File

@ -0,0 +1,297 @@
import 'dotenv/config';
import { createHash, randomUUID } from 'crypto';
import { config, loadDynamicConfig } from '../src/config/index.js';
import { normalizeOrderAction, normalizeTradeSide } from '../src/domain/tradingEnums.js';
import { healthTracker } from '../src/services/healthTracker.js';
import {
ReconciliationBackfillAuditInsert,
ReconciliationBackfillOrderInsert,
supabaseService
} from '../src/services/SupabaseService.js';
import { buildAlpacaSubTag } from '../src/utils/alpacaSubTag.js';
type CliOptions = {
apply: boolean;
tradeIds: string[];
};
type TradeSnapshot = {
profileId: string;
userId: string;
tradeId: string;
symbol: string;
entrySide: 'BUY' | 'SELL';
entryQty: number;
exitQty: number;
openQty: number;
entryAvgPrice: number;
};
const EPSILON = 1e-8;
const ORDER_ID_PREFIX = 'MANOVR';
const parseOptions = (argv: string[]): CliOptions => {
const tradeIds = new Set<string>();
let apply = false;
for (const arg of argv) {
if (arg === '--apply') {
apply = true;
continue;
}
if (arg.startsWith('--trade=')) {
const value = String(arg.slice('--trade='.length) || '').trim();
if (value) tradeIds.add(value);
}
}
return {
apply,
tradeIds: Array.from(tradeIds)
};
};
const toNumber = (value: unknown): number => {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : 0;
};
const expectedExitSide = (entrySide: 'BUY' | 'SELL'): 'BUY' | 'SELL' => {
return entrySide === 'BUY' ? 'SELL' : 'BUY';
};
const buildManualOverrideOrderId = (profileId: string, tradeId: string): string => {
const digest = createHash('md5')
.update(`${profileId}:${tradeId}:manual_override_v1`)
.digest('hex');
return `${ORDER_ID_PREFIX}-${digest}`;
};
const buildTradeSnapshots = (rows: any[]): Map<string, TradeSnapshot> => {
const byTrade = new Map<string, TradeSnapshot>();
for (const row of rows || []) {
const tradeId = String(row.trade_id || '').trim();
const profileId = String(row.profile_id || '').trim();
if (!tradeId || !profileId) continue;
const key = `${profileId}::${tradeId}`;
const qty = toNumber(row.qty ?? row.quantity);
if (!(qty > EPSILON)) continue;
const side = normalizeTradeSide(String(row.side || 'BUY'));
const action = normalizeOrderAction(row.action || undefined);
const symbol = String(row.symbol || '').trim();
const userId = String(row.user_id || '').trim();
const price = toNumber(row.price);
let snapshot = byTrade.get(key);
if (!snapshot) {
snapshot = {
profileId,
userId,
tradeId,
symbol,
entrySide: side,
entryQty: 0,
exitQty: 0,
openQty: 0,
entryAvgPrice: 0
};
byTrade.set(key, snapshot);
}
const resolvedAction = action || (side === snapshot.entrySide ? 'ENTRY' : 'EXIT');
if (resolvedAction === 'ENTRY') {
if (!(snapshot.entryQty > EPSILON)) {
snapshot.entrySide = side;
}
const nextQty = snapshot.entryQty + qty;
snapshot.entryAvgPrice = nextQty > EPSILON
? ((snapshot.entryAvgPrice * snapshot.entryQty) + (price * qty)) / nextQty
: snapshot.entryAvgPrice;
snapshot.entryQty = nextQty;
} else {
snapshot.exitQty += qty;
}
if (!snapshot.symbol && symbol) snapshot.symbol = symbol;
if (!snapshot.userId && userId) snapshot.userId = userId;
snapshot.openQty = Number((snapshot.entryQty - snapshot.exitQty).toFixed(8));
}
for (const [key, snapshot] of Array.from(byTrade.entries())) {
if (!(snapshot.openQty > EPSILON)) {
byTrade.delete(key);
}
}
return byTrade;
};
const run = async (): Promise<void> => {
const options = parseOptions(process.argv.slice(2));
if (options.tradeIds.length === 0) {
throw new Error('Provide at least one --trade=<TRADE_ID>.');
}
await loadDynamicConfig(supabaseService);
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'maintenance-script',
lastChangedAt: Date.now(),
reason: 'Manual override close cycle'
});
const client = supabaseService.getClient();
if (!client) {
throw new Error('Supabase client is not available.');
}
const { data: lifecycleRows, error: lifecycleError } = await client
.from('orders')
.select('profile_id,user_id,trade_id,symbol,side,action,qty,quantity,price,status')
.in('trade_id', options.tradeIds)
.in('status', ['filled', 'partially_filled', 'partially-filled']);
if (lifecycleError) {
throw new Error(`Failed to fetch lifecycle rows: ${lifecycleError.message}`);
}
const snapshots = buildTradeSnapshots(lifecycleRows || []);
const batchId = `MANOVR-BATCH-${randomUUID()}`;
const nowIso = new Date().toISOString();
const nowTs = Date.now();
const candidateOrders: ReconciliationBackfillOrderInsert[] = [];
const preAuditRows: ReconciliationBackfillAuditInsert[] = [];
const skipped: Array<Record<string, any>> = [];
for (const tradeId of options.tradeIds) {
const matching = Array.from(snapshots.values()).filter((row) => row.tradeId === tradeId);
if (matching.length === 0) {
skipped.push({ tradeId, reason: 'trade_not_open_or_not_found' });
continue;
}
if (matching.length > 1) {
skipped.push({ tradeId, reason: 'ambiguous_trade_multiple_profiles' });
continue;
}
const trade = matching[0];
if (!(trade.openQty > EPSILON)) {
skipped.push({ tradeId, reason: 'trade_not_open' });
continue;
}
const orderId = buildManualOverrideOrderId(trade.profileId, trade.tradeId);
const side = expectedExitSide(trade.entrySide);
const subTag = buildAlpacaSubTag({
profileId: trade.profileId,
tradeId: trade.tradeId,
intent: 'EXIT'
}) || undefined;
const fillPrice = trade.entryAvgPrice > EPSILON ? trade.entryAvgPrice : 0;
const order: ReconciliationBackfillOrderInsert = {
user_id: trade.userId,
profile_id: trade.profileId,
order_id: orderId,
symbol: trade.symbol,
type: 'market',
side,
qty: Number(trade.openQty.toFixed(8)),
quantity: Number(trade.openQty.toFixed(8)),
price: Number(fillPrice.toFixed(8)),
status: 'filled',
timestamp: nowTs,
filled_at: nowIso,
trade_id: trade.tradeId,
action: 'EXIT',
source: 'BOT',
sub_tag: subTag
};
candidateOrders.push(order);
preAuditRows.push({
batch_id: batchId,
profile_id: trade.profileId,
symbol: trade.symbol,
trade_id: trade.tradeId,
exchange_order_id: null,
exchange_client_order_id: null,
backfill_order_id: orderId,
filled_qty: order.qty,
filled_price: order.price,
filled_at: order.filled_at || null,
dry_run: !options.apply,
decision: options.apply ? 'MANUAL_OVERRIDE_PENDING' : 'MANUAL_OVERRIDE_DRY',
reason: 'manual_override_user_approved_no_exchange_evidence',
metadata: {
openQtyBefore: trade.openQty,
entryQty: trade.entryQty,
exitQty: trade.exitQty,
fillPriceBasis: 'entry_weighted_avg_price'
}
});
}
if (preAuditRows.length > 0) {
const preSaved = await supabaseService.insertReconciliationBackfillAuditRows(preAuditRows);
if (!preSaved) {
throw new Error('Failed to save manual override pre-audit rows.');
}
}
let insertedRows = 0;
if (options.apply && candidateOrders.length > 0) {
const orderIds = candidateOrders.map((row) => row.order_id);
const existingBefore = await supabaseService.getExistingOrderIds(orderIds);
const ok = await supabaseService.upsertReconciliationBackfillOrders(candidateOrders);
if (!ok) {
throw new Error('Failed to apply manual override rows.');
}
const existingAfter = await supabaseService.getExistingOrderIds(orderIds);
insertedRows = candidateOrders.filter((row) => !existingBefore.has(row.order_id) && existingAfter.has(row.order_id)).length;
const postAuditRows: ReconciliationBackfillAuditInsert[] = candidateOrders.map((row) => ({
batch_id: batchId,
profile_id: row.profile_id,
symbol: row.symbol,
trade_id: row.trade_id,
exchange_order_id: null,
exchange_client_order_id: null,
backfill_order_id: row.order_id,
filled_qty: row.qty,
filled_price: row.price,
filled_at: row.filled_at || null,
dry_run: false,
decision: existingBefore.has(row.order_id) ? 'MANUAL_OVERRIDE_SKIP_EXISTING' : 'MANUAL_OVERRIDE_APPLIED',
reason: existingBefore.has(row.order_id) ? 'already_exists' : 'manual_override_inserted',
metadata: {
matchedBy: 'manual_override'
},
applied_at: !existingBefore.has(row.order_id) ? new Date().toISOString() : null
}));
const postSaved = await supabaseService.insertReconciliationBackfillAuditRows(postAuditRows);
if (!postSaved) {
throw new Error('Failed to save manual override post-audit rows.');
}
}
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
batchId,
requestedTrades: options.tradeIds,
proposedRows: candidateOrders.length,
insertedRows,
skipped
}, null, 2));
};
run().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
console.error(JSON.stringify({ error: message }, null, 2));
process.exit(1);
});

218
backend/monitorFreshWindow.ts Normal file
View File

@ -0,0 +1,218 @@
import fs from 'node:fs';
import path from 'node:path';
type MonitorOptions = {
hours: number;
intervalSec: number;
outFile: string;
healthUrl: string;
stateFile: string;
};
type HealthSnapshot = {
tradingLoopHealthy?: boolean;
monitorLoopHealthy?: boolean;
reconciliationLoopHealthy?: boolean;
reconciliationMismatchCount?: number;
reconciliationMissingFromExchange?: number;
reconciliationMissingInDb?: number;
reconciliationNoGoTrades?: number;
reconciliationIntegrityWatchdogTriggered?: boolean;
tradingControl?: {
mode?: string;
lastChangedBy?: string;
lastChangedAt?: number;
reason?: string;
};
};
const toNumber = (value: unknown, fallback: number): number => {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : fallback;
};
const parseArgs = (argv: string[]): MonitorOptions => {
let hours = 12;
let intervalSec = 60;
let outFile = '';
let healthUrl = 'http://127.0.0.1:5000/internal/health';
let stateFile = path.resolve(process.cwd(), 'bot_state.json');
for (const arg of argv) {
if (arg.startsWith('--hours=')) {
hours = Math.max(1, Math.floor(toNumber(arg.slice('--hours='.length), 12)));
continue;
}
if (arg.startsWith('--interval-sec=')) {
intervalSec = Math.max(15, Math.floor(toNumber(arg.slice('--interval-sec='.length), 60)));
continue;
}
if (arg.startsWith('--out=')) {
outFile = String(arg.slice('--out='.length) || '').trim();
continue;
}
if (arg.startsWith('--health-url=')) {
healthUrl = String(arg.slice('--health-url='.length) || '').trim() || healthUrl;
continue;
}
if (arg.startsWith('--state-file=')) {
const candidate = String(arg.slice('--state-file='.length) || '').trim();
if (candidate) stateFile = path.resolve(process.cwd(), candidate);
}
}
if (!outFile) {
const stamp = new Date().toISOString().replace(/[:]/g, '-');
outFile = path.resolve(process.cwd(), `logs/fresh-window-monitor-${stamp}.jsonl`);
} else {
outFile = path.resolve(process.cwd(), outFile);
}
return {
hours,
intervalSec,
outFile,
healthUrl,
stateFile
};
};
const wait = async (ms: number): Promise<void> => {
await new Promise((resolve) => setTimeout(resolve, ms));
};
const readStateEventSummary = (stateFile: string, fromTimestampMs: number): {
operationalEventsCount: number;
latestEventType: string;
latestEventSeverity: string;
latestEventMessage: string;
latestEventAt: number;
} => {
try {
if (!fs.existsSync(stateFile)) {
return {
operationalEventsCount: 0,
latestEventType: '',
latestEventSeverity: '',
latestEventMessage: '',
latestEventAt: 0
};
}
const raw = fs.readFileSync(stateFile, 'utf8');
const parsed = JSON.parse(raw);
const allEvents = Array.isArray(parsed?.operationalEvents) ? parsed.operationalEvents : [];
const events = allEvents.filter((row: any) => {
const ts = Number(row?.timestamp || 0);
return ts >= fromTimestampMs;
});
const latest = events.length > 0 ? events[events.length - 1] : null;
return {
operationalEventsCount: events.length,
latestEventType: String(latest?.type || '').trim(),
latestEventSeverity: String(latest?.severity || '').trim(),
latestEventMessage: String(latest?.message || '').trim(),
latestEventAt: Number(latest?.timestamp || 0) || 0
};
} catch {
return {
operationalEventsCount: 0,
latestEventType: 'STATE_READ_ERROR',
latestEventSeverity: 'WARN',
latestEventMessage: 'Failed to parse bot_state.json',
latestEventAt: Date.now()
};
}
};
const appendJsonLine = (file: string, row: Record<string, unknown>): void => {
const dir = path.dirname(file);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
fs.appendFileSync(file, `${JSON.stringify(row)}\n`, 'utf8');
};
const fetchHealth = async (healthUrl: string): Promise<HealthSnapshot | null> => {
try {
const response = await fetch(healthUrl, { method: 'GET' });
if (!response.ok) return null;
const payload = await response.json();
return payload as HealthSnapshot;
} catch {
return null;
}
};
const run = async (): Promise<void> => {
const options = parseArgs(process.argv.slice(2));
const startedAtMs = Date.now();
const endAtMs = startedAtMs + (options.hours * 60 * 60 * 1000);
appendJsonLine(options.outFile, {
type: 'monitor_started',
at: startedAtMs,
iso: new Date(startedAtMs).toISOString(),
options
});
while (Date.now() < endAtMs) {
const now = Date.now();
const health = await fetchHealth(options.healthUrl);
const stateSummary = readStateEventSummary(options.stateFile, startedAtMs);
const mode = String(health?.tradingControl?.mode || 'UNKNOWN').toUpperCase();
const mismatch = Number(health?.reconciliationMismatchCount || 0);
const missingDb = Number(health?.reconciliationMissingInDb || 0);
const noGo = Number(health?.reconciliationNoGoTrades || 0);
const watchdog = Boolean(health?.reconciliationIntegrityWatchdogTriggered);
const healthyLoops = Boolean(health?.tradingLoopHealthy) && Boolean(health?.monitorLoopHealthy) && Boolean(health?.reconciliationLoopHealthy);
const severity = (mode !== 'RUNNING' || mismatch > 0 || missingDb > 0 || noGo > 0 || watchdog || !healthyLoops) ? 'WARN' : 'INFO';
appendJsonLine(options.outFile, {
type: 'monitor_tick',
severity,
at: now,
iso: new Date(now).toISOString(),
mode,
tradingLoopHealthy: Boolean(health?.tradingLoopHealthy),
monitorLoopHealthy: Boolean(health?.monitorLoopHealthy),
reconciliationLoopHealthy: Boolean(health?.reconciliationLoopHealthy),
reconciliationMismatchCount: mismatch,
reconciliationMissingFromExchange: Number(health?.reconciliationMissingFromExchange || 0),
reconciliationMissingInDb: missingDb,
reconciliationNoGoTrades: noGo,
reconciliationIntegrityWatchdogTriggered: watchdog,
tradingControlChangedBy: String(health?.tradingControl?.lastChangedBy || ''),
tradingControlReason: String(health?.tradingControl?.reason || ''),
operationalEventsCount: stateSummary.operationalEventsCount,
latestEventType: stateSummary.latestEventType,
latestEventSeverity: stateSummary.latestEventSeverity,
latestEventMessage: stateSummary.latestEventMessage,
latestEventAt: stateSummary.latestEventAt
});
await wait(options.intervalSec * 1000);
}
const endedAtMs = Date.now();
appendJsonLine(options.outFile, {
type: 'monitor_finished',
at: endedAtMs,
iso: new Date(endedAtMs).toISOString(),
elapsedSec: Math.floor((endedAtMs - startedAtMs) / 1000)
});
console.log(JSON.stringify({
success: true,
outFile: options.outFile,
startedAt: new Date(startedAtMs).toISOString(),
endedAt: new Date(endedAtMs).toISOString()
}, null, 2));
};
run().catch((error) => {
console.error(JSON.stringify({
success: false,
error: error instanceof Error ? error.message : String(error)
}, null, 2));
process.exit(1);
});

74
backend/package.json Normal file
View File

@ -0,0 +1,74 @@
{
"name": "@bytelyst/trading-backend",
"version": "0.1.0",
"type": "module",
"description": "ByteLyst Trading backend and execution control service",
"main": "index.js",
"scripts": {
"test": "npm run check:websocket-contract && npm run check:session-rule-normalization",
"dev": "tsx src/index.ts",
"build": "tsc",
"typecheck": "tsc --noEmit",
"start": "node dist/index.js",
"check:schema-contract": "tsx verifySchemaContract.ts",
"check:rls-policies": "tsx verifyRlsPolicies.ts",
"check:secret-hygiene": "tsx verifySecretHygiene.ts",
"check:security-guards": "tsx verifySecurityGuards.ts",
"check:tenant-isolation": "tsx verifyTenantIsolation.ts",
"check:trade-executor-lifecycle": "tsx testTradeExecutorLifecycle.ts",
"check:lifecycle-regressions": "tsx testLifecycleRegressions.ts",
"check:order-sync-regressions": "tsx testOrderStatusSyncRegressions.ts",
"check:supabase-order-persistence-regressions": "tsx testSupabaseOrderPersistenceRegressions.ts",
"check:failure-injection": "tsx testFailureInjection.ts",
"check:alpaca-subtag": "tsx testAlpacaSubTag.ts",
"check:strict-capital-guard": "tsx testStrictCapitalGuard.ts",
"check:reconciliation-parity-heartbeat": "tsx testReconciliationParityHeartbeat.ts",
"check:reconciliation-watchdog-auto-resume": "tsx testReconciliationWatchdogAutoResume.ts",
"check:reconciliation-exit-backfill-evidence-guard": "tsx testReconciliationExitBackfillEvidenceGuard.ts",
"check:backtest-isolation": "tsx testBacktestIsolation.ts",
"check:session-rule-normalization": "tsx testSessionRuleNormalization.ts",
"check:websocket-contract": "tsx src/scripts/verifyWebsocketContract.ts",
"coverage:run": "node --loader ts-node/esm runCoverageSuite.ts",
"coverage:full": "npm run coverage:integration",
"coverage:integration": "c8 --all --include=src/**/*.ts --exclude=src/index.ts --exclude=src/scripts/** --reporter=text-summary --reporter=json-summary --reporter=lcov node --loader ts-node/esm runCoverageSuite.ts",
"coverage": "c8 --all --include=src/domain/tradingEnums.ts --include=src/utils/symbolMapper.ts --include=src/connectors/factory.ts --check-coverage --lines 80 --functions 80 --branches 80 --statements 80 --reporter=text-summary --reporter=json-summary --reporter=lcov npx tsx runCriticalCoverageSuite.ts",
"reconcile:lifecycle-history": "node --loader ts-node/esm reconcileTradeHistoryLifecycle.ts",
"reconcile:exit-backfill-once": "node --loader ts-node/esm reconcileExitBackfillOnce.ts",
"reconcile:missing-order-coverage": "node --loader ts-node/esm reconcileMissingOrderCoverage.ts",
"reconcile:closed-order-fill-data": "node --loader ts-node/esm reconcileClosedOrderFillData.ts",
"reconcile:subtag-repair": "node --loader ts-node/esm reconcileSubTagRepair.ts",
"reconcile:attribution-repair": "node --loader ts-node/esm reconcileAttributionRepair.ts",
"reconcile:capital-ledger-state": "node --loader ts-node/esm reconcileCapitalLedgerState.ts",
"lint": "npm run check:schema-contract && npm run check:rls-policies && npm run check:secret-hygiene && npm run check:security-guards && npm run check:tenant-isolation",
"format": "npm run check:trade-executor-lifecycle && npm run check:lifecycle-regressions && npm run check:order-sync-regressions && npm run check:supabase-order-persistence-regressions && npm run check:failure-injection && npm run check:alpaca-subtag && npm run check:strict-capital-guard && npm run check:reconciliation-parity-heartbeat && npm run check:reconciliation-watchdog-auto-resume && npm run check:reconciliation-exit-backfill-evidence-guard && npm run check:backtest-isolation && npm run check:session-rule-normalization && npm run check:websocket-contract",
"check": "npm run build && npm run lint && npm run format",
"pre-deploy": "npm run check",
"cleanup-stale-orders": "tsx src/scripts/cleanupStaleOrders.ts",
"revert-expired-orders": "tsx src/scripts/revertExpiredOrders.ts"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@alpacahq/alpaca-trade-api": "^3.1.3",
"@supabase/supabase-js": "^2.90.1",
"@types/cors": "^2.8.19",
"@types/express": "^5.0.6",
"axios": "^1.13.2",
"ccxt": "^4.5.31",
"cors": "^2.8.5",
"dotenv": "^17.2.3",
"express": "^5.2.1",
"prom-client": "^15.1.3",
"socket.io": "^4.8.3",
"winston": "^3.19.0"
},
"devDependencies": {
"@types/axios": "^0.14.4",
"@types/node": "^25.0.3",
"c8": "^10.1.3",
"ts-node": "^10.9.2",
"tsx": "^4.21.0",
"typescript": "^5.9.3"
}
}

60
backend/phase8-validation.md Normal file
View File

@ -0,0 +1,60 @@
# Phase 8 Enterprise Validation Summary
## 1. Completion Confirmation for Phases 1–7
Each phase below has been validated, the enforcement mechanisms are operational, and failure modes have been documented for operators.
### Phase 1 — Tenant Isolation
- **Validation criteria:** WebSocket broadcasts partitioned by authenticated user, Supabase RLS policies on orders and trade_history, and filtered runtime states tied to user_id.
- **Evidence:** RLS policies, tenant metrics, and failure scenarios covering cross-tenant attempts documented in runbooks.
### Phase 2 — Restart Durability
- **Validation criteria:** Startup routine reloads active profiles, exchange open orders/positions, and rebuilds lifecycle/capital maps deterministically.
- **Evidence:** Log traces of profile reload, ledger rebuild metrics, and restart-recovery runbook coverage.
### Phase 3 — Capital Ledger
- **Validation criteria:** Ledger invariant `allocated - reserved_for_orders - reserved_for_positions + realized_pnl >= 0` holds, and ledger rebuilds on restart while supporting entry, fill, cancel, and exit adjustments.
- **Evidence:** Ledger RPCs, capital-invariant metrics, reconciliation ledger sync, and capital-invariant runbook handling violations.
### Phase 4 — Transactional Lifecycle
- **Validation criteria:** ENTRY/EXIT persistence occurs inside single DB transactions with idempotent constraints, ensuring lifecycle data only appears after exchange confirmation.
- **Evidence:** fn_persist_entry_lifecycle, UNIQUE constraints, lifecycle metrics, and lifecycle-incident runbook.
### Phase 5 — Reconciliation
- **Validation criteria:** Loop compares full DB/exchange sets, obtains profile locks, routes discrepancies through lifecycle handlers, and resets mismatch metrics automatically.
- **Evidence:** reconciliation lock table, mismatch metrics, reconciliation runbook, and failure scenarios for missing/excess orders.
### Phase 6 — Distributed Safety
- **Validation criteria:** Row-based entry locks with TTL and owner tokens combined with deterministic clientOrderId prevent duplicate entries and survive multi-instance deployments.
- **Evidence:** entry_locks table, lock metrics (contention and latency), and lock-contention runbook.
### Phase 7 — Observability & Health
- **Validation criteria:** /metrics and /internal/health track trading, monitor, reconciliation loops alongside lock, capital, and exchange health, permitting rapid detection of failures.
- **Evidence:** health endpoint fields, Prometheus metrics, structured logs, and loop-health runbook.
## 2. Final Checklist Mapping
| Requirement | Enforcement | Detection | Recovery |
|-------------|-------------|-----------|----------|
| Tenant isolation | Supabase RLS, WebSocket filters | RLS audit logs, tenant breach alerts | Reject broadcast, block execution, notify compliance |
| Capital invariant | Ledger RPCs + watcher | `capitalInvariantViolations`, gauge dips | Reconciliation rebuild, trading loop pause for profile |
| Lifecycle atomicity | Entry/exit RPC transactions + UNIQUE constraints | DB transaction failures, reconciliation mismatch | RPC retry, lifecycle handler repair |
| Exchange truth | Exchange-first ordering before persistence | Reconciliation mismatch count, missing metrics | Lifecycle handler corrections |
| Entry exclusivity | Row-based locks + deterministic clientOrderId | `lockContentionCount`, lock TTL expiration | Wait TTL, reattempt, escalate via lock-contention runbook |
| Reconciliation parity | Lock + handler routing | `reconciliationMismatchCount`, missing metrics | Handler-guided correction, metrics reset |
| Observability coverage | Prometheus counters + health endpoint | `/internal/health` flags, metric spikes | Alerts trigger runbooks, loops restart if needed |
## 3. System Guarantees
- Capital never goes negative for active profiles thanks to ledger invariant enforcement and capital watchdog metrics.
- Only one ENTRY per profile/symbol proceeds at a time due to distributed row locks and deterministic clientOrderId.
- Lifecycle persistence is atomic, always triggered after confirmed exchange events, and idempotent on retries.
- Database state converges to exchange truth through reconciliation and structured health metrics, providing eventual consistency.
- Observability ensures loops, locks, and capital invariants are monitored and documented for operators.
## 4. Explicit Non-Guarantees
- Does not guarantee immediate dashboard synchronization; reconciliation is eventual.
- Does not guarantee tolerance for prolonged Supabase outages without ops intervention—trading pauses until Supabase is writable.
- Does not permit ad-hoc manual ledger or lifecycle edits; such edits require formal runbook approval.
## 5. Future Change Safety
- Any future agent must review docs/invariants.md and these runbooks before touching lifecycle RPCs, ledger services, reconciliation logic, or locking mechanisms.
- Changes must preserve the phase-by-phase guarantees and maintain all health/metrics fields referenced herein.
- Operators should verify that metrics and health indicators remain functional after updates, updating runbooks if detection/recovery steps change.

142
backend/pre-deploy.md Normal file
View File

@ -0,0 +1,142 @@
# Pre-Deploy Validation
## Contract
Running `npm run check` guarantees:
✅ **Build succeeds** - TypeScript compilation passes with no errors
✅ **Security checks pass** - Schema, RLS, secrets, guards, tenant isolation verified
✅ **Lifecycle checks pass** - Trade executor, lifecycle, order sync, persistence, failure injection, WebSocket contract verified
❌ **Does NOT guarantee**:
- Unit test coverage (no unit tests configured)
- Runtime behavior
- Database connectivity
- External API availability
---
## Pre-Deploy Command
```bash
npm run check
```
**What it runs**:
1. `npm run build` - TypeScript compilation
2. `npm run lint` - Security checks (schema, RLS, secrets, guards, tenant)
3. `npm run format` - Lifecycle checks (executor, sync, persistence, injection, WebSocket)
---
## Individual Check Commands
### Build Check
```bash
npm run build
```
**Pass criteria**: TypeScript compiles with 0 errors
**Fail criteria**: Any TypeScript error
### Security Checks (Lint)
```bash
npm run lint
```
**Runs**:
- `check:schema-contract` - Database schema matches code expectations
- `check:rls-policies` - Row-level security policies are correct
- `check:secret-hygiene` - No secrets in code, env vars configured
- `check:security-guards` - Auth guards in place
- `check:tenant-isolation` - Multi-tenant data isolation verified
**Pass criteria**: All 5 checks pass
**Fail criteria**: Any check fails
### Lifecycle Checks (Format)
```bash
npm run format
```
**Runs**:
- `check:trade-executor-lifecycle` - Trade execution flow works
- `check:lifecycle-regressions` - No regressions in trade lifecycle
- `check:order-sync-regressions` - Order status sync works
- `check:supabase-order-persistence-regressions` - Orders persist correctly
- `check:failure-injection` - System handles failures gracefully
- `check:websocket-contract` - WebSocket events match contract
**Pass criteria**: All 6 checks pass
**Fail criteria**: Any check fails
---
## Pass/Fail Rules
### ✅ SAFE TO DEPLOY
All of the following must be true:
- `npm run build` exits with code 0
- `npm run lint` exits with code 0
- `npm run format` exits with code 0
### ❌ DO NOT DEPLOY
If any of the following are true:
- `npm run build` fails (TypeScript errors)
- `npm run lint` fails (security check failed)
- `npm run format` fails (lifecycle check failed)
---
## Current Issues (Must Fix Before Deploy)
⚠️ **Build is currently failing**:
```
src/scripts/verifyWebsocketContract.ts:83:5 - error TS2741:
Property 'health' is missing in type 'BotState'
```
**Action required**: Fix TypeScript error in `verifyWebsocketContract.ts` before deploying
---
## Quick Reference
| Command | Purpose | Time | Critical |
|---------|---------|------|----------|
| `npm run build` | Compile TypeScript | ~10s | YES |
| `npm run lint` | Security checks | ~30s | YES |
| `npm run format` | Lifecycle checks | ~45s | YES |
| `npm run check` | All checks | ~90s | YES |
---
## Troubleshooting
### Build fails
- Check TypeScript errors in output
- Fix type errors in code
- Ensure all imports are correct
### Lint fails
- Check which security check failed
- Review error output
- Fix schema/RLS/secret/guard/tenant issue
### Format fails
- Check which lifecycle check failed
- Review error output
- Fix trade executor/sync/persistence issue
---
## Notes
- **No unit tests**: This project uses integration-style checks instead of unit tests
- **Checks are mandatory**: All checks must pass before deploy
- **No shortcuts**: Do not skip checks or deploy with failures
- **Independent**: This project's checks are independent of the frontend
---
## Related
- Frontend pre-deploy: `../bytelyst-trading-dashboard-web/docs/pre-deploy.md`
- Deployment script: `deploy.ps1`

5
backend/print_btc_rules.ts Normal file
View File

@ -0,0 +1,5 @@
import fs from 'fs';
const state = JSON.parse(fs.readFileSync('bot_state.json', 'utf8'));
const btc = state.symbols['BTC/USDT'];
console.log(JSON.stringify(btc.rules, null, 2));

9
backend/print_symbols.js Normal file
View File

@ -0,0 +1,9 @@
import fs from 'fs';
try {
const data = JSON.parse(fs.readFileSync('full_status.json', 'utf8'));
console.log('--- LIVE SYMBOLS IN BOT ---');
console.log(Object.keys(data.symbols).join(', '));
} catch (e) {
console.log('Failed to parse status file');
}

66
backend/prometheus-metrics.md Normal file
View File

@ -0,0 +1,66 @@
# Prometheus Metrics Guide
The Bytelyst Trading Bot exposes a `/metrics` Prometheus endpoint for advanced monitoring and Grafana integration.
## Scraping Configuration
The metrics are available at `http://<bot-ip>:5000/metrics`.
## Exported Metrics
### System Health & Events
- `bytelyst_bot_operational_events_total` (Counter)
- **Description**: Cumulative count of system events (orders, errors, warnings).
- **Labels**: `severity`, `type`, `profile_id`, `symbol`, `env`, `mode`.
- **Usage**: Monitor for spikes in `severity="ERROR"` or `type="ORDER_FAILURE"`.
### Subsystem Performance
- `bytelyst_bot_subsystem_duration_seconds` (Histogram)
- **Description**: Execution time for core loops (trading, monitor, reconciliation).
- **Labels**: `subsystem`, `env`, `mode`.
- **Buckets**: `[0.1, 0.25, 0.5, 1, 2, 5, 10]`.
- **Usage**: Identify slow execution cycles or database contention.
- `bytelyst_bot_subsystem_last_run_timestamp` (Gauge)
- **Description**: Unix timestamp of the last successful subsystem run.
- **Labels**: `subsystem`, `env`, `mode`.
- **Usage**: Verify how recently each process checked in.
- `bytelyst_bot_subsystem_alive` (Gauge)
- **Description**: Binary flag indicating if a subsystem is fresh (1) or stalled (0).
- **Labels**: `subsystem`, `env`, `mode`.
- **Usage**: Critical dashboard "Traffic Light" indicator.
### Exchange Connectivity
- `bytelyst_bot_exchange_api_latency_seconds` (Histogram)
- **Description**: Latency of external API calls to the exchange.
- **Labels**: `exchange`, `operation`, `env`, `mode`.
- **Usage**: Distinguish between internal bot lag and exchange infrastructure lag.
### Risk & Capital Invariants
- `bytelyst_bot_capital_invariant_violations_total` (Counter)
- **Description**: Count of times available capital fell below zero for a profile.
- **Labels**: `profile_id`, `env`, `mode`.
- **Usage**: Critical alert metric. Should always be 0.
- `bytelyst_bot_profile_utilization_percent` (Gauge)
- **Description**: Percentage of allocated capital currently in use (positions + open orders).
- **Labels**: `profile_id`, `env`, `mode`.
- **Usage**: Monitor capital efficiency and exposure.
### Data Integrity (Reconciliation)
- `bytelyst_bot_reconciliation_mismatches_total` (Counter)
- **Description**: Count of detected mismatches between local state and exchange state.
- **Labels**: `env`, `mode`.
- **Usage**: Track consistency of the "single source of truth."
- `bytelyst_bot_reconciliation_missing_items_count` (Gauge)
- **Description**: Number of missing orders/positions in the last sync cycle.
- **Labels**: `source` (db/exchange), `env`, `mode`.
- **Usage**: Identify synchronization drift.
## Default Labels
Every metric is automatically tagged with:
- `env`: `development` or `production` (from `NODE_ENV`).
- `mode`: `paper` or `live` (from `PAPER_TRADING`).
- `app`: `bytelyst-trading-bot-service`.

10
backend/prometheus.yml Normal file
View File

@ -0,0 +1,10 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: 'alert_bot'
static_configs:
- targets: ['alert_bot:5000']
metrics_path: '/metrics'

417
backend/proposed_risk_profiles.json Normal file
View File

@ -0,0 +1,417 @@
{
"Aggressive_70_Percent_Voting": {
"name": "Aggressive Test (70% Voting)",
"riskLevel": 5,
"allocatedCapital": 5000,
"strategy_config": {
"rules": [
{
"ruleId": "TrendBiasRule",
"enabled": true,
"params": {
"emaFast": 50,
"emaSlow": 200
}
},
{
"ruleId": "SessionRule",
"enabled": true,
"params": {
"allowedSessions": [
"NY",
"LDN"
]
}
},
{
"ruleId": "ZoneRule",
"enabled": true,
"params": {
"emaPeriod": 20
}
},
{
"ruleId": "MomentumRule",
"enabled": true,
"params": {
"rsiPeriod": 14
}
},
{
"ruleId": "EntryTriggerRule",
"enabled": true,
"params": {}
},
{
"ruleId": "RiskManagementRule",
"enabled": true,
"params": {
"atrPeriod": 14
}
},
{
"ruleId": "AIAnalysisRule",
"enabled": false,
"params": {
"minConfidence": 80
}
}
],
"riskLimits": {
"maxDailyLossUsd": 50,
"dailyProfitTargetUsd": 100,
"maxOpenTrades": 2,
"maxConsecutiveLosses": 2
},
"execution": {
"orderType": "market",
"cooldownMinutes": 30,
"minRulePassRatio": 0.7,
"entryMode": "both"
}
}
},
"Risk5_VeryAggressive": {
"name": "Alpha Scalper",
"riskLevel": 5,
"allocatedCapital": 5000,
"strategy_config": {
"rules": [
{
"ruleId": "TrendBiasRule",
"enabled": true,
"params": {
"timeframe": "1h",
"emaFast": 20,
"emaSlow": 50
}
},
{
"ruleId": "SessionRule",
"enabled": false
},
{
"ruleId": "ZoneRule",
"enabled": true,
"params": {
"emaPeriod": 20,
"tolerancePercent": 1.5
}
},
{
"ruleId": "MomentumRule",
"enabled": true,
"params": {
"timeframe": "15m",
"rsiPeriod": 7,
"overbought": 80,
"oversold": 20
}
},
{
"ruleId": "EntryTriggerRule",
"enabled": true,
"params": {
"timeframe": "15m",
"wickRatioThreshold": 0.3
}
},
{
"ruleId": "RiskManagementRule",
"enabled": true,
"params": {
"slMultiplier": 1.0,
"maxRisk": 5.0
}
}
],
"riskLimits": {
"maxDailyLossUsd": 500,
"maxConsecutiveLosses": 5,
"maxOpenTrades": 10
},
"execution": {
"orderType": "market",
"cooldownMinutes": 5,
"entryMode": "both"
}
}
},
"Risk4_Aggressive": {
"name": "Active Swing",
"riskLevel": 4,
"allocatedCapital": 5000,
"strategy_config": {
"rules": [
{
"ruleId": "TrendBiasRule",
"enabled": true,
"params": {
"timeframe": "1h",
"emaFast": 50,
"emaSlow": 100
}
},
{
"ruleId": "SessionRule",
"enabled": false
},
{
"ruleId": "ZoneRule",
"enabled": true,
"params": {
"emaPeriod": 20,
"tolerancePercent": 1.0
}
},
{
"ruleId": "MomentumRule",
"enabled": true,
"params": {
"timeframe": "15m",
"rsiPeriod": 14,
"overbought": 75,
"oversold": 25
}
},
{
"ruleId": "EntryTriggerRule",
"enabled": true,
"params": {
"timeframe": "15m"
}
},
{
"ruleId": "RiskManagementRule",
"enabled": true,
"params": {
"slMultiplier": 1.2,
"maxRisk": 3.0
}
}
],
"riskLimits": {
"maxDailyLossUsd": 300,
"maxConsecutiveLosses": 4,
"maxOpenTrades": 7
},
"execution": {
"orderType": "market",
"cooldownMinutes": 15,
"entryMode": "both"
}
}
},
"Risk3_Balanced": {
"name": "Balanced Core",
"riskLevel": 3,
"allocatedCapital": 5000,
"strategy_config": {
"rules": [
{
"ruleId": "TrendBiasRule",
"enabled": true,
"params": {
"timeframe": "4h",
"emaFast": 50,
"emaSlow": 200
}
},
{
"ruleId": "SessionRule",
"enabled": true,
"params": {
"allowedSessions": [
"NY",
"LDN"
]
}
},
{
"ruleId": "ZoneRule",
"enabled": true,
"params": {
"emaPeriod": 20,
"tolerancePercent": 0.5
}
},
{
"ruleId": "MomentumRule",
"enabled": true,
"params": {
"timeframe": "1h",
"rsiPeriod": 14,
"overbought": 70,
"oversold": 30
}
},
{
"ruleId": "EntryTriggerRule",
"enabled": true,
"params": {
"triggerType": "ema_cross"
}
},
{
"ruleId": "RiskManagementRule",
"enabled": true,
"params": {
"slMultiplier": 1.5,
"maxRisk": 2.0
}
}
],
"riskLimits": {
"maxDailyLossUsd": 150,
"maxConsecutiveLosses": 3,
"maxOpenTrades": 5
},
"execution": {
"orderType": "market",
"cooldownMinutes": 30,
"entryMode": "both"
}
}
},
"Risk2_Conservative": {
"name": "Conservative Guard",
"riskLevel": 2,
"allocatedCapital": 5000,
"strategy_config": {
"rules": [
{
"ruleId": "TrendBiasRule",
"enabled": true,
"params": {
"timeframe": "4h",
"emaFast": 50,
"emaSlow": 200
}
},
{
"ruleId": "SessionRule",
"enabled": true,
"params": {
"allowedSessions": [
"LDN",
"NY"
]
}
},
{
"ruleId": "ZoneRule",
"enabled": true,
"params": {
"emaPeriod": 50,
"tolerancePercent": 0.3
}
},
{
"ruleId": "MomentumRule",
"enabled": true,
"params": {
"timeframe": "1h",
"overbought": 65,
"oversold": 35
}
},
{
"ruleId": "EntryTriggerRule",
"enabled": true,
"params": {
"wickRatioThreshold": 0.6
}
},
{
"ruleId": "RiskManagementRule",
"enabled": true,
"params": {
"slMultiplier": 2.0,
"maxRisk": 1.0
}
}
],
"riskLimits": {
"maxDailyLossUsd": 100,
"maxConsecutiveLosses": 2,
"maxOpenTrades": 3
},
"execution": {
"orderType": "market",
"cooldownMinutes": 60,
"entryMode": "both"
}
}
},
"Risk1_VeryConservative": {
"name": "Ultra Defense",
"riskLevel": 1,
"allocatedCapital": 5000,
"strategy_config": {
"rules": [
{
"ruleId": "TrendBiasRule",
"enabled": true,
"params": {
"timeframe": "4h",
"emaFast": 100,
"emaSlow": 200
}
},
{
"ruleId": "SessionRule",
"enabled": true,
"params": {
"allowedSessions": [
"LDN",
"NY"
]
}
},
{
"ruleId": "ZoneRule",
"enabled": true,
"params": {
"emaPeriod": 50,
"tolerancePercent": 0.2
}
},
{
"ruleId": "MomentumRule",
"enabled": true,
"params": {
"timeframe": "1h",
"rsiPeriod": 21,
"overbought": 60,
"oversold": 40
}
},
{
"ruleId": "EntryTriggerRule",
"enabled": true,
"params": {
"wickRatioThreshold": 0.7
}
},
{
"ruleId": "RiskManagementRule",
"enabled": true,
"params": {
"slMultiplier": 2.5,
"maxRisk": 0.5
}
}
],
"riskLimits": {
"maxDailyLossUsd": 50,
"maxConsecutiveLosses": 1,
"maxOpenTrades": 2
},
"execution": {
"orderType": "market",
"cooldownMinutes": 120,
"entryMode": "both"
}
}
}
}

19
backend/quick_eth.ts Normal file
View File

@ -0,0 +1,19 @@
import { ConnectorFactory } from '../src/connectors/factory.js';
import { Indicators } from '../src/utils/indicators.js';
async function verifyETH() {
const exchange = ConnectorFactory.getCustomConnector('ccxt', '', '');
const symbol = 'ETH/USDT';
console.log(`\n--- 📊 ETH ANALYSIS ---`);
const candles4h = await exchange.fetchOHLCV(symbol, '4h', 100);
const ema50_4h = Indicators.calculateEMA(candles4h.map(c => c.close), 50);
const last4h = candles4h[candles4h.length - 1];
console.log(`ETH 4H Price: $${last4h.close}`);
console.log(`ETH 4H EMA50: $${ema50_4h.toFixed(2)}`);
console.log(`Trend: ${last4h.close > ema50_4h ? 'BULLISH' : 'BEARISH'}`);
}
verifyETH();

847
backend/reconcileAlpacaVsSupabase.ts Normal file
View File

@ -0,0 +1,847 @@
import 'dotenv/config';
import Alpaca from '@alpacahq/alpaca-trade-api';
import { createClient } from '@supabase/supabase-js';
type OrderRow = {
id?: string;
order_id?: string;
profile_id?: string | null;
symbol?: string;
side?: string;
type?: string;
qty?: number | string | null;
quantity?: number | string | null;
price?: number | string | null;
status?: string;
timestamp?: number | string | null;
created_at?: string | null;
trade_id?: string | null;
action?: string | null;
source?: string | null;
};
type HistoryRow = {
id?: string;
profile_id?: string | null;
trade_id?: string | null;
symbol?: string;
side?: string;
size?: number | string | null;
entry_price?: number | string | null;
exit_price?: number | string | null;
pnl?: number | string | null;
pnl_percent?: number | string | null;
reason?: string | null;
source?: string | null;
timestamp?: number | string | null;
created_at?: string | null;
};
type AlpacaOrder = {
id?: string;
client_order_id?: string;
symbol?: string;
side?: string;
qty?: number | string;
filled_qty?: number | string;
type?: string;
limit_price?: number | string | null;
filled_avg_price?: number | string | null;
status?: string;
submitted_at?: string;
filled_at?: string | null;
};
type ReconciliationSample = Record<string, any>;
const cliArgs = process.argv.slice(2);
const positionalArgs = cliArgs.filter((arg) => !arg.startsWith('--'));
const LOOKBACK_DAYS = Number(positionalArgs[0] || '7');
const MAX_SAMPLES = Number(positionalArgs[1] || '12');
const carryLookbackArg = cliArgs.find((arg) => arg.startsWith('--carry-lookback-days='));
const CARRY_IN_LOOKBACK_DAYS = Number(
(carryLookbackArg ? carryLookbackArg.split('=')[1] : '')
|| process.env.RECONCILE_CARRY_IN_LOOKBACK_DAYS
|| '90'
);
const PAGE_SIZE = 1000;
const ALPACA_PAGE_LIMIT = 500;
const MAX_ALPACA_PAGES = 60;
const supabaseUrl = String(process.env.SUPABASE_URL || '').trim();
const supabaseKey = String(
process.env.SUPABASE_KEY ||
process.env.SUPABASE_SERVICE_ROLE_KEY ||
process.env.SUPABASE_ANON_KEY ||
''
).trim();
const alpacaApiKey = String(process.env.ALPACA_API_KEY || '').trim();
const alpacaApiSecret = String(process.env.ALPACA_API_SECRET || '').trim();
const paperTrading = String(process.env.PAPER_TRADING || 'true').toLowerCase() === 'true';
if (!supabaseUrl || !supabaseKey) {
throw new Error('Missing Supabase credentials. Expected SUPABASE_URL + SUPABASE_KEY/SUPABASE_SERVICE_ROLE_KEY.');
}
if (!alpacaApiKey || !alpacaApiSecret) {
throw new Error('Missing Alpaca credentials. Expected ALPACA_API_KEY + ALPACA_API_SECRET.');
}
if (!Number.isFinite(LOOKBACK_DAYS) || LOOKBACK_DAYS <= 0) {
throw new Error(`Invalid lookback days: ${positionalArgs[0] || process.argv[2]}`);
}
if (!Number.isFinite(CARRY_IN_LOOKBACK_DAYS) || CARRY_IN_LOOKBACK_DAYS <= 0) {
throw new Error(`Invalid carry lookback days: ${carryLookbackArg || process.env.RECONCILE_CARRY_IN_LOOKBACK_DAYS}`);
}
const supabase = createClient(supabaseUrl, supabaseKey);
const alpaca = new (Alpaca as any)({
keyId: alpacaApiKey,
secretKey: alpacaApiSecret,
paper: paperTrading
});
const now = new Date();
const startDate = new Date(now.getTime() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000);
const carryStartDate = new Date(startDate.getTime() - CARRY_IN_LOOKBACK_DAYS * 24 * 60 * 60 * 1000);
const startIso = startDate.toISOString();
const carryStartIso = carryStartDate.toISOString();
const nowIso = now.toISOString();
const toNumber = (value: unknown): number => {
const num = Number(value);
return Number.isFinite(num) ? num : 0;
};
const toTimestampMs = (value: unknown, fallback: number = 0): number => {
if (typeof value === 'number') {
return value > 1_000_000_000_000 ? value : value * 1000;
}
if (typeof value === 'string') {
if (/^\d+(\.\d+)?$/.test(value.trim())) {
return toTimestampMs(Number(value.trim()), fallback);
}
const parsed = Date.parse(value);
if (Number.isFinite(parsed) && parsed > 0) return parsed;
}
return fallback;
};
const normalizeStatus = (status: string | undefined | null): string => {
const s = String(status || '').trim().toLowerCase();
if (s === 'filled') return 'filled';
if (s === 'partially_filled' || s === 'partiallyfilled' || s === 'partial_fill') return 'partially_filled';
if (s === 'canceled' || s === 'cancelled') return 'canceled';
if (s === 'expired') return 'expired';
if (s === 'rejected') return 'rejected';
if (s === 'unknown') return 'unknown';
return 'pending_new';
};
const normalizeSide = (side: string | undefined | null): 'BUY' | 'SELL' => {
const s = String(side || '').trim().toUpperCase();
return s === 'SELL' || s === 'SHORT' ? 'SELL' : 'BUY';
};
const normalizeSymbol = (symbol: string | undefined | null): string => {
const raw = String(symbol || '').trim().toUpperCase().replace(/[\/\-_]/g, '');
if (raw.endsWith('USDT')) {
return `${raw.slice(0, -4)}USD`;
}
return raw;
};
const getOrderQty = (row: OrderRow): number => {
const qty = toNumber(row.qty);
if (qty > 0) return qty;
return toNumber(row.quantity);
};
const pctDiff = (left: number, right: number): number => {
const denom = Math.max(Math.abs(left), Math.abs(right), 1e-9);
return Math.abs(left - right) / denom;
};
const pushSample = (bucket: ReconciliationSample[], sample: ReconciliationSample) => {
if (bucket.length < MAX_SAMPLES) bucket.push(sample);
};
const isQuarantinedHistoryReason = (reason: string | undefined | null): boolean => {
const normalized = String(reason || '').trim().toUpperCase();
return normalized.startsWith('[INVALID_')
|| normalized.startsWith('[DUPLICATE_')
|| normalized.startsWith('[RECONCILED_TO_');
};
const getSubmittedTsMs = (order: AlpacaOrder): number => toTimestampMs(order.submitted_at || 0, 0);
const getFillTsMs = (order: AlpacaOrder): number => toTimestampMs(order.filled_at || order.submitted_at || 0, 0);
const isFilledExecutionOrder = (order: AlpacaOrder): boolean => {
const filledQty = toNumber(order.filled_qty);
const filledPrice = toNumber(order.filled_avg_price);
if (filledQty <= 0 || filledPrice <= 0) return false;
const status = normalizeStatus(order.status);
return status === 'filled' || status === 'partially_filled';
};
const statusBreakdown = (rows: Array<{ status?: string | null }>): Record<string, number> => {
const out: Record<string, number> = {};
for (const row of rows) {
const status = normalizeStatus(row.status || undefined);
out[status] = (out[status] || 0) + 1;
}
return out;
};
const fetchPaged = async <T>(
table: 'orders' | 'trade_history',
columns: string,
startIsoFilter: string
): Promise<T[]> => {
const rows: T[] = [];
let offset = 0;
while (true) {
const { data, error } = await supabase
.from(table)
.select(columns)
.gte('created_at', startIsoFilter)
.order('created_at', { ascending: false })
.range(offset, offset + PAGE_SIZE - 1);
if (error) {
throw error;
}
const chunk = (data || []) as T[];
if (!chunk.length) break;
rows.push(...chunk);
if (chunk.length < PAGE_SIZE) break;
offset += PAGE_SIZE;
}
return rows;
};
const fetchSupabaseOrders = async (startIsoFilter: string = startIso): Promise<OrderRow[]> => {
const v2 = 'id,order_id,profile_id,symbol,type,side,qty,quantity,price,status,timestamp,created_at,trade_id,action,source';
const legacy = 'id,order_id,profile_id,symbol,type,side,qty,price,status,timestamp,created_at,trade_id,action,source';
try {
return await fetchPaged<OrderRow>('orders', v2, startIsoFilter);
} catch (error: any) {
const msg = String(error?.message || '').toLowerCase();
if (msg.includes('column') && msg.includes('quantity')) {
return await fetchPaged<OrderRow>('orders', legacy, startIsoFilter);
}
throw error;
}
};
const fetchTradeHistory = async (): Promise<HistoryRow[]> => {
const v2 = 'id,profile_id,trade_id,symbol,side,size,entry_price,exit_price,pnl,pnl_percent,reason,source,timestamp,created_at';
const v1 = 'id,profile_id,symbol,side,size,entry_price,exit_price,pnl,pnl_percent,reason,timestamp,created_at';
try {
return await fetchPaged<HistoryRow>('trade_history', v2, startIso);
} catch (error: any) {
const msg = String(error?.message || '').toLowerCase();
if (msg.includes('column') && (msg.includes('trade_id') || msg.includes('source'))) {
return await fetchPaged<HistoryRow>('trade_history', v1, startIso);
}
throw error;
}
};
type PositionState = { qty: number; avg: number };
type PositionSnapshot = Record<string, { qty: number; avg: number; notional: number }>;
type AlpacaRealizedPnlResult = {
totalRealized: number;
perSymbol: Record<string, number>;
openingPositions: PositionSnapshot;
endingPositions: PositionSnapshot;
openingExposureNotional: number;
endingExposureNotional: number;
preWindowFillCount: number;
inWindowFillCount: number;
};
const cloneStateMap = (source: Map<string, PositionState>): Map<string, PositionState> => {
const cloned = new Map<string, PositionState>();
for (const [symbol, state] of source.entries()) {
cloned.set(symbol, { qty: state.qty, avg: state.avg });
}
return cloned;
};
const snapshotStateMap = (stateBySymbol: Map<string, PositionState>): {
positions: PositionSnapshot;
exposureNotional: number;
} => {
const positions: PositionSnapshot = {};
let exposureNotional = 0;
for (const [symbol, state] of stateBySymbol.entries()) {
const qty = Number(state.qty || 0);
const avg = Number(state.avg || 0);
if (!Number.isFinite(qty) || !Number.isFinite(avg) || Math.abs(qty) <= 1e-12) continue;
const notional = qty * avg;
exposureNotional += Math.abs(notional);
positions[symbol] = {
qty: Number(qty.toFixed(8)),
avg: Number(avg.toFixed(8)),
notional: Number(notional.toFixed(8))
};
}
return {
positions,
exposureNotional: Number(exposureNotional.toFixed(8))
};
};
const applyFilledOrderToState = (
stateBySymbol: Map<string, PositionState>,
order: AlpacaOrder,
onRealized?: (symbol: string, pnl: number) => void
): void => {
const symbol = normalizeSymbol(order.symbol);
const side = normalizeSide(order.side);
const qty = toNumber(order.filled_qty);
const price = toNumber(order.filled_avg_price);
if (!(qty > 0 && price > 0)) return;
const current = stateBySymbol.get(symbol) || { qty: 0, avg: 0 };
let positionQty = current.qty;
let avg = current.avg;
if (side === 'BUY') {
if (positionQty >= 0) {
const newQty = positionQty + qty;
avg = newQty > 0 ? ((avg * positionQty) + (price * qty)) / newQty : 0;
positionQty = newQty;
} else {
const closeQty = Math.min(qty, Math.abs(positionQty));
if (closeQty > 0) {
onRealized?.(symbol, (avg - price) * closeQty);
positionQty += closeQty;
}
const remainQty = qty - closeQty;
if (remainQty > 0) {
positionQty = remainQty;
avg = price;
} else if (Math.abs(positionQty) < 1e-12) {
positionQty = 0;
avg = 0;
}
}
} else {
if (positionQty <= 0) {
const baseQty = Math.abs(positionQty);
const newBaseQty = baseQty + qty;
avg = newBaseQty > 0 ? ((avg * baseQty) + (price * qty)) / newBaseQty : 0;
positionQty = -newBaseQty;
} else {
const closeQty = Math.min(qty, positionQty);
if (closeQty > 0) {
onRealized?.(symbol, (price - avg) * closeQty);
positionQty -= closeQty;
}
const remainQty = qty - closeQty;
if (remainQty > 0) {
positionQty = -remainQty;
avg = price;
} else if (Math.abs(positionQty) < 1e-12) {
positionQty = 0;
avg = 0;
}
}
}
stateBySymbol.set(symbol, { qty: positionQty, avg });
};
const calculateAlpacaRealizedPnl = (
allOrders: AlpacaOrder[],
windowStartMs: number,
windowEndMs: number
): AlpacaRealizedPnlResult => {
const sortedFills = [...allOrders]
.filter((order) => isFilledExecutionOrder(order))
.sort((a, b) => getFillTsMs(a) - getFillTsMs(b));
const openingState = new Map<string, PositionState>();
let preWindowFillCount = 0;
for (const order of sortedFills) {
const fillTs = getFillTsMs(order);
if (fillTs <= 0 || fillTs >= windowStartMs) continue;
applyFilledOrderToState(openingState, order);
preWindowFillCount += 1;
}
const openingSnapshot = snapshotStateMap(openingState);
const currentState = cloneStateMap(openingState);
let totalRealized = 0;
const perSymbol: Record<string, number> = {};
let inWindowFillCount = 0;
for (const order of sortedFills) {
const fillTs = getFillTsMs(order);
if (fillTs < windowStartMs || fillTs > windowEndMs) continue;
inWindowFillCount += 1;
applyFilledOrderToState(currentState, order, (symbol, pnl) => {
totalRealized += pnl;
perSymbol[symbol] = (perSymbol[symbol] || 0) + pnl;
});
}
const endingSnapshot = snapshotStateMap(currentState);
return {
totalRealized,
perSymbol,
openingPositions: openingSnapshot.positions,
endingPositions: endingSnapshot.positions,
openingExposureNotional: openingSnapshot.exposureNotional,
endingExposureNotional: endingSnapshot.exposureNotional,
preWindowFillCount,
inWindowFillCount
};
};
const fetchAlpacaOrdersPaged = async (after: Date, until: Date): Promise<AlpacaOrder[]> => {
const out: AlpacaOrder[] = [];
const seen = new Set<string>();
let cursorUntil = new Date(until);
const afterMs = after.getTime();
for (let page = 0; page < MAX_ALPACA_PAGES; page++) {
const raw = await (alpaca as any).getOrders({
status: 'all',
after,
until: cursorUntil,
direction: 'desc',
limit: ALPACA_PAGE_LIMIT
});
const batch: AlpacaOrder[] = Array.isArray(raw) ? raw : [];
if (!batch.length) break;
let oldestTs = Number.POSITIVE_INFINITY;
for (const order of batch) {
const id = String(order.id || '').trim();
if (id && seen.has(id)) continue;
if (id) seen.add(id);
out.push(order);
const ts = getSubmittedTsMs(order) || getFillTsMs(order);
if (ts > 0 && ts < oldestTs) oldestTs = ts;
}
if (batch.length < ALPACA_PAGE_LIMIT) break;
if (!Number.isFinite(oldestTs) || oldestTs <= 0) break;
const nextUntilMs = oldestTs - 1;
if (nextUntilMs <= afterMs) break;
if (nextUntilMs >= cursorUntil.getTime()) break;
cursorUntil = new Date(nextUntilMs);
}
return out;
};
const calculateSupabaseRealizedPnl = (historyRows: HistoryRow[]) => {
let totalPnl = 0;
const perSymbol: Record<string, number> = {};
const formulaMismatches: ReconciliationSample[] = [];
const tradeIdCounts = new Map<string, number>();
const rankedRows: Array<HistoryRow & { __absPnl: number }> = [];
for (const row of historyRows) {
const symbol = normalizeSymbol(row.symbol);
const side = normalizeSide(row.side);
const size = toNumber(row.size);
const entry = toNumber(row.entry_price);
const exit = toNumber(row.exit_price);
const pnl = toNumber(row.pnl);
const reason = String(row.reason || '');
totalPnl += pnl;
perSymbol[symbol] = (perSymbol[symbol] || 0) + pnl;
rankedRows.push({ ...row, __absPnl: Math.abs(pnl) });
const tradeId = String(row.trade_id || '').trim();
if (tradeId) {
tradeIdCounts.set(tradeId, (tradeIdCounts.get(tradeId) || 0) + 1);
}
if (!isQuarantinedHistoryReason(reason) && size > 0 && entry > 0 && exit > 0) {
const expected = side === 'BUY'
? (exit - entry) * size
: (entry - exit) * size;
const absDiff = Math.abs(expected - pnl);
if (absDiff > 0.02) {
pushSample(formulaMismatches, {
id: row.id,
trade_id: row.trade_id,
symbol: row.symbol,
side: row.side,
size,
entry_price: entry,
exit_price: exit,
pnl_recorded: pnl,
pnl_expected: Number(expected.toFixed(8)),
pnl_abs_diff: Number(absDiff.toFixed(8))
});
}
}
}
const duplicateTradeIds = Array.from(tradeIdCounts.entries())
.filter(([, count]) => count > 1)
.sort((a, b) => b[1] - a[1])
.slice(0, MAX_SAMPLES)
.map(([tradeId, count]) => ({ trade_id: tradeId, count }));
const topRows = rankedRows
.sort((a, b) => b.__absPnl - a.__absPnl)
.slice(0, MAX_SAMPLES)
.map((row) => ({
id: row.id,
trade_id: row.trade_id,
profile_id: row.profile_id,
symbol: row.symbol,
side: row.side,
size: toNumber(row.size),
entry_price: toNumber(row.entry_price),
exit_price: toNumber(row.exit_price),
pnl: toNumber(row.pnl),
reason: row.reason,
created_at: row.created_at
}));
return {
totalPnl,
perSymbol,
formulaMismatches,
duplicateTradeIds,
topRows
};
};
const sortRecordByAbsValueDesc = (record: Record<string, number>) => (
Object.entries(record)
.sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
.map(([key, value]) => ({ key, value: Number(value.toFixed(8)) }))
);
const run = async () => {
const windowStartMs = startDate.getTime();
const windowEndMs = now.getTime();
const alpacaAllOrders = await fetchAlpacaOrdersPaged(carryStartDate, now);
const alpacaOrders = alpacaAllOrders.filter((order) => {
const submittedTs = getSubmittedTsMs(order);
if (submittedTs > 0) {
return submittedTs >= windowStartMs && submittedTs <= windowEndMs;
}
const fillTs = getFillTsMs(order);
return fillTs >= windowStartMs && fillTs <= windowEndMs;
});
const [dbOrders, dbOrdersCarryScope, tradeHistory] = await Promise.all([
fetchSupabaseOrders(startIso),
fetchSupabaseOrders(carryStartIso),
fetchTradeHistory()
]);
const dbPreWindowFilledOrderCount = dbOrdersCarryScope.filter((order) => {
const createdAtTs = toTimestampMs(order.created_at || 0, 0);
if (createdAtTs <= 0 || createdAtTs >= windowStartMs) return false;
const status = normalizeStatus(order.status);
return status === 'filled' || status === 'partially_filled';
}).length;
const comparableSymbols = new Set(
dbOrders
.map((order) => normalizeSymbol(order.symbol))
.filter((symbol) => !!symbol)
);
const useComparableScope = comparableSymbols.size > 0;
const alpacaOrdersForPnlAll = useComparableScope
? alpacaAllOrders.filter((order) => comparableSymbols.has(normalizeSymbol(order.symbol)))
: alpacaAllOrders;
const alpacaOrdersForPnlWindow = useComparableScope
? alpacaOrders.filter((order) => comparableSymbols.has(normalizeSymbol(order.symbol)))
: alpacaOrders;
let alpacaAccountSummary: Record<string, number> | null = null;
try {
const account = await (alpaca as any).getAccount();
alpacaAccountSummary = {
equity: toNumber((account as any)?.equity),
cash: toNumber((account as any)?.cash),
portfolio_value: toNumber((account as any)?.portfolio_value),
buying_power: toNumber((account as any)?.buying_power),
last_equity: toNumber((account as any)?.last_equity)
};
} catch {
alpacaAccountSummary = null;
}
let alpacaPortfolioSummary: Record<string, any> | null = null;
try {
const portfolioHistory = await (alpaca as any).getPortfolioHistory({
date_start: startIso.slice(0, 10),
date_end: nowIso.slice(0, 10),
timeframe: '1D',
extended_hours: true
});
const timestamps = Array.isArray((portfolioHistory as any)?.timestamp)
? (portfolioHistory as any).timestamp
: [];
const equity = Array.isArray((portfolioHistory as any)?.equity)
? (portfolioHistory as any).equity
: [];
const profitLoss = Array.isArray((portfolioHistory as any)?.profit_loss)
? (portfolioHistory as any).profit_loss
: [];
const profitLossPct = Array.isArray((portfolioHistory as any)?.profit_loss_pct)
? (portfolioHistory as any).profit_loss_pct
: [];
const firstEquity = equity.length ? toNumber(equity[0]) : 0;
const lastEquity = equity.length ? toNumber(equity[equity.length - 1]) : 0;
const lastProfitLoss = profitLoss.length ? toNumber(profitLoss[profitLoss.length - 1]) : 0;
const lastProfitLossPct = profitLossPct.length ? toNumber(profitLossPct[profitLossPct.length - 1]) : 0;
const firstTsRaw = timestamps.length ? Number(timestamps[0]) : 0;
const lastTsRaw = timestamps.length ? Number(timestamps[timestamps.length - 1]) : 0;
const firstTsMs = firstTsRaw > 1_000_000_000_000 ? firstTsRaw : firstTsRaw * 1000;
const lastTsMs = lastTsRaw > 1_000_000_000_000 ? lastTsRaw : lastTsRaw * 1000;
alpacaPortfolioSummary = {
points: equity.length,
first_timestamp_utc: firstTsMs > 0 ? new Date(firstTsMs).toISOString() : null,
last_timestamp_utc: lastTsMs > 0 ? new Date(lastTsMs).toISOString() : null,
first_equity: Number(firstEquity.toFixed(8)),
last_equity: Number(lastEquity.toFixed(8)),
equity_change: Number((lastEquity - firstEquity).toFixed(8)),
latest_profit_loss: Number(lastProfitLoss.toFixed(8)),
latest_profit_loss_pct: Number(lastProfitLossPct.toFixed(8))
};
} catch {
alpacaPortfolioSummary = null;
}
const alpacaById = new Map<string, AlpacaOrder>();
for (const order of alpacaOrders) {
const id = String(order.id || '').trim();
if (id) alpacaById.set(id, order);
}
const dbByOrderId = new Map<string, OrderRow[]>();
for (const order of dbOrders) {
const orderId = String(order.order_id || '').trim();
if (!orderId) continue;
if (!dbByOrderId.has(orderId)) dbByOrderId.set(orderId, []);
dbByOrderId.get(orderId)!.push(order);
}
const missingInDb: ReconciliationSample[] = [];
const missingInAlpaca: ReconciliationSample[] = [];
const statusMismatches: ReconciliationSample[] = [];
const qtyMismatches: ReconciliationSample[] = [];
const sideMismatches: ReconciliationSample[] = [];
const symbolMismatches: ReconciliationSample[] = [];
const priceMismatches: ReconciliationSample[] = [];
for (const alpacaOrder of alpacaOrders) {
const alpacaId = String(alpacaOrder.id || '').trim();
if (!alpacaId) continue;
const matches = dbByOrderId.get(alpacaId) || [];
if (!matches.length) {
pushSample(missingInDb, {
alpaca_order_id: alpacaId,
symbol: alpacaOrder.symbol,
side: alpacaOrder.side,
status: alpacaOrder.status,
submitted_at: alpacaOrder.submitted_at
});
continue;
}
const dbOrder = matches[0];
const alpacaStatus = normalizeStatus(alpacaOrder.status);
const dbStatus = normalizeStatus(dbOrder.status);
if (alpacaStatus !== dbStatus) {
pushSample(statusMismatches, {
order_id: alpacaId,
alpaca_status: alpacaOrder.status,
db_status: dbOrder.status,
alpaca_status_normalized: alpacaStatus,
db_status_normalized: dbStatus
});
}
const alpacaQty = toNumber(alpacaOrder.qty);
const dbQty = getOrderQty(dbOrder);
if (alpacaQty > 0 && dbQty > 0 && pctDiff(alpacaQty, dbQty) > 0.000001) {
pushSample(qtyMismatches, {
order_id: alpacaId,
symbol: alpacaOrder.symbol,
alpaca_qty: alpacaQty,
db_qty: dbQty
});
}
const alpacaSide = normalizeSide(alpacaOrder.side);
const dbSide = normalizeSide(dbOrder.side);
if (alpacaSide !== dbSide) {
pushSample(sideMismatches, {
order_id: alpacaId,
symbol: alpacaOrder.symbol,
alpaca_side: alpacaOrder.side,
db_side: dbOrder.side
});
}
const alpacaSymbol = normalizeSymbol(alpacaOrder.symbol);
const dbSymbol = normalizeSymbol(dbOrder.symbol);
if (alpacaSymbol !== dbSymbol) {
pushSample(symbolMismatches, {
order_id: alpacaId,
alpaca_symbol: alpacaOrder.symbol,
db_symbol: dbOrder.symbol,
alpaca_symbol_normalized: alpacaSymbol,
db_symbol_normalized: dbSymbol
});
}
const alpacaFilledPrice = toNumber(alpacaOrder.filled_avg_price);
const dbPrice = toNumber(dbOrder.price);
if (alpacaFilledPrice > 0 && dbPrice > 0 && pctDiff(alpacaFilledPrice, dbPrice) > 0.005) {
pushSample(priceMismatches, {
order_id: alpacaId,
symbol: alpacaOrder.symbol,
alpaca_filled_avg_price: alpacaFilledPrice,
db_price: dbPrice,
diff_percent: Number((pctDiff(alpacaFilledPrice, dbPrice) * 100).toFixed(4))
});
}
}
for (const dbOrder of dbOrders) {
const orderId = String(dbOrder.order_id || '').trim();
if (!orderId) continue;
if (!alpacaById.has(orderId)) {
pushSample(missingInAlpaca, {
db_order_id: orderId,
symbol: dbOrder.symbol,
side: dbOrder.side,
status: dbOrder.status,
profile_id: dbOrder.profile_id,
created_at: dbOrder.created_at
});
}
}
const alpacaRealizedWithCarry = calculateAlpacaRealizedPnl(alpacaOrdersForPnlAll, windowStartMs, windowEndMs);
const alpacaRealizedFlatStart = calculateAlpacaRealizedPnl(alpacaOrdersForPnlWindow, windowStartMs, windowEndMs);
const carryCoverageGap = Math.max(0, alpacaRealizedWithCarry.preWindowFillCount - dbPreWindowFilledOrderCount);
const carryCoverageThreshold = Math.max(20, dbPreWindowFilledOrderCount * 3);
const useCarryAsPrimary = dbPreWindowFilledOrderCount > 0
&& carryCoverageGap <= carryCoverageThreshold;
const alpacaRealizedPrimary = useCarryAsPrimary
? alpacaRealizedWithCarry
: alpacaRealizedFlatStart;
const dbRealized = calculateSupabaseRealizedPnl(tradeHistory);
const pnlDiff = dbRealized.totalPnl - alpacaRealizedPrimary.totalRealized;
const alpacaFillCashFlow = alpacaOrdersForPnlWindow.reduce((sum, order) => {
const status = normalizeStatus(order.status);
if (status !== 'filled' && status !== 'partially_filled') return sum;
const qty = toNumber(order.filled_qty);
const price = toNumber(order.filled_avg_price);
if (!(qty > 0 && price > 0)) return sum;
const side = normalizeSide(order.side);
const signedNotional = side === 'BUY' ? -qty * price : qty * price;
return sum + signedNotional;
}, 0);
const output = {
meta: {
window_start_utc: startIso,
window_end_utc: nowIso,
lookback_days: LOOKBACK_DAYS,
carry_in_lookback_days: CARRY_IN_LOOKBACK_DAYS,
carry_in_start_utc: carryStartIso,
pnl_symbol_scope: useComparableScope ? 'db-order-symbols' : 'all-symbols',
paper_trading: paperTrading
},
counts: {
alpaca_orders_total: alpacaOrders.length,
alpaca_orders_total_with_carry_window: alpacaAllOrders.length,
alpaca_orders_total_in_pnl_scope: alpacaOrdersForPnlWindow.length,
alpaca_orders_total_with_carry_in_pnl_scope: alpacaOrdersForPnlAll.length,
supabase_orders_total: dbOrders.length,
supabase_trade_history_total: tradeHistory.length
},
alpaca_account: alpacaAccountSummary,
alpaca_portfolio_history: alpacaPortfolioSummary,
status_breakdown: {
alpaca_orders: statusBreakdown(alpacaOrders.map((o) => ({ status: o.status }))),
supabase_orders: statusBreakdown(dbOrders.map((o) => ({ status: o.status })))
},
order_conflicts: {
missing_in_supabase_count: missingInDb.length,
missing_in_alpaca_count: missingInAlpaca.length,
status_mismatch_count: statusMismatches.length,
qty_mismatch_count: qtyMismatches.length,
side_mismatch_count: sideMismatches.length,
symbol_mismatch_count: symbolMismatches.length,
price_mismatch_count: priceMismatches.length,
samples: {
missing_in_supabase: missingInDb,
missing_in_alpaca: missingInAlpaca,
status_mismatches: statusMismatches,
qty_mismatches: qtyMismatches,
side_mismatches: sideMismatches,
symbol_mismatches: symbolMismatches,
price_mismatches: priceMismatches
}
},
pnl_comparison: {
caveat: 'Alpaca fill-derived realized PnL publishes both flat_start and with_carry_in. Primary metric auto-selects carry only when pre-window Alpaca fill volume is consistent with Supabase pre-window coverage.',
supabase_trade_history_realized_pnl: Number(dbRealized.totalPnl.toFixed(8)),
alpaca_fill_derived_realized_pnl: Number(alpacaRealizedPrimary.totalRealized.toFixed(8)),
alpaca_fill_derived_realized_pnl_with_carry_in: Number(alpacaRealizedWithCarry.totalRealized.toFixed(8)),
alpaca_fill_derived_realized_pnl_flat_start: Number(alpacaRealizedFlatStart.totalRealized.toFixed(8)),
alpaca_fill_realized_carry_minus_flat: Number((alpacaRealizedWithCarry.totalRealized - alpacaRealizedFlatStart.totalRealized).toFixed(8)),
alpaca_fill_cash_flow: Number(alpacaFillCashFlow.toFixed(8)),
pnl_diff_supabase_minus_alpaca: Number(pnlDiff.toFixed(8)),
supabase_per_symbol_realized_pnl: sortRecordByAbsValueDesc(dbRealized.perSymbol),
alpaca_per_symbol_realized_pnl: sortRecordByAbsValueDesc(alpacaRealizedPrimary.perSymbol),
alpaca_per_symbol_realized_pnl_flat_start: sortRecordByAbsValueDesc(alpacaRealizedFlatStart.perSymbol),
alpaca_per_symbol_realized_pnl_with_carry_in: sortRecordByAbsValueDesc(alpacaRealizedWithCarry.perSymbol),
alpaca_opening_positions_at_window_start: alpacaRealizedWithCarry.openingPositions,
alpaca_opening_exposure_notional: Number(alpacaRealizedWithCarry.openingExposureNotional.toFixed(8)),
alpaca_ending_positions_after_window: alpacaRealizedWithCarry.endingPositions,
alpaca_ending_exposure_notional: Number(alpacaRealizedWithCarry.endingExposureNotional.toFixed(8)),
alpaca_carry_in_bootstrap: {
pre_window_fill_count: alpacaRealizedWithCarry.preWindowFillCount,
in_window_fill_count: alpacaRealizedWithCarry.inWindowFillCount,
has_pre_window_fills: alpacaRealizedWithCarry.preWindowFillCount > 0,
symbol_scope: useComparableScope ? 'db-order-symbols' : 'all-symbols',
supabase_pre_window_filled_order_count: dbPreWindowFilledOrderCount,
carry_coverage_gap: carryCoverageGap,
carry_coverage_threshold: carryCoverageThreshold,
primary_mode: useCarryAsPrimary ? 'with_carry_in' : 'flat_start'
},
trade_history_formula_mismatch_count: dbRealized.formulaMismatches.length,
trade_history_formula_mismatch_samples: dbRealized.formulaMismatches,
duplicate_trade_id_samples: dbRealized.duplicateTradeIds,
top_trade_history_pnl_rows: dbRealized.topRows
}
};
console.log(JSON.stringify(output, null, 2));
};
run().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
console.error(JSON.stringify({ error: message }, null, 2));
process.exit(1);
});

761
backend/reconcileAttributionRepair.ts Normal file
View File

@ -0,0 +1,761 @@
import 'dotenv/config';
import { randomUUID } from 'crypto';
import { createClient } from '@supabase/supabase-js';
import { normalizeOrderAction, normalizeTradeSide } from '../src/domain/tradingEnums.js';
type CliOptions = {
apply: boolean;
tradeIds: string[];
restoreBatchId?: string;
allowCrossTrade: boolean;
allowPartial: boolean;
};
type OrderRow = {
id?: string;
order_id?: string | null;
user_id?: string | null;
profile_id?: string | null;
trade_id?: string | null;
symbol?: string | null;
action?: string | null;
side?: string | null;
qty?: number | string | null;
quantity?: number | string | null;
price?: number | string | null;
status?: string | null;
source?: string | null;
created_at?: string | null;
updated_at?: string | null;
};
type AuditRow = {
id: number;
batch_id?: string | null;
profile_id?: string | null;
symbol?: string | null;
trade_id?: string | null;
exchange_order_id?: string | null;
backfill_order_id?: string | null;
decision?: string | null;
reason?: string | null;
metadata?: Record<string, any> | null;
created_at?: string | null;
reverted_at?: string | null;
};
type CandidateRow = {
order: OrderRow;
qty: number;
exchangeOrderId: string;
realOrder: OrderRow | null;
safe: boolean;
unsafeReason?: string;
};
type ChosenSubset = {
orderIds: string[];
sumQty: number;
nextOpenQty: number;
withinDust: boolean;
};
type TradeEvaluation = {
tradeId: string;
profileId: string;
symbol: string;
entryQty: number;
exitQty: number;
openQtyBefore: number;
openQtyAfter: number;
dustThreshold: number;
candidates: CandidateRow[];
selectedOrderIds: string[];
decision: 'SKIP' | 'DRY_RUN' | 'APPLIED' | 'NO_GO';
reason: string;
};
const EPSILON = 1e-8;
const DUST_ABS_QTY = 0.001;
const DUST_REL_PCT = 0.002; // 0.2%
const FILLED_STATUSES = new Set(['filled', 'partially_filled', 'partially-filled']);
const APPLIED_DECISIONS = new Set([
'APPLIED',
'MANUAL_OVERRIDE_APPLIED',
'SKIP_EXISTING',
'MANUAL_OVERRIDE_SKIP_EXISTING',
'ATTRIB_REPAIR_APPLIED'
]);
const STATUS_REVERT_TARGET = 'canceled';
const parseArgs = (argv: string[]): CliOptions => {
const tradeIds = new Set<string>();
let apply = false;
let restoreBatchId: string | undefined;
let allowCrossTrade = false;
let allowPartial = false;
for (const raw of argv) {
const arg = String(raw || '').trim();
if (!arg) continue;
if (arg === '--apply') {
apply = true;
continue;
}
if (arg === '--allow-cross-trade') {
allowCrossTrade = true;
continue;
}
if (arg === '--allow-partial') {
allowPartial = true;
continue;
}
if (arg.startsWith('--restore-batch=')) {
const value = String(arg.slice('--restore-batch='.length) || '').trim();
if (value) restoreBatchId = value;
continue;
}
if (arg.startsWith('--trade=')) {
const value = String(arg.slice('--trade='.length) || '').trim();
if (value) tradeIds.add(value);
continue;
}
}
return {
apply,
tradeIds: Array.from(tradeIds),
restoreBatchId,
allowCrossTrade,
allowPartial
};
};
const toNumber = (value: unknown): number => {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : 0;
};
const normalizeStatus = (value: unknown): string => String(value || '').trim().toLowerCase();
const inferAction = (actionRaw?: string | null, sideRaw?: string | null): 'ENTRY' | 'EXIT' | undefined => {
const explicit = normalizeOrderAction(actionRaw || undefined);
if (explicit) return explicit;
const side = normalizeTradeSide(sideRaw || 'BUY');
return side === 'BUY' ? 'ENTRY' : 'EXIT';
};
const parseMetadata = (value: unknown): Record<string, any> => {
if (!value) return {};
if (typeof value === 'object') return value as Record<string, any>;
if (typeof value === 'string') {
try {
const parsed = JSON.parse(value);
if (parsed && typeof parsed === 'object') return parsed as Record<string, any>;
} catch {
return {};
}
}
return {};
};
const pickBestSubset = (
openQtyBefore: number,
candidates: CandidateRow[],
dustThreshold: number,
allowPartial: boolean
): ChosenSubset | null => {
if (candidates.length === 0) return null;
if (!(openQtyBefore < -EPSILON)) return null;
if (candidates.length > 20) return null;
const n = candidates.length;
const totalMasks = 1 << n;
let best: ChosenSubset | null = null;
for (let mask = 1; mask < totalMasks; mask += 1) {
let sumQty = 0;
const orderIds: string[] = [];
for (let idx = 0; idx < n; idx += 1) {
if ((mask & (1 << idx)) === 0) continue;
const candidate = candidates[idx];
sumQty += candidate.qty;
orderIds.push(String(candidate.order.order_id || '').trim());
}
const nextOpenQty = Number((openQtyBefore + sumQty).toFixed(8));
const improved = Math.abs(nextOpenQty) + EPSILON < Math.abs(openQtyBefore);
if (!improved) continue;
const withinDust = Math.abs(nextOpenQty) <= dustThreshold + EPSILON;
if (!withinDust && !allowPartial) continue;
// Never move from over-closed to materially under-closed in strict mode.
if (!allowPartial && nextOpenQty > dustThreshold + EPSILON) continue;
const candidateBest: ChosenSubset = {
orderIds,
sumQty: Number(sumQty.toFixed(8)),
nextOpenQty,
withinDust
};
if (!best) {
best = candidateBest;
continue;
}
const bestAbs = Math.abs(best.nextOpenQty);
const candAbs = Math.abs(candidateBest.nextOpenQty);
if (candAbs + EPSILON < bestAbs) {
best = candidateBest;
continue;
}
if (Math.abs(candAbs - bestAbs) <= EPSILON && candidateBest.orderIds.length < best.orderIds.length) {
best = candidateBest;
}
}
return best;
};
const buildTradeLedger = (rows: OrderRow[]): {
entryQty: number;
exitQty: number;
openQty: number;
profileId: string;
symbol: string;
} => {
let entryQty = 0;
let exitQty = 0;
let profileId = '';
let symbol = '';
for (const row of rows) {
const status = normalizeStatus(row.status);
if (!FILLED_STATUSES.has(status)) continue;
const qty = toNumber(row.qty ?? row.quantity);
if (!(qty > EPSILON)) continue;
const action = inferAction(row.action, row.side);
if (!action) continue;
if (!profileId) profileId = String(row.profile_id || '').trim();
if (!symbol) symbol = String(row.symbol || '').trim();
if (action === 'ENTRY') entryQty += qty;
if (action === 'EXIT') exitQty += qty;
}
return {
entryQty: Number(entryQty.toFixed(8)),
exitQty: Number(exitQty.toFixed(8)),
openQty: Number((entryQty - exitQty).toFixed(8)),
profileId,
symbol
};
};
const runRestoreMode = async (
supabase: ReturnType<typeof createClient>,
options: CliOptions
): Promise<void> => {
const restoreBatchId = String(options.restoreBatchId || '').trim();
if (!restoreBatchId) {
throw new Error('Missing --restore-batch=<BATCH_ID>.');
}
const { data: rows, error } = await supabase
.from('reconciliation_backfill_audit')
.select('id,batch_id,profile_id,symbol,trade_id,backfill_order_id,decision,reason,metadata,created_at,reverted_at')
.eq('batch_id', restoreBatchId)
.in('decision', ['ATTRIB_REPAIR_APPLIED'])
.order('created_at', { ascending: true });
if (error) throw error;
const restoreTargets = ((rows || []) as AuditRow[])
.filter((row) => String(row.backfill_order_id || '').trim().length > 0);
const batchId = `ATTRIB-RESTORE-${randomUUID()}`;
const nowIso = new Date().toISOString();
const updates = restoreTargets.map((row) => {
const metadata = parseMetadata(row.metadata);
const restoreStatus = normalizeStatus(metadata.prevStatus || 'filled') || 'filled';
return {
row,
restoreStatus
};
});
let restoredRows = 0;
if (options.apply && updates.length > 0) {
for (const target of updates) {
const orderId = String(target.row.backfill_order_id || '').trim();
const { error: updateError } = await supabase
.from('orders')
.update({
status: target.restoreStatus,
updated_at: nowIso
})
.eq('order_id', orderId);
if (updateError) throw updateError;
restoredRows += 1;
}
const { error: markRevertedError } = await supabase
.from('reconciliation_backfill_audit')
.update({ reverted_at: nowIso })
.eq('batch_id', restoreBatchId)
.in('decision', ['ATTRIB_REPAIR_APPLIED']);
if (markRevertedError) throw markRevertedError;
const restoreAuditRows = updates.map((target) => ({
batch_id: batchId,
profile_id: String(target.row.profile_id || '').trim(),
symbol: String(target.row.symbol || '').trim(),
trade_id: String(target.row.trade_id || '').trim(),
exchange_order_id: null,
exchange_client_order_id: null,
backfill_order_id: String(target.row.backfill_order_id || '').trim(),
filled_qty: null,
filled_price: null,
filled_at: null,
dry_run: false,
decision: 'ATTRIB_RESTORE_APPLIED',
reason: `status_restored_from_${restoreBatchId}`,
metadata: {
restoredFromBatch: restoreBatchId,
restoredFromAuditId: target.row.id,
restoredToStatus: target.restoreStatus
},
applied_at: nowIso
}));
const { error: restoreAuditError } = await supabase
.from('reconciliation_backfill_audit')
.insert(restoreAuditRows);
if (restoreAuditError) throw restoreAuditError;
}
console.log(JSON.stringify({
mode: options.apply ? 'restore-apply' : 'restore-dry-run',
restoreBatchId,
batchId,
targetRows: updates.length,
restoredRows,
targets: updates.map((target) => ({
backfillOrderId: String(target.row.backfill_order_id || '').trim(),
profileId: String(target.row.profile_id || '').trim(),
tradeId: String(target.row.trade_id || '').trim(),
restoreStatus: target.restoreStatus
}))
}, null, 2));
};
const runRepairMode = async (
supabase: ReturnType<typeof createClient>,
options: CliOptions
): Promise<void> => {
if (options.tradeIds.length === 0) {
throw new Error('Provide at least one --trade=<TRADE_ID> or use --restore-batch=<BATCH_ID>.');
}
const { data: tradeRows, error: tradeError } = await supabase
.from('orders')
.select('id,order_id,user_id,profile_id,trade_id,symbol,action,side,qty,quantity,price,status,source,created_at,updated_at')
.in('trade_id', options.tradeIds)
.order('created_at', { ascending: true });
if (tradeError) throw tradeError;
const rows = (tradeRows || []) as OrderRow[];
const rowsByTrade = new Map<string, OrderRow[]>();
for (const row of rows) {
const tradeId = String(row.trade_id || '').trim();
if (!tradeId) continue;
const list = rowsByTrade.get(tradeId) || [];
list.push(row);
rowsByTrade.set(tradeId, list);
}
const candidateBfillOrderIds = Array.from(new Set(
rows
.map((row) => String(row.order_id || '').trim())
.filter((orderId) => orderId.startsWith('BFILL-'))
));
const preferredAuditByBackfill = new Map<string, AuditRow>();
if (candidateBfillOrderIds.length > 0) {
const { data: auditRows, error: auditError } = await supabase
.from('reconciliation_backfill_audit')
.select('id,batch_id,profile_id,symbol,trade_id,exchange_order_id,backfill_order_id,decision,reason,metadata,created_at,reverted_at')
.in('backfill_order_id', candidateBfillOrderIds)
.order('created_at', { ascending: false });
if (auditError) throw auditError;
const rowsByBackfill = new Map<string, AuditRow[]>();
for (const row of (auditRows || []) as AuditRow[]) {
const backfillOrderId = String(row.backfill_order_id || '').trim();
if (!backfillOrderId) continue;
const list = rowsByBackfill.get(backfillOrderId) || [];
list.push(row);
rowsByBackfill.set(backfillOrderId, list);
}
for (const [backfillOrderId, backfillAuditRows] of rowsByBackfill.entries()) {
const preferred = backfillAuditRows.find((row) => {
const decision = String(row.decision || '').trim();
const exchangeOrderId = String(row.exchange_order_id || '').trim();
return APPLIED_DECISIONS.has(decision) && exchangeOrderId.length > 0;
}) || backfillAuditRows[0];
if (preferred) {
preferredAuditByBackfill.set(backfillOrderId, preferred);
}
}
}
const exchangeOrderIds = Array.from(new Set(
Array.from(preferredAuditByBackfill.values())
.map((row) => String(row.exchange_order_id || '').trim())
.filter(Boolean)
));
const realOrderByOrderId = new Map<string, OrderRow>();
if (exchangeOrderIds.length > 0) {
const { data: realRows, error: realError } = await supabase
.from('orders')
.select('id,order_id,user_id,profile_id,trade_id,symbol,action,side,qty,quantity,price,status,source,created_at,updated_at')
.in('order_id', exchangeOrderIds);
if (realError) throw realError;
for (const row of (realRows || []) as OrderRow[]) {
const orderId = String(row.order_id || '').trim();
if (!orderId) continue;
if (!realOrderByOrderId.has(orderId)) {
realOrderByOrderId.set(orderId, row);
}
}
}
const batchId = `ATTRIB-REPAIR-${randomUUID()}`;
const nowIso = new Date().toISOString();
const evaluations: TradeEvaluation[] = [];
const auditRowsToInsert: Record<string, any>[] = [];
const orderIdsToRevert = new Set<string>();
for (const tradeId of options.tradeIds) {
const tradeRowsForId = rowsByTrade.get(tradeId) || [];
if (tradeRowsForId.length === 0) {
evaluations.push({
tradeId,
profileId: '',
symbol: '',
entryQty: 0,
exitQty: 0,
openQtyBefore: 0,
openQtyAfter: 0,
dustThreshold: DUST_ABS_QTY,
candidates: [],
selectedOrderIds: [],
decision: 'NO_GO',
reason: 'trade_not_found'
});
continue;
}
const ledger = buildTradeLedger(tradeRowsForId);
const dustThreshold = Math.max(DUST_ABS_QTY, ledger.entryQty * DUST_REL_PCT);
const candidates = tradeRowsForId
.filter((row) => String(row.order_id || '').trim().startsWith('BFILL-'))
.filter((row) => FILLED_STATUSES.has(normalizeStatus(row.status)))
.filter((row) => inferAction(row.action, row.side) === 'EXIT')
.map((row): CandidateRow => {
const orderId = String(row.order_id || '').trim();
const latestAudit = preferredAuditByBackfill.get(orderId);
if (!latestAudit) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId: '',
realOrder: null,
safe: false,
unsafeReason: 'missing_audit_link'
};
}
if (!APPLIED_DECISIONS.has(String(latestAudit.decision || '').trim())) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId: String(latestAudit.exchange_order_id || '').trim(),
realOrder: null,
safe: false,
unsafeReason: `audit_not_applied:${String(latestAudit.decision || '').trim() || 'unknown'}`
};
}
const exchangeOrderId = String(latestAudit.exchange_order_id || '').trim();
if (!exchangeOrderId) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder: null,
safe: false,
unsafeReason: 'missing_exchange_order_id'
};
}
const realOrder = realOrderByOrderId.get(exchangeOrderId) || null;
if (!realOrder) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder: null,
safe: false,
unsafeReason: 'exchange_order_not_persisted_in_orders'
};
}
if (!FILLED_STATUSES.has(normalizeStatus(realOrder.status))) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder,
safe: false,
unsafeReason: `exchange_order_not_filled:${normalizeStatus(realOrder.status)}`
};
}
const realTradeId = String(realOrder.trade_id || '').trim();
if (!options.allowCrossTrade && realTradeId !== tradeId) {
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder,
safe: false,
unsafeReason: `cross_trade_mapping:${realTradeId || 'null'}`
};
}
return {
order: row,
qty: toNumber(row.qty ?? row.quantity),
exchangeOrderId,
realOrder,
safe: true
};
});
if (!(ledger.openQty < -EPSILON)) {
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: ledger.openQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: [],
decision: 'SKIP',
reason: 'trade_not_overclosed'
});
continue;
}
const safeCandidates = candidates.filter((candidate) => candidate.safe && candidate.qty > EPSILON);
if (safeCandidates.length === 0) {
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: ledger.openQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: [],
decision: 'NO_GO',
reason: 'no_safe_candidates'
});
continue;
}
const bestSubset = pickBestSubset(ledger.openQty, safeCandidates, dustThreshold, options.allowPartial);
if (!bestSubset) {
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: ledger.openQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: [],
decision: 'NO_GO',
reason: options.allowPartial ? 'no_improving_subset' : 'no_subset_within_dust'
});
continue;
}
const resolvedDecision: TradeEvaluation['decision'] = options.apply ? 'APPLIED' : 'DRY_RUN';
for (const orderId of bestSubset.orderIds) {
orderIdsToRevert.add(orderId);
}
evaluations.push({
tradeId,
profileId: ledger.profileId,
symbol: ledger.symbol,
entryQty: ledger.entryQty,
exitQty: ledger.exitQty,
openQtyBefore: ledger.openQty,
openQtyAfter: bestSubset.nextOpenQty,
dustThreshold: Number(dustThreshold.toFixed(8)),
candidates,
selectedOrderIds: bestSubset.orderIds,
decision: resolvedDecision,
reason: bestSubset.withinDust ? 'synthetic_duplicate_subset_selected' : 'synthetic_duplicate_subset_selected_partial'
});
}
if (options.apply && orderIdsToRevert.size > 0) {
for (const orderId of orderIdsToRevert) {
const { error: updateError } = await supabase
.from('orders')
.update({
status: STATUS_REVERT_TARGET,
updated_at: nowIso
})
.eq('order_id', orderId)
.in('status', ['filled', 'partially_filled', 'partially-filled']);
if (updateError) throw updateError;
}
}
for (const evaluation of evaluations) {
if (evaluation.selectedOrderIds.length > 0) {
for (const orderId of evaluation.selectedOrderIds) {
const candidate = evaluation.candidates.find((row) => String(row.order.order_id || '').trim() === orderId);
if (!candidate) continue;
const prevStatus = normalizeStatus(candidate.order.status);
auditRowsToInsert.push({
batch_id: batchId,
profile_id: evaluation.profileId,
symbol: evaluation.symbol,
trade_id: evaluation.tradeId,
exchange_order_id: candidate.exchangeOrderId || null,
exchange_client_order_id: null,
backfill_order_id: orderId,
filled_qty: Number(candidate.qty.toFixed(8)),
filled_price: toNumber(candidate.order.price),
filled_at: null,
dry_run: !options.apply,
decision: options.apply ? 'ATTRIB_REPAIR_APPLIED' : 'ATTRIB_REPAIR_DRY',
reason: 'synthetic_duplicate_status_revert',
metadata: {
prevStatus: prevStatus || 'filled',
nextStatus: options.apply ? STATUS_REVERT_TARGET : `would_${STATUS_REVERT_TARGET}`,
openQtyBefore: evaluation.openQtyBefore,
openQtyAfter: evaluation.openQtyAfter,
dustThreshold: evaluation.dustThreshold,
realOrderTradeId: String(candidate.realOrder?.trade_id || '').trim() || null,
allowCrossTrade: options.allowCrossTrade,
allowPartial: options.allowPartial
},
applied_at: options.apply ? nowIso : null
});
}
} else if (evaluation.decision === 'NO_GO') {
auditRowsToInsert.push({
batch_id: batchId,
profile_id: evaluation.profileId || null,
symbol: evaluation.symbol || null,
trade_id: evaluation.tradeId,
exchange_order_id: null,
exchange_client_order_id: null,
backfill_order_id: null,
filled_qty: null,
filled_price: null,
filled_at: null,
dry_run: !options.apply,
decision: options.apply ? 'ATTRIB_REPAIR_NO_GO' : 'ATTRIB_REPAIR_NO_GO_DRY',
reason: evaluation.reason,
metadata: {
openQtyBefore: evaluation.openQtyBefore,
dustThreshold: evaluation.dustThreshold,
safeCandidates: evaluation.candidates.filter((row) => row.safe).length,
unsafeCandidates: evaluation.candidates.filter((row) => !row.safe).map((row) => ({
orderId: row.order.order_id,
reason: row.unsafeReason
}))
},
applied_at: null
});
}
}
if (auditRowsToInsert.length > 0) {
const { error: auditInsertError } = await supabase
.from('reconciliation_backfill_audit')
.insert(auditRowsToInsert);
if (auditInsertError) throw auditInsertError;
}
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
batchId,
tradeIds: options.tradeIds,
allowCrossTrade: options.allowCrossTrade,
allowPartial: options.allowPartial,
totalTrades: evaluations.length,
appliedOrderStatusReverts: options.apply ? orderIdsToRevert.size : 0,
evaluations: evaluations.map((evaluation) => ({
tradeId: evaluation.tradeId,
profileId: evaluation.profileId,
symbol: evaluation.symbol,
decision: evaluation.decision,
reason: evaluation.reason,
entryQty: evaluation.entryQty,
exitQty: evaluation.exitQty,
openQtyBefore: evaluation.openQtyBefore,
openQtyAfter: evaluation.openQtyAfter,
dustThreshold: evaluation.dustThreshold,
selectedOrderIds: evaluation.selectedOrderIds,
safeCandidateCount: evaluation.candidates.filter((row) => row.safe).length,
unsafeCandidateCount: evaluation.candidates.filter((row) => !row.safe).length
}))
}, null, 2));
};
const run = async (): Promise<void> => {
const options = parseArgs(process.argv.slice(2));
const supabaseUrl = String(process.env.SUPABASE_URL || '').trim();
const supabaseKey = String(
process.env.SUPABASE_KEY
|| process.env.SUPABASE_SERVICE_ROLE_KEY
|| process.env.SUPABASE_ANON_KEY
|| ''
).trim();
if (!supabaseUrl || !supabaseKey) {
throw new Error('Missing Supabase credentials. Expected SUPABASE_URL + SUPABASE_KEY/SUPABASE_SERVICE_ROLE_KEY.');
}
const supabase = createClient(supabaseUrl, supabaseKey);
if (options.restoreBatchId) {
await runRestoreMode(supabase, options);
return;
}
await runRepairMode(supabase, options);
};
run().catch((error) => {
console.error(JSON.stringify({
error: error instanceof Error ? error.message : String(error)
}, null, 2));
process.exit(1);
});

636
backend/reconcileCapitalLedgerState.ts Normal file
View File

@ -0,0 +1,636 @@
import 'dotenv/config';
import { createClient } from '@supabase/supabase-js';
type CliOptions = {
apply: boolean;
includeInactive: boolean;
includeQuarantinedHistory: boolean;
profileIds: string[];
};
type ProfileRow = {
id: string;
name?: string | null;
is_active?: boolean | null;
allocated_capital?: number | string | null;
};
type LedgerRow = {
profile_id: string;
allocated_capital?: number | string | null;
reserved_for_orders?: number | string | null;
reserved_for_positions?: number | string | null;
realized_pnl?: number | string | null;
updated_at?: string | null;
};
type TradeHistoryRow = {
pnl?: number | string | null;
reason?: string | null;
};
type OrderRow = {
symbol?: string | null;
trade_id?: string | null;
action?: string | null;
side?: string | null;
qty?: number | string | null;
quantity?: number | string | null;
price?: number | string | null;
status?: string | null;
timestamp?: number | string | null;
created_at?: string | null;
};
type ProfileRepairReport = {
profileId: string;
profileName: string;
isActive: boolean;
allocatedCapital: number;
ordersAnalyzed: number;
historyRowsAnalyzed: number;
openOrderRowsWithoutPrice: number;
current: {
allocatedCapital: number;
reservedForOrders: number;
reservedForPositions: number;
realizedPnl: number;
};
target: {
allocatedCapital: number;
reservedForOrders: number;
reservedForPositions: number;
realizedPnl: number;
};
delta: {
allocatedCapital: number;
reservedForOrders: number;
reservedForPositions: number;
realizedPnl: number;
};
rawUtilizationBeforePct: number | null;
rawUtilizationAfterPct: number | null;
overAllocatedBefore: boolean;
overAllocatedAfter: boolean;
changed: boolean;
};
const PAGE_SIZE = 1000;
const EPSILON = 1e-8;
const OPEN_ORDER_STATUSES = new Set([
'pending_new',
'accepted',
'pending',
'new',
'partially_filled',
'partially-filled',
'partiallyfilled',
'partial_fill'
]);
const FILLED_STATUSES = new Set([
'filled',
'partially_filled',
'partially-filled',
'partiallyfilled',
'partial_fill'
]);
const parseArgs = (argv: string[]): CliOptions => {
const profileIds = new Set<string>();
let apply = false;
let includeInactive = false;
let includeQuarantinedHistory = false;
for (const raw of argv) {
const arg = String(raw || '').trim();
if (!arg) continue;
if (arg === '--apply') {
apply = true;
continue;
}
if (arg === '--include-inactive') {
includeInactive = true;
continue;
}
if (arg === '--include-quarantined-history') {
includeQuarantinedHistory = true;
continue;
}
if (arg.startsWith('--profile=')) {
const value = String(arg.slice('--profile='.length) || '').trim();
if (value) profileIds.add(value);
}
}
return {
apply,
includeInactive,
includeQuarantinedHistory,
profileIds: Array.from(profileIds)
};
};
const toNumber = (value: unknown): number => {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : 0;
};
const round8 = (value: number): number => Number(value.toFixed(8));
const formatErrorPayload = (error: any): string => {
if (!error) return 'unknown_error';
if (error instanceof Error) {
return JSON.stringify({
message: error.message,
stack: error.stack || null
});
}
if (typeof error === 'object') {
return JSON.stringify({
message: String(error.message || ''),
code: String(error.code || ''),
details: String(error.details || ''),
hint: String(error.hint || ''),
raw: error
});
}
return JSON.stringify({ message: String(error) });
};
const failIfError = (error: any, context: string): void => {
if (!error) return;
throw new Error(`${context}: ${formatErrorPayload(error)}`);
};
const normalizeStatus = (status: unknown): string => {
const normalized = String(status || '').trim().toLowerCase();
if (normalized === 'partially-filled') return 'partially_filled';
if (normalized === 'partiallyfilled') return 'partially_filled';
if (normalized === 'partial_fill') return 'partially_filled';
return normalized;
};
const normalizeSide = (side: unknown): 'BUY' | 'SELL' => {
const normalized = String(side || '').trim().toUpperCase();
return normalized === 'SELL' || normalized === 'SHORT' ? 'SELL' : 'BUY';
};
const normalizeAction = (action: unknown): 'ENTRY' | 'EXIT' | undefined => {
const normalized = String(action || '').trim().toUpperCase();
if (normalized === 'ENTRY' || normalized === 'EXIT') return normalized;
return undefined;
};
const inferAction = (row: OrderRow, knownEntrySide?: 'BUY' | 'SELL'): 'ENTRY' | 'EXIT' => {
const explicit = normalizeAction(row.action);
if (explicit) return explicit;
const side = normalizeSide(row.side);
if (knownEntrySide) {
return side === knownEntrySide ? 'ENTRY' : 'EXIT';
}
return side === 'BUY' ? 'ENTRY' : 'EXIT';
};
const normalizeExecutionScopeSymbol = (symbol: unknown, provider: string): string => {
const raw = String(symbol || '')
.trim()
.toUpperCase()
.replace(/[\/_\-\s]/g, '');
if (!raw) return '';
if (provider === 'alpaca' && raw.endsWith('USDT')) {
return `${raw.slice(0, -4)}USD`;
}
return raw;
};
const toTimestamp = (row: OrderRow, fallback: number): number => {
const ts = Number(row.timestamp);
if (Number.isFinite(ts) && ts > 0) {
return ts > 1_000_000_000_000 ? ts : ts * 1000;
}
const created = Date.parse(String(row.created_at || ''));
if (Number.isFinite(created) && created > 0) return created;
return fallback;
};
const isQuarantinedHistoryReason = (reason: unknown): boolean => {
const normalized = String(reason || '').trim().toUpperCase();
return normalized.startsWith('[INVALID_')
|| normalized.startsWith('[DUPLICATE_')
|| normalized.startsWith('[RECONCILED_TO_');
};
const fetchPagedByProfile = async <T>(
supabase: any,
table: 'orders' | 'trade_history',
columns: string,
profileId: string
): Promise<T[]> => {
const rows: T[] = [];
let offset = 0;
for (;;) {
const { data, error } = await supabase
.from(table)
.select(columns)
.eq('profile_id', profileId)
.order('created_at', { ascending: true })
.range(offset, offset + PAGE_SIZE - 1);
failIfError(error, `fetchPagedByProfile:${table}:${profileId}`);
const chunk = (data || []) as T[];
if (!chunk.length) break;
rows.push(...chunk);
if (chunk.length < PAGE_SIZE) break;
offset += PAGE_SIZE;
}
return rows;
};
const fetchOrdersForProfile = async (
supabase: any,
profileId: string
): Promise<OrderRow[]> => {
const withQuantity = 'symbol,trade_id,action,side,qty,quantity,price,status,timestamp,created_at';
const withoutQuantity = 'symbol,trade_id,action,side,qty,price,status,timestamp,created_at';
try {
return await fetchPagedByProfile<OrderRow>(supabase, 'orders', withQuantity, profileId);
} catch (error: any) {
const message = String(error?.message || '').toLowerCase();
if (message.includes('column') && message.includes('quantity')) {
return await fetchPagedByProfile<OrderRow>(supabase, 'orders', withoutQuantity, profileId);
}
throw error;
}
};
const computeReservedForOpenEntryOrders = (orders: OrderRow[]): {
reserved: number;
rowsWithoutPrice: number;
} => {
let reserved = 0;
let rowsWithoutPrice = 0;
for (const row of orders) {
const status = normalizeStatus(row.status);
if (!OPEN_ORDER_STATUSES.has(status)) continue;
const action = inferAction(row);
if (action !== 'ENTRY') continue;
const qty = toNumber(row.qty);
const quantity = qty > 0 ? qty : toNumber(row.quantity);
if (!(quantity > 0)) continue;
const price = toNumber(row.price);
if (!(price > 0)) {
rowsWithoutPrice += 1;
continue;
}
reserved += quantity * price;
}
return {
reserved: round8(reserved),
rowsWithoutPrice
};
};
const computeOpenPositionNotional = (orders: OrderRow[], provider: string): number => {
type TradeLedger = {
side: 'BUY' | 'SELL';
entryQty: number;
entryNotional: number;
entryLastPrice: number;
exitQty: number;
};
const byExecutionSymbol = new Map<string, Array<{ row: OrderRow; ts: number; idx: number }>>();
let rowIndex = 0;
for (const row of orders) {
const status = normalizeStatus(row.status);
if (!FILLED_STATUSES.has(status)) continue;
const qty = toNumber(row.qty);
const quantity = qty > 0 ? qty : toNumber(row.quantity);
if (!(quantity > 0)) continue;
const symbolKey = normalizeExecutionScopeSymbol(row.symbol, provider);
if (!symbolKey) continue;
const bucket = byExecutionSymbol.get(symbolKey) || [];
bucket.push({
row,
ts: toTimestamp(row, rowIndex),
idx: rowIndex
});
byExecutionSymbol.set(symbolKey, bucket);
rowIndex += 1;
}
let reservedNotional = 0;
for (const [symbolKey, bucket] of Array.from(byExecutionSymbol.entries())) {
const ordered = [...bucket].sort((a, b) => (a.ts - b.ts) || (a.idx - b.idx));
const ledgerByTrade = new Map<string, TradeLedger>();
const entrySideByTrade = new Map<string, 'BUY' | 'SELL'>();
const openQueueBySide: Record<'BUY' | 'SELL', string[]> = { BUY: [], SELL: [] };
let syntheticCounter = 0;
const buildSyntheticTradeId = (side: 'BUY' | 'SELL', ts: number): string => {
syntheticCounter += 1;
const tsToken = Number.isFinite(ts) && ts > 0 ? Math.trunc(ts) : syntheticCounter;
return `__legacy__-${symbolKey}-${side}-${tsToken}-${String(syntheticCounter).padStart(4, '0')}`;
};
for (const wrapped of ordered) {
const row = wrapped.row;
const qtyRaw = toNumber(row.qty);
const qty = qtyRaw > 0 ? qtyRaw : toNumber(row.quantity);
if (!(qty > 0)) continue;
const rowSide = normalizeSide(row.side);
const oppositeSide: 'BUY' | 'SELL' = rowSide === 'BUY' ? 'SELL' : 'BUY';
const explicitAction = normalizeAction(row.action);
let action = explicitAction;
let tradeId = String(row.trade_id || '').trim();
if (!action && !tradeId) {
action = openQueueBySide[oppositeSide].length > 0 ? 'EXIT' : 'ENTRY';
}
if (!tradeId) {
if (action === 'EXIT' && openQueueBySide[oppositeSide].length > 0) {
tradeId = openQueueBySide[oppositeSide][0];
} else {
tradeId = buildSyntheticTradeId(action === 'EXIT' ? oppositeSide : rowSide, wrapped.ts);
}
}
if (!action) {
action = inferAction(row, entrySideByTrade.get(tradeId));
}
let tradeLedger = ledgerByTrade.get(tradeId);
if (!tradeLedger) {
tradeLedger = {
side: rowSide,
entryQty: 0,
entryNotional: 0,
entryLastPrice: 0,
exitQty: 0
};
ledgerByTrade.set(tradeId, tradeLedger);
}
if (action === 'ENTRY') {
if (tradeLedger.entryQty <= EPSILON) {
tradeLedger.side = rowSide;
}
tradeLedger.entryQty += qty;
entrySideByTrade.set(tradeId, tradeLedger.side);
if (!openQueueBySide[tradeLedger.side].includes(tradeId)) {
openQueueBySide[tradeLedger.side].push(tradeId);
}
const price = toNumber(row.price);
if (price > 0) {
tradeLedger.entryNotional += price * qty;
tradeLedger.entryLastPrice = price;
}
} else {
tradeLedger.exitQty += qty;
const queue = openQueueBySide[oppositeSide];
const idx = queue.findIndex((queuedTradeId) => queuedTradeId === tradeId);
if (idx >= 0) {
queue.splice(idx, 1);
} else if (queue.length > 0) {
queue.shift();
}
}
}
for (const tradeLedger of Array.from(ledgerByTrade.values())) {
const remainingQty = tradeLedger.entryQty - tradeLedger.exitQty;
if (!(remainingQty > EPSILON)) continue;
const weightedEntryPrice = tradeLedger.entryQty > EPSILON && tradeLedger.entryNotional > EPSILON
? tradeLedger.entryNotional / tradeLedger.entryQty
: tradeLedger.entryLastPrice;
if (!(weightedEntryPrice > 0)) continue;
reservedNotional += remainingQty * weightedEntryPrice;
}
}
return round8(reservedNotional);
};
const run = async (): Promise<void> => {
const options = parseArgs(process.argv.slice(2));
const supabaseUrl = String(process.env.SUPABASE_URL || '').trim();
const supabaseKey = String(
process.env.SUPABASE_KEY
|| process.env.SUPABASE_SERVICE_ROLE_KEY
|| process.env.SUPABASE_ANON_KEY
|| ''
).trim();
if (!supabaseUrl || !supabaseKey) {
throw new Error('Missing Supabase credentials. Expected SUPABASE_URL + SUPABASE_KEY/SUPABASE_SERVICE_ROLE_KEY.');
}
const provider = String(process.env.EXECUTION_PROVIDER || process.env.PROVIDER || 'alpaca').trim().toLowerCase() || 'alpaca';
const supabase = createClient(supabaseUrl, supabaseKey);
let profileQuery = supabase
.from('trade_profiles')
.select('id,name,is_active,allocated_capital')
.order('created_at', { ascending: true });
if (!options.includeInactive) {
profileQuery = profileQuery.eq('is_active', true);
}
if (options.profileIds.length > 0) {
profileQuery = profileQuery.in('id', options.profileIds);
}
const { data: profileData, error: profileError } = await profileQuery;
failIfError(profileError, 'load_profiles');
const profiles = (profileData || []) as ProfileRow[];
const profileIds = profiles.map((profile) => String(profile.id || '').trim()).filter(Boolean);
if (profileIds.length === 0) {
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
provider,
profilesProcessed: 0,
message: 'No matching profiles found.'
}, null, 2));
return;
}
const { data: ledgerData, error: ledgerError } = await supabase
.from('capital_ledgers')
.select('profile_id,allocated_capital,reserved_for_orders,reserved_for_positions,realized_pnl,updated_at')
.in('profile_id', profileIds);
failIfError(ledgerError, 'load_capital_ledgers');
const ledgerByProfile = new Map<string, LedgerRow>();
for (const row of (ledgerData || []) as LedgerRow[]) {
const profileId = String(row.profile_id || '').trim();
if (!profileId) continue;
ledgerByProfile.set(profileId, row);
}
const reports: ProfileRepairReport[] = [];
for (const profile of profiles) {
const profileId = String(profile.id || '').trim();
if (!profileId) continue;
const [historyRows, orders] = await Promise.all([
fetchPagedByProfile<TradeHistoryRow>(supabase, 'trade_history', 'pnl,reason', profileId),
fetchOrdersForProfile(supabase, profileId)
]);
const realizedFromHistory = historyRows.reduce((sum, row) => {
if (!options.includeQuarantinedHistory && isQuarantinedHistoryReason(row.reason)) {
return sum;
}
return sum + toNumber(row.pnl);
}, 0);
const openOrders = computeReservedForOpenEntryOrders(orders);
const reservedPositions = computeOpenPositionNotional(orders, provider);
const allocatedCapital = toNumber(profile.allocated_capital);
const currentLedger = ledgerByProfile.get(profileId);
const currentAllocated = toNumber(currentLedger?.allocated_capital);
const currentReservedOrders = toNumber(currentLedger?.reserved_for_orders);
const currentReservedPositions = toNumber(currentLedger?.reserved_for_positions);
const currentRealized = toNumber(currentLedger?.realized_pnl);
const targetAllocated = round8(allocatedCapital);
const targetReservedOrders = round8(openOrders.reserved);
const targetReservedPositions = round8(reservedPositions);
const targetRealized = round8(realizedFromHistory);
const deltaAllocated = round8(targetAllocated - currentAllocated);
const deltaReservedOrders = round8(targetReservedOrders - currentReservedOrders);
const deltaReservedPositions = round8(targetReservedPositions - currentReservedPositions);
const deltaRealized = round8(targetRealized - currentRealized);
const changed = !currentLedger
|| Math.abs(deltaAllocated) > 0.01
|| Math.abs(deltaReservedOrders) > 0.01
|| Math.abs(deltaReservedPositions) > 0.01
|| Math.abs(deltaRealized) > 0.01;
const utilizationBefore = currentAllocated > 0
? ((currentReservedOrders + currentReservedPositions) / currentAllocated) * 100
: null;
const utilizationAfter = targetAllocated > 0
? ((targetReservedOrders + targetReservedPositions) / targetAllocated) * 100
: null;
const overAllocatedBefore = currentAllocated > 0
? currentReservedOrders + currentReservedPositions > currentAllocated + EPSILON
: false;
const overAllocatedAfter = targetAllocated > 0
? targetReservedOrders + targetReservedPositions > targetAllocated + EPSILON
: false;
reports.push({
profileId,
profileName: String(profile.name || ''),
isActive: Boolean(profile.is_active),
allocatedCapital: targetAllocated,
ordersAnalyzed: orders.length,
historyRowsAnalyzed: historyRows.length,
openOrderRowsWithoutPrice: openOrders.rowsWithoutPrice,
current: {
allocatedCapital: round8(currentAllocated),
reservedForOrders: round8(currentReservedOrders),
reservedForPositions: round8(currentReservedPositions),
realizedPnl: round8(currentRealized)
},
target: {
allocatedCapital: targetAllocated,
reservedForOrders: targetReservedOrders,
reservedForPositions: targetReservedPositions,
realizedPnl: targetRealized
},
delta: {
allocatedCapital: deltaAllocated,
reservedForOrders: deltaReservedOrders,
reservedForPositions: deltaReservedPositions,
realizedPnl: deltaRealized
},
rawUtilizationBeforePct: utilizationBefore === null ? null : round8(utilizationBefore),
rawUtilizationAfterPct: utilizationAfter === null ? null : round8(utilizationAfter),
overAllocatedBefore,
overAllocatedAfter,
changed
});
}
const changedReports = reports.filter((report) => report.changed);
const payload = changedReports.map((report) => ({
profile_id: report.profileId,
allocated_capital: report.target.allocatedCapital,
reserved_for_orders: report.target.reservedForOrders,
reserved_for_positions: report.target.reservedForPositions,
realized_pnl: report.target.realizedPnl,
updated_at: new Date().toISOString()
}));
let appliedRows = 0;
if (options.apply && payload.length > 0) {
const { error: upsertError } = await supabase
.from('capital_ledgers')
.upsert(payload, { onConflict: 'profile_id' });
failIfError(upsertError, 'upsert_capital_ledgers');
appliedRows = payload.length;
}
const summary = {
mode: options.apply ? 'apply' : 'dry-run',
provider,
includeInactive: options.includeInactive,
includeQuarantinedHistory: options.includeQuarantinedHistory,
profilesProcessed: reports.length,
profilesChanged: changedReports.length,
appliedRows,
overAllocatedBeforeCount: reports.filter((report) => report.overAllocatedBefore).length,
overAllocatedAfterCount: reports.filter((report) => report.overAllocatedAfter).length,
totals: {
realizedDeltaApplied: round8(changedReports.reduce((sum, report) => sum + report.delta.realizedPnl, 0)),
reservedOrdersDeltaApplied: round8(changedReports.reduce((sum, report) => sum + report.delta.reservedForOrders, 0)),
reservedPositionsDeltaApplied: round8(changedReports.reduce((sum, report) => sum + report.delta.reservedForPositions, 0))
},
reports: reports.sort((left, right) => {
const leftScore = Math.abs(left.delta.realizedPnl) + Math.abs(left.delta.reservedForPositions);
const rightScore = Math.abs(right.delta.realizedPnl) + Math.abs(right.delta.reservedForPositions);
return rightScore - leftScore;
})
};
console.log(JSON.stringify(summary, null, 2));
};
run().catch((error) => {
console.error(JSON.stringify({
error: formatErrorPayload(error)
}, null, 2));
process.exit(1);
});

196
backend/reconcileClosedOrderFillData.ts Normal file
View File

@ -0,0 +1,196 @@
import 'dotenv/config';
import Alpaca from '@alpacahq/alpaca-trade-api';
import { createClient } from '@supabase/supabase-js';
type OrderRow = {
id?: string;
order_id?: string | null;
symbol?: string | null;
status?: string | null;
qty?: number | string | null;
price?: number | string | null;
source?: string | null;
created_at?: string | null;
updated_at?: string | null;
filled_at?: string | null;
};
const args = process.argv.slice(2);
const applyMode = args.includes('--apply');
const lookbackArg = args.find((arg) => arg.startsWith('--lookback-hours='));
const LOOKBACK_HOURS = Number((lookbackArg ? lookbackArg.split('=')[1] : '') || '48');
const PAGE_SIZE = 1000;
const SYNTHETIC_PREFIXES = ['BFILL-', 'MANOVR-', 'RECON-BF', 'RECON-', 'SYNC-'];
const supabaseUrl = String(process.env.SUPABASE_URL || '').trim();
const supabaseKey = String(
process.env.SUPABASE_KEY
|| process.env.SUPABASE_SERVICE_ROLE_KEY
|| process.env.SUPABASE_ANON_KEY
|| ''
).trim();
const alpacaApiKey = String(process.env.ALPACA_API_KEY || '').trim();
const alpacaApiSecret = String(process.env.ALPACA_API_SECRET || '').trim();
const paperTrading = String(process.env.PAPER_TRADING || 'true').toLowerCase() === 'true';
if (!supabaseUrl || !supabaseKey) {
throw new Error('Missing Supabase credentials.');
}
if (!alpacaApiKey || !alpacaApiSecret) {
throw new Error('Missing Alpaca credentials.');
}
if (!Number.isFinite(LOOKBACK_HOURS) || LOOKBACK_HOURS <= 0) {
throw new Error(`Invalid --lookback-hours value: ${lookbackArg}`);
}
const supabase = createClient(supabaseUrl, supabaseKey);
const alpaca = new (Alpaca as any)({
keyId: alpacaApiKey,
secretKey: alpacaApiSecret,
paper: paperTrading
});
const toNumber = (value: unknown): number => {
const numeric = Number(value);
return Number.isFinite(numeric) ? numeric : 0;
};
const normalizeStatus = (value: unknown): string => {
const normalized = String(value || '').trim().toLowerCase();
if (normalized === 'partially-filled') return 'partially_filled';
return normalized;
};
const isSyntheticOrderId = (orderId: string): boolean => {
const normalized = String(orderId || '').trim().toUpperCase();
if (!normalized) return false;
return SYNTHETIC_PREFIXES.some((prefix) => normalized.startsWith(prefix));
};
const pctDiff = (left: number, right: number): number => {
const denom = Math.max(Math.abs(left), Math.abs(right), 1e-9);
return Math.abs(left - right) / denom;
};
const run = async () => {
const sinceIso = new Date(Date.now() - LOOKBACK_HOURS * 60 * 60 * 1000).toISOString();
const rows: OrderRow[] = [];
let offset = 0;
for (;;) {
const { data, error } = await supabase
.from('orders')
.select('id,order_id,symbol,status,qty,price,source,created_at,updated_at,filled_at')
.in('status', ['filled', 'partially_filled', 'partially-filled'])
.gte('created_at', sinceIso)
.order('created_at', { ascending: false })
.range(offset, offset + PAGE_SIZE - 1);
if (error) throw error;
const chunk = (data || []) as OrderRow[];
if (!chunk.length) break;
rows.push(...chunk);
if (chunk.length < PAGE_SIZE) break;
offset += PAGE_SIZE;
}
const candidates = rows.filter((row) => {
const orderId = String(row.order_id || '').trim();
if (!orderId) return false;
if (isSyntheticOrderId(orderId)) return false;
const source = String(row.source || '').trim().toUpperCase();
if (source === 'MANUAL') return false;
return true;
});
const summary = {
mode: applyMode ? 'apply' : 'dry-run',
lookbackHours: LOOKBACK_HOURS,
sinceIso,
totalRows: rows.length,
candidates: candidates.length,
checked: 0,
exchangeMissing: 0,
noFillOnExchange: 0,
driftDetected: 0,
updated: 0,
samples: [] as Array<Record<string, unknown>>
};
for (const row of candidates) {
const orderId = String(row.order_id || '').trim();
summary.checked += 1;
let exchangeOrder: any;
try {
exchangeOrder = await (alpaca as any).getOrder(orderId);
} catch {
summary.exchangeMissing += 1;
continue;
}
if (!exchangeOrder) {
summary.exchangeMissing += 1;
continue;
}
const exchangeStatus = normalizeStatus(exchangeOrder.status);
const exchangeQty = toNumber(exchangeOrder.filled_qty ?? exchangeOrder.filledQty ?? exchangeOrder.filled_quantity);
const exchangePrice = toNumber(exchangeOrder.filled_avg_price);
if (!(exchangeQty > 0) || !(exchangePrice > 0)) {
summary.noFillOnExchange += 1;
continue;
}
const dbStatus = normalizeStatus(row.status);
const dbQty = toNumber(row.qty);
const dbPrice = toNumber(row.price);
const qtyDrift = dbQty <= 0 || pctDiff(exchangeQty, dbQty) > 1e-6;
const priceDrift = dbPrice <= 0 || pctDiff(exchangePrice, dbPrice) > 5e-4;
const statusDrift = exchangeStatus && exchangeStatus !== dbStatus;
if (!qtyDrift && !priceDrift && !statusDrift) continue;
summary.driftDetected += 1;
if (summary.samples.length < 20) {
summary.samples.push({
order_id: orderId,
symbol: row.symbol,
db_status: row.status,
ex_status: exchangeOrder.status,
db_qty: dbQty,
ex_qty: exchangeQty,
db_price: dbPrice,
ex_price: exchangePrice
});
}
if (!applyMode) continue;
const filledAtRaw = String(exchangeOrder.filled_at || '').trim();
const parsedFilledAt = filledAtRaw ? Date.parse(filledAtRaw) : NaN;
const filledAtIso = Number.isFinite(parsedFilledAt) && parsedFilledAt > 0
? new Date(parsedFilledAt).toISOString()
: new Date().toISOString();
const { error: updateError } = await supabase
.from('orders')
.update({
status: exchangeStatus || 'filled',
qty: exchangeQty,
price: exchangePrice,
filled_at: filledAtIso,
updated_at: new Date().toISOString()
})
.eq('order_id', orderId);
if (updateError) throw updateError;
summary.updated += 1;
}
console.log(JSON.stringify(summary, null, 2));
};
run().catch((error) => {
console.error(JSON.stringify({
error: error instanceof Error ? error.message : String(error)
}, null, 2));
process.exit(1);
});

217
backend/reconcileExitBackfillOnce.ts Normal file
View File

@ -0,0 +1,217 @@
import 'dotenv/config';
import { config, loadDynamicConfig } from '../src/config/index.js';
import { ConnectorFactory } from '../src/connectors/factory.js';
import { TradeExecutor } from '../src/services/TradeExecutor.js';
import { healthTracker } from '../src/services/healthTracker.js';
import { reconciliationExitBackfillService } from '../src/services/reconciliationExitBackfillService.js';
import { supabaseService } from '../src/services/SupabaseService.js';
type BackfillCliOptions = {
apply: boolean;
profileIds: Set<string>;
ignoreAllowlist: boolean;
};
type ProfileSummary = {
profileId: string;
userId: string;
attempted: boolean;
skippedReason?: string;
batchId?: string;
dryRun: boolean;
openTradeCandidates: number;
proposedRows: number;
insertedRows: number;
noGoTrades: number;
};
const parseOptions = (argv: string[]): BackfillCliOptions => {
const options: BackfillCliOptions = {
apply: false,
profileIds: new Set<string>(),
ignoreAllowlist: false
};
for (const arg of argv) {
if (arg === '--apply') {
options.apply = true;
continue;
}
if (arg === '--ignore-allowlist') {
options.ignoreAllowlist = true;
continue;
}
if (arg.startsWith('--profile=')) {
const value = String(arg.slice('--profile='.length) || '').trim();
if (value) options.profileIds.add(value);
continue;
}
}
return options;
};
const isPlaceholder = (value: string | undefined): boolean => {
const normalized = String(value || '').trim();
if (!normalized) return true;
return normalized === 'your_key' || normalized === 'your_secret';
};
const normalizeProfileIds = (profileIds: Set<string>): string[] => {
return Array.from(profileIds)
.map((value) => String(value || '').trim())
.filter(Boolean);
};
const run = async (): Promise<void> => {
const options = parseOptions(process.argv.slice(2));
await loadDynamicConfig(supabaseService);
const originalDryRun = config.RECON_EXIT_BACKFILL_DRY_RUN;
const originalAllowlist = [...config.RECON_EXIT_BACKFILL_PROFILE_ALLOWLIST];
if (!config.ENABLE_RECON_EXIT_BACKFILL) {
throw new Error('ENABLE_RECON_EXIT_BACKFILL=false. Enable it before running reconciliation EXIT backfill.');
}
config.RECON_EXIT_BACKFILL_DRY_RUN = !options.apply;
if (options.ignoreAllowlist) {
config.RECON_EXIT_BACKFILL_PROFILE_ALLOWLIST = [];
} else if (options.profileIds.size > 0) {
config.RECON_EXIT_BACKFILL_PROFILE_ALLOWLIST = normalizeProfileIds(options.profileIds);
}
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'maintenance-script',
lastChangedAt: Date.now(),
reason: 'Offline reconciliation EXIT backfill cycle'
});
const [users, profiles] = await Promise.all([
supabaseService.getActiveUsers(),
supabaseService.getActiveProfiles()
]);
const userById = new Map<string, any>();
for (const user of users || []) {
const userId = String((user as any)?.user_id || '').trim();
if (!userId) continue;
userById.set(userId, user);
}
const selectedProfiles = (profiles || []).filter((profile: any) => {
const profileId = String(profile?.id || '').trim();
if (!profileId) return false;
if (options.profileIds.size === 0) return true;
return options.profileIds.has(profileId);
});
const results: ProfileSummary[] = [];
for (const profile of selectedProfiles) {
const profileId = String(profile?.id || '').trim();
const userId = String(profile?.user_id || '').trim();
if (!profileId || !userId) continue;
const user = userById.get(userId);
if (!user) {
results.push({
profileId,
userId,
attempted: false,
skippedReason: 'user_not_found',
dryRun: config.RECON_EXIT_BACKFILL_DRY_RUN,
openTradeCandidates: 0,
proposedRows: 0,
insertedRows: 0,
noGoTrades: 0
});
continue;
}
const apiKey = config.PAPER_TRADING ? user.ALPACA_API_KEY : user.REAL_ALPACA_API_KEY;
const apiSecret = config.PAPER_TRADING ? user.ALPACA_SECRET_KEY : user.REAL_ALPACA_SECRET_KEY;
if (isPlaceholder(apiKey) || isPlaceholder(apiSecret)) {
results.push({
profileId,
userId,
attempted: false,
skippedReason: 'missing_exchange_credentials',
dryRun: config.RECON_EXIT_BACKFILL_DRY_RUN,
openTradeCandidates: 0,
proposedRows: 0,
insertedRows: 0,
noGoTrades: 0
});
continue;
}
const connector = ConnectorFactory.getCustomConnector(config.EXECUTION_PROVIDER, apiKey, apiSecret);
const executor = new TradeExecutor(connector, undefined, userId, profileId);
executor.setProfileSettings(profile);
try {
const result = await reconciliationExitBackfillService.runProfile({
profileId,
userId,
executor
});
results.push({
profileId,
userId,
attempted: result.attempted,
skippedReason: result.skippedReason,
batchId: result.batchId,
dryRun: result.dryRun,
openTradeCandidates: result.openTradeCandidates,
proposedRows: result.proposedRows,
insertedRows: result.insertedRows,
noGoTrades: result.noGoTrades
});
} finally {
executor.dispose();
}
}
config.RECON_EXIT_BACKFILL_DRY_RUN = originalDryRun;
config.RECON_EXIT_BACKFILL_PROFILE_ALLOWLIST = originalAllowlist;
const aggregate = results.reduce(
(acc, row) => {
if (row.attempted) acc.attemptedProfiles += 1;
if (!row.attempted && row.skippedReason) {
acc.skippedProfiles[row.skippedReason] = (acc.skippedProfiles[row.skippedReason] || 0) + 1;
}
acc.proposedRows += row.proposedRows;
acc.insertedRows += row.insertedRows;
acc.noGoTrades += row.noGoTrades;
return acc;
},
{
attemptedProfiles: 0,
skippedProfiles: {} as Record<string, number>,
proposedRows: 0,
insertedRows: 0,
noGoTrades: 0
}
);
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
profileFilter: normalizeProfileIds(options.profileIds),
ignoreAllowlist: options.ignoreAllowlist,
requirePause: config.RECON_EXIT_BACKFILL_REQUIRE_PAUSE,
dryRunFlagUsed: !options.apply,
aggregate,
results
}, null, 2));
};
run().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
console.error(JSON.stringify({ error: message }, null, 2));
process.exit(1);
});

308
backend/reconcileMissingOrderCoverage.ts Normal file
View File

@ -0,0 +1,308 @@
import 'dotenv/config';
import { config, loadDynamicConfig } from '../src/config/index.js';
import { AlpacaConnector } from '../src/connectors/alpaca.js';
import { TradeExecutor } from '../src/services/TradeExecutor.js';
import { reconciliationOrderCoverageService } from '../src/services/reconciliationOrderCoverageService.js';
import { supabaseService } from '../src/services/SupabaseService.js';
type CliOptions = {
apply: boolean;
profileIds: Set<string>;
lookbackHours?: number;
fetchLimitPerPage?: number;
maxFetchPages?: number;
maxInsertsPerProfile?: number;
ignoreFeatureFlag: boolean;
};
type ProfileSummary = {
profileId: string;
userId: string;
attempted: boolean;
skippedReason?: string;
dryRun: boolean;
scannedOrders: number;
filledLikeOrders: number;
botOwnedOrders: number;
eligibleOrders: number;
missingInDb: number;
insertedRows: number;
skippedNotBotOwned: number;
skippedUnmappedTrade: number;
skippedUnmappedAction: number;
skippedMissingFillData: number;
skippedMissingOrderId: number;
skippedExisting: number;
skippedMaxInsertLimit: number;
};
const parseOptions = (argv: string[]): CliOptions => {
const options: CliOptions = {
apply: false,
profileIds: new Set<string>(),
ignoreFeatureFlag: false
};
for (const arg of argv) {
if (arg === '--apply') {
options.apply = true;
continue;
}
if (arg === '--ignore-feature-flag') {
options.ignoreFeatureFlag = true;
continue;
}
if (arg.startsWith('--profile=')) {
const profileId = String(arg.slice('--profile='.length) || '').trim();
if (profileId) options.profileIds.add(profileId);
continue;
}
if (arg.startsWith('--lookback-hours=')) {
const parsed = Number(arg.slice('--lookback-hours='.length));
if (Number.isFinite(parsed) && parsed > 0) options.lookbackHours = Math.floor(parsed);
continue;
}
if (arg.startsWith('--max-inserts-per-profile=')) {
const parsed = Number(arg.slice('--max-inserts-per-profile='.length));
if (Number.isFinite(parsed) && parsed > 0) options.maxInsertsPerProfile = Math.floor(parsed);
continue;
}
if (arg.startsWith('--fetch-limit-per-page=')) {
const parsed = Number(arg.slice('--fetch-limit-per-page='.length));
if (Number.isFinite(parsed) && parsed > 0) options.fetchLimitPerPage = Math.floor(parsed);
continue;
}
if (arg.startsWith('--max-fetch-pages=')) {
const parsed = Number(arg.slice('--max-fetch-pages='.length));
if (Number.isFinite(parsed) && parsed > 0) options.maxFetchPages = Math.floor(parsed);
continue;
}
}
return options;
};
const isPlaceholder = (value: string | undefined): boolean => {
const normalized = String(value || '').trim();
if (!normalized) return true;
return normalized === 'your_key' || normalized === 'your_secret';
};
const normalizeProfileIds = (profileIds: Set<string>): string[] => {
return Array.from(profileIds)
.map((value) => String(value || '').trim())
.filter(Boolean);
};
const run = async (): Promise<void> => {
const options = parseOptions(process.argv.slice(2));
await loadDynamicConfig(supabaseService);
const originalEnabled = config.ENABLE_RECON_ORDER_COVERAGE_SYNC;
const originalDryRun = config.RECON_ORDER_COVERAGE_DRY_RUN;
const originalLookback = config.RECON_ORDER_COVERAGE_LOOKBACK_HOURS;
const originalFetchLimitPerPage = config.RECON_ORDER_COVERAGE_FETCH_LIMIT_PER_PAGE;
const originalMaxFetchPages = config.RECON_ORDER_COVERAGE_MAX_FETCH_PAGES;
const originalMaxInserts = config.RECON_ORDER_COVERAGE_MAX_INSERTS_PER_PROFILE;
if (!config.ENABLE_RECON_ORDER_COVERAGE_SYNC && !options.ignoreFeatureFlag) {
throw new Error('ENABLE_RECON_ORDER_COVERAGE_SYNC=false. Enable it or pass --ignore-feature-flag for one-shot run.');
}
if (options.ignoreFeatureFlag) {
config.ENABLE_RECON_ORDER_COVERAGE_SYNC = true;
}
config.RECON_ORDER_COVERAGE_DRY_RUN = !options.apply;
if (Number.isFinite(options.lookbackHours)) {
config.RECON_ORDER_COVERAGE_LOOKBACK_HOURS = Number(options.lookbackHours);
}
if (Number.isFinite(options.fetchLimitPerPage)) {
config.RECON_ORDER_COVERAGE_FETCH_LIMIT_PER_PAGE = Number(options.fetchLimitPerPage);
}
if (Number.isFinite(options.maxFetchPages)) {
config.RECON_ORDER_COVERAGE_MAX_FETCH_PAGES = Number(options.maxFetchPages);
}
if (Number.isFinite(options.maxInsertsPerProfile)) {
config.RECON_ORDER_COVERAGE_MAX_INSERTS_PER_PROFILE = Number(options.maxInsertsPerProfile);
}
const effectiveLookbackHours = config.RECON_ORDER_COVERAGE_LOOKBACK_HOURS;
const effectiveFetchLimitPerPage = config.RECON_ORDER_COVERAGE_FETCH_LIMIT_PER_PAGE;
const effectiveMaxFetchPages = config.RECON_ORDER_COVERAGE_MAX_FETCH_PAGES;
const effectiveMaxInsertsPerProfile = config.RECON_ORDER_COVERAGE_MAX_INSERTS_PER_PROFILE;
const effectiveDryRunFlag = config.RECON_ORDER_COVERAGE_DRY_RUN;
const [users, profiles] = await Promise.all([
supabaseService.getActiveUsers(),
supabaseService.getActiveProfiles()
]);
const userById = new Map<string, any>();
for (const user of users || []) {
const userId = String((user as any)?.user_id || '').trim();
if (!userId) continue;
userById.set(userId, user);
}
const selectedProfiles = (profiles || []).filter((profile: any) => {
const profileId = String(profile?.id || '').trim();
if (!profileId) return false;
if (options.profileIds.size === 0) return true;
return options.profileIds.has(profileId);
});
const results: ProfileSummary[] = [];
for (const profile of selectedProfiles) {
const profileId = String(profile?.id || '').trim();
const userId = String(profile?.user_id || '').trim();
if (!profileId || !userId) continue;
const user = userById.get(userId);
if (!user) {
results.push({
profileId,
userId,
attempted: false,
skippedReason: 'user_not_found',
dryRun: config.RECON_ORDER_COVERAGE_DRY_RUN,
scannedOrders: 0,
filledLikeOrders: 0,
botOwnedOrders: 0,
eligibleOrders: 0,
missingInDb: 0,
insertedRows: 0,
skippedNotBotOwned: 0,
skippedUnmappedTrade: 0,
skippedUnmappedAction: 0,
skippedMissingFillData: 0,
skippedMissingOrderId: 0,
skippedExisting: 0,
skippedMaxInsertLimit: 0
});
continue;
}
const apiKey = config.PAPER_TRADING ? user.ALPACA_API_KEY : user.REAL_ALPACA_API_KEY;
const apiSecret = config.PAPER_TRADING ? user.ALPACA_SECRET_KEY : user.REAL_ALPACA_SECRET_KEY;
if (isPlaceholder(apiKey) || isPlaceholder(apiSecret)) {
results.push({
profileId,
userId,
attempted: false,
skippedReason: 'missing_exchange_credentials',
dryRun: config.RECON_ORDER_COVERAGE_DRY_RUN,
scannedOrders: 0,
filledLikeOrders: 0,
botOwnedOrders: 0,
eligibleOrders: 0,
missingInDb: 0,
insertedRows: 0,
skippedNotBotOwned: 0,
skippedUnmappedTrade: 0,
skippedUnmappedAction: 0,
skippedMissingFillData: 0,
skippedMissingOrderId: 0,
skippedExisting: 0,
skippedMaxInsertLimit: 0
});
continue;
}
const connector = new AlpacaConnector(apiKey, apiSecret);
const executor = new TradeExecutor(connector, undefined, userId, profileId);
executor.setProfileSettings(profile);
try {
const result = await reconciliationOrderCoverageService.runProfile({
profileId,
userId,
executor
});
results.push({
profileId,
userId,
attempted: result.attempted,
skippedReason: result.skippedReason,
dryRun: result.dryRun,
scannedOrders: result.scannedOrders,
filledLikeOrders: result.filledLikeOrders,
botOwnedOrders: result.botOwnedOrders,
eligibleOrders: result.eligibleOrders,
missingInDb: result.missingInDb,
insertedRows: result.insertedRows,
skippedNotBotOwned: result.skippedNotBotOwned,
skippedUnmappedTrade: result.skippedUnmappedTrade,
skippedUnmappedAction: result.skippedUnmappedAction,
skippedMissingFillData: result.skippedMissingFillData,
skippedMissingOrderId: result.skippedMissingOrderId,
skippedExisting: result.skippedExisting,
skippedMaxInsertLimit: result.skippedMaxInsertLimit
});
} finally {
executor.dispose();
}
}
config.ENABLE_RECON_ORDER_COVERAGE_SYNC = originalEnabled;
config.RECON_ORDER_COVERAGE_DRY_RUN = originalDryRun;
config.RECON_ORDER_COVERAGE_LOOKBACK_HOURS = originalLookback;
config.RECON_ORDER_COVERAGE_FETCH_LIMIT_PER_PAGE = originalFetchLimitPerPage;
config.RECON_ORDER_COVERAGE_MAX_FETCH_PAGES = originalMaxFetchPages;
config.RECON_ORDER_COVERAGE_MAX_INSERTS_PER_PROFILE = originalMaxInserts;
const aggregate = results.reduce((acc, row) => {
if (row.attempted) acc.attemptedProfiles += 1;
if (!row.attempted && row.skippedReason) {
acc.skippedProfiles[row.skippedReason] = (acc.skippedProfiles[row.skippedReason] || 0) + 1;
}
acc.scannedOrders += row.scannedOrders;
acc.filledLikeOrders += row.filledLikeOrders;
acc.botOwnedOrders += row.botOwnedOrders;
acc.eligibleOrders += row.eligibleOrders;
acc.missingInDb += row.missingInDb;
acc.insertedRows += row.insertedRows;
acc.skippedNotBotOwned += row.skippedNotBotOwned;
acc.skippedUnmappedTrade += row.skippedUnmappedTrade;
acc.skippedUnmappedAction += row.skippedUnmappedAction;
acc.skippedMissingFillData += row.skippedMissingFillData;
acc.skippedMissingOrderId += row.skippedMissingOrderId;
acc.skippedExisting += row.skippedExisting;
acc.skippedMaxInsertLimit += row.skippedMaxInsertLimit;
return acc;
}, {
attemptedProfiles: 0,
skippedProfiles: {} as Record<string, number>,
scannedOrders: 0,
filledLikeOrders: 0,
botOwnedOrders: 0,
eligibleOrders: 0,
missingInDb: 0,
insertedRows: 0,
skippedNotBotOwned: 0,
skippedUnmappedTrade: 0,
skippedUnmappedAction: 0,
skippedMissingFillData: 0,
skippedMissingOrderId: 0,
skippedExisting: 0,
skippedMaxInsertLimit: 0
});
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
profileFilter: normalizeProfileIds(options.profileIds),
ignoreFeatureFlag: options.ignoreFeatureFlag,
configuredLookbackHours: effectiveLookbackHours,
configuredFetchLimitPerPage: effectiveFetchLimitPerPage,
configuredMaxFetchPages: effectiveMaxFetchPages,
configuredMaxInsertsPerProfile: effectiveMaxInsertsPerProfile,
dryRunFlagUsed: effectiveDryRunFlag,
aggregate,
results
}, null, 2));
};
run().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
console.error(JSON.stringify({ error: message }, null, 2));
process.exit(1);
});

425
backend/reconcileNoGoQtyMismatchOnce.ts Normal file
View File

@ -0,0 +1,425 @@
import 'dotenv/config';
import { createHash, randomUUID } from 'crypto';
import { config, loadDynamicConfig } from '../src/config/index.js';
import { ConnectorFactory } from '../src/connectors/factory.js';
import { normalizeOrderAction, normalizeTradeSide } from '../src/domain/tradingEnums.js';
import { healthTracker } from '../src/services/healthTracker.js';
import {
ReconciliationBackfillAuditInsert,
ReconciliationBackfillOrderInsert,
supabaseService
} from '../src/services/SupabaseService.js';
import { TradeExecutor } from '../src/services/TradeExecutor.js';
import logger from '../src/utils/logger.js';
import { buildAlpacaSubTag } from '../src/utils/alpacaSubTag.js';
type CliOptions = {
apply: boolean;
profileIds: Set<string>;
};
type ExchangeEvidence = {
orderId: string;
side: 'BUY' | 'SELL';
qty: number;
price: number;
filledAtIso: string;
};
type TradeSlice = {
symbol: string;
tradeId: string;
entrySide: 'BUY' | 'SELL';
entryQty: number;
exitQty: number;
openQty: number;
};
type ProposedCandidate = {
order: ReconciliationBackfillOrderInsert;
exchangeOrderId: string;
};
const EPSILON = 1e-8;
const MAX_LOOKBACK_HOURS = 240;
const NO_GO_REASON = 'missing_fill_evidence_for_large_remainder';
const parseOptions = (argv: string[]): CliOptions => {
const out: CliOptions = {
apply: false,
profileIds: new Set<string>()
};
for (const arg of argv) {
if (arg === '--apply') {
out.apply = true;
continue;
}
if (arg.startsWith('--profile=')) {
const value = String(arg.slice('--profile='.length) || '').trim();
if (value) out.profileIds.add(value);
}
}
return out;
};
const toNumber = (value: unknown): number => {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : 0;
};
const toTimestampMs = (value: unknown): number => {
if (typeof value === 'number') {
if (Number.isFinite(value) && value > 1_000_000_000_000) return value;
if (Number.isFinite(value) && value > 0) return value * 1000;
return Date.now();
}
if (typeof value === 'string') {
const trimmed = value.trim();
if (!trimmed) return Date.now();
const numeric = Number(trimmed);
if (Number.isFinite(numeric) && numeric > 0) return toTimestampMs(numeric);
const parsed = Date.parse(trimmed);
if (Number.isFinite(parsed) && parsed > 0) return parsed;
}
return Date.now();
};
const buildBackfillOrderId = (
profileId: string,
tradeId: string,
exchangeOrderId: string,
filledAtIso: string
): string => {
const digest = createHash('md5')
.update(`${profileId}:${tradeId}:${exchangeOrderId}:${filledAtIso}`)
.digest('hex');
return `BFILL-${digest}`;
};
const expectedExitSide = (entrySide: 'BUY' | 'SELL'): 'BUY' | 'SELL' => {
return entrySide === 'BUY' ? 'SELL' : 'BUY';
};
const isPlaceholder = (value: string | undefined): boolean => {
const normalized = String(value || '').trim();
if (!normalized) return true;
return normalized === 'your_key' || normalized === 'your_secret';
};
const buildOpenTradeSlices = (rows: any[]): Map<string, TradeSlice> => {
const out = new Map<string, TradeSlice>();
const sorted = [...rows].sort((a, b) => {
const ats = toTimestampMs(a.timestamp ?? a.created_at ?? 0);
const bts = toTimestampMs(b.timestamp ?? b.created_at ?? 0);
return ats - bts;
});
for (const row of sorted) {
const tradeId = String(row.trade_id || '').trim();
if (!tradeId) continue;
const symbol = String(row.symbol || '').trim();
if (!symbol) continue;
const qty = toNumber(row.qty ?? row.quantity);
if (!(qty > EPSILON)) continue;
const side = normalizeTradeSide(String(row.side || 'BUY'));
const explicitAction = normalizeOrderAction(row.action || undefined);
let slice = out.get(tradeId);
if (!slice) {
slice = {
symbol,
tradeId,
entrySide: side,
entryQty: 0,
exitQty: 0,
openQty: 0
};
out.set(tradeId, slice);
}
const action = explicitAction || (side === slice.entrySide ? 'ENTRY' : 'EXIT');
if (action === 'ENTRY') {
if (!(slice.entryQty > EPSILON)) {
slice.entrySide = side;
}
slice.entryQty += qty;
} else {
slice.exitQty += qty;
}
slice.openQty = Number((slice.entryQty - slice.exitQty).toFixed(8));
}
for (const [tradeId, slice] of Array.from(out.entries())) {
if (!(slice.openQty > EPSILON)) out.delete(tradeId);
}
return out;
};
const normalizeExchangeEvidence = (rows: any[]): Map<string, ExchangeEvidence> => {
const out = new Map<string, ExchangeEvidence>();
for (const row of rows || []) {
const orderId = String(row?.id || row?.order_id || '').trim();
if (!orderId) continue;
const side = normalizeTradeSide(String(row?.side || 'BUY'));
const qty = toNumber(row?.filled_qty ?? row?.filledQty ?? row?.qty ?? row?.amount ?? row?.size);
const price = toNumber(row?.filled_avg_price ?? row?.avg_price ?? row?.price ?? row?.limit_price);
if (!(qty > EPSILON)) continue;
const filledAtMs = toTimestampMs(
row?.filled_at ?? row?.filledAt ?? row?.updated_at ?? row?.closed_at ?? row?.submitted_at ?? row?.timestamp
);
out.set(orderId, {
orderId,
side,
qty,
price,
filledAtIso: new Date(filledAtMs).toISOString()
});
}
return out;
};
const run = async (): Promise<void> => {
const options = parseOptions(process.argv.slice(2));
logger.silent = true;
await loadDynamicConfig(supabaseService);
healthTracker.recordTradingControl({
mode: 'PAUSED',
lastChangedBy: 'maintenance-script',
lastChangedAt: Date.now(),
reason: 'NO_GO mismatch repair cycle'
});
const [users, profiles] = await Promise.all([
supabaseService.getActiveUsers(),
supabaseService.getActiveProfiles()
]);
const userById = new Map<string, any>();
for (const user of users || []) {
const userId = String((user as any)?.user_id || '').trim();
if (!userId) continue;
userById.set(userId, user);
}
const noGoAudit = await supabaseService.getReconciliationBackfillAuditRows({
decisions: ['NO_GO'],
limit: 1000,
offset: 0
});
const seenNoGoTrade = new Set<string>();
const noGoTradeByProfile = new Map<string, Set<string>>();
for (const row of noGoAudit.rows || []) {
const reason = String(row.reason || '').trim();
if (reason !== NO_GO_REASON) continue;
const tradeId = String(row.trade_id || '').trim();
const profileId = String(row.profile_id || '').trim();
if (!tradeId || !profileId) continue;
if (options.profileIds.size > 0 && !options.profileIds.has(profileId)) continue;
const dedupe = `${profileId}::${tradeId}`;
if (seenNoGoTrade.has(dedupe)) continue;
seenNoGoTrade.add(dedupe);
const list = noGoTradeByProfile.get(profileId) || new Set<string>();
list.add(tradeId);
noGoTradeByProfile.set(profileId, list);
}
const batchId = `RECON-BFILL-NOGO-MISMATCH-${randomUUID()}`;
const proposedRows: ProposedCandidate[] = [];
const auditRows: ReconciliationBackfillAuditInsert[] = [];
const skipped: Array<Record<string, any>> = [];
for (const profile of profiles || []) {
const profileId = String(profile?.id || '').trim();
if (!profileId) continue;
const targetTrades = noGoTradeByProfile.get(profileId);
if (!targetTrades || targetTrades.size === 0) continue;
const userId = String(profile?.user_id || '').trim();
const user = userById.get(userId);
if (!user) {
skipped.push({ profileId, reason: 'user_not_found' });
continue;
}
const apiKey = config.PAPER_TRADING ? user.ALPACA_API_KEY : user.REAL_ALPACA_API_KEY;
const apiSecret = config.PAPER_TRADING ? user.ALPACA_SECRET_KEY : user.REAL_ALPACA_SECRET_KEY;
if (isPlaceholder(apiKey) || isPlaceholder(apiSecret)) {
skipped.push({ profileId, reason: 'missing_exchange_credentials' });
continue;
}
const lifecycleRows = await supabaseService.getFilledLifecycleOrdersForProfile(profileId);
const slices = buildOpenTradeSlices(lifecycleRows);
const symbols = Array.from(new Set(
Array.from(targetTrades.values())
.map((tradeId) => slices.get(tradeId)?.symbol)
.filter(Boolean) as string[]
));
if (symbols.length === 0) continue;
const connector = ConnectorFactory.getCustomConnector(config.EXECUTION_PROVIDER, apiKey, apiSecret);
const executor = new TradeExecutor(connector, undefined, userId, profileId);
executor.setProfileSettings(profile);
try {
const exchangeRows = await executor.fetchExchangeClosedOrders(symbols, MAX_LOOKBACK_HOURS);
const evidenceByOrderId = normalizeExchangeEvidence(exchangeRows || []);
for (const tradeId of targetTrades.values()) {
const slice = slices.get(tradeId);
if (!slice || !(slice.openQty > EPSILON)) continue;
const expectedSide = expectedExitSide(slice.entrySide);
const exitRows = lifecycleRows.filter((row) => {
return String(row.trade_id || '').trim() === tradeId
&& normalizeOrderAction(row.action || undefined) === 'EXIT'
&& (normalizeTradeSide(String(row.side || '')) === expectedSide);
});
let remaining = slice.openQty;
let evidenceUsed = 0;
for (const exitRow of exitRows) {
if (!(remaining > EPSILON)) break;
const orderId = String(exitRow.order_id || '').trim();
if (!orderId) continue;
const evidence = evidenceByOrderId.get(orderId);
if (!evidence) continue;
if (evidence.side !== expectedSide) continue;
const dbQty = toNumber(exitRow.qty ?? exitRow.quantity);
const exchangeQty = toNumber(evidence.qty);
const missingFromOrder = Number((exchangeQty - dbQty).toFixed(8));
if (!(missingFromOrder > EPSILON)) continue;
const applyQty = Math.min(remaining, missingFromOrder);
if (!(applyQty > EPSILON)) continue;
const filledAtIso = evidence.filledAtIso;
const backfillOrderId = buildBackfillOrderId(profileId, tradeId, orderId, filledAtIso);
const subTag = buildAlpacaSubTag({
profileId,
tradeId,
intent: 'EXIT'
}) || undefined;
proposedRows.push({
order: {
user_id: userId,
profile_id: profileId,
order_id: backfillOrderId,
symbol: slice.symbol,
type: 'market',
side: expectedSide,
qty: Number(applyQty.toFixed(8)),
quantity: Number(applyQty.toFixed(8)),
price: Number(toNumber(evidence.price).toFixed(8)),
status: 'filled',
timestamp: toTimestampMs(filledAtIso),
filled_at: filledAtIso,
trade_id: tradeId,
action: 'EXIT',
source: 'BOT',
sub_tag: subTag
},
exchangeOrderId: orderId
});
auditRows.push({
batch_id: batchId,
profile_id: profileId,
symbol: slice.symbol,
trade_id: tradeId,
exchange_order_id: orderId,
exchange_client_order_id: null,
backfill_order_id: backfillOrderId,
filled_qty: Number(applyQty.toFixed(8)),
filled_price: Number(toNumber(evidence.price).toFixed(8)),
filled_at: filledAtIso,
dry_run: !options.apply,
decision: options.apply ? 'PENDING_APPLY' : 'DRY_RUN',
reason: 'existing_exit_order_qty_mismatch',
metadata: {
openQtyBefore: slice.openQty,
expectedSide,
dbExitQty: dbQty,
exchangeFilledQty: exchangeQty,
missingFromOrder,
applyQty
}
});
remaining = Number((remaining - applyQty).toFixed(8));
evidenceUsed += 1;
}
if (remaining > EPSILON) {
skipped.push({
profileId,
tradeId,
symbol: slice.symbol,
reason: 'insufficient_same_order_evidence',
remaining,
evidenceUsed
});
}
}
} finally {
executor.dispose();
}
}
await supabaseService.insertReconciliationBackfillAuditRows(auditRows);
let insertedRows = 0;
if (options.apply && proposedRows.length > 0) {
const proposedOrderIds = proposedRows.map((row) => row.order.order_id);
const existingBefore = await supabaseService.getExistingOrderIds(proposedOrderIds);
const ok = await supabaseService.upsertReconciliationBackfillOrders(proposedRows.map((row) => row.order));
if (!ok) throw new Error('Failed to upsert mismatch backfill rows.');
const existingAfter = await supabaseService.getExistingOrderIds(proposedOrderIds);
insertedRows = proposedRows.filter((row) => !existingBefore.has(row.order.order_id) && existingAfter.has(row.order.order_id)).length;
const postAuditRows: ReconciliationBackfillAuditInsert[] = proposedRows.map((row) => ({
batch_id: batchId,
profile_id: row.order.profile_id,
symbol: row.order.symbol,
trade_id: row.order.trade_id,
exchange_order_id: row.exchangeOrderId,
exchange_client_order_id: null,
backfill_order_id: row.order.order_id,
filled_qty: row.order.qty,
filled_price: row.order.price,
filled_at: row.order.filled_at || null,
dry_run: false,
decision: existingBefore.has(row.order.order_id) ? 'SKIP_EXISTING' : 'APPLIED',
reason: existingBefore.has(row.order.order_id) ? 'already_exists' : 'inserted_existing_exit_order_qty_mismatch',
metadata: {
matchedBy: 'existing_exit_order_qty_mismatch'
},
applied_at: !existingBefore.has(row.order.order_id) ? new Date().toISOString() : null
}));
await supabaseService.insertReconciliationBackfillAuditRows(postAuditRows);
}
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
batchId,
proposedRows: proposedRows.length,
insertedRows,
skipped
}, null, 2));
};
run().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
console.error(JSON.stringify({ error: message }, null, 2));
process.exit(1);
});

183
backend/reconcileSubTagRepair.ts Normal file
View File

@ -0,0 +1,183 @@
import 'dotenv/config';
import { config, loadDynamicConfig } from '../src/config/index.js';
import { reconciliationSubTagRepairService } from '../src/services/reconciliationSubTagRepairService.js';
import { supabaseService } from '../src/services/SupabaseService.js';
type CliOptions = {
apply: boolean;
profileIds: Set<string>;
lookbackHours?: number;
maxUpdatesPerProfile?: number;
ignoreFeatureFlag: boolean;
};
type ProfileSummary = {
profileId: string;
userId: string;
attempted: boolean;
skippedReason?: string;
unsupported?: boolean;
dryRun: boolean;
scannedRows: number;
eligibleRows: number;
updatedRows: number;
skippedNoProfile: number;
skippedNoTrade: number;
skippedTagDisabled: number;
skippedAlreadyTagged: number;
};
const parseOptions = (argv: string[]): CliOptions => {
const options: CliOptions = {
apply: false,
profileIds: new Set<string>(),
ignoreFeatureFlag: false
};
for (const arg of argv) {
if (arg === '--apply') {
options.apply = true;
continue;
}
if (arg === '--ignore-feature-flag') {
options.ignoreFeatureFlag = true;
continue;
}
if (arg.startsWith('--profile=')) {
const profileId = String(arg.slice('--profile='.length) || '').trim();
if (profileId) options.profileIds.add(profileId);
continue;
}
if (arg.startsWith('--lookback-hours=')) {
const parsed = Number(arg.slice('--lookback-hours='.length));
if (Number.isFinite(parsed) && parsed > 0) options.lookbackHours = Math.floor(parsed);
continue;
}
if (arg.startsWith('--max-updates-per-profile=')) {
const parsed = Number(arg.slice('--max-updates-per-profile='.length));
if (Number.isFinite(parsed) && parsed > 0) options.maxUpdatesPerProfile = Math.floor(parsed);
continue;
}
}
return options;
};
const normalizeProfileIds = (profileIds: Set<string>): string[] => {
return Array.from(profileIds)
.map((value) => String(value || '').trim())
.filter(Boolean);
};
const run = async (): Promise<void> => {
const options = parseOptions(process.argv.slice(2));
await loadDynamicConfig(supabaseService);
const originalEnabled = config.ENABLE_RECON_SUBTAG_REPAIR;
const originalDryRun = config.RECON_SUBTAG_REPAIR_DRY_RUN;
const originalLookback = config.RECON_SUBTAG_REPAIR_LOOKBACK_HOURS;
const originalMaxUpdates = config.RECON_SUBTAG_REPAIR_MAX_UPDATES_PER_PROFILE;
if (!config.ENABLE_RECON_SUBTAG_REPAIR && !options.ignoreFeatureFlag) {
throw new Error('ENABLE_RECON_SUBTAG_REPAIR=false. Enable it or pass --ignore-feature-flag for one-shot run.');
}
if (options.ignoreFeatureFlag) {
config.ENABLE_RECON_SUBTAG_REPAIR = true;
}
config.RECON_SUBTAG_REPAIR_DRY_RUN = !options.apply;
if (Number.isFinite(options.lookbackHours)) {
config.RECON_SUBTAG_REPAIR_LOOKBACK_HOURS = Number(options.lookbackHours);
}
if (Number.isFinite(options.maxUpdatesPerProfile)) {
config.RECON_SUBTAG_REPAIR_MAX_UPDATES_PER_PROFILE = Number(options.maxUpdatesPerProfile);
}
const effectiveDryRun = config.RECON_SUBTAG_REPAIR_DRY_RUN;
const effectiveLookback = config.RECON_SUBTAG_REPAIR_LOOKBACK_HOURS;
const effectiveMaxUpdates = config.RECON_SUBTAG_REPAIR_MAX_UPDATES_PER_PROFILE;
const profiles = await supabaseService.getActiveProfiles();
const selectedProfiles = (profiles || []).filter((profile: any) => {
const profileId = String(profile?.id || '').trim();
if (!profileId) return false;
if (options.profileIds.size === 0) return true;
return options.profileIds.has(profileId);
});
const results: ProfileSummary[] = [];
for (const profile of selectedProfiles) {
const profileId = String(profile?.id || '').trim();
const userId = String(profile?.user_id || '').trim();
if (!profileId || !userId) continue;
const result = await reconciliationSubTagRepairService.runProfile({
profileId,
userId
});
results.push({
profileId,
userId,
attempted: result.attempted,
skippedReason: result.skippedReason,
unsupported: result.unsupported,
dryRun: result.dryRun,
scannedRows: result.scannedRows,
eligibleRows: result.eligibleRows,
updatedRows: result.updatedRows,
skippedNoProfile: result.skippedNoProfile,
skippedNoTrade: result.skippedNoTrade,
skippedTagDisabled: result.skippedTagDisabled,
skippedAlreadyTagged: result.skippedAlreadyTagged
});
}
config.ENABLE_RECON_SUBTAG_REPAIR = originalEnabled;
config.RECON_SUBTAG_REPAIR_DRY_RUN = originalDryRun;
config.RECON_SUBTAG_REPAIR_LOOKBACK_HOURS = originalLookback;
config.RECON_SUBTAG_REPAIR_MAX_UPDATES_PER_PROFILE = originalMaxUpdates;
const aggregate = results.reduce((acc, row) => {
if (row.attempted) acc.attemptedProfiles += 1;
if (!row.attempted && row.skippedReason) {
acc.skippedProfiles[row.skippedReason] = (acc.skippedProfiles[row.skippedReason] || 0) + 1;
}
if (row.unsupported) acc.unsupportedProfiles += 1;
acc.scannedRows += row.scannedRows;
acc.eligibleRows += row.eligibleRows;
acc.updatedRows += row.updatedRows;
acc.skippedNoProfile += row.skippedNoProfile;
acc.skippedNoTrade += row.skippedNoTrade;
acc.skippedTagDisabled += row.skippedTagDisabled;
acc.skippedAlreadyTagged += row.skippedAlreadyTagged;
return acc;
}, {
attemptedProfiles: 0,
skippedProfiles: {} as Record<string, number>,
unsupportedProfiles: 0,
scannedRows: 0,
eligibleRows: 0,
updatedRows: 0,
skippedNoProfile: 0,
skippedNoTrade: 0,
skippedTagDisabled: 0,
skippedAlreadyTagged: 0
});
console.log(JSON.stringify({
mode: options.apply ? 'apply' : 'dry-run',
profileFilter: normalizeProfileIds(options.profileIds),
ignoreFeatureFlag: options.ignoreFeatureFlag,
configuredLookbackHours: effectiveLookback,
configuredMaxUpdatesPerProfile: effectiveMaxUpdates,
dryRunFlagUsed: effectiveDryRun,
aggregate,
results
}, null, 2));
};
run().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
console.error(JSON.stringify({ error: message }, null, 2));
process.exit(1);
});

372
backend/reconcileTradeHistoryLifecycle.ts Normal file
View File

@ -0,0 +1,372 @@
import 'dotenv/config';
import { createClient } from '@supabase/supabase-js';
type OrderRow = {
id: string;
order_id?: string | null;
user_id?: string | null;
profile_id?: string | null;
symbol?: string | null;
trade_id?: string | null;
action?: string | null;
side?: string | null;
qty?: number | string | null;
price?: number | string | null;
status?: string | null;
created_at?: string | null;
};
type HistoryRow = {
id: string;
user_id?: string | null;
profile_id?: string | null;
symbol?: string | null;
trade_id?: string | null;
side?: string | null;
size?: number | string | null;
entry_price?: number | string | null;
exit_price?: number | string | null;
pnl?: number | string | null;
pnl_percent?: number | string | null;
reason?: string | null;
source?: string | null;
timestamp?: number | string | null;
created_at?: string | null;
};
type CanonicalTrade = {
tradeId: string;
userId: string | null;
profileId: string | null;
symbol: string;
side: 'BUY' | 'SELL';
closedQty: number;
avgEntry: number;
avgExit: number;
pnl: number;
pnlPercent: number;
};
const PAGE_SIZE = 1000;
const EPS = 1e-8;
const DEFAULT_START = '2026-02-12T00:00:00.000Z';
const CANONICAL_REASON = '[RECONCILED_CANONICAL] Order lifecycle reconciled from orders';
const ZEROED_PREFIX = '[RECONCILED_TO_ORDERS]';
const args = process.argv.slice(2);
const applyMode = args.includes('--apply');
const startArg = args.find((arg) => arg.startsWith('--start='));
const startIso = startArg ? String(startArg.split('=')[1] || '').trim() : DEFAULT_START;
const tradeIds = args.filter((arg) => !arg.startsWith('--'));
const supabaseUrl = String(process.env.SUPABASE_URL || '').trim();
const supabaseKey = String(
process.env.SUPABASE_KEY
|| process.env.SUPABASE_SERVICE_ROLE_KEY
|| process.env.SUPABASE_ANON_KEY
|| ''
).trim();
if (!supabaseUrl || !supabaseKey) {
throw new Error('Missing Supabase credentials. Expected SUPABASE_URL + SUPABASE_KEY/SUPABASE_SERVICE_ROLE_KEY.');
}
const supabase = createClient(supabaseUrl, supabaseKey);
const toNumber = (value: unknown): number => {
const num = Number(value);
return Number.isFinite(num) ? num : 0;
};
const normalizeSide = (side: string | null | undefined): 'BUY' | 'SELL' => {
const normalized = String(side || '').trim().toUpperCase();
return normalized === 'SELL' || normalized === 'SHORT' ? 'SELL' : 'BUY';
};
const normalizeAction = (action: string | null | undefined): 'ENTRY' | 'EXIT' | undefined => {
const normalized = String(action || '').trim().toUpperCase();
if (normalized === 'ENTRY' || normalized === 'EXIT') return normalized;
return undefined;
};
const inferAction = (row: OrderRow): 'ENTRY' | 'EXIT' | undefined => {
const explicit = normalizeAction(row.action);
if (explicit) return explicit;
if (String(row.trade_id || '').trim().length === 0) return undefined;
return normalizeSide(row.side) === 'BUY' ? 'ENTRY' : 'EXIT';
};
const startsWithIgnoreCase = (value: string | null | undefined, prefix: string): boolean => {
return String(value || '').toLowerCase().startsWith(prefix.toLowerCase());
};
const sortByCreatedAt = <T extends { created_at?: string | null }>(rows: T[]): T[] => {
return [...rows].sort((a, b) => {
const aTs = Date.parse(String(a.created_at || '')) || 0;
const bTs = Date.parse(String(b.created_at || '')) || 0;
return aTs - bTs;
});
};
const fetchPaged = async <T>(table: 'orders' | 'trade_history', columns: string): Promise<T[]> => {
const out: T[] = [];
let offset = 0;
for (;;) {
const { data, error } = await supabase
.from(table)
.select(columns)
.gte('created_at', startIso)
.order('created_at', { ascending: true })
.range(offset, offset + PAGE_SIZE - 1);
if (error) throw error;
const chunk = (data || []) as T[];
if (!chunk.length) break;
out.push(...chunk);
if (chunk.length < PAGE_SIZE) break;
offset += PAGE_SIZE;
}
return out;
};
const getCanonicalFromOrders = (tradeId: string, rows: OrderRow[]): CanonicalTrade | null => {
const ordered = sortByCreatedAt(rows);
if (!ordered.length) return null;
type Lot = { qty: number; price: number };
const lots: Lot[] = [];
let entrySide: 'BUY' | 'SELL' | null = null;
let closedQty = 0;
let closedEntryNotional = 0;
let closedExitNotional = 0;
let realizedPnl = 0;
let userId: string | null = null;
let profileId: string | null = null;
let symbol = '';
for (const row of ordered) {
const qty = toNumber(row.qty);
const price = toNumber(row.price);
if (!(qty > 0) || !(price > 0)) continue;
if (!userId && row.user_id) userId = row.user_id;
if (profileId === null && row.profile_id !== undefined) profileId = row.profile_id || null;
if (!symbol) symbol = String(row.symbol || '').trim();
const action = inferAction(row);
if (!action) continue;
const side = normalizeSide(row.side);
if (action === 'ENTRY') {
if (!entrySide) entrySide = side;
if (side !== entrySide) continue;
lots.push({ qty, price });
continue;
}
if (!entrySide) continue;
const expectedExitSide = entrySide === 'BUY' ? 'SELL' : 'BUY';
if (side !== expectedExitSide) continue;
let remaining = qty;
while (remaining > EPS && lots.length > 0) {
const lot = lots[0];
const closeQty = Math.min(remaining, lot.qty);
if (closeQty <= EPS) break;
lot.qty -= closeQty;
remaining -= closeQty;
closedQty += closeQty;
closedEntryNotional += closeQty * lot.price;
closedExitNotional += closeQty * price;
realizedPnl += entrySide === 'BUY'
? (price - lot.price) * closeQty
: (lot.price - price) * closeQty;
if (lot.qty <= EPS) lots.shift();
}
}
if (!(closedQty > EPS) || !(closedEntryNotional > 0) || !(closedExitNotional > 0) || !entrySide) {
return null;
}
const avgEntry = closedEntryNotional / closedQty;
const avgExit = closedExitNotional / closedQty;
const pnlPercent = avgEntry > 0
? ((avgExit - avgEntry) / avgEntry) * 100 * (entrySide === 'BUY' ? 1 : -1)
: 0;
return {
tradeId,
userId,
profileId,
symbol,
side: entrySide,
closedQty,
avgEntry,
avgExit,
pnl: realizedPnl,
pnlPercent
};
};
const run = async (): Promise<void> => {
const orderColumns = 'id,order_id,user_id,profile_id,symbol,trade_id,action,side,qty,price,status,created_at';
const historyColumns = 'id,user_id,profile_id,symbol,trade_id,side,size,entry_price,exit_price,pnl,pnl_percent,reason,source,timestamp,created_at';
let orderRows: OrderRow[] = [];
let historyRows: HistoryRow[] = [];
if (tradeIds.length > 0) {
for (const tradeId of tradeIds) {
const { data: oData, error: oError } = await supabase
.from('orders')
.select(orderColumns)
.eq('trade_id', tradeId)
.in('status', ['filled', 'partially_filled'])
.order('created_at', { ascending: true })
.limit(5000);
if (oError) throw oError;
orderRows.push(...((oData || []) as OrderRow[]));
const { data: hData, error: hError } = await supabase
.from('trade_history')
.select(historyColumns)
.eq('trade_id', tradeId)
.order('created_at', { ascending: true })
.limit(5000);
if (hError) throw hError;
historyRows.push(...((hData || []) as HistoryRow[]));
}
} else {
orderRows = await fetchPaged<OrderRow>('orders', orderColumns);
orderRows = orderRows.filter((row) => ['filled', 'partially_filled'].includes(String(row.status || '').toLowerCase()));
historyRows = await fetchPaged<HistoryRow>('trade_history', historyColumns);
}
const ordersByTrade = new Map<string, OrderRow[]>();
for (const row of orderRows) {
const tradeId = String(row.trade_id || '').trim();
if (!tradeId) continue;
const list = ordersByTrade.get(tradeId) || [];
list.push(row);
ordersByTrade.set(tradeId, list);
}
const historyByTrade = new Map<string, HistoryRow[]>();
for (const row of historyRows) {
const tradeId = String(row.trade_id || '').trim();
if (!tradeId) continue;
const list = historyByTrade.get(tradeId) || [];
list.push(row);
historyByTrade.set(tradeId, list);
}
const targets: Array<{
tradeId: string;
canonical: CanonicalTrade;
currentPnl: number;
diff: number;
history: HistoryRow[];
}> = [];
for (const [tradeId, rows] of ordersByTrade.entries()) {
const canonical = getCanonicalFromOrders(tradeId, rows);
if (!canonical) continue;
const history = historyByTrade.get(tradeId) || [];
const currentPnl = history.reduce((sum, row) => sum + toNumber(row.pnl), 0);
const diff = Number((canonical.pnl - currentPnl).toFixed(8));
if (Math.abs(diff) <= 0.02) continue;
targets.push({
tradeId,
canonical,
currentPnl,
diff,
history
});
}
targets.sort((a, b) => Math.abs(b.diff) - Math.abs(a.diff));
console.log(JSON.stringify({
mode: applyMode ? 'apply' : 'dry-run',
startIso,
explicitTradeIds: tradeIds.length > 0 ? tradeIds : null,
totalOrders: orderRows.length,
totalHistory: historyRows.length,
targets: targets.map((target) => ({
trade_id: target.tradeId,
current_history_pnl: Number(target.currentPnl.toFixed(8)),
canonical_order_pnl: Number(target.canonical.pnl.toFixed(8)),
diff_order_minus_history: target.diff,
history_rows: target.history.length
}))
}, null, 2));
if (!applyMode || targets.length === 0) return;
for (const target of targets) {
const rowsToNeutralize = target.history.filter((row) => {
const pnl = toNumber(row.pnl);
if (Math.abs(pnl) <= EPS) return false;
return !startsWithIgnoreCase(row.reason, ZEROED_PREFIX)
&& !startsWithIgnoreCase(row.reason, CANONICAL_REASON);
});
for (const row of rowsToNeutralize) {
const nextReason = startsWithIgnoreCase(row.reason, ZEROED_PREFIX)
? String(row.reason || '')
: `${ZEROED_PREFIX} ${String(row.reason || 'Lifecycle row superseded by canonical order-derived aggregate').trim()}`;
const { error } = await supabase
.from('trade_history')
.update({
pnl: 0,
pnl_percent: 0,
reason: nextReason
})
.eq('id', row.id);
if (error) throw error;
}
const canonicalPayload = {
user_id: target.canonical.userId,
profile_id: target.canonical.profileId,
symbol: target.canonical.symbol,
trade_id: target.canonical.tradeId,
side: target.canonical.side,
size: Number(target.canonical.closedQty.toFixed(8)),
entry_price: Number(target.canonical.avgEntry.toFixed(8)),
exit_price: Number(target.canonical.avgExit.toFixed(8)),
pnl: Number(target.canonical.pnl.toFixed(8)),
pnl_percent: Number(target.canonical.pnlPercent.toFixed(8)),
reason: CANONICAL_REASON,
source: 'BOT',
timestamp: Date.now()
};
const existingCanonical = target.history.find((row) => startsWithIgnoreCase(row.reason, CANONICAL_REASON));
if (existingCanonical) {
const { error } = await supabase
.from('trade_history')
.update(canonicalPayload)
.eq('id', existingCanonical.id);
if (error) throw error;
} else {
const { error } = await supabase
.from('trade_history')
.insert([canonicalPayload]);
if (error) throw error;
}
}
};
run().catch((error) => {
console.error('[reconcileTradeHistoryLifecycle] failed:', error);
process.exit(1);
});

33
backend/rename_profile.ts Normal file
View File

@ -0,0 +1,33 @@
import { supabaseService } from '../src/services/SupabaseService.js';
import logger from '../src/utils/logger.js';
async function updateProfileName() {
const profiles = await supabaseService.getActiveProfiles();
if (profiles.length === 0) return;
const targetProfile = profiles[0]; // The one we modified
const newName = "High Risk Scalper ⚡";
// Update Name in DB
// @ts-ignore
await supabaseService.client
.from('trade_profiles')
.update({ name: newName })
.eq('id', targetProfile.id);
logger.info(`✅ Profile Renamed to: [${newName}]`);
logger.info(`📋 Active Rules for this Profile:`);
// Display Rules
const config = targetProfile.strategy_config;
if (config && config.rules) {
config.rules.forEach((r: any) => {
logger.info(` - ${r.ruleId}: ${r.enabled ? '🟢 ENABLED' : '🔴 OFF'}`);
});
} else {
logger.info(' (Using Default Global Rules)');
}
}
updateProfileName();

67
backend/runCoverageSuite.ts Normal file
View File

@ -0,0 +1,67 @@
import path from 'node:path';
import { spawn } from 'node:child_process';
const suite = [
'scripts/testTradeExecutorLifecycle.ts',
'scripts/testLifecycleRegressions.ts',
'scripts/testOrderStatusSyncRegressions.ts',
'scripts/testSupabaseOrderPersistenceRegressions.ts',
'scripts/testFailureInjection.ts',
'src/scripts/verifyWebsocketContract.ts',
'scripts/testManualTraderCapitalGuard.ts',
'scripts/testSupabaseTradeHistorySourceFallback.ts',
'scripts/testStateMergeCoverage.ts',
'scripts/testBacktestIsolation.ts',
'scripts/testCoreModuleCoverage.ts',
'scripts/testConnectorAndAiCoverage.ts'
];
const runScript = (relativePath: string): Promise<number> =>
new Promise((resolve) => {
const scriptPath = path.resolve(process.cwd(), relativePath);
const child = spawn(
process.execPath,
['--loader', 'ts-node/esm', scriptPath],
{
stdio: 'inherit',
shell: false,
env: {
...process.env,
TS_NODE_TRANSPILE_ONLY: '1'
}
}
);
child.on('close', (code) => resolve(code ?? 1));
child.on('error', () => resolve(1));
});
const main = async () => {
console.log('\n[coverage-suite] Running coverage-friendly regression suite...\n');
const failed: string[] = [];
for (const item of suite) {
console.log(`\n[coverage-suite] ▶ ${item}`);
const code = await runScript(item);
if (code === 0) {
console.log(`[coverage-suite] ✅ ${item} passed`);
} else {
failed.push(item);
console.log(`[coverage-suite] ❌ ${item} failed with exit code ${code}`);
}
}
console.log('\n[coverage-suite] Summary');
console.log(`[coverage-suite] Passed: ${suite.length - failed.length}`);
console.log(`[coverage-suite] Failed: ${failed.length}`);
if (failed.length > 0) {
console.log('[coverage-suite] Failed scripts:');
failed.forEach((item) => console.log(`- ${item}`));
process.exit(1);
}
};
main().catch((error) => {
console.error('[coverage-suite] Unhandled error', error);
process.exit(1);
});

68
backend/runCriticalCoverageSuite.ts Normal file
View File

@ -0,0 +1,68 @@
import assert from 'node:assert/strict';
import { ConnectorFactory } from '../src/connectors/factory.js';
import {
normalizeOrderAction,
normalizeOrderStatus,
normalizeOrderType,
normalizeTradeSide
} from '../src/domain/tradingEnums.js';
import { SymbolMapper } from '../src/utils/symbolMapper.js';
const assertTradingEnums = () => {
assert.equal(normalizeTradeSide('SELL'), 'SELL');
assert.equal(normalizeTradeSide('short'), 'SELL');
assert.equal(normalizeTradeSide('BUY'), 'BUY');
assert.equal(normalizeTradeSide('unknown'), 'BUY');
assert.equal(normalizeOrderStatus('filled'), 'filled');
assert.equal(normalizeOrderStatus('partially_filled'), 'partially_filled');
assert.equal(normalizeOrderStatus('partial_fill'), 'partially_filled');
assert.equal(normalizeOrderStatus('partiallyfilled'), 'partially_filled');
assert.equal(normalizeOrderStatus('cancelled'), 'canceled');
assert.equal(normalizeOrderStatus('canceled'), 'canceled');
assert.equal(normalizeOrderStatus('expired'), 'expired');
assert.equal(normalizeOrderStatus('rejected'), 'rejected');
assert.equal(normalizeOrderStatus('unknown'), 'unknown');
assert.equal(normalizeOrderStatus('new'), 'pending_new');
assert.equal(normalizeOrderAction('entry'), 'ENTRY');
assert.equal(normalizeOrderAction('EXIT'), 'EXIT');
assert.equal(normalizeOrderAction('invalid'), undefined);
assert.equal(normalizeOrderAction(undefined), undefined);
assert.equal(normalizeOrderType('limit'), 'Limit');
assert.equal(normalizeOrderType('stop'), 'Stop');
assert.equal(normalizeOrderType('market'), 'Market');
assert.equal(normalizeOrderType('other'), 'Market');
};
const assertSymbolMapper = () => {
assert.equal(SymbolMapper.toTradeSymbol('BTC/USDT', 'alpaca'), 'BTC/USD');
assert.equal(SymbolMapper.toTradeSymbol('BTC/USD', 'alpaca'), 'BTC/USD');
assert.equal(SymbolMapper.toTradeSymbol('BTC/USDT', 'ccxt'), 'BTC/USDT');
assert.equal(SymbolMapper.toDataSymbol('BTC/USD', 'alpaca'), 'BTC/USDT');
assert.equal(SymbolMapper.toDataSymbol('BTC/USDT', 'ccxt'), 'BTC/USDT');
};
const assertConnectorFactory = () => {
const alpacaConnector = ConnectorFactory.getCustomConnector('alpaca', 'key', 'secret');
assert.equal(alpacaConnector.constructor.name, 'AlpacaConnector');
const ccxtConnector = ConnectorFactory.getCustomConnector('ccxt', 'key', 'secret');
assert.equal(ccxtConnector.constructor.name, 'CCXTConnector');
assert.throws(
() => ConnectorFactory.getCustomConnector('unsupported'),
/is not supported/
);
};
const main = () => {
assertTradingEnums();
assertSymbolMapper();
assertConnectorFactory();
console.log('[critical-coverage] PASS');
};
main();

43
backend/run_all_tests.ts Normal file
View File

@ -0,0 +1,43 @@
import fs from 'fs';
import path from 'path';
import { spawn } from 'child_process';
const testsDir = path.resolve(process.cwd(), 'tests');
async function runAllTests() {
console.log('🧪 Running all tests in tests/ directory...\n');
const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.ts'));
let passed = 0;
let failed = 0;
for (const file of files) {
console.log(`\n---------------------------------------------------------`);
console.log(`▶️ Running ${file}...`);
console.log(`---------------------------------------------------------`);
await new Promise<void>((resolve) => {
const proc = spawn('npx', ['tsx', path.join(testsDir, file)], {
stdio: 'inherit',
shell: true
});
proc.on('close', (code) => {
if (code === 0) {
console.log(`✅ ${file} PASSED`);
passed++;
} else {
console.log(`❌ ${file} FAILED (Exit Code: ${code})`);
failed++;
}
resolve();
});
});
}
console.log(`\n=========================================================`);
console.log(`Summary: ${passed} Passed, ${failed} Failed`);
console.log(`=========================================================\n`);
}
runAllTests();

34
backend/runbooks/capital-invariant.md Normal file
View File

@ -0,0 +1,34 @@
# Capital Invariant Violation Runbook
## Incident description
The ledger calculation `allocated - reserved_for_orders - reserved_for_positions + realized_pnl` dropped below zero for a profile, indicating over-allocation or stale reservations.
## Symptoms
- `capitalInvariantViolations` metric increments in `/metrics` and `/internal/health`.
- Critical log entry from the capital watchdog describing the negative delta and profile_id.
- Dashboard may show unexpected negative available capital or trading loop halting entries for that profile.
## Metrics to check
- `/internal/health` ? `capitalInvariantViolations`, `tradingLoopHealthy`, `lockContentionCount`.
- `/metrics` ? `capital_invariant_violations_total`, `available_capital_gauge`, `reconciliation_mismatch_count`.
## Immediate mitigation
1. Identify affected profile_id from the log or metric labels.
2. Confirm no new ENTRY orders are being placed for that profile (tradingLoopHealthy may be false).
3. Trigger a reconciliation cycle for the profile (if manual trigger exists, otherwise wait for scheduled run). Ensure reconciliation lock is available before forcing.
4. Notify downstream ops that trading for the profile is paused until invariant clears.
## Expected self-recovery
- Reconciliation loop recalculates reservations from exchange open orders/positions and updates ledger fields, restoring the invariant.
- Capital watchdog logs the repair and metrics return to zero once `available_capital` is non-negative.
## When to escalate
- Metric stays non-zero after three reconciliation cycles or 15 minutes.
- Trading loop remains unhealthy while other profiles are functioning.
- Capital watchdog logs repeated violations for different profiles within an hour.
Refer to docs/runbooks/invariant-violation.md for escalation steps.
## What NOT to do
- Do not manually edit ledger rows via Supabase without consulting runbooks.
- Do not restart the trading loop: the rebuild is deterministic; restarting may reintroduce transient errors.
- Do not ignore the metric; unresolved violations can lead to over-leveraging other profiles.

34
backend/runbooks/database-outage.md Normal file
View File

@ -0,0 +1,34 @@
# Supabase / RPC Outage Runbook
## Incident description
Supabase rejects writes or RPCs fail, blocking lifecycle persistence, ledger updates, or reconciliation state.
## Symptoms
- RPC errors logged (e.g., `timeout`, `503`, `service_role` rejection).
- `/internal/health` reports `monitorLoopHealthy` or `tradingLoopHealthy` false.
- `bot_state.json` writes or Supabase insert logs fail.
## Metrics to check
- `/internal/health` ? `tradingLoopHealthy`, `monitorLoopHealthy`, `lockContentionCount`.
- `/metrics` ? `supabase_rpc_errors_total`, `lifecycle_persist_failures_total`.
## Immediate mitigation
1. Pause new ENTRY submissions until Supabase is writable.
2. Queue lifecycle persistence retries and avoid dropping them; persistent message store may be required.
3. Notify Supabase support and confirm any service-wide issues.
4. Monitor whether reconciliation loops can still run with read-only access.
## Expected self-recovery
- Supabase recovers; pending RPC retries succeed, and reconciliation cleans up backlog.
- Metrics for RPC errors fall back to zero.
## When to escalate
- Outage exceeds 10 minutes with STATUS still failing.
- Supabase confirms incident impacting service_role writes.
- Lost trades or ledger inconsistencies appear post-recovery.
Escalate via docs/runbooks/database-outage.md to platform reliability.
## What NOT to do
- Do not switch to a secondary database manually; the system expects single Supabase project.
- Do not delete pending lifecycle entries; they must be replayed once writes succeed.
- Do not ignore RPC failure alerts; they indicate blocked capital and lifecycle flows.

34
backend/runbooks/exchange-degradation.md Normal file
View File

@ -0,0 +1,34 @@
# Exchange API Degradation Runbook
## Incident description
The exchange API responds slowly or returns errors, affecting ENTRY/EXIT execution and reconciliation data.
## Symptoms
- Exchange latency histogram in `/metrics` shows spikes; errors logged from exchange connector.
- `tradingLoopHealthy` or `monitorLoopHealthy` flag false because loops hit timeouts.
- Logs show `exchange timeout` or repeated `429`/`503` responses.
## Metrics to check
- `/internal/health` ? `tradingLoopHealthy`, `monitorLoopHealthy`, `exchangeLatencyHistogram`.
- `/metrics` ? `exchange_api_latency_seconds`, `exchange_api_errors_total`, `entry_orders_rejections_total`.
## Immediate mitigation
1. Back off new ENTRY signals for profiles if exchange is unreachable.
2. Ensure deterministic clientOrderId is ready before retries; do not reissue new orders.
3. Activate retry/backoff logic in connectors; log each retry with correlation IDs.
4. Inform downstream systems (dashboard, ops) about degraded state.
## Expected self-recovery
- Exchange recovers and accepts pending requests; trading loop resumes once latency normalizes.
- Reconciliation loop eventually runs against fresh data; metrics fall back to baseline.
## When to escalate
- Errors persist beyond 5 minutes despite retries.
- Exchange reports credential or rate-limit problems requiring intervention.
- Business-critical trading windows are missed.
Escalate to the Exchange Account Manager and Cloud Ops; reference docs/runbooks/exchange-degradation.md.
## What NOT to do
- Do not flood the exchange with retries; respect backoff policies.
- Do not change API keys mid-incident without direction from the exchange team.
- Do not pause reconciliation; accurate state is needed to diagnose missing fills.

34
backend/runbooks/lock-contention.md Normal file
View File

@ -0,0 +1,34 @@
# ENTRY Lock Contention Spike Runbook
## Incident description
A profile experiences repeated failures to acquire the row-based entry lock, blocking ENTRY signals and indicating pressure on horizontal scaling.
## Symptoms
- `lockContentionCount` increments in `/internal/health` and `/metrics`.
- Logs show fn_try_acquire_entry_lock_row returning false with owner tokens different from the caller.
- Trading loop reports `lock acquisition failed` warnings and may skip signals.
## Metrics to check
- `/internal/health` ? `lockContentionCount`, `tradingLoopHealthy`, `reconciliationLoopHealthy`.
- `/metrics` ? `entry_lock_contention_total`, `lock_acquisition_latency_seconds`, `entry_lock_holder_info` (if available).
## Immediate mitigation
1. Identify the profile_id and symbol from logs; confirm if another worker legitimately holds the lock.
2. Ensure the existing lock owner is still alive or has not crashed; use Supabase to inspect `entry_locks` TTL.
3. Wait for TTL expiry (default 30s) before retrying if owner appears stuck.
4. Avoid forcing lock release unless owner is confirmed dead; manual deletion risks concurrent exchange submission.
## Expected self-recovery
- The TTL expires, the lock row updates or deletes itself, and the next signal acquires the lock.
- Metrics return to baseline if contention was transient.
## When to escalate
- Contention persists beyond three TTL cycles (90s).
- Multiple profiles report contention simultaneously.
- Lock rows show expired timestamps but fail to refresh.
Escalate to Platform Ops and refer to docs/runbooks/lock-timeout.md (if it exists) for lock escalation.
## What NOT to do
- Do not delete lock rows manually while other workers are active.
- Do not restart all workers; indiscriminate restarts magnify contention.
- Do not trigger new ENTRY signals for the affected profile until lock clears.

34
backend/runbooks/loop-health.md Normal file
View File

@ -0,0 +1,34 @@
# Trading & Reconciliation Loop Health Runbook
## Incident description
The trading loop or reconciliation loop is not executing within expected cadence, threatening data freshness and trading responsiveness.
## Symptoms
- `/internal/health` fields `tradingLoopHealthy` or `reconciliationLoopHealthy` flip to false.
- `reconciliationLastRun` or loop duration metrics show no updates for longer than twice the expected interval.
- Prometheus metrics show loop duration stuck or not emitted.
## Metrics to check
- `/internal/health` ? `tradingLoopHealthy`, `reconciliationLoopHealthy`, `monitorLoopHealthy`, `reconciliationLastRun`.
- `/metrics` ? `trading_loop_duration_seconds`, `reconciliation_loop_duration_seconds`, `loop_run_failure_total`.
## Immediate mitigation
1. Inspect last log timestamp to determine if the loop crashed or is still running slow.
2. Verify that the process is still up and not stuck on external calls (e.g., exchange or Supabase). Use debugger or profiling if needed.
3. If a loop is hung, send kill signal to the specific loop worker (do not restart the entire service) and allow auto-restart if configured.
4. Ensure lock contention or capital invariant metrics are not blocking the loop.
## Expected self-recovery
- Auto-recovery restarts the loop or continues once blocked resource clears (e.g., exchange call returns or Supabase writes succeed).
- `/internal/health` marks the loop healthy after successful iteration.
## When to escalate
- Loops fail consecutively more than twice within 10 minutes.
- Loop restarts exceed the configured threshold without recovery.
- Business trades miss critical windows due to loop downtime.
Escalate via docs/runbooks/loop-health.md and notify the reliability team.
## What NOT to do
- Do not disable the health endpoint; it is the single source of truth.
- Do not restart unrelated services; focus on the affected loop.
- Do not skip verification of capital or lock metrics before concluding the loop itself is broken.

194
backend/runbooks/reconciliation-exit-backfill.md Normal file
View File

@ -0,0 +1,194 @@
# Reconciliation EXIT Backfill Rollout Checklist
## Scope Guardrails
- No entry/exit/risk/signal logic changes.
- Data-only repair: insert compensating `EXIT` rows only.
- No deletes. Rollback uses status updates only.
- Backfill must be paused-mode gated and auditable by `batch_id`.
## Feature Flags (default safe state)
- `ENABLE_RECON_EXIT_BACKFILL=false`
- `RECON_EXIT_BACKFILL_DRY_RUN=true`
- `RECON_EXIT_BACKFILL_REQUIRE_PAUSE=true`
- `RECON_EXIT_BACKFILL_DUST_ABS_QTY=0.001`
- `RECON_EXIT_BACKFILL_DUST_REL_PCT=0.002`
- `RECON_EXIT_BACKFILL_LOOKBACK_HOURS=72`
- `EXCHANGE_STATE_MISMATCH_THROTTLE_MS=300000`
## 1. Detect Affected `trade_id` Values
```sql
WITH lifecycle AS (
SELECT
profile_id,
symbol,
trade_id,
SUM(CASE WHEN action = 'ENTRY' THEN COALESCE(quantity, qty, 0) ELSE 0 END) AS entry_qty,
SUM(CASE WHEN action = 'EXIT' THEN COALESCE(quantity, qty, 0) ELSE 0 END) AS exit_qty
FROM orders
WHERE profile_id = '<<PROFILE_ID>>'
AND status IN ('filled', 'partially_filled', 'partially-filled')
AND trade_id IS NOT NULL
GROUP BY profile_id, symbol, trade_id
)
SELECT
profile_id,
symbol,
trade_id,
entry_qty,
exit_qty,
(entry_qty - exit_qty) AS open_qty
FROM lifecycle
WHERE (entry_qty - exit_qty) > 0
ORDER BY symbol, trade_id;
```
## 2. Allow Criteria for Backfill `EXIT` Row
Backfill is allowed only when all are true:
- Profile is paused (`/internal/trading/status` returns `PAUSED`).
- Exchange is flat for symbol.
- No pending lifecycle blocker row exists for that `trade_id`.
- Close qty is supported by exchange fill evidence OR unresolved remainder is dust.
- Idempotency key resolves to unique deterministic `order_id`.
## 3. Idempotency Rules
- Deterministic `order_id`:
- `BFILL-<md5(profile_id:trade_id:exchange_order_id:filled_at)>`
- Insert path:
- `ON CONFLICT (order_id) DO NOTHING` (implemented via upsert + `ignoreDuplicates`).
- Re-runs are safe: existing backfill `order_id` rows are skipped.
- Exchange `client_order_id` (if present) is stored in `reconciliation_backfill_audit.exchange_client_order_id`.
## 4. Dust Threshold Handling
- Dust threshold per trade:
- `MAX(dust_abs_qty, open_qty * dust_rel_pct)`
- Default:
- `dust_abs_qty = 0.001`
- `dust_rel_pct = 0.2%`
- Auto-close allowed:
- evidence-covered qty always
- remainder only if `remainder <= threshold`
- No-Go:
- `remainder > threshold` with missing exchange fill evidence.
## 5. Dry-Run Execution and Verification
1. Set:
- `ENABLE_RECON_EXIT_BACKFILL=true`
- `RECON_EXIT_BACKFILL_DRY_RUN=true`
- `RECON_EXIT_BACKFILL_REQUIRE_PAUSE=true`
2. Pause trading from admin.
3. Let one reconciliation cycle run.
4. Verify audit-only effect:
```sql
SELECT
batch_id,
profile_id,
symbol,
trade_id,
decision,
reason,
filled_qty,
backfill_order_id,
created_at
FROM reconciliation_backfill_audit
WHERE profile_id = '<<PROFILE_ID>>'
ORDER BY created_at DESC
LIMIT 200;
```
5. Confirm `orders` table unchanged in dry-run:
```sql
SELECT COUNT(*) AS dry_run_backfill_rows
FROM orders
WHERE profile_id = '<<PROFILE_ID>>'
AND order_id LIKE 'BFILL-%';
```
## 6. Go / No-Go Before Apply
Go only if:
- Pause state confirmed.
- Dry-run shows expected `DRY_RUN` or `PENDING_APPLY` candidates.
- No unexpected `NO_GO` due to non-flat exchange or pending blockers.
- Candidate qty aligns with known exchange fills.
- Audit table writes succeed.
No-Go if any:
- Exchange not flat for target symbol.
- Large unmatched remainder (`missing_fill_evidence_for_large_remainder`).
- Audit table unavailable/write failure.
- Pending blocker rows for target `trade_id`.
## 7. Apply Step (still paused)
1. Set `RECON_EXIT_BACKFILL_DRY_RUN=false`.
2. Run one reconciliation cycle.
3. Validate inserted rows:
```sql
SELECT
order_id,
profile_id,
symbol,
trade_id,
action,
status,
qty,
price,
filled_at,
created_at
FROM orders
WHERE profile_id = '<<PROFILE_ID>>'
AND order_id LIKE 'BFILL-%'
ORDER BY created_at DESC;
```
## 8. Post-Fix Validation (ghost positions removed)
Open lifecycle should be zero/near-zero:
```sql
WITH lifecycle AS (
SELECT
profile_id,
symbol,
trade_id,
SUM(CASE WHEN action = 'ENTRY' THEN COALESCE(quantity, qty, 0) ELSE 0 END) AS entry_qty,
SUM(CASE WHEN action = 'EXIT' THEN COALESCE(quantity, qty, 0) ELSE 0 END) AS exit_qty
FROM orders
WHERE profile_id = '<<PROFILE_ID>>'
AND status IN ('filled', 'partially_filled', 'partially-filled')
AND trade_id IS NOT NULL
GROUP BY profile_id, symbol, trade_id
)
SELECT *
FROM lifecycle
WHERE (entry_qty - exit_qty) > 0.000001
ORDER BY symbol, trade_id;
```
Mismatch noise check (audit + events):
```sql
SELECT
decision,
COUNT(*) AS rows
FROM reconciliation_backfill_audit
WHERE profile_id = '<<PROFILE_ID>>'
GROUP BY decision
ORDER BY decision;
```
## 9. Rollback (Reversible, Non-Destructive)
Rollback is status-only, never delete:
```sql
UPDATE orders
SET
status = 'canceled',
updated_at = now()
WHERE order_id IN (
SELECT backfill_order_id
FROM reconciliation_backfill_audit
WHERE batch_id = '<<BATCH_ID>>'
AND decision IN ('APPLIED', 'SKIP_EXISTING')
AND backfill_order_id IS NOT NULL
);
```
Record rollback marker:
```sql
UPDATE reconciliation_backfill_audit
SET reverted_at = now()
WHERE batch_id = '<<BATCH_ID>>';
```

70
backend/runbooks/reconciliation.md Normal file
View File

@ -0,0 +1,70 @@
# Reconciliation Divergence Runbook
## Incident description
The reconciliation loop detects mismatches between Supabase orders/positions and exchange open orders, meaning the database drifted from exchange truth.
## Symptoms
- `reconciliationMismatchCount`, `reconciliationMissingFromExchange`, or `reconciliationMissingInDb` metrics rise.
- Logs show lifecycle-safe handlers executing to correct entry/exit states.
- Dashboard shows active orders or positions that do not exist on the exchange.
## Metrics to check
- `/internal/health` ? `reconciliationLoopHealthy`, `reconciliationMismatchCount`, `reconciliationLastRun`.
- `/metrics` ? `reconciliation_mismatch_total`, `reconciliation_missing_from_exchange_total`, `reconciliation_missing_in_db_total`.
- `/internal/health` runtime fields:
- `reconciliationParityMismatchTrades`
- `reconciliationParityQuarantinedTrades`
- `reconciliationParityAutoClosedTrades`
- `reconciliationParityMaxMismatchNotionalUsd`
- `reconciliationParityTotalMismatchNotionalUsd`
- `reconciliationIntegrityWatchdogTriggered`
## Automated parity heartbeat (ghost self-healing)
- Feature flag: `ENABLE_RECON_POSITION_PARITY_HEARTBEAT=true` (default is `true`; set `false` only for controlled rollback).
- Confirmation gate: `RECON_POSITION_PARITY_CONFIRMATIONS` (default `3` consecutive checks).
- Attribution safety gate: `RECON_POSITION_PARITY_REQUIRE_SUBTAG_ATTRIBUTION` (default `true`).
- Watchdog threshold: `RECON_POSITION_PARITY_MAX_NOTIONAL_PCT` (default `0.5` of allocated capital).
- Auto-resume gate: `ENABLE_RECON_WATCHDOG_AUTO_RESUME=true`.
- Auto-resume delay: `RECON_WATCHDOG_AUTO_RESUME_MIN_PAUSE_MS` (default `900000`).
- Auto-resume clean streak: `RECON_WATCHDOG_AUTO_RESUME_CLEAN_CYCLES` (default `2`).
- Auto-resume cooldown: `RECON_WATCHDOG_AUTO_RESUME_COOLDOWN_MS` (default `1800000`).
- Dry-run mode: `RECON_POSITION_PARITY_DRY_RUN=true` to observe without applying synthetic exits.
Heartbeat behavior:
- Detects ghost lifecycle slices where virtual open qty remains but exchange position is effectively zero.
- Requires consecutive mismatch confirmations before synthetic EXIT reconciliation.
- Enforces sub-tag attribution before any synthetic close; unattributed slices are quarantined.
- Triggers integrity watchdog pause when cumulative mismatch notional exceeds configured capital ratio.
- Auto-resumes trading only when pause source is parity watchdog and reconciliation stays clean for required consecutive cycles.
## EXIT backfill safety gates
- `RECON_EXIT_BACKFILL_REQUIRE_STRONG_ATTRIBUTION=true`:
- only uses exchange fills that are attributable to the profile/trade (`sub_tag`, deterministic `client_order_id`, or explicit `trade_id` hint).
- prevents auto-backfill from consuming unrelated account activity.
- `RECON_EXIT_BACKFILL_ALLOW_HEURISTIC_MATCH=false`:
- disables heuristic assignment modes (`single_open_trade`, `qty_unique`) by default.
- keeps unmatched rows in `NO_GO` for operator review instead of auto-closing.
- `RECON_EXIT_BACKFILL_FILL_AFTER_TRADE_GRACE_MINUTES=5`:
- rejects stale fill evidence that predates the lifecycle slice timestamp beyond grace.
- blocks historical fills from being attached to newer open trades.
## Immediate mitigation
1. Confirm the reconciliation lock is available for the affected profile to avoid double processing.
2. Allow the reconciliation loop to run; it will route mismatches through lifecycle-safe handlers (`reconcileEntryFill`, `reconcileExitFill`, `reconcileCancel`).
3. If divergence persists, manually inspect trade_history and positions for inconsistent state.
4. Notify stakeholders that reconciliation is running and that no manual edits should occur during the fix.
## Expected self-recovery
- Handler corrections align DB orders/positions with exchange data, and metrics return to zero.
- The capital ledger recalculates reservations, and dashboard data becomes consistent.
## When to escalate
- Mismatch metrics stay elevated after two reconciliation runs.
- Reconciliation lock contention prevents the loop from running.
- Exchanges report stale or unknown fills after reconciliation.
Escalate to the trading engineering lead and reference docs/runbooks/reconciliation.md and docs/runbooks/lifecycle-incident.md for follow-up.
## What NOT to do
- Do not manually patch `orders` or `positions` tables while reconciliation is active.
- Do not disable the reconciliation loop; divergence will only grow.
- Do not trigger new ENTRY/EXIT flows for the affected profile until reconciliation completes.

28
backend/schema/002_add_strategy_config.sql Normal file
View File

@ -0,0 +1,28 @@
-- Migration: Add JSON Strategy Config to Trade Profiles
-- Date: 2026-02-04
-- Purpose: Enable Per-Profile Strategy Rules & Risk Configuration
ALTER TABLE trade_profiles
ADD COLUMN IF NOT EXISTS strategy_config jsonb DEFAULT '{
"rules": [
{ "ruleId": "TrendBiasRule", "enabled": true, "params": { "timeframe": "4h", "emaFast": 50, "emaSlow": 200 } },
{ "ruleId": "SessionRule", "enabled": true, "params": { "allowedSessions": ["NY", "LDN"] } },
{ "ruleId": "ZoneRule", "enabled": true, "params": { "emaPeriod": 20, "tolerancePercent": 0.5 } },
{ "ruleId": "MomentumRule", "enabled": true, "params": { "timeframe": "1h", "rsiPeriod": 14, "rsiOverbought": 70, "rsiOversold": 30 } },
{ "ruleId": "EntryTriggerRule", "enabled": true, "params": { "triggerType": "ema_cross" } },
{ "ruleId": "RiskManagementRule", "enabled": true, "params": { "atrPeriod": 14, "riskRewardRatio": 1.5 } },
{ "ruleId": "AIAnalysisRule", "enabled": false, "params": { "minConfidence": 80 } }
],
"riskLimits": {
"maxDailyLossUsd": 50,
"maxConsecutiveLosses": 2,
"maxOpenTrades": 3
},
"execution": {
"orderType": "market",
"cooldownMinutes": 30,
"entryMode": "both"
}
}'::jsonb;
COMMENT ON COLUMN trade_profiles.strategy_config IS 'JSON configuration for active strategy rules, risk limits, and execution settings per profile.';

18
backend/schema/003_add_profile_id_to_orders_and_history.sql Normal file
View File

@ -0,0 +1,18 @@
-- Migration: Add profile_id to orders and trade_history
-- Date: 2026-02-07
-- Purpose: Map every order and trade to its originating trade_profile for per-profile analytics
-- Orders table
ALTER TABLE orders
ADD COLUMN IF NOT EXISTS profile_id uuid REFERENCES trade_profiles(id) ON DELETE SET NULL;
CREATE INDEX IF NOT EXISTS idx_orders_profile_id ON orders(profile_id);
-- Trade history table
ALTER TABLE trade_history
ADD COLUMN IF NOT EXISTS profile_id uuid REFERENCES trade_profiles(id) ON DELETE SET NULL;
CREATE INDEX IF NOT EXISTS idx_trade_history_profile_id ON trade_history(profile_id);
COMMENT ON COLUMN orders.profile_id IS 'The trade profile that triggered this order';
COMMENT ON COLUMN trade_history.profile_id IS 'The trade profile that triggered this trade';

161
backend/schema/004_full_schema_sync.sql Normal file
View File

@ -0,0 +1,161 @@
-- ============================================================
-- Migration 004: Full Schema Sync
-- Date: 2026-02-07
-- Purpose: Create missing tables, add missing columns,
-- and align DB schema with application code.
-- ============================================================
-- ────────────────────────────────────────────────────────────
-- 1. TRADE_PROFILES TABLE (Missing CREATE TABLE)
-- ────────────────────────────────────────────────────────────
CREATE TABLE IF NOT EXISTS trade_profiles (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
user_id uuid REFERENCES auth.users NOT NULL,
name text NOT NULL,
allocated_capital numeric DEFAULT 1000,
risk_per_trade_percent numeric DEFAULT 1.0,
symbols text DEFAULT 'BTC/USDT, ETH/USDT',
is_active boolean DEFAULT true,
strategy_config jsonb DEFAULT '{
"rules": [
{ "ruleId": "TrendBiasRule", "enabled": true, "params": { "timeframe": "4h", "emaFast": 50, "emaSlow": 200 } },
{ "ruleId": "SessionRule", "enabled": true, "params": { "allowedSessions": ["NY", "LDN"] } },
{ "ruleId": "ZoneRule", "enabled": true, "params": { "emaPeriod": 20, "tolerancePercent": 0.5 } },
{ "ruleId": "MomentumRule", "enabled": true, "params": { "timeframe": "1h", "rsiPeriod": 14, "rsiOverbought": 70, "rsiOversold": 30 } },
{ "ruleId": "EntryTriggerRule", "enabled": true, "params": { "triggerType": "ema_cross" } },
{ "ruleId": "RiskManagementRule", "enabled": true, "params": { "atrPeriod": 14, "riskRewardRatio": 1.5 } },
{ "ruleId": "AIAnalysisRule", "enabled": false, "params": { "minConfidence": 80 } }
],
"riskLimits": {
"maxDailyLossUsd": 50,
"maxConsecutiveLosses": 2,
"maxOpenTrades": 3
},
"execution": {
"orderType": "market",
"cooldownMinutes": 30,
"entryMode": "both"
}
}'::jsonb,
created_at timestamptz DEFAULT now()
);
COMMENT ON TABLE trade_profiles IS 'Trading strategy profiles with per-profile rule config, risk limits, and execution settings';
COMMENT ON COLUMN trade_profiles.strategy_config IS 'JSON configuration: { rules[], riskLimits{}, execution{} }';
CREATE INDEX IF NOT EXISTS idx_trade_profiles_user_id ON trade_profiles(user_id);
CREATE INDEX IF NOT EXISTS idx_trade_profiles_is_active ON trade_profiles(is_active);
-- RLS
ALTER TABLE trade_profiles ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS "Users can manage own profiles" ON trade_profiles;
CREATE POLICY "Users can manage own profiles" ON trade_profiles
USING (auth.uid() = user_id)
WITH CHECK (auth.uid() = user_id);
-- ────────────────────────────────────────────────────────────
-- 2. ORDERS TABLE — Add missing columns
-- ────────────────────────────────────────────────────────────
-- profile_id: links order to the trade profile that triggered it
ALTER TABLE orders
ADD COLUMN IF NOT EXISTS profile_id uuid REFERENCES trade_profiles(id) ON DELETE SET NULL;
-- stop_loss & take_profit: stored by TradeExecutor.logOrderToDb()
ALTER TABLE orders
ADD COLUMN IF NOT EXISTS stop_loss numeric;
ALTER TABLE orders
ADD COLUMN IF NOT EXISTS take_profit numeric;
CREATE INDEX IF NOT EXISTS idx_orders_profile_id ON orders(profile_id);
COMMENT ON COLUMN orders.profile_id IS 'Trade profile that triggered this order (NULL = manual)';
COMMENT ON COLUMN orders.stop_loss IS 'Stop loss price set at order time';
COMMENT ON COLUMN orders.take_profit IS 'Take profit price set at order time';
-- ────────────────────────────────────────────────────────────
-- 3. TRADE_HISTORY TABLE — Add missing columns
-- ────────────────────────────────────────────────────────────
-- profile_id: links trade to the profile that triggered it
ALTER TABLE trade_history
ADD COLUMN IF NOT EXISTS profile_id uuid REFERENCES trade_profiles(id) ON DELETE SET NULL;
-- stop_loss & take_profit: for post-trade analysis
ALTER TABLE trade_history
ADD COLUMN IF NOT EXISTS stop_loss numeric;
ALTER TABLE trade_history
ADD COLUMN IF NOT EXISTS take_profit numeric;
-- rules_metadata: stores which rules passed/failed for this trade
ALTER TABLE trade_history
ADD COLUMN IF NOT EXISTS rules_metadata jsonb;
CREATE INDEX IF NOT EXISTS idx_trade_history_profile_id ON trade_history(profile_id);
COMMENT ON COLUMN trade_history.profile_id IS 'Trade profile that triggered this trade (NULL = manual)';
COMMENT ON COLUMN trade_history.stop_loss IS 'Stop loss price at trade entry';
COMMENT ON COLUMN trade_history.take_profit IS 'Take profit price at trade entry';
COMMENT ON COLUMN trade_history.rules_metadata IS 'JSON snapshot of rule statuses at trade time: { ruleName: { passed, reason } }';
-- ────────────────────────────────────────────────────────────
-- 4. BOT_CONFIG TABLE (Missing CREATE TABLE)
-- Used by: ConfigTab.tsx, GlobalConfigManager.tsx
-- ────────────────────────────────────────────────────────────
CREATE TABLE IF NOT EXISTS bot_config (
key text PRIMARY KEY,
value text,
description text,
updated_at timestamptz DEFAULT now()
);
COMMENT ON TABLE bot_config IS 'Key-value store for global bot configuration (editable from dashboard)';
-- RLS: allow authenticated users to read, admins to write
ALTER TABLE bot_config ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS "Authenticated users can read bot_config" ON bot_config;
CREATE POLICY "Authenticated users can read bot_config" ON bot_config
FOR SELECT
USING (auth.role() = 'authenticated');
DROP POLICY IF EXISTS "Admins can manage bot_config" ON bot_config;
CREATE POLICY "Admins can manage bot_config" ON bot_config
USING (
EXISTS (SELECT 1 FROM users WHERE user_id = auth.uid() AND role = 'admin')
)
WITH CHECK (
EXISTS (SELECT 1 FROM users WHERE user_id = auth.uid() AND role = 'admin')
);
-- ────────────────────────────────────────────────────────────
-- 5. DYNAMIC_CONFIG TABLE (Missing CREATE TABLE)
-- Used by: bot-service config/index.ts (loadDynamicConfig)
-- ────────────────────────────────────────────────────────────
CREATE TABLE IF NOT EXISTS dynamic_config (
key text PRIMARY KEY,
value text,
updated_at timestamptz DEFAULT now()
);
COMMENT ON TABLE dynamic_config IS 'Runtime-overridable bot settings loaded at startup (SYMBOLS, intervals, etc.)';
-- ────────────────────────────────────────────────────────────
-- 6. UPDATE schema_reference.sql sync comment
-- ────────────────────────────────────────────────────────────
-- After running this migration, all 7 tables are fully defined:
-- 1. users (auth trigger-created)
-- 2. entries (watchlist & manual positions)
-- 3. trade_history (completed trade ledger)
-- 4. orders (active/pending orders)
-- 5. trade_profiles (strategy profiles with rule config)
-- 6. bot_config (global config key-value store)
-- 7. dynamic_config (runtime config overrides)

24
backend/schema/005_add_trade_id_tracking.sql Normal file
View File

@ -0,0 +1,24 @@
-- ────────────────────────────────────────────────────────────
-- 005: Add trade_id for full trade cycle tracing
-- Links entry order → exit order → trade_history record
-- ────────────────────────────────────────────────────────────
-- Add trade_id to orders table (links BUY and SELL of same trade)
ALTER TABLE orders
ADD COLUMN IF NOT EXISTS trade_id text;
-- Add action column to orders (ENTRY or EXIT)
ALTER TABLE orders
ADD COLUMN IF NOT EXISTS action text;
-- Add trade_id to trade_history (links to the orders)
ALTER TABLE trade_history
ADD COLUMN IF NOT EXISTS trade_id text;
-- Index for fast lookups by trade_id
CREATE INDEX IF NOT EXISTS idx_orders_trade_id ON orders(trade_id);
CREATE INDEX IF NOT EXISTS idx_trade_history_trade_id ON trade_history(trade_id);
COMMENT ON COLUMN orders.trade_id IS 'Unique identifier linking entry and exit orders of the same trade cycle (e.g., TRD-1707000000-abc123)';
COMMENT ON COLUMN orders.action IS 'Order action type: ENTRY (opening) or EXIT (closing)';
COMMENT ON COLUMN trade_history.trade_id IS 'Links this trade record to its entry/exit orders in the orders table';

25
backend/schema/006_add_trade_source.sql Normal file
View File

@ -0,0 +1,25 @@
-- ============================================================
-- Migration 006: Trade Source Normalization
-- Date: 2026-02-14
-- Purpose: Separate trade source semantics from trade side.
-- ============================================================
ALTER TABLE trade_history
ADD COLUMN IF NOT EXISTS source text DEFAULT 'BOT';
-- Normalize legacy rows where side was overloaded with MANUAL.
UPDATE trade_history
SET side = 'BUY'
WHERE upper(side) = 'MANUAL';
-- Backfill source for historical/manual records where possible.
UPDATE trade_history
SET source = 'MANUAL'
WHERE source IS NULL
OR upper(source) NOT IN ('BOT', 'MANUAL');
ALTER TABLE trade_history
ALTER COLUMN source SET NOT NULL;
CREATE INDEX IF NOT EXISTS idx_trade_history_source ON trade_history(source);

Some files were not shown because too many files have changed in this diff Show More