learning_ai_common_plat/scripts/encrypt-migrate.ts
saravanakumardb1 c252cfd198 feat(devops): encryption migration CLI with embedded product configs
- scripts/encrypt-migrate.ts — batch-encrypt existing plaintext Cosmos docs
- scripts/ added as pnpm workspace member for clean @bytelyst/* imports
- 10 product configs, 20 containers, 40+ fields
- --dry-run, --product, --container, --batch-size, --verbose flags
- Idempotent via __encrypted sentinel (migrateDocuments helper)
- Updated E2EE roadmap Sprint 6.2 as complete
2026-03-21 13:19:55 -07:00

645 lines
22 KiB
TypeScript

#!/usr/bin/env npx tsx
/* eslint-disable no-console */
/**
* Encryption Migration CLI — Batch-encrypt existing plaintext Cosmos documents.
*
* Uses @bytelyst/field-encrypt migrateDocuments() to idempotently encrypt
* plaintext fields across all product backends.
*
* Usage:
* npx tsx scripts/encrypt-migrate.ts --product lysnrai --dry-run
* npx tsx scripts/encrypt-migrate.ts --product lysnrai --container transcripts
* npx tsx scripts/encrypt-migrate.ts --product all --dry-run
* npx tsx scripts/encrypt-migrate.ts --product chronomind --batch-size 50
*
* Required env vars:
* COSMOS_ENDPOINT — Azure Cosmos DB endpoint
* COSMOS_KEY — Azure Cosmos DB key
* COSMOS_DATABASE — Database name (default: lysnrai)
* FIELD_ENCRYPT_KEY_PROVIDER — 'memory' | 'env' | 'akv' (default: memory)
* FIELD_ENCRYPT_KEY — Hex key (for 'env' provider)
* AZURE_KEYVAULT_URL — AKV URL (for 'akv' provider)
*
* Per-product MEK names are defined in migration configs.
*/
import { CosmosClient, type Container } from '@azure/cosmos';
import {
createFieldEncryptor,
migrateDocuments,
type MigrationResult,
type FieldEncryptor,
type EncryptedField,
} from '@bytelyst/field-encrypt';
// ── Types ──────────────────────────────────────────────
interface FieldSpec {
/** The field path (dot-separated for nested, e.g. 'triageResult.summary'). */
field: string;
/** Encryption context label. */
context: string;
/** The property on the doc that holds the userId for DEK scoping. Default: 'userId'. */
userIdField?: string;
/** If true, the field is a string[] or Record — JSON-serialize before encrypting. */
jsonSerialize?: boolean;
}
interface ContainerMigrationConfig {
/** Cosmos container name. */
container: string;
/** Partition key path (e.g. '/userId'). */
partitionKey: string;
/** Fields to encrypt in this container. */
fields: FieldSpec[];
}
interface ProductMigrationConfig {
/** Product ID. */
productId: string;
/** MEK name for this product. */
mekName: string;
/** Containers to migrate. */
containers: ContainerMigrationConfig[];
}
// ── Field accessor helpers ────────────────────────────
function getNestedField(doc: Record<string, unknown>, path: string): unknown {
const parts = path.split('.');
let current: unknown = doc;
for (const part of parts) {
if (current == null || typeof current !== 'object') return undefined;
current = (current as Record<string, unknown>)[part];
}
return current;
}
function setNestedField(
doc: Record<string, unknown>,
path: string,
value: unknown
): Record<string, unknown> {
const parts = path.split('.');
const result = { ...doc };
if (parts.length === 1) {
result[parts[0]] = value;
return result;
}
// Deep clone the nested path
let current = result;
for (let i = 0; i < parts.length - 1; i++) {
const nested = current[parts[i]];
if (nested == null || typeof nested !== 'object') return result;
current[parts[i]] = { ...(nested as Record<string, unknown>) };
current = current[parts[i]] as Record<string, unknown>;
}
current[parts[parts.length - 1]] = value;
return result;
}
// ── Product migration configs ─────────────────────────
const PRODUCT_CONFIGS: ProductMigrationConfig[] = [
// ── LysnrAI (port 4015) ──
{
productId: 'lysnrai',
mekName: 'lysnr-mek',
containers: [
{
container: 'transcripts',
partitionKey: '/userId',
fields: [{ field: 'transcriptText', context: 'transcripts' }],
},
{
container: 'sessions',
partitionKey: '/userId',
fields: [{ field: 'notes', context: 'sessions' }],
},
],
},
// ── JarvisJr (port 4012) ──
{
productId: 'jarvisjr',
mekName: 'jarvisjr-mek',
containers: [
{
container: 'jarvis_sessions',
partitionKey: '/userId',
fields: [{ field: 'notes', context: 'jarvis-sessions' }],
},
{
container: 'jarvis_memory',
partitionKey: '/agentId',
fields: [{ field: 'content', context: 'jarvis-memory', userIdField: 'agentId' }],
},
],
},
// ── NoteLett (port 4016) ──
{
productId: 'notelett',
mekName: 'notelett-mek',
containers: [
{
container: 'notes',
partitionKey: '/workspaceId',
fields: [{ field: 'body', context: 'notes', userIdField: 'createdBy' }],
},
],
},
// ── NomGap (port 4013) ──
{
productId: 'nomgap',
mekName: 'nomgap-mek',
containers: [
{
container: 'meal_logs',
partitionKey: '/userId',
fields: [{ field: 'notes', context: 'meal-logs' }],
},
{
container: 'fasting_sessions',
partitionKey: '/userId',
fields: [{ field: 'notes', context: 'fasting-sessions' }],
},
{
container: 'weight_log',
partitionKey: '/userId',
fields: [{ field: 'notes', context: 'weight-log' }],
},
],
},
// ── ActionTrail (port 4018) ──
{
productId: 'actiontrail',
mekName: 'actiontrail-mek',
containers: [
{
container: 'trail_actions',
partitionKey: '/userId',
fields: [
{ field: 'before', context: 'trail-actions', jsonSerialize: true },
{ field: 'after', context: 'trail-actions-after', jsonSerialize: true },
],
},
{
container: 'trail_reverts',
partitionKey: '/userId',
fields: [
{ field: 'beforeState', context: 'trail-reverts', jsonSerialize: true },
{ field: 'afterState', context: 'trail-reverts-after', jsonSerialize: true },
{ field: 'revertPayload', context: 'trail-reverts-payload', jsonSerialize: true },
],
},
],
},
// ── FlowMonk (port 4017) ──
{
productId: 'flowmonk',
mekName: 'flowmonk-mek',
containers: [
{
container: 'tasks',
partitionKey: '/userId',
fields: [{ field: 'description', context: 'tasks' }],
},
],
},
// ── ChronoMind (port 4011) ──
{
productId: 'chronomind',
mekName: 'chronomind-mek',
containers: [
{
container: 'timers',
partitionKey: '/userId',
fields: [{ field: 'description', context: 'timers' }],
},
{
container: 'routines',
partitionKey: '/userId',
fields: [
{ field: 'description', context: 'routines' },
// steps[].notes handled differently — see note below
],
},
{
container: 'shared_timers',
partitionKey: '/householdId',
fields: [{ field: 'description', context: 'shared-timers', userIdField: 'createdBy' }],
},
{
container: 'webhook_subscriptions',
partitionKey: '/userId',
fields: [
{ field: 'secret', context: 'webhooks' },
{ field: 'description', context: 'webhook-desc' },
],
},
],
},
// ── PeakPulse (port 4010) ──
{
productId: 'peakpulse',
mekName: 'peakpulse-mek',
containers: [
{
container: 'peak_sessions',
partitionKey: '/userId',
fields: [{ field: 'notes', context: 'peak-sessions' }],
},
],
},
// ── MindLyst (port 4014) ──
{
productId: 'mindlyst',
mekName: 'mindlyst-mek',
containers: [
{
container: 'memory_items',
partitionKey: '/userId',
fields: [
{ field: 'rawContent', context: 'memory' },
{ field: 'triageResult.summary', context: 'memory-triage' },
],
},
{
container: 'reflections',
partitionKey: '/userId',
fields: [
{ field: 'repeatedThemes', context: 'reflection-themes', jsonSerialize: true },
{ field: 'postponedItems', context: 'reflection-postponed', jsonSerialize: true },
{ field: 'roleImbalanceSignals', context: 'reflection-imbalance', jsonSerialize: true },
{ field: 'suggestedAdjustments', context: 'reflection-adjustments', jsonSerialize: true },
{ field: 'vsLastWeek.summary', context: 'reflection-vs' },
],
},
{
container: 'daily_briefs',
partitionKey: '/userId',
fields: [
{ field: 'greeting', context: 'daily-briefs' },
{ field: 'priorityItems', context: 'brief-priorities', jsonSerialize: true },
{ field: 'brainSummaries', context: 'brief-summaries', jsonSerialize: true },
{ field: 'streakMessage', context: 'brief-streak' },
{ field: 'motivationalQuote', context: 'brief-quote' },
],
},
{
container: 'brains',
partitionKey: '/userId',
fields: [{ field: 'rolePrompt', context: 'brains' }],
},
],
},
];
// ── CLI argument parsing ──────────────────────────────
interface CliArgs {
product: string;
container?: string;
batchSize: number;
dryRun: boolean;
verbose: boolean;
}
function parseArgs(): CliArgs {
const args = process.argv.slice(2);
const result: CliArgs = {
product: '',
batchSize: 100,
dryRun: false,
verbose: false,
};
for (let i = 0; i < args.length; i++) {
switch (args[i]) {
case '--product':
case '-p':
result.product = args[++i] ?? '';
break;
case '--container':
case '-c':
result.container = args[++i];
break;
case '--batch-size':
case '-b':
result.batchSize = parseInt(args[++i] ?? '100', 10);
break;
case '--dry-run':
case '-n':
result.dryRun = true;
break;
case '--verbose':
case '-v':
result.verbose = true;
break;
case '--help':
case '-h':
printUsage();
process.exit(0);
break;
default:
console.error(`Unknown argument: ${args[i]}`);
printUsage();
process.exit(1);
}
}
if (!result.product) {
console.error('Error: --product is required\n');
printUsage();
process.exit(1);
}
return result;
}
function printUsage(): void {
console.log(`
╔══════════════════════════════════════════════════════════════╗
║ ByteLyst Encryption Migration CLI ║
║ Batch-encrypt existing plaintext Cosmos documents ║
╚══════════════════════════════════════════════════════════════╝
Usage:
npx tsx scripts/encrypt-migrate.ts --product <id> [options]
Options:
--product, -p <id> Product ID or 'all' (required)
--container, -c <name> Migrate only this container (optional)
--batch-size, -b <n> Documents per batch (default: 100)
--dry-run, -n Count without encrypting
--verbose, -v Show per-document progress
--help, -h Show this help
Products: ${PRODUCT_CONFIGS.map(c => c.productId).join(', ')}
Environment:
COSMOS_ENDPOINT Azure Cosmos DB endpoint
COSMOS_KEY Azure Cosmos DB key
COSMOS_DATABASE Database name (default: lysnrai)
FIELD_ENCRYPT_KEY_PROVIDER 'memory' | 'env' | 'akv' (default: memory)
FIELD_ENCRYPT_KEY Hex key (for 'env' provider)
AZURE_KEYVAULT_URL AKV URL (for 'akv' provider)
Examples:
npx tsx scripts/encrypt-migrate.ts --product lysnrai --dry-run
npx tsx scripts/encrypt-migrate.ts --product all --dry-run
npx tsx scripts/encrypt-migrate.ts --product chronomind --container timers
npx tsx scripts/encrypt-migrate.ts --product mindlyst --batch-size 50
`);
}
// ── Cosmos helpers ────────────────────────────────────
function createCosmosClient(): CosmosClient {
const endpoint = process.env.COSMOS_ENDPOINT;
const key = process.env.COSMOS_KEY;
if (!endpoint || !key) {
console.error('Error: COSMOS_ENDPOINT and COSMOS_KEY environment variables are required');
process.exit(1);
}
return new CosmosClient({ endpoint, key });
}
function getPartitionKeyValue(doc: Record<string, unknown>, partitionKeyPath: string): string {
const field = partitionKeyPath.replace(/^\//, '');
return String(doc[field] ?? '');
}
// ── Migration engine ──────────────────────────────────
interface MigrateFieldResult {
containerName: string;
field: string;
result: MigrationResult;
}
async function migrateContainerField(
cosmosContainer: Container,
partitionKey: string,
fieldSpec: FieldSpec,
encryptor: FieldEncryptor,
batchSize: number,
dryRun: boolean,
verbose: boolean
): Promise<MigrateFieldResult> {
const containerName = cosmosContainer.id;
const userIdProp = fieldSpec.userIdField ?? 'userId';
const result = await migrateDocuments<Record<string, unknown>>({
batchSize,
dryRun,
fetchBatch: async (offset: number, size: number) => {
const query = `SELECT * FROM c ORDER BY c._ts OFFSET ${offset} LIMIT ${size}`;
const { resources } = await cosmosContainer.items.query(query).fetchAll();
return resources;
},
getId: (doc: Record<string, unknown>) => String(doc.id ?? 'unknown'),
getField: (doc: Record<string, unknown>) => getNestedField(doc, fieldSpec.field),
encryptValue: async (plaintext: string, doc: Record<string, unknown>) => {
const userId = String(doc[userIdProp] ?? 'unknown');
return encryptor.encrypt(plaintext, {
userId,
context: fieldSpec.context,
});
},
writeBack: async (doc: Record<string, unknown>, encrypted: EncryptedField) => {
const updated = setNestedField(doc, fieldSpec.field, encrypted);
const pk = getPartitionKeyValue(doc, partitionKey);
await cosmosContainer.item(String(doc.id), pk).replace(updated);
},
onProgress: (progress: MigrationResult) => {
if (verbose) {
const mode = dryRun ? '[DRY-RUN]' : '[LIVE]';
process.stdout.write(
`\r ${mode} ${containerName}.${fieldSpec.field}: scanned=${progress.scanned} encrypted=${progress.encrypted} skipped=${progress.skipped} errors=${progress.errors}`
);
}
},
});
if (verbose) {
process.stdout.write('\n');
}
return { containerName, field: fieldSpec.field, result };
}
async function migrateProduct(
config: ProductMigrationConfig,
cosmosClient: CosmosClient,
args: CliArgs
): Promise<MigrateFieldResult[]> {
const dbName = process.env.COSMOS_DATABASE || 'lysnrai';
const database = cosmosClient.database(dbName);
const encryptor = createFieldEncryptor({
keyProvider: (process.env.FIELD_ENCRYPT_KEY_PROVIDER as 'memory' | 'env' | 'akv') ?? 'memory',
encryptionKey: process.env.FIELD_ENCRYPT_KEY || undefined,
keyVaultUrl: process.env.AZURE_KEYVAULT_URL || undefined,
mekName: config.mekName,
});
const results: MigrateFieldResult[] = [];
for (const containerConfig of config.containers) {
// Skip if user specified a specific container
if (args.container && containerConfig.container !== args.container) {
continue;
}
const cosmosContainer = database.container(containerConfig.container);
const mode = args.dryRun ? 'DRY-RUN' : 'LIVE';
console.log(`\n 📦 ${containerConfig.container} (${mode})`);
for (const fieldSpec of containerConfig.fields) {
console.log(` 🔐 Field: ${fieldSpec.field} (context: ${fieldSpec.context})`);
try {
const fieldResult = await migrateContainerField(
cosmosContainer,
containerConfig.partitionKey,
fieldSpec,
encryptor,
args.batchSize,
args.dryRun,
args.verbose
);
results.push(fieldResult);
const r = fieldResult.result;
console.log(
` scanned=${r.scanned} encrypted=${r.encrypted} skipped=${r.skipped} errors=${r.errors}`
);
if (r.errorDetails.length > 0) {
console.log(' Errors:');
for (const err of r.errorDetails) {
console.log(` - ${err.id}: ${err.error}`);
}
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error(` ERROR: ${message}`);
results.push({
containerName: containerConfig.container,
field: fieldSpec.field,
result: {
scanned: 0,
encrypted: 0,
skipped: 0,
errors: 1,
errorDetails: [{ id: 'container', error: message }],
},
});
}
}
}
return results;
}
// ── Main ──────────────────────────────────────────────
async function main(): Promise<void> {
const args = parseArgs();
const startTime = Date.now();
console.log('\n════════════════════════════════════════════════════');
console.log(' ByteLyst Encryption Migration CLI');
console.log('════════════════════════════════════════════════════');
console.log(` Mode: ${args.dryRun ? 'DRY-RUN (no writes)' : 'LIVE (encrypting!)'}`);
console.log(` Product: ${args.product}`);
console.log(` Container: ${args.container ?? 'all'}`);
console.log(` Batch size: ${args.batchSize}`);
console.log(` Database: ${process.env.COSMOS_DATABASE || 'lysnrai'}`);
console.log(` Key provider: ${process.env.FIELD_ENCRYPT_KEY_PROVIDER || 'memory'}`);
console.log('════════════════════════════════════════════════════');
const cosmosClient = createCosmosClient();
const allResults: MigrateFieldResult[] = [];
const productsToMigrate =
args.product === 'all'
? PRODUCT_CONFIGS
: PRODUCT_CONFIGS.filter(c => c.productId === args.product);
if (productsToMigrate.length === 0) {
console.error(`\nError: Unknown product '${args.product}'`);
console.error(`Available: ${PRODUCT_CONFIGS.map(c => c.productId).join(', ')}`);
process.exit(1);
}
for (const productConfig of productsToMigrate) {
console.log(`\n🏢 ${productConfig.productId} (MEK: ${productConfig.mekName})`);
const results = await migrateProduct(productConfig, cosmosClient, args);
allResults.push(...results);
}
// ── Summary ──
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
const totals = allResults.reduce(
(acc, r) => ({
scanned: acc.scanned + r.result.scanned,
encrypted: acc.encrypted + r.result.encrypted,
skipped: acc.skipped + r.result.skipped,
errors: acc.errors + r.result.errors,
}),
{ scanned: 0, encrypted: 0, skipped: 0, errors: 0 }
);
console.log('\n════════════════════════════════════════════════════');
console.log(' Migration Summary');
console.log('════════════════════════════════════════════════════');
console.log(` Products: ${productsToMigrate.length}`);
console.log(` Fields: ${allResults.length}`);
console.log(` Scanned: ${totals.scanned}`);
console.log(` Encrypted: ${totals.encrypted}`);
console.log(` Skipped: ${totals.skipped}`);
console.log(` Errors: ${totals.errors}`);
console.log(` Duration: ${elapsed}s`);
console.log(` Mode: ${args.dryRun ? 'DRY-RUN' : 'LIVE'}`);
console.log('════════════════════════════════════════════════════\n');
// Per-field breakdown
if (allResults.length > 0) {
console.log(' Per-field breakdown:');
console.log(' ┌─────────────────────────────┬────────┬───────────┬─────────┬────────┐');
console.log(' │ Container.Field │ Scanned│ Encrypted │ Skipped │ Errors │');
console.log(' ├─────────────────────────────┼────────┼───────────┼─────────┼────────┤');
for (const r of allResults) {
const name = `${r.containerName}.${r.field}`.padEnd(29);
const sc = String(r.result.scanned).padStart(7);
const en = String(r.result.encrypted).padStart(9);
const sk = String(r.result.skipped).padStart(7);
const er = String(r.result.errors).padStart(6);
console.log(`${name}${sc}${en}${sk}${er}`);
}
console.log(' └─────────────────────────────┴────────┴───────────┴─────────┴────────┘');
}
if (totals.errors > 0) {
process.exit(1);
}
}
main().catch(err => {
console.error('\nFatal error:', err instanceof Error ? err.message : err);
process.exit(1);
});