From 44d8867aa5cb0280ef59e0a887a03d7da5645ff7 Mon Sep 17 00:00:00 2001
From: saravanakumardb1 <saravanakumardb1@users.noreply.github.com>
Date: Fri, 10 Apr 2026 01:15:41 -0700
Subject: [PATCH] feat(palace): add user-isolated repository with CRUD, search,
 dedup, prune, encryption

---
 backend/src/modules/palace/repository.ts | 540 +++++++++++++++++++++++
 1 file changed, 540 insertions(+)
 create mode 100644 backend/src/modules/palace/repository.ts

diff --git a/backend/src/modules/palace/repository.ts b/backend/src/modules/palace/repository.ts
new file mode 100644
index 0000000..c6701fe
--- /dev/null
+++ b/backend/src/modules/palace/repository.ts
@@ -0,0 +1,540 @@
+import { getCollection } from '../../lib/datastore.js';
+import { getEncryptor } from '../../lib/field-encrypt.js';
+import { isEncryptedField, type EncryptedField } from '@bytelyst/field-encrypt';
+import { embedText } from '../../lib/embeddings.js';
+import { config } from '../../lib/config.js';
+import { cosineSimilarity, topKByCosine, isContentDuplicate, computeDecayedRelevance } from '@bytelyst/palace';
+import type { FilterMap } from '@bytelyst/datastore';
+import type {
+  PalaceWingDoc,
+  PalaceRoomDoc,
+  PalaceMemoryDoc,
+  PalaceTunnelDoc,
+  PalaceKGTripleDoc,
+  PalaceDiaryDoc,
+  HallType,
+} from './types.js';
+
+const ENCRYPT_CONTEXT = 'palace-memory';
+
+// ── Collection accessors ────────────────────────────────────────────
+
+function wingsCollection() {
+  return getCollection<PalaceWingDoc>('palace_wings', '/userId');
+}
+function roomsCollection() {
+  return getCollection<PalaceRoomDoc>('palace_rooms', '/userId');
+}
+function memoriesCollection() {
+  return getCollection<PalaceMemoryDoc>('palace_memories', '/userId');
+}
+function tunnelsCollection() {
+  return getCollection<PalaceTunnelDoc>('palace_tunnels', '/userId');
+}
+function kgCollection() {
+  return getCollection<PalaceKGTripleDoc>('palace_kg', '/userId');
+}
+function diariesCollection() {
+  return getCollection<PalaceDiaryDoc>('palace_diaries', '/userId');
+}
+
+// ── Field Encryption ────────────────────────────────────────────────
+
+async function encryptMemoryContent(doc: PalaceMemoryDoc): Promise<PalaceMemoryDoc> {
+  const enc = getEncryptor();
+  const ctx = { userId: doc.userId, context: ENCRYPT_CONTEXT };
+  const encrypted = await enc.encrypt(doc.content, ctx);
+  return { ...doc, content: encrypted as unknown as string };
+}
+
+async function decryptMemoryContent(doc: PalaceMemoryDoc): Promise<PalaceMemoryDoc> {
+  const enc = getEncryptor();
+  const ctx = { userId: doc.userId, context: ENCRYPT_CONTEXT };
+  let content = doc.content;
+  if (isEncryptedField(content)) {
+    content = await enc.decrypt(content as unknown as EncryptedField, ctx);
+  }
+  return { ...doc, content };
+}
+
+async function decryptMemoryBatch(docs: PalaceMemoryDoc[]): Promise<PalaceMemoryDoc[]> {
+  return Promise.all(docs.map(decryptMemoryContent));
+}
+
+// ── Wings ───────────────────────────────────────────────────────────
+
+export async function ensureWing(
+  userId: string,
+  productId: string,
+  sourceWorkspaceId: string,
+  name: string,
+): Promise<PalaceWingDoc> {
+  const existing = await wingsCollection().findMany({
+    filter: { userId, productId, sourceWorkspaceId },
+    limit: 1,
+  });
+  if (existing.length > 0) return existing[0];
+
+  const now = new Date().toISOString();
+  const wing: PalaceWingDoc = {
+    id: `wing-${sourceWorkspaceId}`,
+    productId,
+    userId,
+    name,
+    sourceWorkspaceId,
+    memoryCount: 0,
+    createdAt: now,
+    updatedAt: now,
+  };
+  return wingsCollection().create(wing);
+}
+
+export async function getWing(
+  userId: string,
+  productId: string,
+  wingId: string,
+): Promise<PalaceWingDoc | null> {
+  const doc = await wingsCollection().findById(wingId, userId);
+  if (!doc || doc.productId !== productId) return null;
+  return doc;
+}
+
+export async function listWings(
+  userId: string,
+  productId: string,
+): Promise<PalaceWingDoc[]> {
+  return wingsCollection().findMany({
+    filter: { userId, productId },
+    sort: { updatedAt: -1 },
+  });
+}
+
+export async function deleteWing(
+  userId: string,
+  productId: string,
+  wingId: string,
+): Promise<void> {
+  const wing = await getWing(userId, productId, wingId);
+  if (!wing) return;
+
+  // Cascade delete: rooms, memories, tunnels, KG, diaries
+  const rooms = await roomsCollection().findMany({ filter: { userId, productId, wingId } });
+  for (const room of rooms) {
+    await roomsCollection().delete(room.id, userId);
+  }
+
+  const memories = await memoriesCollection().findMany({ filter: { userId, productId, wingId } });
+  for (const mem of memories) {
+    await memoriesCollection().delete(mem.id, userId);
+  }
+
+  const tunnels = await tunnelsCollection().findMany({ filter: { userId, productId } });
+  for (const tunnel of tunnels) {
+    if (tunnel.fromWingId === wingId || tunnel.toWingId === wingId) {
+      await tunnelsCollection().delete(tunnel.id, userId);
+    }
+  }
+
+  const triples = await kgCollection().findMany({ filter: { userId, productId, wingId } });
+  for (const triple of triples) {
+    await kgCollection().delete(triple.id, userId);
+  }
+
+  const diaries = await diariesCollection().findMany({ filter: { userId, productId, wingId } });
+  for (const diary of diaries) {
+    await diariesCollection().delete(diary.id, userId);
+  }
+
+  await wingsCollection().delete(wingId, userId);
+}
+
+// ── Rooms ───────────────────────────────────────────────────────────
+
+export async function ensureRoom(
+  userId: string,
+  productId: string,
+  wingId: string,
+  name: string,
+  description?: string,
+): Promise<PalaceRoomDoc> {
+  const existing = await roomsCollection().findMany({
+    filter: { userId, productId, wingId, name },
+    limit: 1,
+  });
+  if (existing.length > 0) return existing[0];
+
+  const now = new Date().toISOString();
+  const room: PalaceRoomDoc = {
+    id: `room-${wingId}-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-')}`,
+    productId,
+    userId,
+    wingId,
+    name,
+    description,
+    memoryCount: 0,
+    createdAt: now,
+    updatedAt: now,
+  };
+  return roomsCollection().create(room);
+}
+
+export async function listRooms(
+  userId: string,
+  productId: string,
+  wingId: string,
+): Promise<PalaceRoomDoc[]> {
+  return roomsCollection().findMany({
+    filter: { userId, productId, wingId },
+    sort: { createdAt: -1 },
+  });
+}
+
+// ── Memories ────────────────────────────────────────────────────────
+
+export async function storeMemory(
+  userId: string,
+  productId: string,
+  wingId: string,
+  roomId: string,
+  hall: HallType,
+  content: string,
+  sourceNoteId?: string,
+  embedding?: number[] | null,
+): Promise<PalaceMemoryDoc> {
+  const now = new Date().toISOString();
+  const doc: PalaceMemoryDoc = {
+    id: `mem-${crypto.randomUUID()}`,
+    productId,
+    userId,
+    wingId,
+    roomId,
+    hall,
+    content,
+    relevance: 1.0,
+    embedding: embedding ?? undefined,
+    sourceNoteId,
+    createdAt: now,
+    updatedAt: now,
+  };
+
+  const encrypted = await encryptMemoryContent(doc);
+  const created = await memoriesCollection().create(encrypted);
+
+  // Increment memory counts
+  await incrementMemoryCount(userId, wingId, roomId);
+
+  return decryptMemoryContent(created);
+}
+
+async function incrementMemoryCount(userId: string, wingId: string, roomId: string): Promise<void> {
+  const wing = await wingsCollection().findById(wingId, userId);
+  if (wing) {
+    await wingsCollection().update(wingId, userId, {
+      memoryCount: (wing.memoryCount || 0) + 1,
+      updatedAt: new Date().toISOString(),
+    });
+  }
+  const room = await roomsCollection().findById(roomId, userId);
+  if (room) {
+    await roomsCollection().update(roomId, userId, {
+      memoryCount: (room.memoryCount || 0) + 1,
+      updatedAt: new Date().toISOString(),
+    });
+  }
+}
+
+export async function getMemory(
+  userId: string,
+  productId: string,
+  memoryId: string,
+): Promise<PalaceMemoryDoc | null> {
+  const doc = await memoriesCollection().findById(memoryId, userId);
+  if (!doc || doc.productId !== productId) return null;
+  return decryptMemoryContent(doc);
+}
+
+export async function deleteMemory(
+  userId: string,
+  productId: string,
+  memoryId: string,
+): Promise<boolean> {
+  const doc = await memoriesCollection().findById(memoryId, userId);
+  if (!doc || doc.productId !== productId) return false;
+  await memoriesCollection().delete(memoryId, userId);
+  return true;
+}
+
+export async function listMemories(
+  userId: string,
+  productId: string,
+  opts: { wingId?: string; roomId?: string; hall?: HallType; limit?: number },
+): Promise<PalaceMemoryDoc[]> {
+  const filter: FilterMap = { userId, productId };
+  if (opts.wingId) filter.wingId = opts.wingId;
+  if (opts.roomId) filter.roomId = opts.roomId;
+  if (opts.hall) filter.hall = opts.hall;
+
+  const docs = await memoriesCollection().findMany({
+    filter,
+    sort: { updatedAt: -1 },
+    limit: opts.limit ?? 50,
+  });
+  return decryptMemoryBatch(docs);
+}
+
+// ── Search ──────────────────────────────────────────────────────────
+
+export async function searchSemantic(
+  userId: string,
+  productId: string,
+  _query: string,
+  embedding: number[],
+  wingId?: string,
+  limit = 10,
+): Promise<PalaceMemoryDoc[]> {
+  const filter: FilterMap = { userId, productId };
+  if (wingId) filter.wingId = wingId;
+
+  const candidates = await memoriesCollection().findMany({
+    filter,
+    sort: { updatedAt: -1 },
+    limit: limit * 5,
+  });
+
+  const decrypted = await decryptMemoryBatch(candidates);
+  const ranked = topKByCosine(
+    embedding,
+    decrypted.filter(m => m.embedding && m.embedding.length > 0),
+    (m: PalaceMemoryDoc) => m.embedding,
+    limit,
+  );
+
+  return ranked.map(r => r.item);
+}
+
+export async function searchText(
+  userId: string,
+  productId: string,
+  query: string,
+  wingId?: string,
+  limit = 10,
+): Promise<PalaceMemoryDoc[]> {
+  const filter: FilterMap = { userId, productId };
+  if (wingId) filter.wingId = wingId;
+
+  // Text search: fetch all, decrypt, filter in-memory
+  const candidates = await memoriesCollection().findMany({
+    filter,
+    sort: { updatedAt: -1 },
+    limit: limit * 10,
+  });
+
+  const decrypted = await decryptMemoryBatch(candidates);
+  const lowerQ = query.toLowerCase();
+  return decrypted
+    .filter(m => m.content.toLowerCase().includes(lowerQ))
+    .slice(0, limit);
+}
+
+export async function searchHybrid(
+  userId: string,
+  productId: string,
+  query: string,
+  embedding: number[],
+  wingId?: string,
+  limit = 10,
+): Promise<PalaceMemoryDoc[]> {
+  const filter: FilterMap = { userId, productId };
+  if (wingId) filter.wingId = wingId;
+
+  // Fetch candidates
+  const candidates = await memoriesCollection().findMany({
+    filter,
+    sort: { updatedAt: -1 },
+    limit: limit * 10,
+  });
+
+  const decrypted = await decryptMemoryBatch(candidates);
+  const lowerQ = query.toLowerCase();
+
+  // Text filter first
+  const textMatches = decrypted.filter(m => m.content.toLowerCase().includes(lowerQ));
+
+  // Re-rank by cosine similarity
+  const withEmbeddings = textMatches.filter(m => m.embedding && m.embedding.length > 0);
+  if (withEmbeddings.length === 0) return textMatches.slice(0, limit);
+
+  const ranked = topKByCosine(embedding, withEmbeddings, (m: PalaceMemoryDoc) => m.embedding, limit);
+  return ranked.map(r => r.item);
+}
+
+// ── Wing Summary ────────────────────────────────────────────────────
+
+export async function getWingSummary(
+  userId: string,
+  productId: string,
+  wingId: string,
+): Promise<{ wing: PalaceWingDoc | null; rooms: PalaceRoomDoc[]; totalMemories: number }> {
+  const wing = await getWing(userId, productId, wingId);
+  if (!wing) return { wing: null, rooms: [], totalMemories: 0 };
+
+  const rooms = await listRooms(userId, productId, wingId);
+  const totalMemories = await memoriesCollection().count({ userId, productId, wingId });
+
+  return { wing, rooms, totalMemories };
+}
+
+// ── Deduplication ───────────────────────────────────────────────────
+
+export async function isNearDuplicate(
+  userId: string,
+  productId: string,
+  roomId: string,
+  hall: HallType,
+  content: string,
+  embedding: number[] | null,
+  threshold?: number,
+): Promise<boolean> {
+  const dupThreshold = threshold ?? config.PALACE_DEDUP_THRESHOLD;
+
+  // 1. Exact match (cheap)
+  const filter: FilterMap = { userId, productId, roomId, hall };
+  const candidates = await memoriesCollection().findMany({
+    filter,
+    sort: { createdAt: -1 },
+    limit: 20,
+  });
+
+  const decrypted = await decryptMemoryBatch(candidates);
+
+  // Check exact
+  if (decrypted.some(c => c.content === content)) return true;
+
+  // 2. Semantic dedup via shared @bytelyst/palace helper
+  if (embedding) {
+    const candidateEmbeddings = decrypted
+      .map(c => c.embedding)
+      .filter((e): e is number[] => !!e && e.length > 0);
+
+    return isContentDuplicate(embedding, candidateEmbeddings, dupThreshold);
+  }
+
+  return false;
+}
+
+// ── Maintenance ─────────────────────────────────────────────────────
+
+export async function pruneOldMemories(
+  userId: string,
+  productId: string,
+  wingId: string | undefined,
+  olderThanDays: number,
+  minRelevance: number,
+): Promise<number> {
+  const filter: FilterMap = { userId, productId };
+  if (wingId) filter.wingId = wingId;
+
+  const all = await memoriesCollection().findMany({ filter, limit: 1000 });
+  const cutoff = new Date(Date.now() - olderThanDays * 86_400_000).toISOString();
+  let deleted = 0;
+
+  for (const mem of all) {
+    if (mem.createdAt < cutoff && mem.relevance < minRelevance) {
+      await memoriesCollection().delete(mem.id, userId);
+      deleted++;
+    }
+  }
+
+  return deleted;
+}
+
+export async function decayRelevance(
+  userId: string,
+  productId: string,
+  halfLifeDays?: number,
+): Promise<number> {
+  const halfLife = halfLifeDays ?? config.PALACE_RELEVANCE_HALF_LIFE_DAYS;
+  const filter: FilterMap = { userId, productId };
+
+  const all = await memoriesCollection().findMany({ filter, limit: 5000 });
+  let updated = 0;
+
+  for (const mem of all) {
+    const decayed = computeDecayedRelevance(mem.relevance, mem.createdAt, halfLife);
+    if (Math.abs(decayed - mem.relevance) > 0.001) {
+      await memoriesCollection().update(mem.id, userId, {
+        relevance: decayed,
+        updatedAt: new Date().toISOString(),
+      });
+      updated++;
+    }
+  }
+
+  return updated;
+}
+
+export async function backfillEmbeddings(
+  userId: string,
+  productId: string,
+): Promise<number> {
+  const filter: FilterMap = { userId, productId };
+  const all = await memoriesCollection().findMany({ filter, limit: 500 });
+  const decrypted = await decryptMemoryBatch(all);
+  let count = 0;
+
+  for (const mem of decrypted) {
+    if (!mem.embedding || mem.embedding.length === 0) {
+      const emb = await embedText(mem.content);
+      if (emb) {
+        await memoriesCollection().update(mem.id, userId, {
+          embedding: emb,
+          updatedAt: new Date().toISOString(),
+        });
+        count++;
+      }
+    }
+  }
+
+  return count;
+}
+
+// ── Stats ───────────────────────────────────────────────────────────
+
+export async function getPalaceStats(
+  userId: string,
+  productId: string,
+): Promise<{
+  wings: number;
+  rooms: number;
+  memories: number;
+  kgTriples: number;
+  tunnels: number;
+  diaries: number;
+}> {
+  const [wings, rooms, memories, kgTriples, tunnels, diaries] = await Promise.all([
+    wingsCollection().count({ userId, productId }),
+    roomsCollection().count({ userId, productId }),
+    memoriesCollection().count({ userId, productId }),
+    kgCollection().count({ userId, productId }),
+    tunnelsCollection().count({ userId, productId }),
+    diariesCollection().count({ userId, productId }),
+  ]);
+  return { wings, rooms, memories, kgTriples, tunnels, diaries };
+}
+
+export async function healthCheck(): Promise<{ cosmos: boolean; llm: boolean }> {
+  let cosmos = false;
+  let llm = false;
+
+  try {
+    await wingsCollection().count({});
+    cosmos = true;
+  } catch { /* cosmos unavailable */ }
+
+  try {
+    const emb = await embedText('health check');
+    llm = emb !== null;
+  } catch { /* llm unavailable */ }
+
+  return { cosmos, llm };
+}