feat(palace): add user-isolated repository with CRUD, search, dedup, prune, encryption
This commit is contained in:
saravanakumardb1
parent
38006af1a3
commit
44d8867aa5
540
backend/src/modules/palace/repository.ts
Normal file
540
backend/src/modules/palace/repository.ts
Normal file
@ -0,0 +1,540 @@
|
||||
import { getCollection } from '../../lib/datastore.js';
|
||||
import { getEncryptor } from '../../lib/field-encrypt.js';
|
||||
import { isEncryptedField, type EncryptedField } from '@bytelyst/field-encrypt';
|
||||
import { embedText } from '../../lib/embeddings.js';
|
||||
import { config } from '../../lib/config.js';
|
||||
import { cosineSimilarity, topKByCosine, isContentDuplicate, computeDecayedRelevance } from '@bytelyst/palace';
|
||||
import type { FilterMap } from '@bytelyst/datastore';
|
||||
import type {
|
||||
PalaceWingDoc,
|
||||
PalaceRoomDoc,
|
||||
PalaceMemoryDoc,
|
||||
PalaceTunnelDoc,
|
||||
PalaceKGTripleDoc,
|
||||
PalaceDiaryDoc,
|
||||
HallType,
|
||||
} from './types.js';
|
||||
|
||||
const ENCRYPT_CONTEXT = 'palace-memory';
|
||||
|
||||
// ── Collection accessors ────────────────────────────────────────────
|
||||
|
||||
function wingsCollection() {
|
||||
return getCollection<PalaceWingDoc>('palace_wings', '/userId');
|
||||
}
|
||||
function roomsCollection() {
|
||||
return getCollection<PalaceRoomDoc>('palace_rooms', '/userId');
|
||||
}
|
||||
function memoriesCollection() {
|
||||
return getCollection<PalaceMemoryDoc>('palace_memories', '/userId');
|
||||
}
|
||||
function tunnelsCollection() {
|
||||
return getCollection<PalaceTunnelDoc>('palace_tunnels', '/userId');
|
||||
}
|
||||
function kgCollection() {
|
||||
return getCollection<PalaceKGTripleDoc>('palace_kg', '/userId');
|
||||
}
|
||||
function diariesCollection() {
|
||||
return getCollection<PalaceDiaryDoc>('palace_diaries', '/userId');
|
||||
}
|
||||
|
||||
// ── Field Encryption ────────────────────────────────────────────────
|
||||
|
||||
async function encryptMemoryContent(doc: PalaceMemoryDoc): Promise<PalaceMemoryDoc> {
|
||||
const enc = getEncryptor();
|
||||
const ctx = { userId: doc.userId, context: ENCRYPT_CONTEXT };
|
||||
const encrypted = await enc.encrypt(doc.content, ctx);
|
||||
return { ...doc, content: encrypted as unknown as string };
|
||||
}
|
||||
|
||||
async function decryptMemoryContent(doc: PalaceMemoryDoc): Promise<PalaceMemoryDoc> {
|
||||
const enc = getEncryptor();
|
||||
const ctx = { userId: doc.userId, context: ENCRYPT_CONTEXT };
|
||||
let content = doc.content;
|
||||
if (isEncryptedField(content)) {
|
||||
content = await enc.decrypt(content as unknown as EncryptedField, ctx);
|
||||
}
|
||||
return { ...doc, content };
|
||||
}
|
||||
|
||||
async function decryptMemoryBatch(docs: PalaceMemoryDoc[]): Promise<PalaceMemoryDoc[]> {
|
||||
return Promise.all(docs.map(decryptMemoryContent));
|
||||
}
|
||||
|
||||
// ── Wings ───────────────────────────────────────────────────────────
|
||||
|
||||
export async function ensureWing(
|
||||
userId: string,
|
||||
productId: string,
|
||||
sourceWorkspaceId: string,
|
||||
name: string,
|
||||
): Promise<PalaceWingDoc> {
|
||||
const existing = await wingsCollection().findMany({
|
||||
filter: { userId, productId, sourceWorkspaceId },
|
||||
limit: 1,
|
||||
});
|
||||
if (existing.length > 0) return existing[0];
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const wing: PalaceWingDoc = {
|
||||
id: `wing-${sourceWorkspaceId}`,
|
||||
productId,
|
||||
userId,
|
||||
name,
|
||||
sourceWorkspaceId,
|
||||
memoryCount: 0,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
return wingsCollection().create(wing);
|
||||
}
|
||||
|
||||
export async function getWing(
|
||||
userId: string,
|
||||
productId: string,
|
||||
wingId: string,
|
||||
): Promise<PalaceWingDoc | null> {
|
||||
const doc = await wingsCollection().findById(wingId, userId);
|
||||
if (!doc || doc.productId !== productId) return null;
|
||||
return doc;
|
||||
}
|
||||
|
||||
export async function listWings(
|
||||
userId: string,
|
||||
productId: string,
|
||||
): Promise<PalaceWingDoc[]> {
|
||||
return wingsCollection().findMany({
|
||||
filter: { userId, productId },
|
||||
sort: { updatedAt: -1 },
|
||||
});
|
||||
}
|
||||
|
||||
export async function deleteWing(
|
||||
userId: string,
|
||||
productId: string,
|
||||
wingId: string,
|
||||
): Promise<void> {
|
||||
const wing = await getWing(userId, productId, wingId);
|
||||
if (!wing) return;
|
||||
|
||||
// Cascade delete: rooms, memories, tunnels, KG, diaries
|
||||
const rooms = await roomsCollection().findMany({ filter: { userId, productId, wingId } });
|
||||
for (const room of rooms) {
|
||||
await roomsCollection().delete(room.id, userId);
|
||||
}
|
||||
|
||||
const memories = await memoriesCollection().findMany({ filter: { userId, productId, wingId } });
|
||||
for (const mem of memories) {
|
||||
await memoriesCollection().delete(mem.id, userId);
|
||||
}
|
||||
|
||||
const tunnels = await tunnelsCollection().findMany({ filter: { userId, productId } });
|
||||
for (const tunnel of tunnels) {
|
||||
if (tunnel.fromWingId === wingId || tunnel.toWingId === wingId) {
|
||||
await tunnelsCollection().delete(tunnel.id, userId);
|
||||
}
|
||||
}
|
||||
|
||||
const triples = await kgCollection().findMany({ filter: { userId, productId, wingId } });
|
||||
for (const triple of triples) {
|
||||
await kgCollection().delete(triple.id, userId);
|
||||
}
|
||||
|
||||
const diaries = await diariesCollection().findMany({ filter: { userId, productId, wingId } });
|
||||
for (const diary of diaries) {
|
||||
await diariesCollection().delete(diary.id, userId);
|
||||
}
|
||||
|
||||
await wingsCollection().delete(wingId, userId);
|
||||
}
|
||||
|
||||
// ── Rooms ───────────────────────────────────────────────────────────
|
||||
|
||||
export async function ensureRoom(
|
||||
userId: string,
|
||||
productId: string,
|
||||
wingId: string,
|
||||
name: string,
|
||||
description?: string,
|
||||
): Promise<PalaceRoomDoc> {
|
||||
const existing = await roomsCollection().findMany({
|
||||
filter: { userId, productId, wingId, name },
|
||||
limit: 1,
|
||||
});
|
||||
if (existing.length > 0) return existing[0];
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const room: PalaceRoomDoc = {
|
||||
id: `room-${wingId}-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-')}`,
|
||||
productId,
|
||||
userId,
|
||||
wingId,
|
||||
name,
|
||||
description,
|
||||
memoryCount: 0,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
return roomsCollection().create(room);
|
||||
}
|
||||
|
||||
export async function listRooms(
|
||||
userId: string,
|
||||
productId: string,
|
||||
wingId: string,
|
||||
): Promise<PalaceRoomDoc[]> {
|
||||
return roomsCollection().findMany({
|
||||
filter: { userId, productId, wingId },
|
||||
sort: { createdAt: -1 },
|
||||
});
|
||||
}
|
||||
|
||||
// ── Memories ────────────────────────────────────────────────────────
|
||||
|
||||
export async function storeMemory(
|
||||
userId: string,
|
||||
productId: string,
|
||||
wingId: string,
|
||||
roomId: string,
|
||||
hall: HallType,
|
||||
content: string,
|
||||
sourceNoteId?: string,
|
||||
embedding?: number[] | null,
|
||||
): Promise<PalaceMemoryDoc> {
|
||||
const now = new Date().toISOString();
|
||||
const doc: PalaceMemoryDoc = {
|
||||
id: `mem-${crypto.randomUUID()}`,
|
||||
productId,
|
||||
userId,
|
||||
wingId,
|
||||
roomId,
|
||||
hall,
|
||||
content,
|
||||
relevance: 1.0,
|
||||
embedding: embedding ?? undefined,
|
||||
sourceNoteId,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
const encrypted = await encryptMemoryContent(doc);
|
||||
const created = await memoriesCollection().create(encrypted);
|
||||
|
||||
// Increment memory counts
|
||||
await incrementMemoryCount(userId, wingId, roomId);
|
||||
|
||||
return decryptMemoryContent(created);
|
||||
}
|
||||
|
||||
async function incrementMemoryCount(userId: string, wingId: string, roomId: string): Promise<void> {
|
||||
const wing = await wingsCollection().findById(wingId, userId);
|
||||
if (wing) {
|
||||
await wingsCollection().update(wingId, userId, {
|
||||
memoryCount: (wing.memoryCount || 0) + 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
const room = await roomsCollection().findById(roomId, userId);
|
||||
if (room) {
|
||||
await roomsCollection().update(roomId, userId, {
|
||||
memoryCount: (room.memoryCount || 0) + 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function getMemory(
|
||||
userId: string,
|
||||
productId: string,
|
||||
memoryId: string,
|
||||
): Promise<PalaceMemoryDoc | null> {
|
||||
const doc = await memoriesCollection().findById(memoryId, userId);
|
||||
if (!doc || doc.productId !== productId) return null;
|
||||
return decryptMemoryContent(doc);
|
||||
}
|
||||
|
||||
export async function deleteMemory(
|
||||
userId: string,
|
||||
productId: string,
|
||||
memoryId: string,
|
||||
): Promise<boolean> {
|
||||
const doc = await memoriesCollection().findById(memoryId, userId);
|
||||
if (!doc || doc.productId !== productId) return false;
|
||||
await memoriesCollection().delete(memoryId, userId);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function listMemories(
|
||||
userId: string,
|
||||
productId: string,
|
||||
opts: { wingId?: string; roomId?: string; hall?: HallType; limit?: number },
|
||||
): Promise<PalaceMemoryDoc[]> {
|
||||
const filter: FilterMap = { userId, productId };
|
||||
if (opts.wingId) filter.wingId = opts.wingId;
|
||||
if (opts.roomId) filter.roomId = opts.roomId;
|
||||
if (opts.hall) filter.hall = opts.hall;
|
||||
|
||||
const docs = await memoriesCollection().findMany({
|
||||
filter,
|
||||
sort: { updatedAt: -1 },
|
||||
limit: opts.limit ?? 50,
|
||||
});
|
||||
return decryptMemoryBatch(docs);
|
||||
}
|
||||
|
||||
// ── Search ──────────────────────────────────────────────────────────
|
||||
|
||||
export async function searchSemantic(
|
||||
userId: string,
|
||||
productId: string,
|
||||
_query: string,
|
||||
embedding: number[],
|
||||
wingId?: string,
|
||||
limit = 10,
|
||||
): Promise<PalaceMemoryDoc[]> {
|
||||
const filter: FilterMap = { userId, productId };
|
||||
if (wingId) filter.wingId = wingId;
|
||||
|
||||
const candidates = await memoriesCollection().findMany({
|
||||
filter,
|
||||
sort: { updatedAt: -1 },
|
||||
limit: limit * 5,
|
||||
});
|
||||
|
||||
const decrypted = await decryptMemoryBatch(candidates);
|
||||
const ranked = topKByCosine(
|
||||
embedding,
|
||||
decrypted.filter(m => m.embedding && m.embedding.length > 0),
|
||||
(m: PalaceMemoryDoc) => m.embedding,
|
||||
limit,
|
||||
);
|
||||
|
||||
return ranked.map(r => r.item);
|
||||
}
|
||||
|
||||
export async function searchText(
|
||||
userId: string,
|
||||
productId: string,
|
||||
query: string,
|
||||
wingId?: string,
|
||||
limit = 10,
|
||||
): Promise<PalaceMemoryDoc[]> {
|
||||
const filter: FilterMap = { userId, productId };
|
||||
if (wingId) filter.wingId = wingId;
|
||||
|
||||
// Text search: fetch all, decrypt, filter in-memory
|
||||
const candidates = await memoriesCollection().findMany({
|
||||
filter,
|
||||
sort: { updatedAt: -1 },
|
||||
limit: limit * 10,
|
||||
});
|
||||
|
||||
const decrypted = await decryptMemoryBatch(candidates);
|
||||
const lowerQ = query.toLowerCase();
|
||||
return decrypted
|
||||
.filter(m => m.content.toLowerCase().includes(lowerQ))
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
||||
export async function searchHybrid(
|
||||
userId: string,
|
||||
productId: string,
|
||||
query: string,
|
||||
embedding: number[],
|
||||
wingId?: string,
|
||||
limit = 10,
|
||||
): Promise<PalaceMemoryDoc[]> {
|
||||
const filter: FilterMap = { userId, productId };
|
||||
if (wingId) filter.wingId = wingId;
|
||||
|
||||
// Fetch candidates
|
||||
const candidates = await memoriesCollection().findMany({
|
||||
filter,
|
||||
sort: { updatedAt: -1 },
|
||||
limit: limit * 10,
|
||||
});
|
||||
|
||||
const decrypted = await decryptMemoryBatch(candidates);
|
||||
const lowerQ = query.toLowerCase();
|
||||
|
||||
// Text filter first
|
||||
const textMatches = decrypted.filter(m => m.content.toLowerCase().includes(lowerQ));
|
||||
|
||||
// Re-rank by cosine similarity
|
||||
const withEmbeddings = textMatches.filter(m => m.embedding && m.embedding.length > 0);
|
||||
if (withEmbeddings.length === 0) return textMatches.slice(0, limit);
|
||||
|
||||
const ranked = topKByCosine(embedding, withEmbeddings, (m: PalaceMemoryDoc) => m.embedding, limit);
|
||||
return ranked.map(r => r.item);
|
||||
}
|
||||
|
||||
// ── Wing Summary ────────────────────────────────────────────────────
|
||||
|
||||
export async function getWingSummary(
|
||||
userId: string,
|
||||
productId: string,
|
||||
wingId: string,
|
||||
): Promise<{ wing: PalaceWingDoc | null; rooms: PalaceRoomDoc[]; totalMemories: number }> {
|
||||
const wing = await getWing(userId, productId, wingId);
|
||||
if (!wing) return { wing: null, rooms: [], totalMemories: 0 };
|
||||
|
||||
const rooms = await listRooms(userId, productId, wingId);
|
||||
const totalMemories = await memoriesCollection().count({ userId, productId, wingId });
|
||||
|
||||
return { wing, rooms, totalMemories };
|
||||
}
|
||||
|
||||
// ── Deduplication ───────────────────────────────────────────────────
|
||||
|
||||
export async function isNearDuplicate(
|
||||
userId: string,
|
||||
productId: string,
|
||||
roomId: string,
|
||||
hall: HallType,
|
||||
content: string,
|
||||
embedding: number[] | null,
|
||||
threshold?: number,
|
||||
): Promise<boolean> {
|
||||
const dupThreshold = threshold ?? config.PALACE_DEDUP_THRESHOLD;
|
||||
|
||||
// 1. Exact match (cheap)
|
||||
const filter: FilterMap = { userId, productId, roomId, hall };
|
||||
const candidates = await memoriesCollection().findMany({
|
||||
filter,
|
||||
sort: { createdAt: -1 },
|
||||
limit: 20,
|
||||
});
|
||||
|
||||
const decrypted = await decryptMemoryBatch(candidates);
|
||||
|
||||
// Check exact
|
||||
if (decrypted.some(c => c.content === content)) return true;
|
||||
|
||||
// 2. Semantic dedup via shared @bytelyst/palace helper
|
||||
if (embedding) {
|
||||
const candidateEmbeddings = decrypted
|
||||
.map(c => c.embedding)
|
||||
.filter((e): e is number[] => !!e && e.length > 0);
|
||||
|
||||
return isContentDuplicate(embedding, candidateEmbeddings, dupThreshold);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── Maintenance ─────────────────────────────────────────────────────
|
||||
|
||||
export async function pruneOldMemories(
|
||||
userId: string,
|
||||
productId: string,
|
||||
wingId: string | undefined,
|
||||
olderThanDays: number,
|
||||
minRelevance: number,
|
||||
): Promise<number> {
|
||||
const filter: FilterMap = { userId, productId };
|
||||
if (wingId) filter.wingId = wingId;
|
||||
|
||||
const all = await memoriesCollection().findMany({ filter, limit: 1000 });
|
||||
const cutoff = new Date(Date.now() - olderThanDays * 86_400_000).toISOString();
|
||||
let deleted = 0;
|
||||
|
||||
for (const mem of all) {
|
||||
if (mem.createdAt < cutoff && mem.relevance < minRelevance) {
|
||||
await memoriesCollection().delete(mem.id, userId);
|
||||
deleted++;
|
||||
}
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
export async function decayRelevance(
|
||||
userId: string,
|
||||
productId: string,
|
||||
halfLifeDays?: number,
|
||||
): Promise<number> {
|
||||
const halfLife = halfLifeDays ?? config.PALACE_RELEVANCE_HALF_LIFE_DAYS;
|
||||
const filter: FilterMap = { userId, productId };
|
||||
|
||||
const all = await memoriesCollection().findMany({ filter, limit: 5000 });
|
||||
let updated = 0;
|
||||
|
||||
for (const mem of all) {
|
||||
const decayed = computeDecayedRelevance(mem.relevance, mem.createdAt, halfLife);
|
||||
if (Math.abs(decayed - mem.relevance) > 0.001) {
|
||||
await memoriesCollection().update(mem.id, userId, {
|
||||
relevance: decayed,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
export async function backfillEmbeddings(
|
||||
userId: string,
|
||||
productId: string,
|
||||
): Promise<number> {
|
||||
const filter: FilterMap = { userId, productId };
|
||||
const all = await memoriesCollection().findMany({ filter, limit: 500 });
|
||||
const decrypted = await decryptMemoryBatch(all);
|
||||
let count = 0;
|
||||
|
||||
for (const mem of decrypted) {
|
||||
if (!mem.embedding || mem.embedding.length === 0) {
|
||||
const emb = await embedText(mem.content);
|
||||
if (emb) {
|
||||
await memoriesCollection().update(mem.id, userId, {
|
||||
embedding: emb,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
// ── Stats ───────────────────────────────────────────────────────────
|
||||
|
||||
export async function getPalaceStats(
|
||||
userId: string,
|
||||
productId: string,
|
||||
): Promise<{
|
||||
wings: number;
|
||||
rooms: number;
|
||||
memories: number;
|
||||
kgTriples: number;
|
||||
tunnels: number;
|
||||
diaries: number;
|
||||
}> {
|
||||
const [wings, rooms, memories, kgTriples, tunnels, diaries] = await Promise.all([
|
||||
wingsCollection().count({ userId, productId }),
|
||||
roomsCollection().count({ userId, productId }),
|
||||
memoriesCollection().count({ userId, productId }),
|
||||
kgCollection().count({ userId, productId }),
|
||||
tunnelsCollection().count({ userId, productId }),
|
||||
diariesCollection().count({ userId, productId }),
|
||||
]);
|
||||
return { wings, rooms, memories, kgTriples, tunnels, diaries };
|
||||
}
|
||||
|
||||
export async function healthCheck(): Promise<{ cosmos: boolean; llm: boolean }> {
|
||||
let cosmos = false;
|
||||
let llm = false;
|
||||
|
||||
try {
|
||||
await wingsCollection().count({});
|
||||
cosmos = true;
|
||||
} catch { /* cosmos unavailable */ }
|
||||
|
||||
try {
|
||||
const emb = await embedText('health check');
|
||||
llm = emb !== null;
|
||||
} catch { /* llm unavailable */ }
|
||||
|
||||
return { cosmos, llm };
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user