bytelyst/learning_ai_common_plat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(storage): migrate to storage abstraction

Browse Source
This commit is contained in:
saravanakumardb1 2026-03-02 09:07:33 -08:00
parent 8315814fd9
commit 7ca2139418
5 changed files with 60 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/blob/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@bytelyst/blob", "name": "@bytelyst/blob",
"version": "0.1.0", "version": "0.2.0",
"type": "module", "type": "module",
"exports": { "exports": {
".": { ".": {
@ -17,7 +17,7 @@
"build": "tsc", "build": "tsc",
"test": "vitest run" "test": "vitest run"
}, },
"peerDependencies": { "dependencies": {
"@azure/storage-blob": ">=12.0.0" "@bytelyst/storage": "workspace:*"
} }
} }

128
packages/blob/src/blob.ts
View File

@ -1,26 +1,22 @@
/** /**
* Shared Azure Blob Storage utilities. * Shared Blob Storage utilities.
* *
* Provides singleton BlobServiceClient, container helpers, and SAS token generation. * Delegates to @bytelyst/storage for provider-agnostic blob operations.
* Containers are lazily created on first access (createIfNotExists). * Keeps the same exported API surface for backward compatibility.
* *
* Expected env vars: * Expected env vars:
* AZURE_BLOB_CONNECTION_STRING — full connection string (preferred) * STORAGE_PROVIDER — 'azure' (default) | 'memory'
* AZURE_BLOB_CONNECTION_STRING — full connection string (preferred, when provider=azure)
* — OR — * — OR —
* AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY * AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY
*/ */
import { import {
BlobServiceClient, getStorage,
ContainerClient, _resetStorage,
StorageSharedKeyCredential, type StorageProvider,
BlobSASPermissions, type StorageBucket,
generateBlobSASQueryParameters, } from '@bytelyst/storage';
SASProtocol,
} from '@azure/storage-blob';
let serviceClient: BlobServiceClient | null = null;
const containerClients = new Map<string, ContainerClient>();
/** /**
* Known blob containers and their purposes. * Known blob containers and their purposes.
@ -40,112 +36,49 @@ export const BLOB_CONTAINERS = {
export type BlobContainerName = (typeof BLOB_CONTAINERS)[keyof typeof BLOB_CONTAINERS]; export type BlobContainerName = (typeof BLOB_CONTAINERS)[keyof typeof BLOB_CONTAINERS];
/** /**
* Get or create the BlobServiceClient singleton. * Get the storage provider singleton.
*/ */
export function getBlobServiceClient(): BlobServiceClient { export async function getStorageProvider(): Promise<StorageProvider> {
if (serviceClient) return serviceClient; return getStorage();
const connectionString = process.env.AZURE_BLOB_CONNECTION_STRING;
if (connectionString) {
serviceClient = BlobServiceClient.fromConnectionString(connectionString);
return serviceClient;
}
const accountName = process.env.AZURE_BLOB_ACCOUNT_NAME;
const accountKey = process.env.AZURE_BLOB_ACCOUNT_KEY;
if (accountName && accountKey) {
const credential = new StorageSharedKeyCredential(accountName, accountKey);
serviceClient = new BlobServiceClient(
`https://${accountName}.blob.core.windows.net`,
credential
);
return serviceClient;
}
throw new Error(
'Azure Blob Storage not configured. Set AZURE_BLOB_CONNECTION_STRING or AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY'
);
} }
/** /**
* Get a container client, creating the container if it doesn't exist. * Get a bucket (container) by name.
*/ */
export async function getContainerClient(containerName: string): Promise<ContainerClient> { export async function getBucket(containerName: string): Promise<StorageBucket> {
const cached = containerClients.get(containerName); const storage = await getStorage();
if (cached) return cached; return storage.getBucket(containerName);
const client = getBlobServiceClient().getContainerClient(containerName);
await client.createIfNotExists({ access: undefined }); // private by default
containerClients.set(containerName, client);
return client;
} }
/** /**
* Generate a SAS URL for direct browser upload (or download). * Generate a signed URL for direct browser upload (or download).
* *
* @param containerName - Target container * @param containerName - Target container
* @param blobName - Full blob path (e.g., "product/user123/audio/recording.wav") * @param blobName - Full blob path (e.g., "product/user123/audio/recording.wav")
* @param permissions - SAS permissions (default: read) * @param permissions - SAS permissions (default: read)
* @param expiresInMinutes - Token lifetime (default: 60) * @param expiresInMinutes - Token lifetime (default: 60)
* @returns Full SAS URL for the blob * @returns Full signed URL for the blob
*/ */
export function generateSasUrl( export async function generateSasUrl(
containerName: string, containerName: string,
blobName: string, blobName: string,
permissions: 'r' | 'w' | 'rw' | 'rwc' | 'rwd' = 'r', permissions: 'r' | 'w' | 'rw' | 'rwc' | 'rwd' = 'r',
expiresInMinutes = 60 expiresInMinutes = 60
): string { ): Promise<string> {
const connectionString = process.env.AZURE_BLOB_CONNECTION_STRING; const bucket = await getBucket(containerName);
const accountName = process.env.AZURE_BLOB_ACCOUNT_NAME; const perm = permissions.includes('w') ? ('write' as const) : ('read' as const);
const accountKey = process.env.AZURE_BLOB_ACCOUNT_KEY; return bucket.getSignedUrl(blobName, {
permissions: perm,
let credAccountName: string; expiresIn: expiresInMinutes * 60,
let credential: StorageSharedKeyCredential; });
if (accountName && accountKey) {
credAccountName = accountName;
credential = new StorageSharedKeyCredential(accountName, accountKey);
} else if (connectionString) {
// Parse account name and key from connection string
const nameMatch = connectionString.match(/AccountName=([^;]+)/);
const keyMatch = connectionString.match(/AccountKey=([^;]+)/);
if (!nameMatch || !keyMatch) {
throw new Error('Cannot parse AccountName/AccountKey from connection string');
}
credAccountName = nameMatch[1];
credential = new StorageSharedKeyCredential(nameMatch[1], keyMatch[1]);
} else {
throw new Error('Blob storage credentials not configured');
}
const sasPermissions = new BlobSASPermissions();
if (permissions.includes('r')) sasPermissions.read = true;
if (permissions.includes('w')) sasPermissions.write = true;
if (permissions.includes('c')) sasPermissions.create = true;
if (permissions.includes('d')) sasPermissions.delete = true;
const now = new Date();
const expiresOn = new Date(now.getTime() + expiresInMinutes * 60 * 1000);
const sasToken = generateBlobSASQueryParameters(
{
containerName,
blobName,
permissions: sasPermissions,
startsOn: now,
expiresOn,
protocol: SASProtocol.Https,
},
credential
).toString();
return `https://${credAccountName}.blob.core.windows.net/${containerName}/${blobName}?${sasToken}`;
} }
/** /**
* Check if blob storage is configured. * Check if blob storage is configured.
*/ */
export function isBlobStorageConfigured(): boolean { export function isBlobStorageConfigured(): boolean {
const provider = process.env.STORAGE_PROVIDER || 'azure';
if (provider === 'memory') return true;
return !!( return !!(
process.env.AZURE_BLOB_CONNECTION_STRING || process.env.AZURE_BLOB_CONNECTION_STRING ||
(process.env.AZURE_BLOB_ACCOUNT_NAME && process.env.AZURE_BLOB_ACCOUNT_KEY) (process.env.AZURE_BLOB_ACCOUNT_NAME && process.env.AZURE_BLOB_ACCOUNT_KEY)
@ -156,6 +89,5 @@ export function isBlobStorageConfigured(): boolean {
* Test helper: reset module singletons/caches. * Test helper: reset module singletons/caches.
*/ */
export function _resetBlobClient(): void { export function _resetBlobClient(): void {
serviceClient = null; _resetStorage();
containerClients.clear();
} }

2
services/platform-service/package.json
View File

@ -14,9 +14,9 @@
}, },
"dependencies": { "dependencies": {
"@azure/cosmos": "^4.2.0", "@azure/cosmos": "^4.2.0",
"@azure/storage-blob": "^12.31.0",
"@bytelyst/auth": "workspace:*", "@bytelyst/auth": "workspace:*",
"@bytelyst/blob": "workspace:*", "@bytelyst/blob": "workspace:*",
"@bytelyst/storage": "workspace:*",
"@bytelyst/config": "workspace:*", "@bytelyst/config": "workspace:*",
"@bytelyst/cosmos": "workspace:*", "@bytelyst/cosmos": "workspace:*",
"@bytelyst/datastore": "workspace:*", "@bytelyst/datastore": "workspace:*",

3
services/platform-service/src/lib/blob.ts
View File

@ -1,8 +1,7 @@
export { export {
BLOB_CONTAINERS, BLOB_CONTAINERS,
type BlobContainerName, type BlobContainerName,
getBlobServiceClient, getBucket,
getContainerClient,
generateSasUrl, generateSasUrl,
isBlobStorageConfigured, isBlobStorageConfigured,
} from '@bytelyst/blob'; } from '@bytelyst/blob';

56
services/platform-service/src/modules/blob/routes.ts
View File

@ -12,7 +12,7 @@ import type { FastifyInstance } from 'fastify';
import { verifyToken } from '../auth/jwt.js'; import { verifyToken } from '../auth/jwt.js';
import { BadRequestError, UnauthorizedError, NotFoundError } from '../../lib/errors.js'; import { BadRequestError, UnauthorizedError, NotFoundError } from '../../lib/errors.js';
import { import {
getContainerClient, getBucket,
generateSasUrl, generateSasUrl,
isBlobStorageConfigured, isBlobStorageConfigured,
BLOB_CONTAINERS, BLOB_CONTAINERS,
@ -83,7 +83,7 @@ export async function blobRoutes(app: FastifyInstance) {
} }
} }
const sasUrl = generateSasUrl(container, blobName, permissions, expiresInMinutes); const sasUrl = await generateSasUrl(container, blobName, permissions, expiresInMinutes);
return { return {
sasUrl, sasUrl,
container, container,
@ -113,23 +113,18 @@ export async function blobRoutes(app: FastifyInstance) {
? requestedPrefix ? requestedPrefix
: `${userPrefix({ productId: auth.productId, sub: auth.sub })}${requestedPrefix ?? ''}`; : `${userPrefix({ productId: auth.productId, sub: auth.sub })}${requestedPrefix ?? ''}`;
const containerClient = await getContainerClient(container); const bucket = await getBucket(container);
const allBlobs = await bucket.list(effectivePrefix || undefined);
const blobs: BlobInfo[] = []; const blobs: BlobInfo[] = allBlobs.slice(0, limit).map(blob => ({
let count = 0; name: blob.key,
for await (const blob of containerClient.listBlobsFlat({ prefix: effectivePrefix || undefined })) { container,
if (count >= limit) break; contentType: blob.contentType,
blobs.push({ size: blob.size ?? 0,
name: blob.name, lastModified: blob.lastModified,
container, url: blob.key,
contentType: blob.properties.contentType, metadata: blob.metadata ?? {},
size: blob.properties.contentLength ?? 0, }));
lastModified: blob.properties.lastModified,
url: `${containerClient.url}/${blob.name}`,
metadata: blob.metadata ?? {},
});
count++;
}
return { blobs, count: blobs.length, container, prefix: effectivePrefix || null }; return { blobs, count: blobs.length, container, prefix: effectivePrefix || null };
}); });
@ -146,15 +141,14 @@ export async function blobRoutes(app: FastifyInstance) {
throw new BadRequestError(parsed.error.issues.map(i => i.message).join('; ')); throw new BadRequestError(parsed.error.issues.map(i => i.message).join('; '));
} }
const { container, blobName } = parsed.data; const { container, blobName } = parsed.data;
const containerClient = await getContainerClient(container); const bucket = await getBucket(container);
const blobClient = containerClient.getBlobClient(blobName);
const exists = await blobClient.exists(); const exists = await bucket.exists(blobName);
if (!exists) { if (!exists) {
throw new NotFoundError(`Blob not found: ${container}/${blobName}`); throw new NotFoundError(`Blob not found: ${container}/${blobName}`);
} }
await blobClient.delete(); await bucket.delete(blobName);
return { success: true, container, blobName }; return { success: true, container, blobName };
}); });
@ -178,23 +172,23 @@ export async function blobRoutes(app: FastifyInstance) {
} }
} }
const containerClient = await getContainerClient(container); const bucket = await getBucket(container);
const blobClient = containerClient.getBlobClient(blobName);
const exists = await blobClient.exists(); const exists = await bucket.exists(blobName);
if (!exists) { if (!exists) {
throw new NotFoundError(`Blob not found: ${container}/${blobName}`); throw new NotFoundError(`Blob not found: ${container}/${blobName}`);
} }
const props = await blobClient.getProperties(); const blobs = await bucket.list(blobName);
const meta = blobs.find(b => b.key === blobName);
return { return {
name: blobName, name: blobName,
container, container,
contentType: props.contentType, contentType: meta?.contentType,
size: props.contentLength, size: meta?.size ?? 0,
lastModified: props.lastModified, lastModified: meta?.lastModified,
url: blobClient.url, url: blobName,
metadata: props.metadata ?? {}, metadata: meta?.metadata ?? {},
}; };
}); });