From 66a11c571374a355790fad1b63f5d58689ef9897 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 14 Mar 2026 05:45:43 +0000 Subject: [PATCH] Add Azurite-backed blob storage for prototype --- .env.example | 19 ++++++-- docker-compose.yml | 21 ++++++++ docs/PROTOTYPE_DEPLOYMENT.md | 6 +++ packages/storage/src/providers/azure-blob.ts | 51 +++++++++++++++++--- 4 files changed, 87 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index a39a3251..394ca02c 100644 --- a/.env.example +++ b/.env.example @@ -16,9 +16,11 @@ COSMOS_DATABASE=lysnrai JWT_SECRET=change-me-prototype-jwt-secret # ── Azure Blob Storage (platform-service) ───────────────────── -AZURE_BLOB_CONNECTION_STRING= -AZURE_BLOB_ACCOUNT_NAME=bytelystblobs -AZURE_BLOB_ACCOUNT_KEY=your-blob-key +STORAGE_PROVIDER=azure +AZURE_BLOB_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=;BlobEndpoint=http://azurite:10000/devstoreaccount1; +AZURE_BLOB_ACCOUNT_NAME=devstoreaccount1 +AZURE_BLOB_ACCOUNT_KEY= +AZURE_BLOB_PUBLIC_ENDPOINT=http://localhost:10000/devstoreaccount1 # ── Stripe (platform-service) ──────────────────────── STRIPE_SECRET_KEY=sk_test_... @@ -26,6 +28,17 @@ STRIPE_WEBHOOK_SECRET=whsec_... STRIPE_PRICE_PRO=price_... STRIPE_PRICE_ENTERPRISE=price_... +# ── Email Delivery (platform-service) ───────────────────────── +# Use `smtp` for a self-hosted SMTP relay such as Mailpit, Postal, Mailcow, etc. +EMAIL_PROVIDER=smtp +EMAIL_FROM_ADDRESS=noreply@bytelyst.local +EMAIL_FROM_NAME=ByteLyst +SMTP_HOST=localhost +SMTP_PORT=1025 +SMTP_SECURE=false +SMTP_USER= +SMTP_PASSWORD= + # ── Extraction Service (port 4005 + Python sidecar 4006) ───── PYTHON_SIDECAR_URL=http://localhost:4006 DEFAULT_MODEL_ID=gemini-2.5-flash diff --git a/docker-compose.yml b/docker-compose.yml index 59f26deb..eb9fa34d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,23 @@ services: + # ── Azurite Blob Storage (prototype only) ───────────────────── + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.35.0 + command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck + ports: + - '10000:10000' + healthcheck: + test: + [ + 'CMD', + 'node', + '-e', + 'const net=require("net");const s=net.connect(10000,"127.0.0.1",()=>{s.end();process.exit(0)});s.on("error",()=>process.exit(1));', + ] + interval: 10s + timeout: 5s + retries: 6 + restart: unless-stopped + # ── Azure Cosmos DB Emulator (prototype only) ───────────────── cosmos-emulator: image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview @@ -97,6 +116,8 @@ services: # Local/dev convenience: ensure Cosmos DB + containers exist. - COSMOS_AUTO_INIT=true depends_on: + azurite: + condition: service_healthy cosmos-emulator: condition: service_healthy labels: diff --git a/docs/PROTOTYPE_DEPLOYMENT.md b/docs/PROTOTYPE_DEPLOYMENT.md index 30fd0330..e5049703 100644 --- a/docs/PROTOTYPE_DEPLOYMENT.md +++ b/docs/PROTOTYPE_DEPLOYMENT.md @@ -8,6 +8,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp - `extraction-service` - `mcp-server` - `cosmos-emulator` +- `azurite` - `gateway` (Traefik) - `loki` - `grafana` @@ -18,6 +19,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp - Any real API credentials such as Stripe or Gemini For this VM prototype, Cosmos is self-hosted through the Linux Cosmos DB Emulator container. Everything else should still stay in `.env` and move to a real secret manager later. +Blob/file uploads are self-hosted through Azurite. ## First-Time Setup @@ -33,6 +35,8 @@ If you want extraction features that call Gemini, also set: - `GEMINI_API_KEY` +If you access generated blob SAS URLs from outside the VM, change `AZURE_BLOB_PUBLIC_ENDPOINT` to a host or IP that your browser can reach. + ## Start The Stack ```bash @@ -53,10 +57,12 @@ docker compose logs -f platform-service docker compose logs -f extraction-service docker compose logs -f mcp-server docker compose logs -f cosmos-emulator +docker compose logs -f azurite docker compose down ``` The Cosmos Data Explorer is exposed on `http://localhost:1234`. +Azurite Blob Storage is exposed on `http://localhost:10000`. ## Notes diff --git a/packages/storage/src/providers/azure-blob.ts b/packages/storage/src/providers/azure-blob.ts index 78e72995..86d3a9b2 100644 --- a/packages/storage/src/providers/azure-blob.ts +++ b/packages/storage/src/providers/azure-blob.ts @@ -16,6 +16,24 @@ export interface AzureBlobProviderConfig { connectionString?: string; accountName?: string; accountKey?: string; + blobEndpoint?: string; + publicBlobEndpoint?: string; +} + +function parseConnectionString(connectionString: string): Partial { + const parts = new Map(); + + for (const segment of connectionString.split(';')) { + const [key, ...rest] = segment.split('='); + if (!key || rest.length === 0) continue; + parts.set(key, rest.join('=')); + } + + return { + accountName: parts.get('AccountName'), + accountKey: parts.get('AccountKey'), + blobEndpoint: parts.get('BlobEndpoint'), + }; } export class AzureBlobStorageProvider implements StorageProvider { @@ -24,10 +42,23 @@ export class AzureBlobStorageProvider implements StorageProvider { private buckets = new Map(); constructor(config?: AzureBlobProviderConfig) { - this.config = config ?? { + const envConfig = config ?? { connectionString: process.env.AZURE_BLOB_CONNECTION_STRING, accountName: process.env.AZURE_BLOB_ACCOUNT_NAME, accountKey: process.env.AZURE_BLOB_ACCOUNT_KEY, + publicBlobEndpoint: process.env.AZURE_BLOB_PUBLIC_ENDPOINT, + }; + + const parsed = envConfig.connectionString + ? parseConnectionString(envConfig.connectionString) + : undefined; + + this.config = { + ...parsed, + ...envConfig, + accountName: envConfig.accountName ?? parsed?.accountName, + accountKey: envConfig.accountKey ?? parsed?.accountKey, + blobEndpoint: envConfig.blobEndpoint ?? parsed?.blobEndpoint, }; } @@ -42,10 +73,9 @@ export class AzureBlobStorageProvider implements StorageProvider { this.config.accountName, this.config.accountKey ); - this.client = new BlobServiceClient( - `https://${this.config.accountName}.blob.core.windows.net`, - cred - ); + const endpoint = + this.config.blobEndpoint ?? `https://${this.config.accountName}.blob.core.windows.net`; + this.client = new BlobServiceClient(endpoint, cred); } else { throw new Error( 'AzureBlobStorageProvider requires AZURE_BLOB_CONNECTION_STRING or AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY' @@ -86,7 +116,9 @@ class AzureBlobBucket implements StorageBucket { private async containerClient() { const client = await this.getClient(); - return client.getContainerClient(this.containerName); + const container = client.getContainerClient(this.containerName); + await container.createIfNotExists(); + return container; } async upload( @@ -165,6 +197,11 @@ class AzureBlobBucket implements StorageBucket { cred ); - return `https://${this.config.accountName}.blob.core.windows.net/${this.containerName}/${key}?${sas.toString()}`; + const baseUrl = + this.config.publicBlobEndpoint ?? + this.config.blobEndpoint ?? + `https://${this.config.accountName}.blob.core.windows.net`; + + return `${baseUrl.replace(/\/$/, '')}/${this.containerName}/${key}?${sas.toString()}`; } }