Add Azurite-backed blob storage for prototype

This commit is contained in:
root 2026-03-14 05:45:43 +00:00
parent 19b58b3ea0
commit 66a11c5713
4 changed files with 87 additions and 10 deletions

View File

@ -16,9 +16,11 @@ COSMOS_DATABASE=lysnrai
JWT_SECRET=change-me-prototype-jwt-secret
# ── Azure Blob Storage (platform-service) ─────────────────────
AZURE_BLOB_CONNECTION_STRING=
AZURE_BLOB_ACCOUNT_NAME=bytelystblobs
AZURE_BLOB_ACCOUNT_KEY=your-blob-key
STORAGE_PROVIDER=azure
AZURE_BLOB_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=<azurite-default-key>;BlobEndpoint=http://azurite:10000/devstoreaccount1;
AZURE_BLOB_ACCOUNT_NAME=devstoreaccount1
AZURE_BLOB_ACCOUNT_KEY=<azurite-default-key>
AZURE_BLOB_PUBLIC_ENDPOINT=http://localhost:10000/devstoreaccount1
# ── Stripe (platform-service) ────────────────────────
STRIPE_SECRET_KEY=sk_test_...
@ -26,6 +28,17 @@ STRIPE_WEBHOOK_SECRET=whsec_...
STRIPE_PRICE_PRO=price_...
STRIPE_PRICE_ENTERPRISE=price_...
# ── Email Delivery (platform-service) ─────────────────────────
# Use `smtp` for a self-hosted SMTP relay such as Mailpit, Postal, Mailcow, etc.
EMAIL_PROVIDER=smtp
EMAIL_FROM_ADDRESS=noreply@bytelyst.local
EMAIL_FROM_NAME=ByteLyst
SMTP_HOST=localhost
SMTP_PORT=1025
SMTP_SECURE=false
SMTP_USER=
SMTP_PASSWORD=
# ── Extraction Service (port 4005 + Python sidecar 4006) ─────
PYTHON_SIDECAR_URL=http://localhost:4006
DEFAULT_MODEL_ID=gemini-2.5-flash

View File

@ -1,4 +1,23 @@
services:
# ── Azurite Blob Storage (prototype only) ─────────────────────
azurite:
image: mcr.microsoft.com/azure-storage/azurite:3.35.0
command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck
ports:
- '10000:10000'
healthcheck:
test:
[
'CMD',
'node',
'-e',
'const net=require("net");const s=net.connect(10000,"127.0.0.1",()=>{s.end();process.exit(0)});s.on("error",()=>process.exit(1));',
]
interval: 10s
timeout: 5s
retries: 6
restart: unless-stopped
# ── Azure Cosmos DB Emulator (prototype only) ─────────────────
cosmos-emulator:
image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview
@ -97,6 +116,8 @@ services:
# Local/dev convenience: ensure Cosmos DB + containers exist.
- COSMOS_AUTO_INIT=true
depends_on:
azurite:
condition: service_healthy
cosmos-emulator:
condition: service_healthy
labels:

View File

@ -8,6 +8,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp
- `extraction-service`
- `mcp-server`
- `cosmos-emulator`
- `azurite`
- `gateway` (Traefik)
- `loki`
- `grafana`
@ -18,6 +19,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp
- Any real API credentials such as Stripe or Gemini
For this VM prototype, Cosmos is self-hosted through the Linux Cosmos DB Emulator container. Everything else should still stay in `.env` and move to a real secret manager later.
Blob/file uploads are self-hosted through Azurite.
## First-Time Setup
@ -33,6 +35,8 @@ If you want extraction features that call Gemini, also set:
- `GEMINI_API_KEY`
If you access generated blob SAS URLs from outside the VM, change `AZURE_BLOB_PUBLIC_ENDPOINT` to a host or IP that your browser can reach.
## Start The Stack
```bash
@ -53,10 +57,12 @@ docker compose logs -f platform-service
docker compose logs -f extraction-service
docker compose logs -f mcp-server
docker compose logs -f cosmos-emulator
docker compose logs -f azurite
docker compose down
```
The Cosmos Data Explorer is exposed on `http://localhost:1234`.
Azurite Blob Storage is exposed on `http://localhost:10000`.
## Notes

View File

@ -16,6 +16,24 @@ export interface AzureBlobProviderConfig {
connectionString?: string;
accountName?: string;
accountKey?: string;
blobEndpoint?: string;
publicBlobEndpoint?: string;
}
function parseConnectionString(connectionString: string): Partial<AzureBlobProviderConfig> {
const parts = new Map<string, string>();
for (const segment of connectionString.split(';')) {
const [key, ...rest] = segment.split('=');
if (!key || rest.length === 0) continue;
parts.set(key, rest.join('='));
}
return {
accountName: parts.get('AccountName'),
accountKey: parts.get('AccountKey'),
blobEndpoint: parts.get('BlobEndpoint'),
};
}
export class AzureBlobStorageProvider implements StorageProvider {
@ -24,10 +42,23 @@ export class AzureBlobStorageProvider implements StorageProvider {
private buckets = new Map<string, AzureBlobBucket>();
constructor(config?: AzureBlobProviderConfig) {
this.config = config ?? {
const envConfig = config ?? {
connectionString: process.env.AZURE_BLOB_CONNECTION_STRING,
accountName: process.env.AZURE_BLOB_ACCOUNT_NAME,
accountKey: process.env.AZURE_BLOB_ACCOUNT_KEY,
publicBlobEndpoint: process.env.AZURE_BLOB_PUBLIC_ENDPOINT,
};
const parsed = envConfig.connectionString
? parseConnectionString(envConfig.connectionString)
: undefined;
this.config = {
...parsed,
...envConfig,
accountName: envConfig.accountName ?? parsed?.accountName,
accountKey: envConfig.accountKey ?? parsed?.accountKey,
blobEndpoint: envConfig.blobEndpoint ?? parsed?.blobEndpoint,
};
}
@ -42,10 +73,9 @@ export class AzureBlobStorageProvider implements StorageProvider {
this.config.accountName,
this.config.accountKey
);
this.client = new BlobServiceClient(
`https://${this.config.accountName}.blob.core.windows.net`,
cred
);
const endpoint =
this.config.blobEndpoint ?? `https://${this.config.accountName}.blob.core.windows.net`;
this.client = new BlobServiceClient(endpoint, cred);
} else {
throw new Error(
'AzureBlobStorageProvider requires AZURE_BLOB_CONNECTION_STRING or AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY'
@ -86,7 +116,9 @@ class AzureBlobBucket implements StorageBucket {
private async containerClient() {
const client = await this.getClient();
return client.getContainerClient(this.containerName);
const container = client.getContainerClient(this.containerName);
await container.createIfNotExists();
return container;
}
async upload(
@ -165,6 +197,11 @@ class AzureBlobBucket implements StorageBucket {
cred
);
return `https://${this.config.accountName}.blob.core.windows.net/${this.containerName}/${key}?${sas.toString()}`;
const baseUrl =
this.config.publicBlobEndpoint ??
this.config.blobEndpoint ??
`https://${this.config.accountName}.blob.core.windows.net`;
return `${baseUrl.replace(/\/$/, '')}/${this.containerName}/${key}?${sas.toString()}`;
}
}