Add Azurite-backed blob storage for prototype
This commit is contained in:
parent
19b58b3ea0
commit
66a11c5713
19
.env.example
19
.env.example
@ -16,9 +16,11 @@ COSMOS_DATABASE=lysnrai
|
||||
JWT_SECRET=change-me-prototype-jwt-secret
|
||||
|
||||
# ── Azure Blob Storage (platform-service) ─────────────────────
|
||||
AZURE_BLOB_CONNECTION_STRING=
|
||||
AZURE_BLOB_ACCOUNT_NAME=bytelystblobs
|
||||
AZURE_BLOB_ACCOUNT_KEY=your-blob-key
|
||||
STORAGE_PROVIDER=azure
|
||||
AZURE_BLOB_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=<azurite-default-key>;BlobEndpoint=http://azurite:10000/devstoreaccount1;
|
||||
AZURE_BLOB_ACCOUNT_NAME=devstoreaccount1
|
||||
AZURE_BLOB_ACCOUNT_KEY=<azurite-default-key>
|
||||
AZURE_BLOB_PUBLIC_ENDPOINT=http://localhost:10000/devstoreaccount1
|
||||
|
||||
# ── Stripe (platform-service) ────────────────────────
|
||||
STRIPE_SECRET_KEY=sk_test_...
|
||||
@ -26,6 +28,17 @@ STRIPE_WEBHOOK_SECRET=whsec_...
|
||||
STRIPE_PRICE_PRO=price_...
|
||||
STRIPE_PRICE_ENTERPRISE=price_...
|
||||
|
||||
# ── Email Delivery (platform-service) ─────────────────────────
|
||||
# Use `smtp` for a self-hosted SMTP relay such as Mailpit, Postal, Mailcow, etc.
|
||||
EMAIL_PROVIDER=smtp
|
||||
EMAIL_FROM_ADDRESS=noreply@bytelyst.local
|
||||
EMAIL_FROM_NAME=ByteLyst
|
||||
SMTP_HOST=localhost
|
||||
SMTP_PORT=1025
|
||||
SMTP_SECURE=false
|
||||
SMTP_USER=
|
||||
SMTP_PASSWORD=
|
||||
|
||||
# ── Extraction Service (port 4005 + Python sidecar 4006) ─────
|
||||
PYTHON_SIDECAR_URL=http://localhost:4006
|
||||
DEFAULT_MODEL_ID=gemini-2.5-flash
|
||||
|
||||
@ -1,4 +1,23 @@
|
||||
services:
|
||||
# ── Azurite Blob Storage (prototype only) ─────────────────────
|
||||
azurite:
|
||||
image: mcr.microsoft.com/azure-storage/azurite:3.35.0
|
||||
command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck
|
||||
ports:
|
||||
- '10000:10000'
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
'CMD',
|
||||
'node',
|
||||
'-e',
|
||||
'const net=require("net");const s=net.connect(10000,"127.0.0.1",()=>{s.end();process.exit(0)});s.on("error",()=>process.exit(1));',
|
||||
]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 6
|
||||
restart: unless-stopped
|
||||
|
||||
# ── Azure Cosmos DB Emulator (prototype only) ─────────────────
|
||||
cosmos-emulator:
|
||||
image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview
|
||||
@ -97,6 +116,8 @@ services:
|
||||
# Local/dev convenience: ensure Cosmos DB + containers exist.
|
||||
- COSMOS_AUTO_INIT=true
|
||||
depends_on:
|
||||
azurite:
|
||||
condition: service_healthy
|
||||
cosmos-emulator:
|
||||
condition: service_healthy
|
||||
labels:
|
||||
|
||||
@ -8,6 +8,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp
|
||||
- `extraction-service`
|
||||
- `mcp-server`
|
||||
- `cosmos-emulator`
|
||||
- `azurite`
|
||||
- `gateway` (Traefik)
|
||||
- `loki`
|
||||
- `grafana`
|
||||
@ -18,6 +19,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp
|
||||
- Any real API credentials such as Stripe or Gemini
|
||||
|
||||
For this VM prototype, Cosmos is self-hosted through the Linux Cosmos DB Emulator container. Everything else should still stay in `.env` and move to a real secret manager later.
|
||||
Blob/file uploads are self-hosted through Azurite.
|
||||
|
||||
## First-Time Setup
|
||||
|
||||
@ -33,6 +35,8 @@ If you want extraction features that call Gemini, also set:
|
||||
|
||||
- `GEMINI_API_KEY`
|
||||
|
||||
If you access generated blob SAS URLs from outside the VM, change `AZURE_BLOB_PUBLIC_ENDPOINT` to a host or IP that your browser can reach.
|
||||
|
||||
## Start The Stack
|
||||
|
||||
```bash
|
||||
@ -53,10 +57,12 @@ docker compose logs -f platform-service
|
||||
docker compose logs -f extraction-service
|
||||
docker compose logs -f mcp-server
|
||||
docker compose logs -f cosmos-emulator
|
||||
docker compose logs -f azurite
|
||||
docker compose down
|
||||
```
|
||||
|
||||
The Cosmos Data Explorer is exposed on `http://localhost:1234`.
|
||||
Azurite Blob Storage is exposed on `http://localhost:10000`.
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
@ -16,6 +16,24 @@ export interface AzureBlobProviderConfig {
|
||||
connectionString?: string;
|
||||
accountName?: string;
|
||||
accountKey?: string;
|
||||
blobEndpoint?: string;
|
||||
publicBlobEndpoint?: string;
|
||||
}
|
||||
|
||||
function parseConnectionString(connectionString: string): Partial<AzureBlobProviderConfig> {
|
||||
const parts = new Map<string, string>();
|
||||
|
||||
for (const segment of connectionString.split(';')) {
|
||||
const [key, ...rest] = segment.split('=');
|
||||
if (!key || rest.length === 0) continue;
|
||||
parts.set(key, rest.join('='));
|
||||
}
|
||||
|
||||
return {
|
||||
accountName: parts.get('AccountName'),
|
||||
accountKey: parts.get('AccountKey'),
|
||||
blobEndpoint: parts.get('BlobEndpoint'),
|
||||
};
|
||||
}
|
||||
|
||||
export class AzureBlobStorageProvider implements StorageProvider {
|
||||
@ -24,10 +42,23 @@ export class AzureBlobStorageProvider implements StorageProvider {
|
||||
private buckets = new Map<string, AzureBlobBucket>();
|
||||
|
||||
constructor(config?: AzureBlobProviderConfig) {
|
||||
this.config = config ?? {
|
||||
const envConfig = config ?? {
|
||||
connectionString: process.env.AZURE_BLOB_CONNECTION_STRING,
|
||||
accountName: process.env.AZURE_BLOB_ACCOUNT_NAME,
|
||||
accountKey: process.env.AZURE_BLOB_ACCOUNT_KEY,
|
||||
publicBlobEndpoint: process.env.AZURE_BLOB_PUBLIC_ENDPOINT,
|
||||
};
|
||||
|
||||
const parsed = envConfig.connectionString
|
||||
? parseConnectionString(envConfig.connectionString)
|
||||
: undefined;
|
||||
|
||||
this.config = {
|
||||
...parsed,
|
||||
...envConfig,
|
||||
accountName: envConfig.accountName ?? parsed?.accountName,
|
||||
accountKey: envConfig.accountKey ?? parsed?.accountKey,
|
||||
blobEndpoint: envConfig.blobEndpoint ?? parsed?.blobEndpoint,
|
||||
};
|
||||
}
|
||||
|
||||
@ -42,10 +73,9 @@ export class AzureBlobStorageProvider implements StorageProvider {
|
||||
this.config.accountName,
|
||||
this.config.accountKey
|
||||
);
|
||||
this.client = new BlobServiceClient(
|
||||
`https://${this.config.accountName}.blob.core.windows.net`,
|
||||
cred
|
||||
);
|
||||
const endpoint =
|
||||
this.config.blobEndpoint ?? `https://${this.config.accountName}.blob.core.windows.net`;
|
||||
this.client = new BlobServiceClient(endpoint, cred);
|
||||
} else {
|
||||
throw new Error(
|
||||
'AzureBlobStorageProvider requires AZURE_BLOB_CONNECTION_STRING or AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY'
|
||||
@ -86,7 +116,9 @@ class AzureBlobBucket implements StorageBucket {
|
||||
|
||||
private async containerClient() {
|
||||
const client = await this.getClient();
|
||||
return client.getContainerClient(this.containerName);
|
||||
const container = client.getContainerClient(this.containerName);
|
||||
await container.createIfNotExists();
|
||||
return container;
|
||||
}
|
||||
|
||||
async upload(
|
||||
@ -165,6 +197,11 @@ class AzureBlobBucket implements StorageBucket {
|
||||
cred
|
||||
);
|
||||
|
||||
return `https://${this.config.accountName}.blob.core.windows.net/${this.containerName}/${key}?${sas.toString()}`;
|
||||
const baseUrl =
|
||||
this.config.publicBlobEndpoint ??
|
||||
this.config.blobEndpoint ??
|
||||
`https://${this.config.accountName}.blob.core.windows.net`;
|
||||
|
||||
return `${baseUrl.replace(/\/$/, '')}/${this.containerName}/${key}?${sas.toString()}`;
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user