Add Azurite-backed blob storage for prototype
This commit is contained in:
parent
19b58b3ea0
commit
66a11c5713
19
.env.example
19
.env.example
@ -16,9 +16,11 @@ COSMOS_DATABASE=lysnrai
|
|||||||
JWT_SECRET=change-me-prototype-jwt-secret
|
JWT_SECRET=change-me-prototype-jwt-secret
|
||||||
|
|
||||||
# ── Azure Blob Storage (platform-service) ─────────────────────
|
# ── Azure Blob Storage (platform-service) ─────────────────────
|
||||||
AZURE_BLOB_CONNECTION_STRING=
|
STORAGE_PROVIDER=azure
|
||||||
AZURE_BLOB_ACCOUNT_NAME=bytelystblobs
|
AZURE_BLOB_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=<azurite-default-key>;BlobEndpoint=http://azurite:10000/devstoreaccount1;
|
||||||
AZURE_BLOB_ACCOUNT_KEY=your-blob-key
|
AZURE_BLOB_ACCOUNT_NAME=devstoreaccount1
|
||||||
|
AZURE_BLOB_ACCOUNT_KEY=<azurite-default-key>
|
||||||
|
AZURE_BLOB_PUBLIC_ENDPOINT=http://localhost:10000/devstoreaccount1
|
||||||
|
|
||||||
# ── Stripe (platform-service) ────────────────────────
|
# ── Stripe (platform-service) ────────────────────────
|
||||||
STRIPE_SECRET_KEY=sk_test_...
|
STRIPE_SECRET_KEY=sk_test_...
|
||||||
@ -26,6 +28,17 @@ STRIPE_WEBHOOK_SECRET=whsec_...
|
|||||||
STRIPE_PRICE_PRO=price_...
|
STRIPE_PRICE_PRO=price_...
|
||||||
STRIPE_PRICE_ENTERPRISE=price_...
|
STRIPE_PRICE_ENTERPRISE=price_...
|
||||||
|
|
||||||
|
# ── Email Delivery (platform-service) ─────────────────────────
|
||||||
|
# Use `smtp` for a self-hosted SMTP relay such as Mailpit, Postal, Mailcow, etc.
|
||||||
|
EMAIL_PROVIDER=smtp
|
||||||
|
EMAIL_FROM_ADDRESS=noreply@bytelyst.local
|
||||||
|
EMAIL_FROM_NAME=ByteLyst
|
||||||
|
SMTP_HOST=localhost
|
||||||
|
SMTP_PORT=1025
|
||||||
|
SMTP_SECURE=false
|
||||||
|
SMTP_USER=
|
||||||
|
SMTP_PASSWORD=
|
||||||
|
|
||||||
# ── Extraction Service (port 4005 + Python sidecar 4006) ─────
|
# ── Extraction Service (port 4005 + Python sidecar 4006) ─────
|
||||||
PYTHON_SIDECAR_URL=http://localhost:4006
|
PYTHON_SIDECAR_URL=http://localhost:4006
|
||||||
DEFAULT_MODEL_ID=gemini-2.5-flash
|
DEFAULT_MODEL_ID=gemini-2.5-flash
|
||||||
|
|||||||
@ -1,4 +1,23 @@
|
|||||||
services:
|
services:
|
||||||
|
# ── Azurite Blob Storage (prototype only) ─────────────────────
|
||||||
|
azurite:
|
||||||
|
image: mcr.microsoft.com/azure-storage/azurite:3.35.0
|
||||||
|
command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck
|
||||||
|
ports:
|
||||||
|
- '10000:10000'
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
'CMD',
|
||||||
|
'node',
|
||||||
|
'-e',
|
||||||
|
'const net=require("net");const s=net.connect(10000,"127.0.0.1",()=>{s.end();process.exit(0)});s.on("error",()=>process.exit(1));',
|
||||||
|
]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 6
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
# ── Azure Cosmos DB Emulator (prototype only) ─────────────────
|
# ── Azure Cosmos DB Emulator (prototype only) ─────────────────
|
||||||
cosmos-emulator:
|
cosmos-emulator:
|
||||||
image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview
|
image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview
|
||||||
@ -97,6 +116,8 @@ services:
|
|||||||
# Local/dev convenience: ensure Cosmos DB + containers exist.
|
# Local/dev convenience: ensure Cosmos DB + containers exist.
|
||||||
- COSMOS_AUTO_INIT=true
|
- COSMOS_AUTO_INIT=true
|
||||||
depends_on:
|
depends_on:
|
||||||
|
azurite:
|
||||||
|
condition: service_healthy
|
||||||
cosmos-emulator:
|
cosmos-emulator:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
labels:
|
labels:
|
||||||
|
|||||||
@ -8,6 +8,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp
|
|||||||
- `extraction-service`
|
- `extraction-service`
|
||||||
- `mcp-server`
|
- `mcp-server`
|
||||||
- `cosmos-emulator`
|
- `cosmos-emulator`
|
||||||
|
- `azurite`
|
||||||
- `gateway` (Traefik)
|
- `gateway` (Traefik)
|
||||||
- `loki`
|
- `loki`
|
||||||
- `grafana`
|
- `grafana`
|
||||||
@ -18,6 +19,7 @@ This repo is currently set up to run as a single-host prototype with Docker Comp
|
|||||||
- Any real API credentials such as Stripe or Gemini
|
- Any real API credentials such as Stripe or Gemini
|
||||||
|
|
||||||
For this VM prototype, Cosmos is self-hosted through the Linux Cosmos DB Emulator container. Everything else should still stay in `.env` and move to a real secret manager later.
|
For this VM prototype, Cosmos is self-hosted through the Linux Cosmos DB Emulator container. Everything else should still stay in `.env` and move to a real secret manager later.
|
||||||
|
Blob/file uploads are self-hosted through Azurite.
|
||||||
|
|
||||||
## First-Time Setup
|
## First-Time Setup
|
||||||
|
|
||||||
@ -33,6 +35,8 @@ If you want extraction features that call Gemini, also set:
|
|||||||
|
|
||||||
- `GEMINI_API_KEY`
|
- `GEMINI_API_KEY`
|
||||||
|
|
||||||
|
If you access generated blob SAS URLs from outside the VM, change `AZURE_BLOB_PUBLIC_ENDPOINT` to a host or IP that your browser can reach.
|
||||||
|
|
||||||
## Start The Stack
|
## Start The Stack
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@ -53,10 +57,12 @@ docker compose logs -f platform-service
|
|||||||
docker compose logs -f extraction-service
|
docker compose logs -f extraction-service
|
||||||
docker compose logs -f mcp-server
|
docker compose logs -f mcp-server
|
||||||
docker compose logs -f cosmos-emulator
|
docker compose logs -f cosmos-emulator
|
||||||
|
docker compose logs -f azurite
|
||||||
docker compose down
|
docker compose down
|
||||||
```
|
```
|
||||||
|
|
||||||
The Cosmos Data Explorer is exposed on `http://localhost:1234`.
|
The Cosmos Data Explorer is exposed on `http://localhost:1234`.
|
||||||
|
Azurite Blob Storage is exposed on `http://localhost:10000`.
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
|
|||||||
@ -16,6 +16,24 @@ export interface AzureBlobProviderConfig {
|
|||||||
connectionString?: string;
|
connectionString?: string;
|
||||||
accountName?: string;
|
accountName?: string;
|
||||||
accountKey?: string;
|
accountKey?: string;
|
||||||
|
blobEndpoint?: string;
|
||||||
|
publicBlobEndpoint?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseConnectionString(connectionString: string): Partial<AzureBlobProviderConfig> {
|
||||||
|
const parts = new Map<string, string>();
|
||||||
|
|
||||||
|
for (const segment of connectionString.split(';')) {
|
||||||
|
const [key, ...rest] = segment.split('=');
|
||||||
|
if (!key || rest.length === 0) continue;
|
||||||
|
parts.set(key, rest.join('='));
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
accountName: parts.get('AccountName'),
|
||||||
|
accountKey: parts.get('AccountKey'),
|
||||||
|
blobEndpoint: parts.get('BlobEndpoint'),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export class AzureBlobStorageProvider implements StorageProvider {
|
export class AzureBlobStorageProvider implements StorageProvider {
|
||||||
@ -24,10 +42,23 @@ export class AzureBlobStorageProvider implements StorageProvider {
|
|||||||
private buckets = new Map<string, AzureBlobBucket>();
|
private buckets = new Map<string, AzureBlobBucket>();
|
||||||
|
|
||||||
constructor(config?: AzureBlobProviderConfig) {
|
constructor(config?: AzureBlobProviderConfig) {
|
||||||
this.config = config ?? {
|
const envConfig = config ?? {
|
||||||
connectionString: process.env.AZURE_BLOB_CONNECTION_STRING,
|
connectionString: process.env.AZURE_BLOB_CONNECTION_STRING,
|
||||||
accountName: process.env.AZURE_BLOB_ACCOUNT_NAME,
|
accountName: process.env.AZURE_BLOB_ACCOUNT_NAME,
|
||||||
accountKey: process.env.AZURE_BLOB_ACCOUNT_KEY,
|
accountKey: process.env.AZURE_BLOB_ACCOUNT_KEY,
|
||||||
|
publicBlobEndpoint: process.env.AZURE_BLOB_PUBLIC_ENDPOINT,
|
||||||
|
};
|
||||||
|
|
||||||
|
const parsed = envConfig.connectionString
|
||||||
|
? parseConnectionString(envConfig.connectionString)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
this.config = {
|
||||||
|
...parsed,
|
||||||
|
...envConfig,
|
||||||
|
accountName: envConfig.accountName ?? parsed?.accountName,
|
||||||
|
accountKey: envConfig.accountKey ?? parsed?.accountKey,
|
||||||
|
blobEndpoint: envConfig.blobEndpoint ?? parsed?.blobEndpoint,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -42,10 +73,9 @@ export class AzureBlobStorageProvider implements StorageProvider {
|
|||||||
this.config.accountName,
|
this.config.accountName,
|
||||||
this.config.accountKey
|
this.config.accountKey
|
||||||
);
|
);
|
||||||
this.client = new BlobServiceClient(
|
const endpoint =
|
||||||
`https://${this.config.accountName}.blob.core.windows.net`,
|
this.config.blobEndpoint ?? `https://${this.config.accountName}.blob.core.windows.net`;
|
||||||
cred
|
this.client = new BlobServiceClient(endpoint, cred);
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
'AzureBlobStorageProvider requires AZURE_BLOB_CONNECTION_STRING or AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY'
|
'AzureBlobStorageProvider requires AZURE_BLOB_CONNECTION_STRING or AZURE_BLOB_ACCOUNT_NAME + AZURE_BLOB_ACCOUNT_KEY'
|
||||||
@ -86,7 +116,9 @@ class AzureBlobBucket implements StorageBucket {
|
|||||||
|
|
||||||
private async containerClient() {
|
private async containerClient() {
|
||||||
const client = await this.getClient();
|
const client = await this.getClient();
|
||||||
return client.getContainerClient(this.containerName);
|
const container = client.getContainerClient(this.containerName);
|
||||||
|
await container.createIfNotExists();
|
||||||
|
return container;
|
||||||
}
|
}
|
||||||
|
|
||||||
async upload(
|
async upload(
|
||||||
@ -165,6 +197,11 @@ class AzureBlobBucket implements StorageBucket {
|
|||||||
cred
|
cred
|
||||||
);
|
);
|
||||||
|
|
||||||
return `https://${this.config.accountName}.blob.core.windows.net/${this.containerName}/${key}?${sas.toString()}`;
|
const baseUrl =
|
||||||
|
this.config.publicBlobEndpoint ??
|
||||||
|
this.config.blobEndpoint ??
|
||||||
|
`https://${this.config.accountName}.blob.core.windows.net`;
|
||||||
|
|
||||||
|
return `${baseUrl.replace(/\/$/, '')}/${this.containerName}/${key}?${sas.toString()}`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user