feat(devops): add Local LLM Lab to ecosystem deployment
- docker-compose.ecosystem.yml: add llmlab-dashboard service (port 3075) - setup.sh: add learning_ai_local_llms as 12th repo - README.md: update to 31 services, 11 products, add Docker vs K8s recommendation - docker/README.md: update port map, phase descriptions - prompt.md: update repo list and service counts
This commit is contained in:
saravanakumardb1
parent
e3f638d609
commit
fc12a8eaa2
@ -39,7 +39,6 @@ secrets:
|
||||
environment: GITEA_NPM_TOKEN
|
||||
|
||||
services:
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════
|
||||
# INFRASTRUCTURE
|
||||
# ═════════════════════════════════════════════════════════════════
|
||||
@ -529,6 +528,10 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_voice_ai_agent
|
||||
dockerfile: user-dashboard-web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003
|
||||
NEXT_PUBLIC_PRODUCT_ID: lysnrai
|
||||
ports:
|
||||
- '3002:3002'
|
||||
environment:
|
||||
@ -548,6 +551,10 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_clock
|
||||
dockerfile: web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_BACKEND_URL: http://localhost:4011
|
||||
NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003
|
||||
ports:
|
||||
- '3030:3030'
|
||||
environment:
|
||||
@ -565,6 +572,9 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_jarvis_jr
|
||||
dockerfile: web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003
|
||||
ports:
|
||||
- '3035:3035'
|
||||
environment:
|
||||
@ -581,6 +591,10 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_flowmonk
|
||||
dockerfile: web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_API_URL: http://localhost:4017
|
||||
NEXT_PUBLIC_PLATFORM_URL: http://localhost:4003/api
|
||||
ports:
|
||||
- '3040:3040'
|
||||
environment:
|
||||
@ -598,6 +612,10 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_notes
|
||||
dockerfile: web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_NOTES_API_URL: http://localhost:4016/api
|
||||
NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003/api
|
||||
ports:
|
||||
- '3045:3045'
|
||||
environment:
|
||||
@ -615,6 +633,9 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_multimodal_memory_agents
|
||||
dockerfile: mindlyst-native/web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003
|
||||
ports:
|
||||
- '3050:3050'
|
||||
environment:
|
||||
@ -631,6 +652,10 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_fastgap
|
||||
dockerfile: web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_NOMGAP_API_URL: http://localhost:4013/api
|
||||
NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003/api
|
||||
ports:
|
||||
- '3055:3055'
|
||||
environment:
|
||||
@ -648,6 +673,10 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_trails
|
||||
dockerfile: web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_API_URL: http://localhost:4018
|
||||
NEXT_PUBLIC_PLATFORM_URL: http://localhost:4003
|
||||
ports:
|
||||
- '3060:3060'
|
||||
environment:
|
||||
@ -665,6 +694,10 @@ services:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_local_memory_gpt
|
||||
dockerfile: web/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
NEXT_PUBLIC_BACKEND_URL: http://localhost:4019
|
||||
NEXT_PUBLIC_PLATFORM_URL: http://localhost:4003
|
||||
ports:
|
||||
- '3070:3070'
|
||||
environment:
|
||||
@ -677,6 +710,32 @@ services:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ── Local LLM Lab (no backend — dashboard talks directly to Ollama) ──
|
||||
|
||||
llmlab-dashboard:
|
||||
build:
|
||||
<<: *product-build
|
||||
context: ../learning_ai_local_llms
|
||||
dockerfile: dashboard/Dockerfile
|
||||
args:
|
||||
GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal}
|
||||
OLLAMA_URL: http://host.docker.internal:11434
|
||||
ports:
|
||||
- '3075:3075'
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- PORT=3075
|
||||
- OLLAMA_URL=http://host.docker.internal:11434
|
||||
- OLLAMA_HOST=http://host.docker.internal:11434
|
||||
extra_hosts:
|
||||
- 'host.docker.internal:host-gateway'
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '-q', '--spider', 'http://127.0.0.1:3075']
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════
|
||||
# VOLUMES
|
||||
# ═════════════════════════════════════════════════════════════════
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# ByteLyst Single-VM Deployment
|
||||
|
||||
> Deploy the **entire ByteLyst ecosystem** (30 services, 10 products) on a single Azure VM.
|
||||
> Deploy the **entire ByteLyst ecosystem** (31 services, 11 products) on a single Azure VM.
|
||||
> Two orchestration approaches — pick one or learn both side by side.
|
||||
|
||||
---
|
||||
@ -11,6 +11,7 @@
|
||||
|
||||
Proven, battle-tested deployment using `docker-compose.ecosystem.yml`.
|
||||
Installs everything from scratch on a raw Ubuntu VM in ~20 minutes.
|
||||
Includes Gitea CI (act_runner) for continuous integration.
|
||||
|
||||
```bash
|
||||
sudo ./docker/setup.sh # Full install
|
||||
@ -22,7 +23,7 @@ sudo ./docker/setup.sh --resume # Resume after disconnect
|
||||
|
||||
### [`k8s/`](k8s/) — Kubernetes via k3s (Learning / Future-ready)
|
||||
|
||||
Same 30 services orchestrated by Kubernetes on a single VM using k3s.
|
||||
Same 31 services orchestrated by Kubernetes on a single VM using k3s.
|
||||
Builds on the same Docker images — no Dockerfile changes needed.
|
||||
|
||||
**Use this if:** You want to learn K8s with real services, practice `kubectl`,
|
||||
@ -35,26 +36,47 @@ and prepare for multi-node scaling later.
|
||||
```
|
||||
Raw Ubuntu 24.04 VM (Standard_D8s_v5: 8 vCPU, 32 GB RAM)
|
||||
├── Ollama (systemd, :11434) ─── local LLM inference
|
||||
├── Gitea (Docker/:3300) ──────── npm package registry
|
||||
└── 30 Services
|
||||
├── Gitea (Docker/:3300) ──────── npm package registry + CI
|
||||
├── act_runner (systemd) ──────── Gitea CI runner (host mode)
|
||||
└── 31 Services
|
||||
├── Infrastructure (6): cosmos-emulator, azurite, mailpit, loki, grafana, traefik
|
||||
├── Platform (3): platform-service, extraction-service, mcp-server
|
||||
├── Dashboards (2): admin-web, tracker-web
|
||||
├── Backends (10): peakpulse, chronomind, jarvisjr, nomgap, mindlyst,
|
||||
│ lysnrai, notelett, flowmonk, actiontrail, localmemgpt
|
||||
└── Web Apps (9): lysnrai-dashboard, chronomind-web, jarvisjr-web, flowmonk-web,
|
||||
notelett-web, mindlyst-web, nomgap-web, actiontrail-web, localmemgpt-web
|
||||
├── Web Apps (9): lysnrai-dashboard, chronomind-web, jarvisjr-web, flowmonk-web,
|
||||
│ notelett-web, mindlyst-web, nomgap-web, actiontrail-web, localmemgpt-web
|
||||
└── Standalone (1): llmlab-dashboard (LLM Lab Mission Control)
|
||||
```
|
||||
|
||||
## Comparison
|
||||
|
||||
| | Docker Compose | K8s (k3s) |
|
||||
|--|----------------|-----------|
|
||||
| **Setup time** | ~20 min | ~30 min |
|
||||
| **RAM overhead** | ~100 MB | ~600 MB |
|
||||
| **Config files** | 1 compose + 1 .env | ~30 manifests (or Helm) |
|
||||
| **Scaling** | Manual | `kubectl scale` / HPA |
|
||||
| **Rolling updates** | Restart-based | Zero-downtime |
|
||||
| **Resource limits** | Basic | Fine-grained per pod |
|
||||
| **Multi-VM ready** | Docker Swarm | Native `kubectl join` |
|
||||
| **Learning value** | Low | High (transferable to AKS/EKS/GKE) |
|
||||
| | Docker Compose | K8s (k3s) |
|
||||
| ------------------- | ------------------ | ---------------------------------- |
|
||||
| **Setup time** | ~20 min | ~30 min |
|
||||
| **RAM overhead** | ~100 MB | ~600 MB |
|
||||
| **Config files** | 1 compose + 1 .env | ~30 manifests (or Helm) |
|
||||
| **Scaling** | Manual | `kubectl scale` / HPA |
|
||||
| **Rolling updates** | Restart-based | Zero-downtime |
|
||||
| **Resource limits** | Basic | Fine-grained per pod |
|
||||
| **Multi-VM ready** | Docker Swarm | Native `kubectl join` |
|
||||
| **Learning value** | Low | High (transferable to AKS/EKS/GKE) |
|
||||
|
||||
## Recommendation
|
||||
|
||||
**For a single Azure VM → use Docker Compose.** Here's why:
|
||||
|
||||
1. **One VM = no cluster benefits.** K8s shines at multi-node scheduling, auto-healing across hosts, and rolling deploys with replica sets. With 1 node, all pods compete for the same resources anyway.
|
||||
2. **RAM matters.** k3s adds ~500-600 MB overhead. On a 32 GB VM running 31 services + Cosmos emulator + Ollama, that headroom is useful.
|
||||
3. **Operational simplicity.** `docker compose logs -f platform-service` vs `kubectl logs deploy/platform-service -n bytelyst-platform -f` — Compose wins for a solo developer.
|
||||
4. **Faster iteration.** `docker compose up --build flowmonk-backend` rebuilds + restarts in seconds. K8s requires image tag bumps, manifest edits, and `kubectl apply`.
|
||||
5. **CI already runs on host.** act_runner uses host mode, not Docker-in-Docker. Compose services are reachable at `localhost:<port>` — K8s NodePort adds a layer.
|
||||
|
||||
**When to switch to K8s:**
|
||||
|
||||
- Scaling beyond 1 VM (add nodes with `k3s agent`)
|
||||
- Need zero-downtime rolling updates for beta users
|
||||
- Want fine-grained resource limits per service
|
||||
- Preparing for AKS/EKS production migration
|
||||
|
||||
The `k8s/` folder is ready for when you need it. Both approaches share the same Docker images and Gitea registry.
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# ByteLyst Single-VM Deployment
|
||||
|
||||
> Deploy the **entire ByteLyst ecosystem** (30 services, 10 products) on a single **raw** Azure VM.
|
||||
> Deploy the **entire ByteLyst ecosystem** (31 services, 11 products) on a single **raw** Azure VM.
|
||||
> Nothing pre-installed required — the script handles everything from a blank Ubuntu machine.
|
||||
> Two files: this README and `setup.sh`. Copy both to the VM and run the script.
|
||||
|
||||
@ -50,81 +50,87 @@ sudo ./setup.sh --help # Show full usage
|
||||
|
||||
### Software installed on the VM (from scratch)
|
||||
|
||||
| Software | Version | Purpose |
|
||||
|----------|---------|----------|
|
||||
| **Docker CE** | latest | Container runtime + Compose + BuildKit |
|
||||
| **Node.js** | 22 LTS | Build toolchain for TypeScript packages |
|
||||
| **pnpm** | 10.6.5 | Package manager (workspace-aware) |
|
||||
| **Gitea** | 1.22 (Docker) | Local npm package registry on `:3300` |
|
||||
| **Ollama** | latest | Local LLM inference for LocalMemGPT on `:11434` |
|
||||
| **git, jq, curl** | latest | System utilities |
|
||||
| Software | Version | Purpose |
|
||||
| ----------------- | ------------- | ----------------------------------------------- |
|
||||
| **Docker CE** | latest | Container runtime + Compose + BuildKit |
|
||||
| **Node.js** | 22 LTS | Build toolchain for TypeScript packages |
|
||||
| **pnpm** | 10.6.5 | Package manager (workspace-aware) |
|
||||
| **Gitea** | 1.22 (Docker) | Local npm package registry on `:3300` |
|
||||
| **Ollama** | latest | Local LLM inference for LocalMemGPT on `:11434` |
|
||||
| **git, jq, curl** | latest | System utilities |
|
||||
|
||||
### Execution phases
|
||||
|
||||
| Phase | Duration | Description |
|
||||
|-------|----------|-------------|
|
||||
| 1. System | ~3 min | Pre-flight checks (disk ≥40 GB, RAM ≥16 GB), install Docker, Node.js 22, pnpm 10.6.5, Ollama, git, jq, build-essential |
|
||||
| 2. Gitea | ~1 min | Start Gitea Docker container, create admin + org + API token |
|
||||
| 3. Clone | ~3 min | Clone all 11 repos to `/opt/bytelyst/` |
|
||||
| 4. Build | ~5 min | `pnpm install && pnpm -r build` all `@bytelyst/*` packages |
|
||||
| 5. Publish | ~3 min | Publish all packages to local Gitea npm registry |
|
||||
| 6. Env | instant | Generate `.env.ecosystem` with Cosmos emulator key, Azurite key, JWT secret |
|
||||
| 7. Deploy | ~10 min | Stop Ollama (free RAM), per-service Docker build + deploy (30 services, with fallback), prune build cache, restart Ollama |
|
||||
| 8. Verify | ~1 min | Health-check all 30+ endpoints + create `/opt/bytelyst/check-health.sh` |
|
||||
| Phase | Duration | Description |
|
||||
| ------------- | -------- | ------------------------------------------------------------------------------------------------------------------------- |
|
||||
| 1. System | ~3 min | Pre-flight checks (disk ≥40 GB, RAM ≥16 GB), install Docker, Node.js 22, pnpm 10.6.5, Ollama, git, jq, build-essential |
|
||||
| 2. Gitea + CI | ~2 min | Start Gitea Docker container, admin + org + token, install act_runner |
|
||||
| 3. Clone | ~3 min | Clone all 12 repos to `/opt/bytelyst/`, push to Gitea for CI |
|
||||
| 4. Build | ~5 min | `pnpm install && pnpm -r build` all `@bytelyst/*` packages |
|
||||
| 5. Publish | ~3 min | Publish all packages to local Gitea npm registry |
|
||||
| 6. Env | instant | Generate `.env.ecosystem` with Cosmos emulator key, Azurite key, JWT secret |
|
||||
| 7. Deploy | ~10 min | Stop Ollama (free RAM), per-service Docker build + deploy (30 services, with fallback), prune build cache, restart Ollama |
|
||||
| 8. Verify | ~1 min | Health-check all 31+ endpoints + create `/opt/bytelyst/check-health.sh` |
|
||||
|
||||
## Port Map (after deployment)
|
||||
|
||||
### Infrastructure (installed by setup.sh)
|
||||
| Service | Port | URL |
|
||||
|---------|------|-----|
|
||||
| Gitea (npm registry) | 3300 | `http://<vm-ip>:3300` |
|
||||
| Ollama (LLM API) | 11434 | `http://<vm-ip>:11434` |
|
||||
| Cosmos Data Explorer | 1234 | `http://<vm-ip>:1234` |
|
||||
| Azurite (Blob) | 10000 | `http://<vm-ip>:10000` |
|
||||
| Mailpit UI | 8025 | `http://<vm-ip>:8025` |
|
||||
| Loki (Logs) | 3100 | `http://<vm-ip>:3100/ready` |
|
||||
| Grafana | 3000 | `http://<vm-ip>:3000` |
|
||||
| Traefik Dashboard | 8080 | `http://<vm-ip>:8080` |
|
||||
|
||||
| Service | Port | URL |
|
||||
| -------------------- | ----- | --------------------------- |
|
||||
| Gitea (npm registry) | 3300 | `http://<vm-ip>:3300` |
|
||||
| Ollama (LLM API) | 11434 | `http://<vm-ip>:11434` |
|
||||
| Cosmos Data Explorer | 1234 | `http://<vm-ip>:1234` |
|
||||
| Azurite (Blob) | 10000 | `http://<vm-ip>:10000` |
|
||||
| Mailpit UI | 8025 | `http://<vm-ip>:8025` |
|
||||
| Loki (Logs) | 3100 | `http://<vm-ip>:3100/ready` |
|
||||
| Grafana | 3000 | `http://<vm-ip>:3000` |
|
||||
| Traefik Dashboard | 8080 | `http://<vm-ip>:8080` |
|
||||
|
||||
### Platform Services
|
||||
| Service | Port | URL |
|
||||
|---------|------|-----|
|
||||
| platform-service | 4003 | `http://<vm-ip>:4003/health` |
|
||||
|
||||
| Service | Port | URL |
|
||||
| ------------------ | ---- | ---------------------------- |
|
||||
| platform-service | 4003 | `http://<vm-ip>:4003/health` |
|
||||
| extraction-service | 4005 | `http://<vm-ip>:4005/health` |
|
||||
| mcp-server | 4007 | `http://<vm-ip>:4007/health` |
|
||||
| mcp-server | 4007 | `http://<vm-ip>:4007/health` |
|
||||
|
||||
### Platform Dashboards
|
||||
| Dashboard | Port | URL |
|
||||
|-----------|------|-----|
|
||||
|
||||
| Dashboard | Port | URL |
|
||||
| ------------- | ---- | --------------------- |
|
||||
| Admin Console | 3001 | `http://<vm-ip>:3001` |
|
||||
| Issue Tracker | 3003 | `http://<vm-ip>:3003` |
|
||||
|
||||
### Product Backends
|
||||
| Product | Port | Health |
|
||||
|---------|------|--------|
|
||||
| PeakPulse | 4010 | `http://<vm-ip>:4010/health` |
|
||||
| ChronoMind | 4011 | `http://<vm-ip>:4011/health` |
|
||||
| JarvisJr | 4012 | `http://<vm-ip>:4012/health` |
|
||||
| NomGap | 4013 | `http://<vm-ip>:4013/health` |
|
||||
| MindLyst | 4014 | `http://<vm-ip>:4014/health` |
|
||||
| LysnrAI | 4015 | `http://<vm-ip>:4015/health` |
|
||||
| NoteLett | 4016 | `http://<vm-ip>:4016/health` |
|
||||
| FlowMonk | 4017 | `http://<vm-ip>:4017/health` |
|
||||
|
||||
| Product | Port | Health |
|
||||
| ----------- | ---- | ---------------------------- |
|
||||
| PeakPulse | 4010 | `http://<vm-ip>:4010/health` |
|
||||
| ChronoMind | 4011 | `http://<vm-ip>:4011/health` |
|
||||
| JarvisJr | 4012 | `http://<vm-ip>:4012/health` |
|
||||
| NomGap | 4013 | `http://<vm-ip>:4013/health` |
|
||||
| MindLyst | 4014 | `http://<vm-ip>:4014/health` |
|
||||
| LysnrAI | 4015 | `http://<vm-ip>:4015/health` |
|
||||
| NoteLett | 4016 | `http://<vm-ip>:4016/health` |
|
||||
| FlowMonk | 4017 | `http://<vm-ip>:4017/health` |
|
||||
| ActionTrail | 4018 | `http://<vm-ip>:4018/health` |
|
||||
| LocalMemGPT | 4019 | `http://<vm-ip>:4019/health` |
|
||||
|
||||
### Product Web Apps
|
||||
| Product | Port | URL |
|
||||
|---------|------|-----|
|
||||
|
||||
| Product | Port | URL |
|
||||
| ----------------- | ---- | --------------------- |
|
||||
| LysnrAI Dashboard | 3002 | `http://<vm-ip>:3002` |
|
||||
| ChronoMind | 3030 | `http://<vm-ip>:3030` |
|
||||
| JarvisJr | 3035 | `http://<vm-ip>:3035` |
|
||||
| FlowMonk | 3040 | `http://<vm-ip>:3040` |
|
||||
| NoteLett | 3045 | `http://<vm-ip>:3045` |
|
||||
| MindLyst | 3050 | `http://<vm-ip>:3050` |
|
||||
| NomGap | 3055 | `http://<vm-ip>:3055` |
|
||||
| ActionTrail | 3060 | `http://<vm-ip>:3060` |
|
||||
| LocalMemGPT | 3070 | `http://<vm-ip>:3070` |
|
||||
| ChronoMind | 3030 | `http://<vm-ip>:3030` |
|
||||
| JarvisJr | 3035 | `http://<vm-ip>:3035` |
|
||||
| FlowMonk | 3040 | `http://<vm-ip>:3040` |
|
||||
| NoteLett | 3045 | `http://<vm-ip>:3045` |
|
||||
| MindLyst | 3050 | `http://<vm-ip>:3050` |
|
||||
| NomGap | 3055 | `http://<vm-ip>:3055` |
|
||||
| ActionTrail | 3060 | `http://<vm-ip>:3060` |
|
||||
| LocalMemGPT | 3070 | `http://<vm-ip>:3070` |
|
||||
| LLM Lab Dashboard | 3075 | `http://<vm-ip>:3075` |
|
||||
|
||||
## Post-Deployment Commands
|
||||
|
||||
@ -151,26 +157,26 @@ docker compose -f /opt/bytelyst/learning_ai_common_plat/docker-compose.ecosystem
|
||||
|
||||
All optional — defaults work for most setups:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `GITHUB_USER` | `saravanakumardb1` | GitHub org/user to clone repos from |
|
||||
| `GITHUB_TOKEN` | (empty) | Set for private repos (HTTPS auth) |
|
||||
| `GITEA_ADMIN` | `bytelyst-admin` | Gitea admin username |
|
||||
| `GITEA_PASS` | `ByteLyst2026!` | Gitea admin password |
|
||||
| `OLLAMA_MODEL` | `llama3.2:3b` | Default LLM model to pull |
|
||||
| `SKIP_CLONE` | `0` | Set `1` to skip cloning (re-runs) |
|
||||
| `SKIP_BUILD` | `0` | Set `1` to skip package build+publish (re-runs) |
|
||||
| Variable | Default | Description |
|
||||
| -------------- | ------------------ | ----------------------------------------------- |
|
||||
| `GITHUB_USER` | `saravanakumardb1` | GitHub org/user to clone repos from |
|
||||
| `GITHUB_TOKEN` | (empty) | Set for private repos (HTTPS auth) |
|
||||
| `GITEA_ADMIN` | `bytelyst-admin` | Gitea admin username |
|
||||
| `GITEA_PASS` | `ByteLyst2026!` | Gitea admin password |
|
||||
| `OLLAMA_MODEL` | `llama3.2:3b` | Default LLM model to pull |
|
||||
| `SKIP_CLONE` | `0` | Set `1` to skip cloning (re-runs) |
|
||||
| `SKIP_BUILD` | `0` | Set `1` to skip package build+publish (re-runs) |
|
||||
|
||||
## CLI Flags
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `--resume` | Auto-resume from last completed phase |
|
||||
| `--resume-from=N` | Resume from phase N (1-8) |
|
||||
| `--phase=N` | Run ONLY phase N (useful for retrying) |
|
||||
| `--reset` | Clear phase markers and start fresh |
|
||||
| `--status` | Show completed phases and exit |
|
||||
| `-h`, `--help` | Show usage help |
|
||||
| Flag | Description |
|
||||
| ----------------- | -------------------------------------- |
|
||||
| `--resume` | Auto-resume from last completed phase |
|
||||
| `--resume-from=N` | Resume from phase N (1-8) |
|
||||
| `--phase=N` | Run ONLY phase N (useful for retrying) |
|
||||
| `--reset` | Clear phase markers and start fresh |
|
||||
| `--status` | Show completed phases and exit |
|
||||
| `-h`, `--help` | Show usage help |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ This folder contains three files you must work with:
|
||||
- **`README.md`** — Deployment guide documenting what the script does, ports, troubleshooting
|
||||
- **`prompt.md`** — This file (agent instructions)
|
||||
|
||||
The script installs everything from scratch (Docker, Node.js, pnpm, Gitea, Ollama) then clones 11 repos, builds + publishes ~49 `@bytelyst/*` npm packages to a local Gitea registry, generates environment config, and deploys 30 Docker Compose services (6 infra + 3 platform + 2 dashboards + 10 backends + 9 webs).
|
||||
The script installs everything from scratch (Docker, Node.js, pnpm, Gitea, act_runner, Ollama) then clones 12 repos, builds + publishes ~57 `@bytelyst/*` npm packages to a local Gitea registry, generates environment config, and deploys 31 Docker Compose services (6 infra + 3 platform + 2 dashboards + 10 backends + 9 webs + 1 standalone).
|
||||
|
||||
### Current State (ALREADY IMPLEMENTED — do NOT redo)
|
||||
|
||||
@ -34,16 +34,16 @@ The following features are already built and tested in `setup.sh`:
|
||||
|
||||
### Key files outside this folder that the script depends on
|
||||
|
||||
| File | Repo | Purpose |
|
||||
|------|------|---------|
|
||||
| `docker-compose.ecosystem.yml` | `learning_ai_common_plat` (root) | Defines all 30 services |
|
||||
| `.env.ecosystem.example` | `learning_ai_common_plat` (root) | Template for env vars |
|
||||
| `packages/*/package.json` | `learning_ai_common_plat` | ~49 `@bytelyst/*` packages to publish |
|
||||
| `backend/Dockerfile` | Each of the 10 product repos | Product backend Docker builds |
|
||||
| `web/Dockerfile` | Each of the 10 product repos | Product web Docker builds |
|
||||
| `.npmrc.docker` | Each of the 10 product repos | Gitea npm registry config for Docker builds |
|
||||
| File | Repo | Purpose |
|
||||
| ------------------------------ | -------------------------------- | ------------------------------------------- |
|
||||
| `docker-compose.ecosystem.yml` | `learning_ai_common_plat` (root) | Defines all 30 services |
|
||||
| `.env.ecosystem.example` | `learning_ai_common_plat` (root) | Template for env vars |
|
||||
| `packages/*/package.json` | `learning_ai_common_plat` | ~49 `@bytelyst/*` packages to publish |
|
||||
| `backend/Dockerfile` | Each of the 10 product repos | Product backend Docker builds |
|
||||
| `web/Dockerfile` | Each of the 10 product repos | Product web Docker builds |
|
||||
| `.npmrc.docker` | Each of the 10 product repos | Gitea npm registry config for Docker builds |
|
||||
|
||||
### Repo list (all 11, cloned to `/opt/bytelyst/`)
|
||||
### Repo list (all 12, cloned to `/opt/bytelyst/`)
|
||||
|
||||
```
|
||||
learning_ai_common_plat # Shared platform: packages, services, dashboards, compose
|
||||
@ -57,6 +57,7 @@ learning_ai_flowmonk # FlowMonk
|
||||
learning_ai_notes # NoteLett
|
||||
learning_ai_trails # ActionTrail
|
||||
learning_ai_local_memory_gpt # LocalMemGPT
|
||||
learning_ai_local_llms # Local LLM Lab (dashboard only, no backend)
|
||||
```
|
||||
|
||||
GitHub org: `saravanakumardb1` (repos are public).
|
||||
@ -67,39 +68,39 @@ GitHub org: `saravanakumardb1` (repos are public).
|
||||
|
||||
The following issues have already been identified and fixed in the current `setup.sh`:
|
||||
|
||||
| Bug | Fix | Commit |
|
||||
|-----|-----|--------|
|
||||
| Docker apt source had extra whitespace from `\` continuation | Single-line echo | `ddd2db84` |
|
||||
| Gitea 1.22 returns token in `.sha1`, newer versions use `.token` | `jq -r '.sha1 // .token'` fallback | `ddd2db84` |
|
||||
| jfrog registry sed didn't handle multi-line `\` continuation | Added `/jfrog-pkg-proxy.*\\$/d` pattern | `ddd2db84` |
|
||||
| `detect_docker_host_ip()` uses `ip` command not in minimal installs | Added `iproute2` to apt deps | `ddd2db84` |
|
||||
| SSH disconnect loses all output | `exec > >(tee -a setup.log) 2>&1` | `ddd2db84` |
|
||||
| `localmemgpt-backend` can't reach Ollama on Linux | `extra_hosts: ['host.docker.internal:host-gateway']` in compose | `3b31709b` |
|
||||
| Dashboard Dockerfiles had hardcoded corporate proxy | Converted to `ARG`-based proxy with empty defaults | `2b9fd717` |
|
||||
| `pnpm install --frozen-lockfile` fails on shallow clones | Removed `--frozen-lockfile` | `3b31709b` |
|
||||
| 3 service Dockerfiles had stale package.json COPY lists | Updated to all 57 packages + workspace members | `85aca553` |
|
||||
| Phase 5 publish counted 409 conflicts as failures | Distinguish real failures from expected conflicts | `c0bc13e1` |
|
||||
| `set -e` + `pipefail` aborted script on `docker compose up` partial failure | Added `|| true` | `a9414218` |
|
||||
| Phase 7 marked done even with partial build failures | Only mark done when all builds succeed | `a9414218` |
|
||||
| `docker compose config --format json` called 30x in loop | Cached once | `a9414218` |
|
||||
| `--phase=7` printed success even with failures | Now exits 1 with build log path | `a9414218` |
|
||||
| `last_completed_phase` didn't enforce sequential order | Stops at first gap | `a3f4c6fa` |
|
||||
| Phase 7 missing `.env.ecosystem` guard | Fail early with helpful message | `a3f4c6fa` |
|
||||
| `ollama pull \| tail` aborted entire setup on slow network | Made non-fatal | `b634708d` |
|
||||
| NodeSource `curl\|bash` deprecated install method | Modern GPG key + apt source method | `c2ca7f53` |
|
||||
| Missing `build-essential python3` for native addons | Added to apt deps | `c2ca7f53` |
|
||||
| `pnpm -r build` fails on workspace members without build script | Added `--if-present` flag | `c2ca7f53` |
|
||||
| `gpg --dearmor` prompts on re-run if keyring exists | Added `--batch --yes` | `1a1f7dd5` |
|
||||
| `jq` aborts script on malformed Gitea token response | Added `2>/dev/null \|\| echo ""` guard | `1a1f7dd5` |
|
||||
| `pnpm install`/`build` failures show no useful message | Wrapped in `if ! ...; then fail("...")` | `1a1f7dd5` |
|
||||
| Docker builds OOM with Ollama + Cosmos (~7 GB combined) | Stop Ollama during Phase 7, restart after | `1a1f7dd5` |
|
||||
| Pre-flight: script runs on tiny VMs with no warning | Added disk (≥40 GB) and RAM (≥16 GB) checks | `1a1f7dd5` |
|
||||
| Azurite + Loki missing from Phase 8 health checks | Added both to check-health.sh | `f78d382d` |
|
||||
| GITEA_NPM_TOKEN silently empty on resume | Added `require_gitea_token()` guard in Phase 4 + 7 | `e928ec60` |
|
||||
| Dashboard Dockerfiles `--frozen-lockfile` fails (incomplete workspace) | Removed from admin-web + tracker-web | `e928ec60` |
|
||||
| Docker build cache exhausts disk (~20-40 GB) | Added `docker builder prune` after Phase 7 | `e928ec60` |
|
||||
| Compose `NEXT_PUBLIC_*` env vars wrong for 8/9 web services | Fixed per-service to match product code | `01f2276a` |
|
||||
| MindLyst web 3 files fallback to production URLs | Changed to `http://localhost:4003` | `09bdda8` |
|
||||
| Bug | Fix | Commit |
|
||||
| --------------------------------------------------------------------------- | --------------------------------------------------------------- | ---------- | ----- | ---------- |
|
||||
| Docker apt source had extra whitespace from `\` continuation | Single-line echo | `ddd2db84` |
|
||||
| Gitea 1.22 returns token in `.sha1`, newer versions use `.token` | `jq -r '.sha1 // .token'` fallback | `ddd2db84` |
|
||||
| jfrog registry sed didn't handle multi-line `\` continuation | Added `/jfrog-pkg-proxy.*\\$/d` pattern | `ddd2db84` |
|
||||
| `detect_docker_host_ip()` uses `ip` command not in minimal installs | Added `iproute2` to apt deps | `ddd2db84` |
|
||||
| SSH disconnect loses all output | `exec > >(tee -a setup.log) 2>&1` | `ddd2db84` |
|
||||
| `localmemgpt-backend` can't reach Ollama on Linux | `extra_hosts: ['host.docker.internal:host-gateway']` in compose | `3b31709b` |
|
||||
| Dashboard Dockerfiles had hardcoded corporate proxy | Converted to `ARG`-based proxy with empty defaults | `2b9fd717` |
|
||||
| `pnpm install --frozen-lockfile` fails on shallow clones | Removed `--frozen-lockfile` | `3b31709b` |
|
||||
| 3 service Dockerfiles had stale package.json COPY lists | Updated to all 57 packages + workspace members | `85aca553` |
|
||||
| Phase 5 publish counted 409 conflicts as failures | Distinguish real failures from expected conflicts | `c0bc13e1` |
|
||||
| `set -e` + `pipefail` aborted script on `docker compose up` partial failure | Added ` | | true` | `a9414218` |
|
||||
| Phase 7 marked done even with partial build failures | Only mark done when all builds succeed | `a9414218` |
|
||||
| `docker compose config --format json` called 30x in loop | Cached once | `a9414218` |
|
||||
| `--phase=7` printed success even with failures | Now exits 1 with build log path | `a9414218` |
|
||||
| `last_completed_phase` didn't enforce sequential order | Stops at first gap | `a3f4c6fa` |
|
||||
| Phase 7 missing `.env.ecosystem` guard | Fail early with helpful message | `a3f4c6fa` |
|
||||
| `ollama pull \| tail` aborted entire setup on slow network | Made non-fatal | `b634708d` |
|
||||
| NodeSource `curl\|bash` deprecated install method | Modern GPG key + apt source method | `c2ca7f53` |
|
||||
| Missing `build-essential python3` for native addons | Added to apt deps | `c2ca7f53` |
|
||||
| `pnpm -r build` fails on workspace members without build script | Added `--if-present` flag | `c2ca7f53` |
|
||||
| `gpg --dearmor` prompts on re-run if keyring exists | Added `--batch --yes` | `1a1f7dd5` |
|
||||
| `jq` aborts script on malformed Gitea token response | Added `2>/dev/null \|\| echo ""` guard | `1a1f7dd5` |
|
||||
| `pnpm install`/`build` failures show no useful message | Wrapped in `if ! ...; then fail("...")` | `1a1f7dd5` |
|
||||
| Docker builds OOM with Ollama + Cosmos (~7 GB combined) | Stop Ollama during Phase 7, restart after | `1a1f7dd5` |
|
||||
| Pre-flight: script runs on tiny VMs with no warning | Added disk (≥40 GB) and RAM (≥16 GB) checks | `1a1f7dd5` |
|
||||
| Azurite + Loki missing from Phase 8 health checks | Added both to check-health.sh | `f78d382d` |
|
||||
| GITEA_NPM_TOKEN silently empty on resume | Added `require_gitea_token()` guard in Phase 4 + 7 | `e928ec60` |
|
||||
| Dashboard Dockerfiles `--frozen-lockfile` fails (incomplete workspace) | Removed from admin-web + tracker-web | `e928ec60` |
|
||||
| Docker build cache exhausts disk (~20-40 GB) | Added `docker builder prune` after Phase 7 | `e928ec60` |
|
||||
| Compose `NEXT_PUBLIC_*` env vars wrong for 8/9 web services | Fixed per-service to match product code | `01f2276a` |
|
||||
| MindLyst web 3 files fallback to production URLs | Changed to `http://localhost:4003` | `09bdda8` |
|
||||
|
||||
---
|
||||
|
||||
@ -111,6 +112,7 @@ The following issues have already been identified and fixed in the current `setu
|
||||
### ~~1. Audit `setup.sh` for correctness~~ ✅ DONE
|
||||
|
||||
The script has been audited and all identified bugs fixed (see table above). Phases 1-8 are tested. Key things already verified:
|
||||
|
||||
- Docker CE install, Node.js 22 (NodeSource), pnpm 10.6.5, Ollama — all idempotent
|
||||
- Gitea token: `.sha1 // .token` fallback in place
|
||||
- Corporate proxy: removed at source in all repos, no runtime `sed` needed
|
||||
@ -127,6 +129,7 @@ All bugs fixed — see the 16-item table in "Bugs Already Fixed" above.
|
||||
### ~~3. Add error recovery and logging~~ ✅ DONE
|
||||
|
||||
Already implemented:
|
||||
|
||||
- **Phase completion markers:** `/opt/bytelyst/.setup-state/phaseN.done`
|
||||
- **Resume:** `--resume` (auto-detect), `--resume-from=N`, `--phase=N` (single), `--reset`, `--status`
|
||||
- **Logging:** `exec > >(tee -a setup.log) 2>&1`
|
||||
@ -136,6 +139,7 @@ Already implemented:
|
||||
### 4. Add a dry-run / validation mode (TODO)
|
||||
|
||||
Add `--dry-run` support that:
|
||||
|
||||
- Checks all prerequisites (disk space, memory, network access to GitHub)
|
||||
- Validates Docker is installed and running
|
||||
- Validates Gitea is reachable
|
||||
@ -157,6 +161,7 @@ Read `docker-compose.ecosystem.yml` (in the repo root) and verify:
|
||||
### 6. Update `README.md`
|
||||
|
||||
After all fixes, update `README.md` to reflect:
|
||||
|
||||
- CLI flags: `--resume`, `--resume-from=N`, `--phase=N`, `--reset`, `--status`, `--help`
|
||||
- Correct service count: 30 (not 27)
|
||||
- Updated duration estimates if phases changed
|
||||
@ -269,6 +274,7 @@ Raw Ubuntu 24.04 VM
|
||||
## How Docker Builds Reach Gitea
|
||||
|
||||
Product Dockerfiles use BuildKit secret mount for the npm token:
|
||||
|
||||
```dockerfile
|
||||
RUN --mount=type=secret,id=gitea_npm_token \
|
||||
cp .npmrc.docker .npmrc && \
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
# - Gitea (Docker container — npm package registry + CI on :3300)
|
||||
# - act_runner (Gitea CI runner — systemd service, host mode)
|
||||
# - Ollama (local LLM inference for LocalMemGPT on :11434)
|
||||
# - All 11 ByteLyst repos (cloned from GitHub)
|
||||
# - All 12 ByteLyst repos (cloned from GitHub)
|
||||
# - All @bytelyst/* packages (built + published to Gitea)
|
||||
# - Full 30-service ecosystem (via docker-compose.ecosystem.yml)
|
||||
#
|
||||
@ -28,7 +28,7 @@
|
||||
# Phases:
|
||||
# 1 System dependencies (Docker, Node, pnpm, Ollama)
|
||||
# 2 Gitea npm registry + CI runner (container on :3300 + act_runner systemd)
|
||||
# 3 Clone 11 repositories from GitHub + push to Gitea
|
||||
# 3 Clone 12 repositories from GitHub + push to Gitea
|
||||
# 4 Build all @bytelyst/* packages
|
||||
# 5 Publish packages to Gitea npm registry
|
||||
# 6 Generate .env.ecosystem config
|
||||
@ -87,6 +87,7 @@ REPOS=(
|
||||
learning_ai_notes
|
||||
learning_ai_trails
|
||||
learning_ai_local_memory_gpt
|
||||
learning_ai_local_llms
|
||||
)
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────
|
||||
@ -471,7 +472,7 @@ phase3_clone() {
|
||||
return
|
||||
fi
|
||||
|
||||
log "Phase 3: Cloning ${#REPOS[@]} repositories..."
|
||||
log "Phase 3: Cloning ${#REPOS[@]} repositories (12 repos)..."
|
||||
|
||||
local clone_base="https://github.com/${GITHUB_USER}"
|
||||
if [ -n "${GITHUB_TOKEN:-}" ]; then
|
||||
|
||||
Loading…
Reference in New Issue
Block a user