diff --git a/docker-compose.ecosystem.yml b/docker-compose.ecosystem.yml index 53ff24ea..52ab7706 100644 --- a/docker-compose.ecosystem.yml +++ b/docker-compose.ecosystem.yml @@ -39,7 +39,6 @@ secrets: environment: GITEA_NPM_TOKEN services: - # ═════════════════════════════════════════════════════════════════ # INFRASTRUCTURE # ═════════════════════════════════════════════════════════════════ @@ -529,6 +528,10 @@ services: <<: *product-build context: ../learning_voice_ai_agent dockerfile: user-dashboard-web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003 + NEXT_PUBLIC_PRODUCT_ID: lysnrai ports: - '3002:3002' environment: @@ -548,6 +551,10 @@ services: <<: *product-build context: ../learning_ai_clock dockerfile: web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_BACKEND_URL: http://localhost:4011 + NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003 ports: - '3030:3030' environment: @@ -565,6 +572,9 @@ services: <<: *product-build context: ../learning_ai_jarvis_jr dockerfile: web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003 ports: - '3035:3035' environment: @@ -581,6 +591,10 @@ services: <<: *product-build context: ../learning_ai_flowmonk dockerfile: web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_API_URL: http://localhost:4017 + NEXT_PUBLIC_PLATFORM_URL: http://localhost:4003/api ports: - '3040:3040' environment: @@ -598,6 +612,10 @@ services: <<: *product-build context: ../learning_ai_notes dockerfile: web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_NOTES_API_URL: http://localhost:4016/api + NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003/api ports: - '3045:3045' environment: @@ -615,6 +633,9 @@ services: <<: *product-build context: ../learning_multimodal_memory_agents dockerfile: mindlyst-native/web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003 ports: - '3050:3050' environment: @@ -631,6 +652,10 @@ services: <<: *product-build context: ../learning_ai_fastgap dockerfile: web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_NOMGAP_API_URL: http://localhost:4013/api + NEXT_PUBLIC_PLATFORM_SERVICE_URL: http://localhost:4003/api ports: - '3055:3055' environment: @@ -648,6 +673,10 @@ services: <<: *product-build context: ../learning_ai_trails dockerfile: web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_API_URL: http://localhost:4018 + NEXT_PUBLIC_PLATFORM_URL: http://localhost:4003 ports: - '3060:3060' environment: @@ -665,6 +694,10 @@ services: <<: *product-build context: ../learning_ai_local_memory_gpt dockerfile: web/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + NEXT_PUBLIC_BACKEND_URL: http://localhost:4019 + NEXT_PUBLIC_PLATFORM_URL: http://localhost:4003 ports: - '3070:3070' environment: @@ -677,6 +710,32 @@ services: condition: service_healthy restart: unless-stopped + # ── Local LLM Lab (no backend — dashboard talks directly to Ollama) ── + + llmlab-dashboard: + build: + <<: *product-build + context: ../learning_ai_local_llms + dockerfile: dashboard/Dockerfile + args: + GITEA_NPM_HOST: ${GITEA_NPM_HOST:-host.docker.internal} + OLLAMA_URL: http://host.docker.internal:11434 + ports: + - '3075:3075' + environment: + - NODE_ENV=production + - PORT=3075 + - OLLAMA_URL=http://host.docker.internal:11434 + - OLLAMA_HOST=http://host.docker.internal:11434 + extra_hosts: + - 'host.docker.internal:host-gateway' + healthcheck: + test: ['CMD', 'wget', '-q', '--spider', 'http://127.0.0.1:3075'] + interval: 15s + timeout: 5s + retries: 3 + restart: unless-stopped + # ═════════════════════════════════════════════════════════════════ # VOLUMES # ═════════════════════════════════════════════════════════════════ diff --git a/docs/devops/single_azure_vm/README.md b/docs/devops/single_azure_vm/README.md index cdfab326..745f6e8b 100644 --- a/docs/devops/single_azure_vm/README.md +++ b/docs/devops/single_azure_vm/README.md @@ -1,6 +1,6 @@ # ByteLyst Single-VM Deployment -> Deploy the **entire ByteLyst ecosystem** (30 services, 10 products) on a single Azure VM. +> Deploy the **entire ByteLyst ecosystem** (31 services, 11 products) on a single Azure VM. > Two orchestration approaches — pick one or learn both side by side. --- @@ -11,6 +11,7 @@ Proven, battle-tested deployment using `docker-compose.ecosystem.yml`. Installs everything from scratch on a raw Ubuntu VM in ~20 minutes. +Includes Gitea CI (act_runner) for continuous integration. ```bash sudo ./docker/setup.sh # Full install @@ -22,7 +23,7 @@ sudo ./docker/setup.sh --resume # Resume after disconnect ### [`k8s/`](k8s/) — Kubernetes via k3s (Learning / Future-ready) -Same 30 services orchestrated by Kubernetes on a single VM using k3s. +Same 31 services orchestrated by Kubernetes on a single VM using k3s. Builds on the same Docker images — no Dockerfile changes needed. **Use this if:** You want to learn K8s with real services, practice `kubectl`, @@ -35,26 +36,47 @@ and prepare for multi-node scaling later. ``` Raw Ubuntu 24.04 VM (Standard_D8s_v5: 8 vCPU, 32 GB RAM) ├── Ollama (systemd, :11434) ─── local LLM inference -├── Gitea (Docker/:3300) ──────── npm package registry -└── 30 Services +├── Gitea (Docker/:3300) ──────── npm package registry + CI +├── act_runner (systemd) ──────── Gitea CI runner (host mode) +└── 31 Services ├── Infrastructure (6): cosmos-emulator, azurite, mailpit, loki, grafana, traefik ├── Platform (3): platform-service, extraction-service, mcp-server ├── Dashboards (2): admin-web, tracker-web ├── Backends (10): peakpulse, chronomind, jarvisjr, nomgap, mindlyst, │ lysnrai, notelett, flowmonk, actiontrail, localmemgpt - └── Web Apps (9): lysnrai-dashboard, chronomind-web, jarvisjr-web, flowmonk-web, - notelett-web, mindlyst-web, nomgap-web, actiontrail-web, localmemgpt-web + ├── Web Apps (9): lysnrai-dashboard, chronomind-web, jarvisjr-web, flowmonk-web, + │ notelett-web, mindlyst-web, nomgap-web, actiontrail-web, localmemgpt-web + └── Standalone (1): llmlab-dashboard (LLM Lab Mission Control) ``` ## Comparison -| | Docker Compose | K8s (k3s) | -|--|----------------|-----------| -| **Setup time** | ~20 min | ~30 min | -| **RAM overhead** | ~100 MB | ~600 MB | -| **Config files** | 1 compose + 1 .env | ~30 manifests (or Helm) | -| **Scaling** | Manual | `kubectl scale` / HPA | -| **Rolling updates** | Restart-based | Zero-downtime | -| **Resource limits** | Basic | Fine-grained per pod | -| **Multi-VM ready** | Docker Swarm | Native `kubectl join` | -| **Learning value** | Low | High (transferable to AKS/EKS/GKE) | +| | Docker Compose | K8s (k3s) | +| ------------------- | ------------------ | ---------------------------------- | +| **Setup time** | ~20 min | ~30 min | +| **RAM overhead** | ~100 MB | ~600 MB | +| **Config files** | 1 compose + 1 .env | ~30 manifests (or Helm) | +| **Scaling** | Manual | `kubectl scale` / HPA | +| **Rolling updates** | Restart-based | Zero-downtime | +| **Resource limits** | Basic | Fine-grained per pod | +| **Multi-VM ready** | Docker Swarm | Native `kubectl join` | +| **Learning value** | Low | High (transferable to AKS/EKS/GKE) | + +## Recommendation + +**For a single Azure VM → use Docker Compose.** Here's why: + +1. **One VM = no cluster benefits.** K8s shines at multi-node scheduling, auto-healing across hosts, and rolling deploys with replica sets. With 1 node, all pods compete for the same resources anyway. +2. **RAM matters.** k3s adds ~500-600 MB overhead. On a 32 GB VM running 31 services + Cosmos emulator + Ollama, that headroom is useful. +3. **Operational simplicity.** `docker compose logs -f platform-service` vs `kubectl logs deploy/platform-service -n bytelyst-platform -f` — Compose wins for a solo developer. +4. **Faster iteration.** `docker compose up --build flowmonk-backend` rebuilds + restarts in seconds. K8s requires image tag bumps, manifest edits, and `kubectl apply`. +5. **CI already runs on host.** act_runner uses host mode, not Docker-in-Docker. Compose services are reachable at `localhost:` — K8s NodePort adds a layer. + +**When to switch to K8s:** + +- Scaling beyond 1 VM (add nodes with `k3s agent`) +- Need zero-downtime rolling updates for beta users +- Want fine-grained resource limits per service +- Preparing for AKS/EKS production migration + +The `k8s/` folder is ready for when you need it. Both approaches share the same Docker images and Gitea registry. diff --git a/docs/devops/single_azure_vm/docker/README.md b/docs/devops/single_azure_vm/docker/README.md index 1e5d3a82..1ef26ef2 100644 --- a/docs/devops/single_azure_vm/docker/README.md +++ b/docs/devops/single_azure_vm/docker/README.md @@ -1,6 +1,6 @@ # ByteLyst Single-VM Deployment -> Deploy the **entire ByteLyst ecosystem** (30 services, 10 products) on a single **raw** Azure VM. +> Deploy the **entire ByteLyst ecosystem** (31 services, 11 products) on a single **raw** Azure VM. > Nothing pre-installed required — the script handles everything from a blank Ubuntu machine. > Two files: this README and `setup.sh`. Copy both to the VM and run the script. @@ -50,81 +50,87 @@ sudo ./setup.sh --help # Show full usage ### Software installed on the VM (from scratch) -| Software | Version | Purpose | -|----------|---------|----------| -| **Docker CE** | latest | Container runtime + Compose + BuildKit | -| **Node.js** | 22 LTS | Build toolchain for TypeScript packages | -| **pnpm** | 10.6.5 | Package manager (workspace-aware) | -| **Gitea** | 1.22 (Docker) | Local npm package registry on `:3300` | -| **Ollama** | latest | Local LLM inference for LocalMemGPT on `:11434` | -| **git, jq, curl** | latest | System utilities | +| Software | Version | Purpose | +| ----------------- | ------------- | ----------------------------------------------- | +| **Docker CE** | latest | Container runtime + Compose + BuildKit | +| **Node.js** | 22 LTS | Build toolchain for TypeScript packages | +| **pnpm** | 10.6.5 | Package manager (workspace-aware) | +| **Gitea** | 1.22 (Docker) | Local npm package registry on `:3300` | +| **Ollama** | latest | Local LLM inference for LocalMemGPT on `:11434` | +| **git, jq, curl** | latest | System utilities | ### Execution phases -| Phase | Duration | Description | -|-------|----------|-------------| -| 1. System | ~3 min | Pre-flight checks (disk ≥40 GB, RAM ≥16 GB), install Docker, Node.js 22, pnpm 10.6.5, Ollama, git, jq, build-essential | -| 2. Gitea | ~1 min | Start Gitea Docker container, create admin + org + API token | -| 3. Clone | ~3 min | Clone all 11 repos to `/opt/bytelyst/` | -| 4. Build | ~5 min | `pnpm install && pnpm -r build` all `@bytelyst/*` packages | -| 5. Publish | ~3 min | Publish all packages to local Gitea npm registry | -| 6. Env | instant | Generate `.env.ecosystem` with Cosmos emulator key, Azurite key, JWT secret | -| 7. Deploy | ~10 min | Stop Ollama (free RAM), per-service Docker build + deploy (30 services, with fallback), prune build cache, restart Ollama | -| 8. Verify | ~1 min | Health-check all 30+ endpoints + create `/opt/bytelyst/check-health.sh` | +| Phase | Duration | Description | +| ------------- | -------- | ------------------------------------------------------------------------------------------------------------------------- | +| 1. System | ~3 min | Pre-flight checks (disk ≥40 GB, RAM ≥16 GB), install Docker, Node.js 22, pnpm 10.6.5, Ollama, git, jq, build-essential | +| 2. Gitea + CI | ~2 min | Start Gitea Docker container, admin + org + token, install act_runner | +| 3. Clone | ~3 min | Clone all 12 repos to `/opt/bytelyst/`, push to Gitea for CI | +| 4. Build | ~5 min | `pnpm install && pnpm -r build` all `@bytelyst/*` packages | +| 5. Publish | ~3 min | Publish all packages to local Gitea npm registry | +| 6. Env | instant | Generate `.env.ecosystem` with Cosmos emulator key, Azurite key, JWT secret | +| 7. Deploy | ~10 min | Stop Ollama (free RAM), per-service Docker build + deploy (30 services, with fallback), prune build cache, restart Ollama | +| 8. Verify | ~1 min | Health-check all 31+ endpoints + create `/opt/bytelyst/check-health.sh` | ## Port Map (after deployment) ### Infrastructure (installed by setup.sh) -| Service | Port | URL | -|---------|------|-----| -| Gitea (npm registry) | 3300 | `http://:3300` | -| Ollama (LLM API) | 11434 | `http://:11434` | -| Cosmos Data Explorer | 1234 | `http://:1234` | -| Azurite (Blob) | 10000 | `http://:10000` | -| Mailpit UI | 8025 | `http://:8025` | -| Loki (Logs) | 3100 | `http://:3100/ready` | -| Grafana | 3000 | `http://:3000` | -| Traefik Dashboard | 8080 | `http://:8080` | + +| Service | Port | URL | +| -------------------- | ----- | --------------------------- | +| Gitea (npm registry) | 3300 | `http://:3300` | +| Ollama (LLM API) | 11434 | `http://:11434` | +| Cosmos Data Explorer | 1234 | `http://:1234` | +| Azurite (Blob) | 10000 | `http://:10000` | +| Mailpit UI | 8025 | `http://:8025` | +| Loki (Logs) | 3100 | `http://:3100/ready` | +| Grafana | 3000 | `http://:3000` | +| Traefik Dashboard | 8080 | `http://:8080` | ### Platform Services -| Service | Port | URL | -|---------|------|-----| -| platform-service | 4003 | `http://:4003/health` | + +| Service | Port | URL | +| ------------------ | ---- | ---------------------------- | +| platform-service | 4003 | `http://:4003/health` | | extraction-service | 4005 | `http://:4005/health` | -| mcp-server | 4007 | `http://:4007/health` | +| mcp-server | 4007 | `http://:4007/health` | ### Platform Dashboards -| Dashboard | Port | URL | -|-----------|------|-----| + +| Dashboard | Port | URL | +| ------------- | ---- | --------------------- | | Admin Console | 3001 | `http://:3001` | | Issue Tracker | 3003 | `http://:3003` | ### Product Backends -| Product | Port | Health | -|---------|------|--------| -| PeakPulse | 4010 | `http://:4010/health` | -| ChronoMind | 4011 | `http://:4011/health` | -| JarvisJr | 4012 | `http://:4012/health` | -| NomGap | 4013 | `http://:4013/health` | -| MindLyst | 4014 | `http://:4014/health` | -| LysnrAI | 4015 | `http://:4015/health` | -| NoteLett | 4016 | `http://:4016/health` | -| FlowMonk | 4017 | `http://:4017/health` | + +| Product | Port | Health | +| ----------- | ---- | ---------------------------- | +| PeakPulse | 4010 | `http://:4010/health` | +| ChronoMind | 4011 | `http://:4011/health` | +| JarvisJr | 4012 | `http://:4012/health` | +| NomGap | 4013 | `http://:4013/health` | +| MindLyst | 4014 | `http://:4014/health` | +| LysnrAI | 4015 | `http://:4015/health` | +| NoteLett | 4016 | `http://:4016/health` | +| FlowMonk | 4017 | `http://:4017/health` | | ActionTrail | 4018 | `http://:4018/health` | | LocalMemGPT | 4019 | `http://:4019/health` | ### Product Web Apps -| Product | Port | URL | -|---------|------|-----| + +| Product | Port | URL | +| ----------------- | ---- | --------------------- | | LysnrAI Dashboard | 3002 | `http://:3002` | -| ChronoMind | 3030 | `http://:3030` | -| JarvisJr | 3035 | `http://:3035` | -| FlowMonk | 3040 | `http://:3040` | -| NoteLett | 3045 | `http://:3045` | -| MindLyst | 3050 | `http://:3050` | -| NomGap | 3055 | `http://:3055` | -| ActionTrail | 3060 | `http://:3060` | -| LocalMemGPT | 3070 | `http://:3070` | +| ChronoMind | 3030 | `http://:3030` | +| JarvisJr | 3035 | `http://:3035` | +| FlowMonk | 3040 | `http://:3040` | +| NoteLett | 3045 | `http://:3045` | +| MindLyst | 3050 | `http://:3050` | +| NomGap | 3055 | `http://:3055` | +| ActionTrail | 3060 | `http://:3060` | +| LocalMemGPT | 3070 | `http://:3070` | +| LLM Lab Dashboard | 3075 | `http://:3075` | ## Post-Deployment Commands @@ -151,26 +157,26 @@ docker compose -f /opt/bytelyst/learning_ai_common_plat/docker-compose.ecosystem All optional — defaults work for most setups: -| Variable | Default | Description | -|----------|---------|-------------| -| `GITHUB_USER` | `saravanakumardb1` | GitHub org/user to clone repos from | -| `GITHUB_TOKEN` | (empty) | Set for private repos (HTTPS auth) | -| `GITEA_ADMIN` | `bytelyst-admin` | Gitea admin username | -| `GITEA_PASS` | `ByteLyst2026!` | Gitea admin password | -| `OLLAMA_MODEL` | `llama3.2:3b` | Default LLM model to pull | -| `SKIP_CLONE` | `0` | Set `1` to skip cloning (re-runs) | -| `SKIP_BUILD` | `0` | Set `1` to skip package build+publish (re-runs) | +| Variable | Default | Description | +| -------------- | ------------------ | ----------------------------------------------- | +| `GITHUB_USER` | `saravanakumardb1` | GitHub org/user to clone repos from | +| `GITHUB_TOKEN` | (empty) | Set for private repos (HTTPS auth) | +| `GITEA_ADMIN` | `bytelyst-admin` | Gitea admin username | +| `GITEA_PASS` | `ByteLyst2026!` | Gitea admin password | +| `OLLAMA_MODEL` | `llama3.2:3b` | Default LLM model to pull | +| `SKIP_CLONE` | `0` | Set `1` to skip cloning (re-runs) | +| `SKIP_BUILD` | `0` | Set `1` to skip package build+publish (re-runs) | ## CLI Flags -| Flag | Description | -|------|-------------| -| `--resume` | Auto-resume from last completed phase | -| `--resume-from=N` | Resume from phase N (1-8) | -| `--phase=N` | Run ONLY phase N (useful for retrying) | -| `--reset` | Clear phase markers and start fresh | -| `--status` | Show completed phases and exit | -| `-h`, `--help` | Show usage help | +| Flag | Description | +| ----------------- | -------------------------------------- | +| `--resume` | Auto-resume from last completed phase | +| `--resume-from=N` | Resume from phase N (1-8) | +| `--phase=N` | Run ONLY phase N (useful for retrying) | +| `--reset` | Clear phase markers and start fresh | +| `--status` | Show completed phases and exit | +| `-h`, `--help` | Show usage help | ## Troubleshooting diff --git a/docs/devops/single_azure_vm/docker/prompt.md b/docs/devops/single_azure_vm/docker/prompt.md index cf51ffb1..0b0512e1 100644 --- a/docs/devops/single_azure_vm/docker/prompt.md +++ b/docs/devops/single_azure_vm/docker/prompt.md @@ -14,7 +14,7 @@ This folder contains three files you must work with: - **`README.md`** — Deployment guide documenting what the script does, ports, troubleshooting - **`prompt.md`** — This file (agent instructions) -The script installs everything from scratch (Docker, Node.js, pnpm, Gitea, Ollama) then clones 11 repos, builds + publishes ~49 `@bytelyst/*` npm packages to a local Gitea registry, generates environment config, and deploys 30 Docker Compose services (6 infra + 3 platform + 2 dashboards + 10 backends + 9 webs). +The script installs everything from scratch (Docker, Node.js, pnpm, Gitea, act_runner, Ollama) then clones 12 repos, builds + publishes ~57 `@bytelyst/*` npm packages to a local Gitea registry, generates environment config, and deploys 31 Docker Compose services (6 infra + 3 platform + 2 dashboards + 10 backends + 9 webs + 1 standalone). ### Current State (ALREADY IMPLEMENTED — do NOT redo) @@ -34,16 +34,16 @@ The following features are already built and tested in `setup.sh`: ### Key files outside this folder that the script depends on -| File | Repo | Purpose | -|------|------|---------| -| `docker-compose.ecosystem.yml` | `learning_ai_common_plat` (root) | Defines all 30 services | -| `.env.ecosystem.example` | `learning_ai_common_plat` (root) | Template for env vars | -| `packages/*/package.json` | `learning_ai_common_plat` | ~49 `@bytelyst/*` packages to publish | -| `backend/Dockerfile` | Each of the 10 product repos | Product backend Docker builds | -| `web/Dockerfile` | Each of the 10 product repos | Product web Docker builds | -| `.npmrc.docker` | Each of the 10 product repos | Gitea npm registry config for Docker builds | +| File | Repo | Purpose | +| ------------------------------ | -------------------------------- | ------------------------------------------- | +| `docker-compose.ecosystem.yml` | `learning_ai_common_plat` (root) | Defines all 30 services | +| `.env.ecosystem.example` | `learning_ai_common_plat` (root) | Template for env vars | +| `packages/*/package.json` | `learning_ai_common_plat` | ~49 `@bytelyst/*` packages to publish | +| `backend/Dockerfile` | Each of the 10 product repos | Product backend Docker builds | +| `web/Dockerfile` | Each of the 10 product repos | Product web Docker builds | +| `.npmrc.docker` | Each of the 10 product repos | Gitea npm registry config for Docker builds | -### Repo list (all 11, cloned to `/opt/bytelyst/`) +### Repo list (all 12, cloned to `/opt/bytelyst/`) ``` learning_ai_common_plat # Shared platform: packages, services, dashboards, compose @@ -57,6 +57,7 @@ learning_ai_flowmonk # FlowMonk learning_ai_notes # NoteLett learning_ai_trails # ActionTrail learning_ai_local_memory_gpt # LocalMemGPT +learning_ai_local_llms # Local LLM Lab (dashboard only, no backend) ``` GitHub org: `saravanakumardb1` (repos are public). @@ -67,39 +68,39 @@ GitHub org: `saravanakumardb1` (repos are public). The following issues have already been identified and fixed in the current `setup.sh`: -| Bug | Fix | Commit | -|-----|-----|--------| -| Docker apt source had extra whitespace from `\` continuation | Single-line echo | `ddd2db84` | -| Gitea 1.22 returns token in `.sha1`, newer versions use `.token` | `jq -r '.sha1 // .token'` fallback | `ddd2db84` | -| jfrog registry sed didn't handle multi-line `\` continuation | Added `/jfrog-pkg-proxy.*\\$/d` pattern | `ddd2db84` | -| `detect_docker_host_ip()` uses `ip` command not in minimal installs | Added `iproute2` to apt deps | `ddd2db84` | -| SSH disconnect loses all output | `exec > >(tee -a setup.log) 2>&1` | `ddd2db84` | -| `localmemgpt-backend` can't reach Ollama on Linux | `extra_hosts: ['host.docker.internal:host-gateway']` in compose | `3b31709b` | -| Dashboard Dockerfiles had hardcoded corporate proxy | Converted to `ARG`-based proxy with empty defaults | `2b9fd717` | -| `pnpm install --frozen-lockfile` fails on shallow clones | Removed `--frozen-lockfile` | `3b31709b` | -| 3 service Dockerfiles had stale package.json COPY lists | Updated to all 57 packages + workspace members | `85aca553` | -| Phase 5 publish counted 409 conflicts as failures | Distinguish real failures from expected conflicts | `c0bc13e1` | -| `set -e` + `pipefail` aborted script on `docker compose up` partial failure | Added `|| true` | `a9414218` | -| Phase 7 marked done even with partial build failures | Only mark done when all builds succeed | `a9414218` | -| `docker compose config --format json` called 30x in loop | Cached once | `a9414218` | -| `--phase=7` printed success even with failures | Now exits 1 with build log path | `a9414218` | -| `last_completed_phase` didn't enforce sequential order | Stops at first gap | `a3f4c6fa` | -| Phase 7 missing `.env.ecosystem` guard | Fail early with helpful message | `a3f4c6fa` | -| `ollama pull \| tail` aborted entire setup on slow network | Made non-fatal | `b634708d` | -| NodeSource `curl\|bash` deprecated install method | Modern GPG key + apt source method | `c2ca7f53` | -| Missing `build-essential python3` for native addons | Added to apt deps | `c2ca7f53` | -| `pnpm -r build` fails on workspace members without build script | Added `--if-present` flag | `c2ca7f53` | -| `gpg --dearmor` prompts on re-run if keyring exists | Added `--batch --yes` | `1a1f7dd5` | -| `jq` aborts script on malformed Gitea token response | Added `2>/dev/null \|\| echo ""` guard | `1a1f7dd5` | -| `pnpm install`/`build` failures show no useful message | Wrapped in `if ! ...; then fail("...")` | `1a1f7dd5` | -| Docker builds OOM with Ollama + Cosmos (~7 GB combined) | Stop Ollama during Phase 7, restart after | `1a1f7dd5` | -| Pre-flight: script runs on tiny VMs with no warning | Added disk (≥40 GB) and RAM (≥16 GB) checks | `1a1f7dd5` | -| Azurite + Loki missing from Phase 8 health checks | Added both to check-health.sh | `f78d382d` | -| GITEA_NPM_TOKEN silently empty on resume | Added `require_gitea_token()` guard in Phase 4 + 7 | `e928ec60` | -| Dashboard Dockerfiles `--frozen-lockfile` fails (incomplete workspace) | Removed from admin-web + tracker-web | `e928ec60` | -| Docker build cache exhausts disk (~20-40 GB) | Added `docker builder prune` after Phase 7 | `e928ec60` | -| Compose `NEXT_PUBLIC_*` env vars wrong for 8/9 web services | Fixed per-service to match product code | `01f2276a` | -| MindLyst web 3 files fallback to production URLs | Changed to `http://localhost:4003` | `09bdda8` | +| Bug | Fix | Commit | +| --------------------------------------------------------------------------- | --------------------------------------------------------------- | ---------- | ----- | ---------- | +| Docker apt source had extra whitespace from `\` continuation | Single-line echo | `ddd2db84` | +| Gitea 1.22 returns token in `.sha1`, newer versions use `.token` | `jq -r '.sha1 // .token'` fallback | `ddd2db84` | +| jfrog registry sed didn't handle multi-line `\` continuation | Added `/jfrog-pkg-proxy.*\\$/d` pattern | `ddd2db84` | +| `detect_docker_host_ip()` uses `ip` command not in minimal installs | Added `iproute2` to apt deps | `ddd2db84` | +| SSH disconnect loses all output | `exec > >(tee -a setup.log) 2>&1` | `ddd2db84` | +| `localmemgpt-backend` can't reach Ollama on Linux | `extra_hosts: ['host.docker.internal:host-gateway']` in compose | `3b31709b` | +| Dashboard Dockerfiles had hardcoded corporate proxy | Converted to `ARG`-based proxy with empty defaults | `2b9fd717` | +| `pnpm install --frozen-lockfile` fails on shallow clones | Removed `--frozen-lockfile` | `3b31709b` | +| 3 service Dockerfiles had stale package.json COPY lists | Updated to all 57 packages + workspace members | `85aca553` | +| Phase 5 publish counted 409 conflicts as failures | Distinguish real failures from expected conflicts | `c0bc13e1` | +| `set -e` + `pipefail` aborted script on `docker compose up` partial failure | Added ` | | true` | `a9414218` | +| Phase 7 marked done even with partial build failures | Only mark done when all builds succeed | `a9414218` | +| `docker compose config --format json` called 30x in loop | Cached once | `a9414218` | +| `--phase=7` printed success even with failures | Now exits 1 with build log path | `a9414218` | +| `last_completed_phase` didn't enforce sequential order | Stops at first gap | `a3f4c6fa` | +| Phase 7 missing `.env.ecosystem` guard | Fail early with helpful message | `a3f4c6fa` | +| `ollama pull \| tail` aborted entire setup on slow network | Made non-fatal | `b634708d` | +| NodeSource `curl\|bash` deprecated install method | Modern GPG key + apt source method | `c2ca7f53` | +| Missing `build-essential python3` for native addons | Added to apt deps | `c2ca7f53` | +| `pnpm -r build` fails on workspace members without build script | Added `--if-present` flag | `c2ca7f53` | +| `gpg --dearmor` prompts on re-run if keyring exists | Added `--batch --yes` | `1a1f7dd5` | +| `jq` aborts script on malformed Gitea token response | Added `2>/dev/null \|\| echo ""` guard | `1a1f7dd5` | +| `pnpm install`/`build` failures show no useful message | Wrapped in `if ! ...; then fail("...")` | `1a1f7dd5` | +| Docker builds OOM with Ollama + Cosmos (~7 GB combined) | Stop Ollama during Phase 7, restart after | `1a1f7dd5` | +| Pre-flight: script runs on tiny VMs with no warning | Added disk (≥40 GB) and RAM (≥16 GB) checks | `1a1f7dd5` | +| Azurite + Loki missing from Phase 8 health checks | Added both to check-health.sh | `f78d382d` | +| GITEA_NPM_TOKEN silently empty on resume | Added `require_gitea_token()` guard in Phase 4 + 7 | `e928ec60` | +| Dashboard Dockerfiles `--frozen-lockfile` fails (incomplete workspace) | Removed from admin-web + tracker-web | `e928ec60` | +| Docker build cache exhausts disk (~20-40 GB) | Added `docker builder prune` after Phase 7 | `e928ec60` | +| Compose `NEXT_PUBLIC_*` env vars wrong for 8/9 web services | Fixed per-service to match product code | `01f2276a` | +| MindLyst web 3 files fallback to production URLs | Changed to `http://localhost:4003` | `09bdda8` | --- @@ -111,6 +112,7 @@ The following issues have already been identified and fixed in the current `setu ### ~~1. Audit `setup.sh` for correctness~~ ✅ DONE The script has been audited and all identified bugs fixed (see table above). Phases 1-8 are tested. Key things already verified: + - Docker CE install, Node.js 22 (NodeSource), pnpm 10.6.5, Ollama — all idempotent - Gitea token: `.sha1 // .token` fallback in place - Corporate proxy: removed at source in all repos, no runtime `sed` needed @@ -127,6 +129,7 @@ All bugs fixed — see the 16-item table in "Bugs Already Fixed" above. ### ~~3. Add error recovery and logging~~ ✅ DONE Already implemented: + - **Phase completion markers:** `/opt/bytelyst/.setup-state/phaseN.done` - **Resume:** `--resume` (auto-detect), `--resume-from=N`, `--phase=N` (single), `--reset`, `--status` - **Logging:** `exec > >(tee -a setup.log) 2>&1` @@ -136,6 +139,7 @@ Already implemented: ### 4. Add a dry-run / validation mode (TODO) Add `--dry-run` support that: + - Checks all prerequisites (disk space, memory, network access to GitHub) - Validates Docker is installed and running - Validates Gitea is reachable @@ -157,6 +161,7 @@ Read `docker-compose.ecosystem.yml` (in the repo root) and verify: ### 6. Update `README.md` After all fixes, update `README.md` to reflect: + - CLI flags: `--resume`, `--resume-from=N`, `--phase=N`, `--reset`, `--status`, `--help` - Correct service count: 30 (not 27) - Updated duration estimates if phases changed @@ -269,6 +274,7 @@ Raw Ubuntu 24.04 VM ## How Docker Builds Reach Gitea Product Dockerfiles use BuildKit secret mount for the npm token: + ```dockerfile RUN --mount=type=secret,id=gitea_npm_token \ cp .npmrc.docker .npmrc && \ diff --git a/docs/devops/single_azure_vm/docker/setup.sh b/docs/devops/single_azure_vm/docker/setup.sh index 0a7a9943..3994f310 100755 --- a/docs/devops/single_azure_vm/docker/setup.sh +++ b/docs/devops/single_azure_vm/docker/setup.sh @@ -11,7 +11,7 @@ # - Gitea (Docker container — npm package registry + CI on :3300) # - act_runner (Gitea CI runner — systemd service, host mode) # - Ollama (local LLM inference for LocalMemGPT on :11434) -# - All 11 ByteLyst repos (cloned from GitHub) +# - All 12 ByteLyst repos (cloned from GitHub) # - All @bytelyst/* packages (built + published to Gitea) # - Full 30-service ecosystem (via docker-compose.ecosystem.yml) # @@ -28,7 +28,7 @@ # Phases: # 1 System dependencies (Docker, Node, pnpm, Ollama) # 2 Gitea npm registry + CI runner (container on :3300 + act_runner systemd) -# 3 Clone 11 repositories from GitHub + push to Gitea +# 3 Clone 12 repositories from GitHub + push to Gitea # 4 Build all @bytelyst/* packages # 5 Publish packages to Gitea npm registry # 6 Generate .env.ecosystem config @@ -87,6 +87,7 @@ REPOS=( learning_ai_notes learning_ai_trails learning_ai_local_memory_gpt + learning_ai_local_llms ) # ── Helpers ────────────────────────────────────────────────────────── @@ -471,7 +472,7 @@ phase3_clone() { return fi - log "Phase 3: Cloning ${#REPOS[@]} repositories..." + log "Phase 3: Cloning ${#REPOS[@]} repositories (12 repos)..." local clone_base="https://github.com/${GITHUB_USER}" if [ -n "${GITHUB_TOKEN:-}" ]; then