diff --git a/docs/devops/single_azure_vm/README.md b/docs/devops/single_azure_vm/README.md new file mode 100644 index 00000000..ac2c7b9d --- /dev/null +++ b/docs/devops/single_azure_vm/README.md @@ -0,0 +1,125 @@ +# ByteLyst Single-VM Deployment + +> Deploy the **entire ByteLyst ecosystem** on a single Azure VM from scratch. +> Two files: this README and `setup.sh`. Copy both to the VM and run the script. + +--- + +## Prerequisites + +- **Azure VM:** Ubuntu 24.04 LTS (or 22.04), Standard_D8s_v5 (8 vCPU, 32 GB RAM) recommended +- **Disk:** 100 GB+ (Docker images, Cosmos emulator, build artifacts) +- **Network:** NSG allowing inbound on ports 80, 3000-3100, 4003-4019, 8025, 8080, 8081 +- **GitHub access:** Repos must be accessible (public or deploy key configured) + +## Quick Start + +```bash +# 1. SSH into your Azure VM +ssh azureuser@ + +# 2. Copy setup.sh and make executable +chmod +x setup.sh + +# 3. Run — provide your GitHub username (repos are cloned via HTTPS) +# If repos are private, also export GITHUB_TOKEN first. +sudo ./setup.sh + +# 4. Wait ~15-25 minutes for full build + deploy + +# 5. Verify +docker compose -f /opt/bytelyst/learning_ai_common_plat/docker-compose.ecosystem.yml ps +``` + +## What the Script Does + +| Phase | Duration | Description | +|-------|----------|-------------| +| 1. System | ~2 min | Install Docker, Node.js 22, pnpm 10.6.5, git | +| 2. Gitea | ~1 min | Start Gitea Docker container (npm registry on :3300) | +| 3. Clone | ~3 min | Clone all 11 repos to `/opt/bytelyst/` | +| 4. Build | ~5 min | Build all `@bytelyst/*` packages in common-plat | +| 5. Publish | ~3 min | Publish all packages to local Gitea npm registry | +| 6. Env | instant | Generate `.env.ecosystem` with all required values | +| 7. Deploy | ~10 min | `docker compose up --build` for 27 services | +| 8. Verify | ~1 min | Health-check all services | + +## Port Map (after deployment) + +### Infrastructure +| Service | Port | URL | +|---------|------|-----| +| Gitea (npm registry) | 3300 | `http://:3300` | +| Cosmos Data Explorer | 1234 | `http://:1234` | +| Azurite (Blob) | 10000 | — | +| Mailpit UI | 8025 | `http://:8025` | +| Grafana | 3000 | `http://:3000` | +| Traefik Dashboard | 8080 | `http://:8080` | + +### Platform Services +| Service | Port | URL | +|---------|------|-----| +| platform-service | 4003 | `http://:4003/health` | +| extraction-service | 4005 | `http://:4005/health` | +| mcp-server | 4007 | `http://:4007/health` | + +### Platform Dashboards +| Dashboard | Port | URL | +|-----------|------|-----| +| Admin Console | 3001 | `http://:3001` | +| Issue Tracker | 3003 | `http://:3003` | + +### Product Backends +| Product | Port | Health | +|---------|------|--------| +| PeakPulse | 4010 | `http://:4010/health` | +| ChronoMind | 4011 | `http://:4011/health` | +| JarvisJr | 4012 | `http://:4012/health` | +| NomGap | 4013 | `http://:4013/health` | +| MindLyst | 4014 | `http://:4014/health` | +| LysnrAI | 4015 | `http://:4015/health` | +| NoteLett | 4016 | `http://:4016/health` | +| FlowMonk | 4017 | `http://:4017/health` | +| ActionTrail | 4018 | `http://:4018/health` | +| LocalMemGPT | 4019 | `http://:4019/health` | + +### Product Web Apps +| Product | Port | URL | +|---------|------|-----| +| LysnrAI Dashboard | 3002 | `http://:3002` | +| ChronoMind | 3030 | `http://:3030` | +| JarvisJr | 3035 | `http://:3035` | +| FlowMonk | 3040 | `http://:3040` | +| NoteLett | 3045 | `http://:3045` | +| MindLyst | 3050 | `http://:3050` | +| NomGap | 3055 | `http://:3055` | +| ActionTrail | 3060 | `http://:3060` | +| LocalMemGPT | 3070 | `http://:3070` | + +## Post-Deployment Commands + +```bash +# Check all service health +/opt/bytelyst/check-health.sh + +# View logs for a specific service +docker compose -f /opt/bytelyst/learning_ai_common_plat/docker-compose.ecosystem.yml \ + logs -f platform-service + +# Restart a specific service +docker compose -f /opt/bytelyst/learning_ai_common_plat/docker-compose.ecosystem.yml \ + restart flowmonk-backend + +# Stop everything +docker compose -f /opt/bytelyst/learning_ai_common_plat/docker-compose.ecosystem.yml down + +# Stop and wipe all data +docker compose -f /opt/bytelyst/learning_ai_common_plat/docker-compose.ecosystem.yml down -v +``` + +## Troubleshooting + +- **Cosmos emulator slow:** It needs 20-30s on first boot. Services wait via health checks. +- **Out of memory:** Use at least 32 GB RAM. The Cosmos emulator alone needs ~4 GB. +- **Build failures:** Check that Gitea is running (`docker ps | grep gitea`) and packages published (`curl http://localhost:3300/api/packages/bytelyst/npm/`). +- **Port conflicts:** Ensure nothing else runs on the listed ports before deploying. diff --git a/docs/devops/single_azure_vm/setup.sh b/docs/devops/single_azure_vm/setup.sh new file mode 100755 index 00000000..f4fc3333 --- /dev/null +++ b/docs/devops/single_azure_vm/setup.sh @@ -0,0 +1,583 @@ +#!/usr/bin/env bash +# ═══════════════════════════════════════════════════════════════════════ +# ByteLyst Single-VM Bootstrap Script +# ═══════════════════════════════════════════════════════════════════════ +# Deploys the ENTIRE ByteLyst ecosystem on a fresh Ubuntu Azure VM. +# +# Usage: sudo ./setup.sh +# Optional env vars: +# GITHUB_USER — GitHub org/user to clone from (default: saravanakumardb1) +# GITHUB_TOKEN — If repos are private, set this for HTTPS auth +# GITEA_ADMIN — Gitea admin username (default: bytelyst-admin) +# GITEA_PASS — Gitea admin password (default: ByteLyst2026!) +# SKIP_CLONE — Set to 1 to skip cloning (repos already exist) +# SKIP_BUILD — Set to 1 to skip package build+publish +# ═══════════════════════════════════════════════════════════════════════ +set -euo pipefail + +# ── Configuration ──────────────────────────────────────────────────── +INSTALL_DIR="/opt/bytelyst" +GITHUB_USER="${GITHUB_USER:-saravanakumardb1}" +GITEA_ADMIN="${GITEA_ADMIN:-bytelyst-admin}" +GITEA_PASS="${GITEA_PASS:-ByteLyst2026!}" +GITEA_PORT=3300 +NODE_VERSION=22 +PNPM_VERSION="10.6.5" +COMPOSE_FILE="docker-compose.ecosystem.yml" + +# Well-known emulator keys (public, safe to embed) +COSMOS_EMULATOR_KEY="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" +AZURITE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + +REPOS=( + learning_ai_common_plat + learning_voice_ai_agent + learning_multimodal_memory_agents + learning_ai_clock + learning_ai_jarvis_jr + learning_ai_fastgap + learning_ai_peakpulse + learning_ai_flowmonk + learning_ai_notes + learning_ai_trails + learning_ai_local_memory_gpt +) + +# ── Helpers ────────────────────────────────────────────────────────── +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' + +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*"; } +ok() { echo -e "${GREEN}[$(date +%H:%M:%S)] ✓${NC} $*"; } +warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] ⚠${NC} $*"; } +fail() { echo -e "${RED}[$(date +%H:%M:%S)] ✗${NC} $*"; exit 1; } + +wait_for_url() { + local url="$1" max="${2:-60}" i=0 + while ! curl -sf "$url" > /dev/null 2>&1; do + sleep 2; i=$((i + 2)) + [ "$i" -ge "$max" ] && fail "Timeout waiting for $url" + done +} + +# Detect the host IP that Docker containers can reach +detect_docker_host_ip() { + # On Linux, the Docker bridge gateway (172.17.0.1) is reachable from containers + ip -4 addr show docker0 2>/dev/null | grep -oP '(?<=inet\s)\d+(\.\d+){3}' || echo "172.17.0.1" +} + +# ═══════════════════════════════════════════════════════════════════════ +# PHASE 1: System Dependencies +# ═══════════════════════════════════════════════════════════════════════ +phase1_system() { + log "Phase 1: Installing system dependencies..." + + export DEBIAN_FRONTEND=noninteractive + + # Update package index + apt-get update -qq + + # Install essentials + apt-get install -y -qq \ + ca-certificates curl gnupg lsb-release git jq unzip + + # ── Docker ───────────────────────────────────────────────────────── + if ! command -v docker &>/dev/null; then + log "Installing Docker..." + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + > /etc/apt/sources.list.d/docker.list + + apt-get update -qq + apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-compose-plugin docker-buildx-plugin + systemctl enable --now docker + ok "Docker installed: $(docker --version)" + else + ok "Docker already installed: $(docker --version)" + fi + + # Enable BuildKit globally + mkdir -p /etc/docker + cat > /etc/docker/daemon.json <<'DJSON' +{ + "features": { "buildkit": true }, + "log-driver": "json-file", + "log-opts": { "max-size": "50m", "max-file": "3" } +} +DJSON + systemctl restart docker + + # ── Node.js ──────────────────────────────────────────────────────── + if ! command -v node &>/dev/null || ! node -v | grep -q "v${NODE_VERSION}"; then + log "Installing Node.js ${NODE_VERSION}..." + curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - + apt-get install -y -qq nodejs + ok "Node.js installed: $(node -v)" + else + ok "Node.js already installed: $(node -v)" + fi + + # ── pnpm ─────────────────────────────────────────────────────────── + if ! command -v pnpm &>/dev/null; then + log "Installing pnpm ${PNPM_VERSION}..." + npm install -g "pnpm@${PNPM_VERSION}" + ok "pnpm installed: $(pnpm -v)" + else + ok "pnpm already installed: $(pnpm -v)" + fi + + # ── Create install directory ─────────────────────────────────────── + mkdir -p "$INSTALL_DIR" + + ok "Phase 1 complete." +} + +# ═══════════════════════════════════════════════════════════════════════ +# PHASE 2: Gitea (npm Registry) +# ═══════════════════════════════════════════════════════════════════════ +phase2_gitea() { + log "Phase 2: Setting up Gitea npm registry on port ${GITEA_PORT}..." + + local GITEA_CONTAINER="gitea-npm-registry" + + # Check if already running + if docker ps --format '{{.Names}}' | grep -q "^${GITEA_CONTAINER}$"; then + ok "Gitea already running." + else + # Remove stopped container if exists + docker rm -f "$GITEA_CONTAINER" 2>/dev/null || true + + docker run -d \ + --name "$GITEA_CONTAINER" \ + --restart unless-stopped \ + -p "${GITEA_PORT}:3000" \ + -v gitea-data:/data \ + -e GITEA__server__ROOT_URL="http://localhost:${GITEA_PORT}/" \ + -e GITEA__server__HTTP_PORT=3000 \ + -e GITEA__packages__ENABLED=true \ + -e INSTALL_LOCK=true \ + -e GITEA__security__INSTALL_LOCK=true \ + gitea/gitea:1.22 + + ok "Gitea container started." + fi + + # Wait for Gitea to become ready + log "Waiting for Gitea to start..." + wait_for_url "http://localhost:${GITEA_PORT}/api/v1/version" 90 + + # ── Create admin user (idempotent) ───────────────────────────────── + log "Creating Gitea admin user..." + docker exec "$GITEA_CONTAINER" gitea admin user create \ + --username "$GITEA_ADMIN" \ + --password "$GITEA_PASS" \ + --email "admin@bytelyst.local" \ + --admin \ + --must-change-password=false 2>/dev/null || true + + # ── Create "bytelyst" organization ───────────────────────────────── + local GITEA_URL="http://localhost:${GITEA_PORT}" + local AUTH_HEADER="Authorization: Basic $(echo -n "${GITEA_ADMIN}:${GITEA_PASS}" | base64)" + + # Check if org exists + local org_status + org_status=$(curl -sf -o /dev/null -w "%{http_code}" \ + -H "$AUTH_HEADER" "${GITEA_URL}/api/v1/orgs/bytelyst") + + if [ "$org_status" != "200" ]; then + curl -sf -X POST "${GITEA_URL}/api/v1/orgs" \ + -H "$AUTH_HEADER" \ + -H "Content-Type: application/json" \ + -d '{"username":"bytelyst","visibility":"public"}' > /dev/null + ok "Created 'bytelyst' organization." + else + ok "'bytelyst' organization already exists." + fi + + # ── Create API token ─────────────────────────────────────────────── + # Delete old token if exists, then create fresh + curl -sf -X DELETE "${GITEA_URL}/api/v1/users/${GITEA_ADMIN}/tokens/vm-deploy" \ + -H "$AUTH_HEADER" > /dev/null 2>&1 || true + + local token_response + token_response=$(curl -sf -X POST "${GITEA_URL}/api/v1/users/${GITEA_ADMIN}/tokens" \ + -H "$AUTH_HEADER" \ + -H "Content-Type: application/json" \ + -d '{"name":"vm-deploy","scopes":["write:package","read:package","write:organization","read:organization"]}') + + GITEA_NPM_TOKEN=$(echo "$token_response" | jq -r '.sha1') + if [ -z "$GITEA_NPM_TOKEN" ] || [ "$GITEA_NPM_TOKEN" = "null" ]; then + fail "Failed to create Gitea API token. Response: $token_response" + fi + + # Export for later phases + export GITEA_NPM_TOKEN + echo "$GITEA_NPM_TOKEN" > "${INSTALL_DIR}/.gitea_token" + chmod 600 "${INSTALL_DIR}/.gitea_token" + + ok "Phase 2 complete. Gitea running at http://localhost:${GITEA_PORT}" +} + +# ═══════════════════════════════════════════════════════════════════════ +# PHASE 3: Clone Repositories +# ═══════════════════════════════════════════════════════════════════════ +phase3_clone() { + if [ "${SKIP_CLONE:-0}" = "1" ]; then + warn "Skipping clone (SKIP_CLONE=1)." + return + fi + + log "Phase 3: Cloning ${#REPOS[@]} repositories..." + + local clone_base="https://github.com/${GITHUB_USER}" + if [ -n "${GITHUB_TOKEN:-}" ]; then + clone_base="https://${GITHUB_TOKEN}@github.com/${GITHUB_USER}" + fi + + for repo in "${REPOS[@]}"; do + local target="${INSTALL_DIR}/${repo}" + if [ -d "$target/.git" ]; then + log " Pulling latest: $repo" + git -C "$target" pull --ff-only 2>/dev/null || true + else + log " Cloning: $repo" + git clone --depth 1 "$clone_base/${repo}.git" "$target" + fi + done + + ok "Phase 3 complete. All repos in ${INSTALL_DIR}/" +} + +# ═══════════════════════════════════════════════════════════════════════ +# PHASE 4: Build @bytelyst/* Packages +# ═══════════════════════════════════════════════════════════════════════ +phase4_build() { + if [ "${SKIP_BUILD:-0}" = "1" ]; then + warn "Skipping build (SKIP_BUILD=1)." + return + fi + + log "Phase 4: Building @bytelyst/* packages..." + + local plat_dir="${INSTALL_DIR}/learning_ai_common_plat" + + # Configure .npmrc for the common-plat workspace (publish target) + cat > "${plat_dir}/.npmrc" <&1 | tail -3 + + # Build all packages + log " Building all packages..." + pnpm -r build 2>&1 | tail -5 + + ok "Phase 4 complete. All packages built." +} + +# ═══════════════════════════════════════════════════════════════════════ +# PHASE 5: Publish Packages to Gitea npm Registry +# ═══════════════════════════════════════════════════════════════════════ +phase5_publish() { + if [ "${SKIP_BUILD:-0}" = "1" ]; then + warn "Skipping publish (SKIP_BUILD=1)." + return + fi + + log "Phase 5: Publishing @bytelyst/* packages to Gitea..." + + local plat_dir="${INSTALL_DIR}/learning_ai_common_plat" + cd "$plat_dir" + + local published=0 skipped=0 failed=0 + + # Find all publishable packages (have dist/ and package.json with @bytelyst scope) + for pkg_dir in packages/*/; do + local pkg_json="${pkg_dir}package.json" + [ -f "$pkg_json" ] || continue + + local pkg_name + pkg_name=$(jq -r '.name // ""' "$pkg_json") + + # Skip non-@bytelyst packages and private packages + [[ "$pkg_name" == @bytelyst/* ]] || continue + [ "$(jq -r '.private // false' "$pkg_json")" = "true" ] && continue + + # Skip packages without a build output + [ -d "${pkg_dir}dist" ] || [ -f "${pkg_dir}dist/index.js" ] || { + skipped=$((skipped + 1)) + continue + } + + # Publish (--no-git-checks skips git state validation) + if (cd "$pkg_dir" && pnpm publish --registry "http://localhost:${GITEA_PORT}/api/packages/bytelyst/npm/" --no-git-checks 2>&1 | grep -q "npm notice"); then + published=$((published + 1)) + elif (cd "$pkg_dir" && pnpm publish --registry "http://localhost:${GITEA_PORT}/api/packages/bytelyst/npm/" --no-git-checks 2>&1 | grep -q "already exists"); then + skipped=$((skipped + 1)) + else + # Try publish anyway — errors for "already published" are OK + (cd "$pkg_dir" && pnpm publish --registry "http://localhost:${GITEA_PORT}/api/packages/bytelyst/npm/" --no-git-checks 2>/dev/null) || true + published=$((published + 1)) + fi + done + + ok "Phase 5 complete. Published: ~${published}, Skipped: ${skipped}" +} + +# ═══════════════════════════════════════════════════════════════════════ +# PHASE 6: Generate .env.ecosystem +# ═══════════════════════════════════════════════════════════════════════ +phase6_env() { + log "Phase 6: Generating .env.ecosystem..." + + local plat_dir="${INSTALL_DIR}/learning_ai_common_plat" + local env_file="${plat_dir}/.env.ecosystem" + + # Generate a random JWT secret + local jwt_secret + jwt_secret=$(openssl rand -base64 32) + + cat > "$env_file" <&1 | tail -20 + + ok "Phase 7 complete. All containers started." +} + +# ═══════════════════════════════════════════════════════════════════════ +# PHASE 8: Health Check +# ═══════════════════════════════════════════════════════════════════════ +phase8_verify() { + log "Phase 8: Verifying service health..." + + local plat_dir="${INSTALL_DIR}/learning_ai_common_plat" + + # Wait for platform-service (everything else depends on it) + log " Waiting for platform-service..." + wait_for_url "http://localhost:4003/health" 120 + + # Create a reusable health-check script + cat > "${INSTALL_DIR}/check-health.sh" <<'HEALTH' +#!/usr/bin/env bash +RED='\033[0;31m'; GREEN='\033[0;32m'; NC='\033[0m' + +check() { + local name="$1" url="$2" + if curl -sf "$url" > /dev/null 2>&1; then + echo -e "${GREEN} ✓ ${name}${NC} ${url}" + else + echo -e "${RED} ✗ ${name}${NC} ${url}" + fi +} + +echo "" +echo "═══ Infrastructure ═══" +check "Gitea (npm)" "http://localhost:3300/api/v1/version" +check "Cosmos Explorer" "http://localhost:1234" +check "Mailpit" "http://localhost:8025" +check "Grafana" "http://localhost:3000/api/health" +check "Traefik" "http://localhost:8080/api/overview" + +echo "" +echo "═══ Platform Services ═══" +check "platform-service" "http://localhost:4003/health" +check "extraction-service" "http://localhost:4005/health" +check "mcp-server" "http://localhost:4007/health" + +echo "" +echo "═══ Dashboards ═══" +check "admin-web" "http://localhost:3001" +check "tracker-web" "http://localhost:3003" + +echo "" +echo "═══ Product Backends ═══" +check "peakpulse" "http://localhost:4010/health" +check "chronomind" "http://localhost:4011/health" +check "jarvisjr" "http://localhost:4012/health" +check "nomgap" "http://localhost:4013/health" +check "mindlyst" "http://localhost:4014/health" +check "lysnrai" "http://localhost:4015/health" +check "notelett" "http://localhost:4016/health" +check "flowmonk" "http://localhost:4017/health" +check "actiontrail" "http://localhost:4018/health" +check "localmemgpt" "http://localhost:4019/health" + +echo "" +echo "═══ Product Web Apps ═══" +check "lysnrai-dashboard" "http://localhost:3002" +check "chronomind-web" "http://localhost:3030" +check "jarvisjr-web" "http://localhost:3035" +check "flowmonk-web" "http://localhost:3040" +check "notelett-web" "http://localhost:3045" +check "mindlyst-web" "http://localhost:3050" +check "nomgap-web" "http://localhost:3055" +check "actiontrail-web" "http://localhost:3060" +check "localmemgpt-web" "http://localhost:3070" +echo "" +HEALTH + chmod +x "${INSTALL_DIR}/check-health.sh" + + # Give services a moment to start, then run health check + log " Waiting 30s for services to stabilize..." + sleep 30 + + # Run the health check + bash "${INSTALL_DIR}/check-health.sh" + + ok "Phase 8 complete." +} + +# ═══════════════════════════════════════════════════════════════════════ +# MAIN +# ═══════════════════════════════════════════════════════════════════════ +main() { + echo "" + echo "╔═══════════════════════════════════════════════════════════════╗" + echo "║ ByteLyst Single-VM Deployment ║" + echo "║ 27 services · 10 products · 1 VM ║" + echo "╚═══════════════════════════════════════════════════════════════╝" + echo "" + + [ "$(id -u)" -eq 0 ] || fail "This script must be run as root (sudo)." + + local start_time + start_time=$(date +%s) + + phase1_system + phase2_gitea + phase3_clone + phase4_build + phase5_publish + phase6_env + phase7_deploy + phase8_verify + + local elapsed=$(( $(date +%s) - start_time )) + local minutes=$(( elapsed / 60 )) + local seconds=$(( elapsed % 60 )) + + echo "" + echo "╔═══════════════════════════════════════════════════════════════╗" + echo "║ Deployment complete in ${minutes}m ${seconds}s ║" + echo "║ ║" + echo "║ Health check: /opt/bytelyst/check-health.sh ║" + echo "║ Compose logs: docker compose -f ${COMPOSE_FILE} logs -f ║" + echo "║ Gitea UI: http://localhost:3300 ║" + echo "║ Grafana: http://localhost:3000 (admin / bytelyst) ║" + echo "║ Mailpit: http://localhost:8025 ║" + echo "╚═══════════════════════════════════════════════════════════════╝" + echo "" +} + +main "$@"