learning_ai_common_plat/docs/devops/single_azure_vm/setup.sh

#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# ByteLyst Single-VM Bootstrap Script
# ═══════════════════════════════════════════════════════════════════════
# Deploys the ENTIRE ByteLyst ecosystem on a fresh Ubuntu Azure VM.
#
# Usage:  sudo ./setup.sh
# Optional env vars:
#   GITHUB_USER   — GitHub org/user to clone from (default: saravanakumardb1)
#   GITHUB_TOKEN  — If repos are private, set this for HTTPS auth
#   GITEA_ADMIN   — Gitea admin username (default: bytelyst-admin)
#   GITEA_PASS    — Gitea admin password (default: ByteLyst2026!)
#   SKIP_CLONE    — Set to 1 to skip cloning (repos already exist)
#   SKIP_BUILD    — Set to 1 to skip package build+publish
# ═══════════════════════════════════════════════════════════════════════
set -euo pipefail

# ── Configuration ────────────────────────────────────────────────────
INSTALL_DIR="/opt/bytelyst"
GITHUB_USER="${GITHUB_USER:-saravanakumardb1}"
GITEA_ADMIN="${GITEA_ADMIN:-bytelyst-admin}"
GITEA_PASS="${GITEA_PASS:-ByteLyst2026!}"
GITEA_PORT=3300
NODE_VERSION=22
PNPM_VERSION="10.6.5"
COMPOSE_FILE="docker-compose.ecosystem.yml"

# Well-known emulator keys (public, safe to embed)
COSMOS_EMULATOR_KEY="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
AZURITE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="

REPOS=(
  learning_ai_common_plat
  learning_voice_ai_agent
  learning_multimodal_memory_agents
  learning_ai_clock
  learning_ai_jarvis_jr
  learning_ai_fastgap
  learning_ai_peakpulse
  learning_ai_flowmonk
  learning_ai_notes
  learning_ai_trails
  learning_ai_local_memory_gpt
)

# ── Helpers ──────────────────────────────────────────────────────────
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'

log()  { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*"; }
ok()   { echo -e "${GREEN}[$(date +%H:%M:%S)] ✓${NC} $*"; }
warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] ⚠${NC} $*"; }
fail() { echo -e "${RED}[$(date +%H:%M:%S)] ✗${NC} $*"; exit 1; }

wait_for_url() {
  local url="$1" max="${2:-60}" i=0
  while ! curl -sf "$url" > /dev/null 2>&1; do
    sleep 2; i=$((i + 2))
    [ "$i" -ge "$max" ] && fail "Timeout waiting for $url"
  done
}

# Detect the host IP that Docker containers can reach
detect_docker_host_ip() {
  # On Linux, the Docker bridge gateway (172.17.0.1) is reachable from containers
  ip -4 addr show docker0 2>/dev/null | grep -oP '(?<=inet\s)\d+(\.\d+){3}' || echo "172.17.0.1"
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 1: System Dependencies
# ═══════════════════════════════════════════════════════════════════════
phase1_system() {
  log "Phase 1: Installing system dependencies..."

  export DEBIAN_FRONTEND=noninteractive

  # Update package index
  apt-get update -qq

  # Install essentials
  apt-get install -y -qq \
    ca-certificates curl gnupg lsb-release git jq unzip

  # ── Docker ─────────────────────────────────────────────────────────
  if ! command -v docker &>/dev/null; then
    log "Installing Docker..."
    install -m 0755 -d /etc/apt/keyrings
    curl -fsSL https://download.docker.com/linux/ubuntu/gpg \
      | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
    chmod a+r /etc/apt/keyrings/docker.gpg

    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
      https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
      > /etc/apt/sources.list.d/docker.list

    apt-get update -qq
    apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-compose-plugin docker-buildx-plugin
    systemctl enable --now docker
    ok "Docker installed: $(docker --version)"
  else
    ok "Docker already installed: $(docker --version)"
  fi

  # Enable BuildKit globally
  mkdir -p /etc/docker
  cat > /etc/docker/daemon.json <<'DJSON'
{
  "features": { "buildkit": true },
  "log-driver": "json-file",
  "log-opts": { "max-size": "50m", "max-file": "3" }
}
DJSON
  systemctl restart docker

  # ── Node.js ────────────────────────────────────────────────────────
  if ! command -v node &>/dev/null || ! node -v | grep -q "v${NODE_VERSION}"; then
    log "Installing Node.js ${NODE_VERSION}..."
    curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
    apt-get install -y -qq nodejs
    ok "Node.js installed: $(node -v)"
  else
    ok "Node.js already installed: $(node -v)"
  fi

  # ── pnpm ───────────────────────────────────────────────────────────
  if ! command -v pnpm &>/dev/null; then
    log "Installing pnpm ${PNPM_VERSION}..."
    npm install -g "pnpm@${PNPM_VERSION}"
    ok "pnpm installed: $(pnpm -v)"
  else
    ok "pnpm already installed: $(pnpm -v)"
  fi

  # ── Create install directory ───────────────────────────────────────
  mkdir -p "$INSTALL_DIR"

  ok "Phase 1 complete."
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 2: Gitea (npm Registry)
# ═══════════════════════════════════════════════════════════════════════
phase2_gitea() {
  log "Phase 2: Setting up Gitea npm registry on port ${GITEA_PORT}..."

  local GITEA_CONTAINER="gitea-npm-registry"

  # Check if already running
  if docker ps --format '{{.Names}}' | grep -q "^${GITEA_CONTAINER}$"; then
    ok "Gitea already running."
  else
    # Remove stopped container if exists
    docker rm -f "$GITEA_CONTAINER" 2>/dev/null || true

    docker run -d \
      --name "$GITEA_CONTAINER" \
      --restart unless-stopped \
      -p "${GITEA_PORT}:3000" \
      -v gitea-data:/data \
      -e GITEA__server__ROOT_URL="http://localhost:${GITEA_PORT}/" \
      -e GITEA__server__HTTP_PORT=3000 \
      -e GITEA__packages__ENABLED=true \
      -e INSTALL_LOCK=true \
      -e GITEA__security__INSTALL_LOCK=true \
      gitea/gitea:1.22

    ok "Gitea container started."
  fi

  # Wait for Gitea to become ready
  log "Waiting for Gitea to start..."
  wait_for_url "http://localhost:${GITEA_PORT}/api/v1/version" 90

  # ── Create admin user (idempotent) ─────────────────────────────────
  log "Creating Gitea admin user..."
  docker exec "$GITEA_CONTAINER" gitea admin user create \
    --username "$GITEA_ADMIN" \
    --password "$GITEA_PASS" \
    --email "admin@bytelyst.local" \
    --admin \
    --must-change-password=false 2>/dev/null || true

  # ── Create "bytelyst" organization ─────────────────────────────────
  local GITEA_URL="http://localhost:${GITEA_PORT}"
  local AUTH_HEADER="Authorization: Basic $(echo -n "${GITEA_ADMIN}:${GITEA_PASS}" | base64)"

  # Check if org exists
  local org_status
  org_status=$(curl -sf -o /dev/null -w "%{http_code}" \
    -H "$AUTH_HEADER" "${GITEA_URL}/api/v1/orgs/bytelyst")

  if [ "$org_status" != "200" ]; then
    curl -sf -X POST "${GITEA_URL}/api/v1/orgs" \
      -H "$AUTH_HEADER" \
      -H "Content-Type: application/json" \
      -d '{"username":"bytelyst","visibility":"public"}' > /dev/null
    ok "Created 'bytelyst' organization."
  else
    ok "'bytelyst' organization already exists."
  fi

  # ── Create API token ───────────────────────────────────────────────
  # Delete old token if exists, then create fresh
  curl -sf -X DELETE "${GITEA_URL}/api/v1/users/${GITEA_ADMIN}/tokens/vm-deploy" \
    -H "$AUTH_HEADER" > /dev/null 2>&1 || true

  local token_response
  token_response=$(curl -sf -X POST "${GITEA_URL}/api/v1/users/${GITEA_ADMIN}/tokens" \
    -H "$AUTH_HEADER" \
    -H "Content-Type: application/json" \
    -d '{"name":"vm-deploy","scopes":["write:package","read:package","write:organization","read:organization"]}')

  GITEA_NPM_TOKEN=$(echo "$token_response" | jq -r '.sha1')
  if [ -z "$GITEA_NPM_TOKEN" ] || [ "$GITEA_NPM_TOKEN" = "null" ]; then
    fail "Failed to create Gitea API token. Response: $token_response"
  fi

  # Export for later phases
  export GITEA_NPM_TOKEN
  echo "$GITEA_NPM_TOKEN" > "${INSTALL_DIR}/.gitea_token"
  chmod 600 "${INSTALL_DIR}/.gitea_token"

  ok "Phase 2 complete. Gitea running at http://localhost:${GITEA_PORT}"
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 3: Clone Repositories
# ═══════════════════════════════════════════════════════════════════════
phase3_clone() {
  if [ "${SKIP_CLONE:-0}" = "1" ]; then
    warn "Skipping clone (SKIP_CLONE=1)."
    return
  fi

  log "Phase 3: Cloning ${#REPOS[@]} repositories..."

  local clone_base="https://github.com/${GITHUB_USER}"
  if [ -n "${GITHUB_TOKEN:-}" ]; then
    clone_base="https://${GITHUB_TOKEN}@github.com/${GITHUB_USER}"
  fi

  for repo in "${REPOS[@]}"; do
    local target="${INSTALL_DIR}/${repo}"
    if [ -d "$target/.git" ]; then
      log "  Pulling latest: $repo"
      git -C "$target" pull --ff-only 2>/dev/null || true
    else
      log "  Cloning: $repo"
      git clone --depth 1 "$clone_base/${repo}.git" "$target"
    fi
  done

  ok "Phase 3 complete. All repos in ${INSTALL_DIR}/"
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 4: Build @bytelyst/* Packages
# ═══════════════════════════════════════════════════════════════════════
phase4_build() {
  if [ "${SKIP_BUILD:-0}" = "1" ]; then
    warn "Skipping build (SKIP_BUILD=1)."
    return
  fi

  log "Phase 4: Building @bytelyst/* packages..."

  local plat_dir="${INSTALL_DIR}/learning_ai_common_plat"

  # Configure .npmrc for the common-plat workspace (publish target)
  cat > "${plat_dir}/.npmrc" <<NPMRC
@bytelyst:registry=http://localhost:${GITEA_PORT}/api/packages/bytelyst/npm/
//localhost:${GITEA_PORT}/api/packages/bytelyst/npm/:_authToken=${GITEA_NPM_TOKEN}
strict-ssl=false
NPMRC

  cd "$plat_dir"

  # Install workspace deps
  log "  Installing workspace dependencies..."
  pnpm install --frozen-lockfile 2>&1 | tail -3

  # Build all packages
  log "  Building all packages..."
  pnpm -r build 2>&1 | tail -5

  ok "Phase 4 complete. All packages built."
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 5: Publish Packages to Gitea npm Registry
# ═══════════════════════════════════════════════════════════════════════
phase5_publish() {
  if [ "${SKIP_BUILD:-0}" = "1" ]; then
    warn "Skipping publish (SKIP_BUILD=1)."
    return
  fi

  log "Phase 5: Publishing @bytelyst/* packages to Gitea..."

  local plat_dir="${INSTALL_DIR}/learning_ai_common_plat"
  cd "$plat_dir"

  local published=0 skipped=0 failed=0

  # Find all publishable packages (have dist/ and package.json with @bytelyst scope)
  for pkg_dir in packages/*/; do
    local pkg_json="${pkg_dir}package.json"
    [ -f "$pkg_json" ] || continue

    local pkg_name
    pkg_name=$(jq -r '.name // ""' "$pkg_json")

    # Skip non-@bytelyst packages and private packages
    [[ "$pkg_name" == @bytelyst/* ]] || continue
    [ "$(jq -r '.private // false' "$pkg_json")" = "true" ] && continue

    # Skip packages without a build output
    [ -d "${pkg_dir}dist" ] || [ -f "${pkg_dir}dist/index.js" ] || {
      skipped=$((skipped + 1))
      continue
    }

    # Publish (--no-git-checks skips git state validation)
    if (cd "$pkg_dir" && pnpm publish --registry "http://localhost:${GITEA_PORT}/api/packages/bytelyst/npm/" --no-git-checks 2>&1 | grep -q "npm notice"); then
      published=$((published + 1))
    elif (cd "$pkg_dir" && pnpm publish --registry "http://localhost:${GITEA_PORT}/api/packages/bytelyst/npm/" --no-git-checks 2>&1 | grep -q "already exists"); then
      skipped=$((skipped + 1))
    else
      # Try publish anyway — errors for "already published" are OK
      (cd "$pkg_dir" && pnpm publish --registry "http://localhost:${GITEA_PORT}/api/packages/bytelyst/npm/" --no-git-checks 2>/dev/null) || true
      published=$((published + 1))
    fi
  done

  ok "Phase 5 complete. Published: ~${published}, Skipped: ${skipped}"
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 6: Generate .env.ecosystem
# ═══════════════════════════════════════════════════════════════════════
phase6_env() {
  log "Phase 6: Generating .env.ecosystem..."

  local plat_dir="${INSTALL_DIR}/learning_ai_common_plat"
  local env_file="${plat_dir}/.env.ecosystem"

  # Generate a random JWT secret
  local jwt_secret
  jwt_secret=$(openssl rand -base64 32)

  cat > "$env_file" <<ENV
# ── Auto-generated by setup.sh on $(date -Iseconds) ──────────────────

# Cosmos DB Emulator
COSMOS_ENDPOINT=http://cosmos-emulator:8081
COSMOS_KEY=${COSMOS_EMULATOR_KEY}
COSMOS_DATABASE=bytelyst

# Auth
JWT_SECRET=${jwt_secret}
RATE_LIMIT_STORE_MODE=datastore

# Azure Blob Storage (Azurite)
STORAGE_PROVIDER=azure
AZURE_BLOB_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=${AZURITE_KEY};BlobEndpoint=http://azurite:10000/devstoreaccount1;
AZURE_BLOB_ACCOUNT_NAME=devstoreaccount1
AZURE_BLOB_ACCOUNT_KEY=${AZURITE_KEY}
AZURE_BLOB_PUBLIC_ENDPOINT=http://localhost:10000/devstoreaccount1

# Email (Mailpit)
EMAIL_PROVIDER=smtp
EMAIL_FROM_ADDRESS=noreply@bytelyst.local
EMAIL_FROM_NAME=ByteLyst
SMTP_HOST=mailpit
SMTP_PORT=1025
SMTP_SECURE=false
SMTP_USER=
SMTP_PASSWORD=

# Stripe (test placeholders)
STRIPE_SECRET_KEY=sk_test_placeholder
STRIPE_WEBHOOK_SECRET=whsec_placeholder
STRIPE_PRICE_PRO=price_placeholder
STRIPE_PRICE_ENTERPRISE=price_placeholder

# Extraction Service
PYTHON_SIDECAR_URL=http://localhost:4006
DEFAULT_MODEL_ID=gemini-2.5-flash
GEMINI_API_KEY=placeholder
EXTRACTION_QUEUE_BACKEND=file
EXTRACTION_QUEUE_FILE=.data/extraction-jobs.json

# Cross-service URLs
PLATFORM_SERVICE_URL=http://platform-service:4003
EXTRACTION_SERVICE_URL=http://extraction-service:4005
MCP_SERVER_URL=http://mcp-server:4007

# Datastore
DB_PROVIDER=cosmos

# Telemetry
TELEMETRY_ENABLED=true

# Event Bus
EVENT_BUS_BACKEND=file
EVENT_BUS_FILE=.data/platform-events.json

# Field Encryption
FIELD_ENCRYPT_KEY_PROVIDER=memory

# Product Identity
DEFAULT_PRODUCT_ID=lysnrai

# Webhooks (disabled)
WEBHOOK_INVITATION_REDEEMED_URL=
WEBHOOK_REFERRAL_STATUS_URL=
WEBHOOK_WAITLIST_JOINED_URL=

# Notifications (disabled)
TELEGRAM_BOT_TOKEN=
TELEGRAM_DEFAULT_CHAT_ID=
SLACK_WEBHOOK_URL=
SLACK_DEFAULT_CHANNEL=
ENV

  ok "Phase 6 complete. Generated ${env_file}"
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 7: Deploy Ecosystem via Docker Compose
# ═══════════════════════════════════════════════════════════════════════
phase7_deploy() {
  log "Phase 7: Deploying 27-service ecosystem..."

  local plat_dir="${INSTALL_DIR}/learning_ai_common_plat"
  cd "$plat_dir"

  # Detect host IP for Docker builds to reach Gitea
  local docker_host_ip
  docker_host_ip=$(detect_docker_host_ip)
  log "  Docker host IP for Gitea access: ${docker_host_ip}"

  # Export vars needed by compose
  export GITEA_NPM_TOKEN
  export GITEA_NPM_HOST="${docker_host_ip}"
  export DOCKER_BUILDKIT=1
  export COMPOSE_DOCKER_CLI_BUILD=1

  # Build and start all services
  log "  Building and starting services (this takes ~10-15 minutes)..."
  docker compose \
    -f "$COMPOSE_FILE" \
    --env-file .env.ecosystem \
    up --build -d 2>&1 | tail -20

  ok "Phase 7 complete. All containers started."
}

# ═══════════════════════════════════════════════════════════════════════
# PHASE 8: Health Check
# ═══════════════════════════════════════════════════════════════════════
phase8_verify() {
  log "Phase 8: Verifying service health..."

  local plat_dir="${INSTALL_DIR}/learning_ai_common_plat"

  # Wait for platform-service (everything else depends on it)
  log "  Waiting for platform-service..."
  wait_for_url "http://localhost:4003/health" 120

  # Create a reusable health-check script
  cat > "${INSTALL_DIR}/check-health.sh" <<'HEALTH'
#!/usr/bin/env bash
RED='\033[0;31m'; GREEN='\033[0;32m'; NC='\033[0m'

check() {
  local name="$1" url="$2"
  if curl -sf "$url" > /dev/null 2>&1; then
    echo -e "${GREEN}  ✓ ${name}${NC}  ${url}"
  else
    echo -e "${RED}  ✗ ${name}${NC}  ${url}"
  fi
}

echo ""
echo "═══ Infrastructure ═══"
check "Gitea (npm)"          "http://localhost:3300/api/v1/version"
check "Cosmos Explorer"      "http://localhost:1234"
check "Mailpit"              "http://localhost:8025"
check "Grafana"              "http://localhost:3000/api/health"
check "Traefik"              "http://localhost:8080/api/overview"

echo ""
echo "═══ Platform Services ═══"
check "platform-service"     "http://localhost:4003/health"
check "extraction-service"   "http://localhost:4005/health"
check "mcp-server"           "http://localhost:4007/health"

echo ""
echo "═══ Dashboards ═══"
check "admin-web"            "http://localhost:3001"
check "tracker-web"          "http://localhost:3003"

echo ""
echo "═══ Product Backends ═══"
check "peakpulse"            "http://localhost:4010/health"
check "chronomind"           "http://localhost:4011/health"
check "jarvisjr"             "http://localhost:4012/health"
check "nomgap"               "http://localhost:4013/health"
check "mindlyst"             "http://localhost:4014/health"
check "lysnrai"              "http://localhost:4015/health"
check "notelett"             "http://localhost:4016/health"
check "flowmonk"             "http://localhost:4017/health"
check "actiontrail"          "http://localhost:4018/health"
check "localmemgpt"          "http://localhost:4019/health"

echo ""
echo "═══ Product Web Apps ═══"
check "lysnrai-dashboard"    "http://localhost:3002"
check "chronomind-web"       "http://localhost:3030"
check "jarvisjr-web"         "http://localhost:3035"
check "flowmonk-web"         "http://localhost:3040"
check "notelett-web"         "http://localhost:3045"
check "mindlyst-web"         "http://localhost:3050"
check "nomgap-web"           "http://localhost:3055"
check "actiontrail-web"      "http://localhost:3060"
check "localmemgpt-web"      "http://localhost:3070"
echo ""
HEALTH
  chmod +x "${INSTALL_DIR}/check-health.sh"

  # Give services a moment to start, then run health check
  log "  Waiting 30s for services to stabilize..."
  sleep 30

  # Run the health check
  bash "${INSTALL_DIR}/check-health.sh"

  ok "Phase 8 complete."
}

# ═══════════════════════════════════════════════════════════════════════
# MAIN
# ═══════════════════════════════════════════════════════════════════════
main() {
  echo ""
  echo "╔═══════════════════════════════════════════════════════════════╗"
  echo "║         ByteLyst Single-VM Deployment                        ║"
  echo "║         27 services · 10 products · 1 VM                     ║"
  echo "╚═══════════════════════════════════════════════════════════════╝"
  echo ""

  [ "$(id -u)" -eq 0 ] || fail "This script must be run as root (sudo)."

  local start_time
  start_time=$(date +%s)

  phase1_system
  phase2_gitea
  phase3_clone
  phase4_build
  phase5_publish
  phase6_env
  phase7_deploy
  phase8_verify

  local elapsed=$(( $(date +%s) - start_time ))
  local minutes=$(( elapsed / 60 ))
  local seconds=$(( elapsed % 60 ))

  echo ""
  echo "╔═══════════════════════════════════════════════════════════════╗"
  echo "║  Deployment complete in ${minutes}m ${seconds}s                            ║"
  echo "║                                                               ║"
  echo "║  Health check:  /opt/bytelyst/check-health.sh                ║"
  echo "║  Compose logs:  docker compose -f ${COMPOSE_FILE} logs -f    ║"
  echo "║  Gitea UI:      http://localhost:3300                        ║"
  echo "║  Grafana:       http://localhost:3000  (admin / bytelyst)    ║"
  echo "║  Mailpit:       http://localhost:8025                        ║"
  echo "╚═══════════════════════════════════════════════════════════════╝"
  echo ""
}

main "$@"