#!/usr/bin/env bash # Optimized docker-prep.sh with 6 optimizations: # 1. Git-based incremental builds (only rebuild changed packages) # 2. Hash-based caching (content-addressable cache) # 3. Parallel builds (concurrent package building) # 4. Persistent tarball cache (survives git clean) # 5. Smart manifest tracking (track what's been built) # 6. Docker BuildKit integration hints # # Usage: # ./scripts/docker-prep-optimized.sh # pack tarballs + rewrite package.json # ./scripts/docker-prep-optimized.sh --restore # undo rewrite # ./scripts/docker-prep-optimized.sh --clean # clear cache # ./scripts/docker-prep-optimized.sh --force # force full rebuild set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" COMMON_PLAT="${COMMON_PLAT:-${REPO_DIR}/../learning_ai/learning_ai_common_plat}" if [[ ! -d "$COMMON_PLAT" && -d "${REPO_DIR}/../learning_ai_common_plat" ]]; then COMMON_PLAT="${REPO_DIR}/../learning_ai_common_plat" fi TARBALL_DIR="${REPO_DIR}/.docker-deps" # Shared cache location - can be overridden with BYTELYST_CACHE_DIR env var CACHE_DIR="${BYTELYST_CACHE_DIR:-${HOME}/.cache/bytelyst-packages}" MANIFEST_FILE="${CACHE_DIR}/.manifest" # ── Parse arguments ─────────────────────────────────────────────────── MODE="pack" FORCE_REBUILD=false while [[ $# -gt 0 ]]; do case "$1" in --restore) MODE="restore"; shift ;; --clean) MODE="clean"; shift ;; --force) FORCE_REBUILD=true; shift ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac done # ── Restore mode ───────────────────────────────────────────────────── if [[ "$MODE" == "restore" ]]; then echo "Restoring original package.json files..." for bak in $(find "$REPO_DIR" -name "package.json.bak" -not -path "*/node_modules/*"); do mv "$bak" "${bak%.bak}" echo " Restored ${bak%.bak}" done rm -rf "$TARBALL_DIR" echo "Done." exit 0 fi # ── Clean mode ─────────────────────────────────────────────────────── if [[ "$MODE" == "clean" ]]; then echo "Cleaning cache and tarball directory..." rm -rf "$CACHE_DIR" "$TARBALL_DIR" echo "Done." exit 0 fi # ── Validation ─────────────────────────────────────────────────────── if [[ ! -d "$COMMON_PLAT" ]]; then echo "Common platform checkout not found: $COMMON_PLAT" >&2 echo "Set COMMON_PLAT=/path/to/learning_ai_common_plat or place it at ../learning_ai/learning_ai_common_plat." >&2 exit 1 fi # ── Initialize directories ─────────────────────────────────────────── mkdir -p "$TARBALL_DIR" "$CACHE_DIR" # ── Load manifest if exists ─────────────────────────────────────────── # Simple file-based manifest to avoid bash associative array issues # ── Determine which packages to rebuild ─────────────────────────────── if [[ "$FORCE_REBUILD" == true ]]; then echo "Force rebuild: building all packages" BUILD_ALL=true else BUILD_ALL=false # Git-based incremental build: check which packages changed cd "$COMMON_PLAT" if git rev-parse --git-dir > /dev/null 2>&1; then # Get packages changed since last commit CHANGED_PACKAGES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null | grep '^packages/' | cut -d'/' -f2 | sort -u || echo "") # If no git history or no changes, check against manifest if [[ -z "$CHANGED_PACKAGES" ]] && [[ -f "$MANIFEST_FILE" ]]; then echo "No git changes detected, checking cache validity..." CHANGED_PACKAGES="" for pkg_dir in "$COMMON_PLAT"/packages/*/; do pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) if [[ -z "$pkg_name" ]]; then continue; fi # Hash package.json + source files HASH=$(find "$pkg_dir" -name "*.ts" -o -name "*.json" -o -name "*.js" 2>/dev/null | xargs cat 2>/dev/null | sha256sum | cut -d' ' -f1 || echo "") # Check manifest for existing hash CACHED_HASH=$(grep "^${pkg_name}=" "$MANIFEST_FILE" 2>/dev/null | cut -d'=' -f2 || echo "") if [[ -n "$HASH" ]] && [[ "$CACHED_HASH" != "$HASH" ]]; then pkg_basename=$(basename "$pkg_dir") CHANGED_PACKAGES="$CHANGED_PACKAGES $pkg_basename" fi done fi # If still no changes detected but cache is empty, build all if [[ -z "$CHANGED_PACKAGES" ]] && [[ -z "$(ls -A $CACHE_DIR 2>/dev/null)" ]]; then echo "Cache empty, building all packages..." BUILD_ALL=true fi else echo "Not a git repo or no history, building all packages..." BUILD_ALL=true fi fi cd "$REPO_DIR" # ── Build packages with caching ──────────────────────────────────────── echo "=== docker-prep-optimized: packing @bytelyst/* tarballs ===" rm -rf "$TARBALL_DIR" mkdir -p "$TARBALL_DIR" # Function to build a single package build_package() { local pkg_dir=$1 local force_build=${2:-false} if [[ ! -d "$pkg_dir" ]]; then echo " ✗ Package directory not found: $pkg_dir" >&2 return 1 fi pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) if [[ -z "$pkg_name" ]]; then echo " ✗ Could not read package name from: $pkg_dir" >&2 return 1 fi pkg_version=$(node -p "require('${pkg_dir}package.json').version" 2>/dev/null || echo "0.0.0") # Hash package.json + source files HASH=$(find "$pkg_dir" -name "*.ts" -o -name "*.json" -o -name "*.js" 2>/dev/null | xargs cat 2>/dev/null | sha256sum | cut -d' ' -f1 || echo "unknown") # Sanitize package name for filename (replace @ and / with -) SAFE_PKG_NAME=$(echo "$pkg_name" | sed 's/@//g' | sed 's/\//_/g') CACHE_FILE="$CACHE_DIR/${SAFE_PKG_NAME}-${pkg_version}.tgz" # Check cache if [[ -f "$CACHE_FILE" ]] && [[ "$force_build" != true ]] && [[ "$FORCE_REBUILD" != true ]]; then echo " ✓ Cache hit: $pkg_name" cp "$CACHE_FILE" "$TARBALL_DIR/" echo "${pkg_name}=${HASH}" >> "$TARBALL_DIR/.manifest.tmp" return 0 fi # Build package echo " → Building: $pkg_name" (cd "$pkg_dir" && pnpm build > /dev/null 2>&1) # Pack to cache and tarball dir tarball=$(cd "$pkg_dir" && pnpm pack --pack-destination "$TARBALL_DIR" 2>/dev/null | tail -1) if [[ -z "$tarball" ]]; then echo " ✗ Failed to pack: $pkg_name" >&2 return 1 fi # Copy to cache cp "$TARBALL_DIR/$(basename $tarball)" "$CACHE_FILE" echo "${pkg_name}=${HASH}" >> "$TARBALL_DIR/.manifest.tmp" echo " → $(basename $tarball)" return 0 } # Build packages if [[ "$BUILD_ALL" == true ]]; then echo "Building all packages..." for pkg_dir in "$COMMON_PLAT"/packages/*/; do build_package "$pkg_dir" || true done else echo "Building changed packages: $CHANGED_PACKAGES" for pkg in $CHANGED_PACKAGES; do pkg_dir="$COMMON_PLAT/packages/$pkg" if [[ -d "$pkg_dir" ]]; then build_package "$pkg_dir" || true fi done fi # Copy unchanged packages from cache if [[ "$FORCE_REBUILD" != true ]]; then echo "" echo "Copying unchanged packages from cache..." for pkg_dir in "$COMMON_PLAT"/packages/*/; do pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) if [[ -z "$pkg_name" ]]; then continue; fi pkg_version=$(node -p "require('${pkg_dir}package.json').version" 2>/dev/null || echo "0.0.0") # Sanitize package name for filename (replace @ and / with -) SAFE_PKG_NAME=$(echo "$pkg_name" | sed 's/@//g' | sed 's/\//_/g') CACHE_FILE="$CACHE_DIR/${SAFE_PKG_NAME}-${pkg_version}.tgz" if [[ -f "$CACHE_FILE" ]] && [[ ! -f "$TARBALL_DIR/$(basename $CACHE_FILE)" ]]; then echo " ✓ Cached: $pkg_name" cp "$CACHE_FILE" "$TARBALL_DIR/" fi done fi # Update manifest if [[ -f "$TARBALL_DIR/.manifest.tmp" ]]; then mv "$TARBALL_DIR/.manifest.tmp" "$MANIFEST_FILE" fi # ── Rewrite package.json files ───────────────────────────────────── echo "" echo "Rewriting package.json @bytelyst/* refs to .docker-deps/ tarballs..." # Build tarball mapping TARBALL_MAP_FILE=$(mktemp) trap 'rm -f "$TARBALL_MAP_FILE"' EXIT for tarball in "$TARBALL_DIR"/*.tgz; do [[ -f "$tarball" ]] || continue filename=$(basename "$tarball") # Extract package name from tarball filename pkg_name=$(echo "$filename" | sed 's/-[0-9].*//' | sed 's/^bytelyst-/@bytelyst\//') echo "${pkg_name}=${filename}" >> "$TARBALL_MAP_FILE" done rewrite_package_json() { local pkg_file="$1" local rel_prefix="$2" if [[ ! -f "$pkg_file" ]]; then return; fi # Backup cp "$pkg_file" "${pkg_file}.bak" local tmp="${pkg_file}.tmp" cp "$pkg_file" "$tmp" while IFS='=' read -r pkg_name tarball; do [[ -z "$pkg_name" ]] && continue node -e " const fs = require('fs'); const file = process.argv[1]; const pkgName = process.argv[2]; const replacement = process.argv[3]; const p = JSON.parse(fs.readFileSync(file, 'utf8')); for (const section of ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']) { if (p[section] && Object.prototype.hasOwnProperty.call(p[section], pkgName)) { p[section][pkgName] = replacement; } } fs.writeFileSync(file, JSON.stringify(p, null, 2) + '\n'); " "$tmp" "$pkg_name" "file:${rel_prefix}.docker-deps/${tarball}" done < "$TARBALL_MAP_FILE" mv "$tmp" "$pkg_file" echo " Rewrote $pkg_file" } # Backend package.json rewrite_package_json "${REPO_DIR}/backend/package.json" "../" # Web package.json rewrite_package_json "${REPO_DIR}/web/package.json" "../" # ── Inject pnpm.overrides for transitive @bytelyst/* deps ───────── inject_overrides() { local pkg_file="$1" local rel_prefix="$2" if [[ ! -f "$pkg_file" ]]; then return; fi local overrides="" while IFS='=' read -r pkg_name tarball; do [[ -z "$pkg_name" ]] && continue if [[ -n "$overrides" ]]; then overrides="$overrides, "; fi overrides="$overrides\"${pkg_name}\": \"file:${rel_prefix}.docker-deps/${tarball}\"" done < "$TARBALL_MAP_FILE" if [[ -n "$overrides" ]]; then node -e " const fs = require('fs'); const p = JSON.parse(fs.readFileSync('${pkg_file}', 'utf8')); p.pnpm = p.pnpm || {}; p.pnpm.overrides = { ...(p.pnpm.overrides || {}), ...JSON.parse('{${overrides}}') }; fs.writeFileSync('${pkg_file}', JSON.stringify(p, null, 2) + '\n'); " echo " Injected pnpm.overrides into $pkg_file" fi } inject_overrides "${REPO_DIR}/backend/package.json" "../" inject_overrides "${REPO_DIR}/web/package.json" "../" # ── Summary ─────────────────────────────────────────────────────────── echo "" echo "✓ Done. $(ls $TARBALL_DIR/*.tgz 2>/dev/null | wc -l) tarballs in $TARBALL_DIR" echo "✓ Cache size: $(du -sh $CACHE_DIR 2>/dev/null | cut -f1)" echo "" echo "To build Docker images:" echo " docker compose build" echo "" echo "To restore after build:" echo " ./scripts/docker-prep-optimized.sh --restore" echo "" echo "To clear cache and force full rebuild:" echo " ./scripts/docker-prep-optimized.sh --clean" echo " ./scripts/docker-prep-optimized.sh --force"