From 216ebb83fb4a42374f7e73cf3dcc7b1097b6c5e8 Mon Sep 17 00:00:00 2001 From: saravanakumardb1 Date: Wed, 27 May 2026 04:05:46 -0700 Subject: [PATCH] feat(ops): roll out canonical docker-prep + docker-doctor + Makefile (Phase D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply Phase D playbook from docker-build-optimization-roadmap.md: - scripts/docker-prep.sh: synced from canonical template (learning_ai_common_plat@a418a23e). Adds --dry-run, --check, --strip-overrides, idempotency guard, trap-based auto-restore, portable sed, .gitkeep preservation. - scripts/_docker-prep-inject.js: helper for pnpm.overrides injection - scripts/_docker-prep-strip.js: helper for --strip-overrides mode - scripts/docker-doctor.sh: thin wrapper to canonical linter - Makefile: make doctor target (gitea-doctor + docker-doctor) Per-repo Dockerfile/compose fixes flagged by docker-doctor are tracked in the roadmap and will land in follow-up commits. Refs: docker-build-optimization-roadmap.md §Phase D --- Makefile | 21 ++ scripts/_docker-prep-inject.js | 9 + scripts/_docker-prep-strip.js | 15 ++ scripts/docker-doctor.sh | 13 ++ scripts/docker-prep.sh | 337 +++++++++++++++++++++++---------- 5 files changed, 300 insertions(+), 95 deletions(-) create mode 100644 Makefile create mode 100644 scripts/_docker-prep-inject.js create mode 100644 scripts/_docker-prep-strip.js create mode 100755 scripts/docker-doctor.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..78a4403 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +# notes — convenience make targets. +# Most development commands use pnpm directly. Targets here are for +# operational concerns (Docker/registry health, linting docker config). + +.PHONY: doctor docker-doctor gitea-doctor help + +help: + @echo "Available targets:" + @echo " make doctor run gitea-doctor + docker-doctor (full pre-flight)" + @echo " make docker-doctor run docker-doctor only" + @echo " make gitea-doctor run gitea-doctor only" + +doctor: gitea-doctor docker-doctor + @echo "" + @echo "✅ All doctors PASS" + +docker-doctor: + @bash scripts/docker-doctor.sh + +gitea-doctor: + @bash ../learning_ai_common_plat/scripts/gitea/doctor.sh diff --git a/scripts/_docker-prep-inject.js b/scripts/_docker-prep-inject.js new file mode 100644 index 0000000..1e4f110 --- /dev/null +++ b/scripts/_docker-prep-inject.js @@ -0,0 +1,9 @@ +// Helper for docker-prep.sh — inject pnpm.overrides for @bytelyst/* tarballs. +// Reads: PKG_FILE_ARG (path), OVERRIDES_ARG (JSON object string) +// Writes: $PKG_FILE_ARG in place +const fs = require('fs'); +const f = process.env.PKG_FILE_ARG; +const p = JSON.parse(fs.readFileSync(f, 'utf8')); +p.pnpm = p.pnpm || {}; +p.pnpm.overrides = Object.assign({}, p.pnpm.overrides || {}, JSON.parse(process.env.OVERRIDES_ARG)); +fs.writeFileSync(f, JSON.stringify(p, null, 2) + '\n'); diff --git a/scripts/_docker-prep-strip.js b/scripts/_docker-prep-strip.js new file mode 100644 index 0000000..838ba50 --- /dev/null +++ b/scripts/_docker-prep-strip.js @@ -0,0 +1,15 @@ +// Helper for docker-prep.sh --strip-overrides — remove @bytelyst/* keys from +// pnpm.overrides. Reads: PKG_FILE_ARG (path). Writes: $PKG_FILE_ARG in place. +const fs = require('fs'); +const f = process.env.PKG_FILE_ARG; +const p = JSON.parse(fs.readFileSync(f, 'utf8')); +if (p.pnpm && p.pnpm.overrides) { + const filtered = {}; + for (const k of Object.keys(p.pnpm.overrides)) { + if (!k.startsWith('@bytelyst/')) filtered[k] = p.pnpm.overrides[k]; + } + if (Object.keys(filtered).length === 0) delete p.pnpm.overrides; + else p.pnpm.overrides = filtered; + if (Object.keys(p.pnpm).length === 0) delete p.pnpm; + fs.writeFileSync(f, JSON.stringify(p, null, 2) + '\n'); +} diff --git a/scripts/docker-doctor.sh b/scripts/docker-doctor.sh new file mode 100755 index 0000000..6769fad --- /dev/null +++ b/scripts/docker-doctor.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Thin wrapper — delegates to canonical docker-doctor in learning_ai_common_plat. +# Do not edit logic here; edit the canonical script. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$HERE/.." && pwd)" +CANONICAL="$REPO_ROOT/../learning_ai_common_plat/scripts/docker-doctor.sh" +if [ ! -x "$CANONICAL" ]; then + echo "✗ Canonical docker-doctor not found at $CANONICAL" >&2 + echo " Clone learning_ai_common_plat as a sibling directory." >&2 + exit 2 +fi +exec bash "$CANONICAL" --repo "$REPO_ROOT" "$@" diff --git a/scripts/docker-prep.sh b/scripts/docker-prep.sh index e16b3ba..6ffa8dc 100755 --- a/scripts/docker-prep.sh +++ b/scripts/docker-prep.sh @@ -1,59 +1,227 @@ #!/usr/bin/env bash -# Pack @bytelyst/* tarballs from the sibling common-plat repo for -# self-contained Docker builds that don't need the Gitea npm registry. +# docker-prep — pack @bytelyst/* tarballs for hermetic Docker builds. +# +# CANONICAL TEMPLATE — do not hand-edit copies in product repos. +# Sync via: bash learning_ai_common_plat/scripts/sync-docker-prep.sh # # Usage: -# ./scripts/docker-prep.sh # pack tarballs + rewrite package.json -# ./scripts/docker-prep.sh --restore # undo rewrite +# bash scripts/docker-prep.sh # pack + rewrite (default) +# bash scripts/docker-prep.sh --restore # undo rewrite, drop .bak/tarballs +# bash scripts/docker-prep.sh --dry-run # list packs/rewrites, no side effects +# bash scripts/docker-prep.sh --check # exit 1 if rewrites still in place +# bash scripts/docker-prep.sh --strip-overrides # remove pnpm.overrides only +# bash scripts/docker-prep.sh --force # bypass idempotency guard +# bash scripts/docker-prep.sh --keep # do NOT auto-restore on error +# +# Behavior: +# - Auto-discovers package.json files containing @bytelyst/* deps (max-depth 3). +# - Idempotent: refuses to run pack mode if any *.bak exists (use --force). +# - Safe: on error, auto-restores .bak files unless --keep is passed. +# - Portable: sed -i '' for macOS, sed -i for Linux. +# +# Phase B refs: B1, B2, B5, B6, B7, B8 of docker-build-optimization-roadmap.md -set -euo pipefail +set -uo pipefail + +# ── CLI parsing ─────────────────────────────────────────────────── +MODE=pack +FORCE=false +KEEP_ON_ERROR=false +DRY_RUN=false +while [ $# -gt 0 ]; do + case "$1" in + --restore) MODE=restore ;; + --check) MODE=check ;; + --strip-overrides) MODE=strip-overrides ;; + --dry-run) DRY_RUN=true ;; + --force) FORCE=true ;; + --keep) KEEP_ON_ERROR=true ;; + -h|--help) sed -n '2,21p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "✗ Unknown arg: $1" >&2; exit 2 ;; + esac + shift +done SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -COMMON_PLAT="${COMMON_PLAT:-${REPO_DIR}/../learning_ai_common_plat}" -if [[ ! -d "$COMMON_PLAT" && -d "${REPO_DIR}/../learning_ai/learning_ai_common_plat" ]]; then - COMMON_PLAT="${REPO_DIR}/../learning_ai/learning_ai_common_plat" -fi - +COMMON_PLAT="${COMMON_PLAT_DIR:-${REPO_DIR}/../learning_ai_common_plat}" TARBALL_DIR="${REPO_DIR}/.docker-deps" -# ── Restore mode ─────────────────────────────────────────────────── -if [[ "${1:-}" == "--restore" ]]; then - echo "Restoring original package.json files..." - for bak in $(find "$REPO_DIR" -name "package.json.bak" -not -path "*/node_modules/*"); do - mv "$bak" "${bak%.bak}" - echo " Restored ${bak%.bak}" +# Portable sed -i (BSD vs GNU) +sed_inplace() { + if [[ "$(uname)" == "Darwin" ]]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + +# ── Discover package.json files with @bytelyst/* refs ──────────── +discover_pkg_files() { + find "$REPO_DIR" -maxdepth 3 -name 'package.json' \ + -not -path '*/node_modules/*' \ + -not -path '*/.docker-deps/*' \ + -not -path '*/dist/*' \ + -not -path '*/.next/*' 2>/dev/null \ + | while read -r f; do + if grep -q '"@bytelyst/' "$f" 2>/dev/null; then + echo "$f" + fi + done +} + +# ── relative prefix from a package.json's dir back to repo root ── +rel_prefix_for() { + local pkg_file="$1" + local pkg_dir + pkg_dir="$(dirname "$pkg_file")" + # If package.json is at repo root, prefix is ./ + if [[ "$pkg_dir" == "$REPO_DIR" ]]; then + echo "" + return + fi + # Count dir depth from repo root → produce ../ * depth + local rel="${pkg_dir#$REPO_DIR/}" + local depth + depth=$(echo "$rel" | awk -F/ '{print NF}') + local prefix="" + for ((i=0; i/dev/null 2>&1; then + echo "✗ Tarballs still in $TARBALL_DIR" + found=1 + fi + for bak in $(find "$REPO_DIR" -name 'package.json.bak' -not -path '*/node_modules/*' 2>/dev/null); do + echo "✗ Backup still present: $bak" + found=1 + done + if [ $found -eq 0 ]; then + echo "✓ docker-prep state clean" + exit 0 + fi + exit 1 +fi + +# ── --restore mode (idempotent) ────────────────────────────────── +restore_all() { + local restored=0 + for bak in $(find "$REPO_DIR" -name 'package.json.bak' -not -path '*/node_modules/*' 2>/dev/null); do + if $DRY_RUN; then + echo " [dry-run] would restore ${bak%.bak}" + else + mv "$bak" "${bak%.bak}" + echo " ✓ Restored ${bak%.bak}" + fi + restored=$((restored+1)) + done + if [ -d "$TARBALL_DIR" ]; then + if $DRY_RUN; then + echo " [dry-run] would clear tarballs from $TARBALL_DIR (preserving .gitkeep)" + else + # Preserve .gitkeep so the dir survives for fresh clones / Dockerfile COPY + find "$TARBALL_DIR" -mindepth 1 ! -name '.gitkeep' -delete 2>/dev/null || true + echo " ✓ Cleared tarballs from $TARBALL_DIR (preserved .gitkeep)" + fi + fi + if [ $restored -eq 0 ] && [ ! -d "$TARBALL_DIR" ]; then + echo " (nothing to restore — already clean)" + fi +} + +if [[ "$MODE" == "restore" ]]; then + echo "=== docker-prep: restoring original state ===" + restore_all echo "Done." exit 0 fi -# ── Pack mode ────────────────────────────────────────────────────── -if [[ ! -d "$COMMON_PLAT" ]]; then - echo "Common platform checkout not found: $COMMON_PLAT" >&2 - echo "Set COMMON_PLAT=/path/to/learning_ai_common_plat or place it at ../learning_ai/learning_ai_common_plat." >&2 +# ── --strip-overrides (B8) ────────────────────────────────────── +if [[ "$MODE" == "strip-overrides" ]]; then + echo "=== docker-prep: stripping pnpm.overrides ===" + for pkg in $(discover_pkg_files); do + if grep -q '"overrides"' "$pkg"; then + if $DRY_RUN; then + echo " [dry-run] would strip overrides from $pkg" + else + PKG_FILE_ARG="$pkg" node "$SCRIPT_DIR/_docker-prep-strip.js" + echo " ✓ Stripped @bytelyst/* overrides from $pkg" + fi + fi + done + echo "Done." + exit 0 +fi + +# ── Pack mode (default) ───────────────────────────────────────── +echo "=== docker-prep: packing @bytelyst/* tarballs ===" + +# B2: idempotency guard +EXISTING_BAKS=$(find "$REPO_DIR" -name 'package.json.bak' -not -path '*/node_modules/*' 2>/dev/null | head -5) +if [[ -n "$EXISTING_BAKS" ]] && ! $FORCE; then + echo "✗ Existing *.bak files detected — previous docker-prep run not cleaned up:" + echo "$EXISTING_BAKS" | sed 's/^/ /' + echo "" + echo "Run one of:" + echo " bash scripts/docker-prep.sh --restore # clean up" + echo " bash scripts/docker-prep.sh --force # ignore guard (DANGEROUS)" exit 1 fi -echo "=== docker-prep: packing @bytelyst/* tarballs ===" +# Verify common-plat sibling +if [ ! -d "$COMMON_PLAT/packages" ]; then + echo "✗ Cannot find common-plat packages at $COMMON_PLAT/packages" + echo " Clone learning_ai_common_plat as a sibling directory, or set COMMON_PLAT_DIR." + exit 1 +fi -rm -rf "$TARBALL_DIR" -mkdir -p "$TARBALL_DIR" +# B5: trap to auto-restore on error +cleanup_on_error() { + local code=$? + if [ $code -ne 0 ] && ! $KEEP_ON_ERROR && ! $DRY_RUN; then + echo "" + echo "⚠ docker-prep failed (exit $code) — auto-restoring (pass --keep to disable)..." + restore_all + fi + return $code +} +trap cleanup_on_error EXIT -# Build all packages first (--filter limits to packages/, skips services/) -echo "Building @bytelyst/* packages..." -(cd "$COMMON_PLAT" && pnpm -r --filter './packages/*' build) +if $DRY_RUN; then + echo " [dry-run] would build @bytelyst/* packages in $COMMON_PLAT" +else + echo "Building @bytelyst/* packages..." + (cd "$COMMON_PLAT" && pnpm -r --filter './packages/*' build) >/dev/null +fi + +# Pack each package +if $DRY_RUN; then + echo " [dry-run] would pack packages to $TARBALL_DIR" +else + mkdir -p "$TARBALL_DIR" + # Clear stale tarballs but preserve .gitkeep + find "$TARBALL_DIR" -mindepth 1 ! -name '.gitkeep' -delete 2>/dev/null || true +fi -# Pack each package and build a mapping of name → tarball filename -# (uses a temp file instead of associative array for bash 3.2 compat) TARBALL_MAP_FILE=$(mktemp) -trap 'rm -f "$TARBALL_MAP_FILE"' EXIT - +# Note: not adding to existing EXIT trap; clean up explicitly at end for pkg_dir in "$COMMON_PLAT"/packages/*/; do pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) - if [[ -z "$pkg_name" ]]; then continue; fi - + [[ -z "$pkg_name" ]] && continue + if $DRY_RUN; then + echo " [dry-run] would pack $pkg_name" + continue + fi echo " Packing $pkg_name..." tarball=$(cd "$pkg_dir" && pnpm pack --pack-destination "$TARBALL_DIR" 2>/dev/null | tail -1) filename=$(basename "$tarball") @@ -61,86 +229,65 @@ for pkg_dir in "$COMMON_PLAT"/packages/*/; do echo " -> $filename" done -# ── Rewrite package.json files ───────────────────────────────────── +if $DRY_RUN; then + echo "" + echo " [dry-run] would rewrite refs in:" + discover_pkg_files | sed 's/^/ /' + rm -f "$TARBALL_MAP_FILE" + trap - EXIT + exit 0 +fi + +# ── Rewrite + inject overrides ─────────────────────────────────── echo "" echo "Rewriting package.json @bytelyst/* refs to .docker-deps/ tarballs..." -rewrite_package_json() { - local pkg_file="$1" - local rel_prefix="$2" # relative path from package.json dir to repo root +PKG_FILES=$(discover_pkg_files) +if [[ -z "$PKG_FILES" ]]; then + echo "✗ No package.json with @bytelyst/* deps found in $REPO_DIR" + rm -f "$TARBALL_MAP_FILE" + trap - EXIT + exit 1 +fi - if [[ ! -f "$pkg_file" ]]; then return; fi - - # Backup +for pkg_file in $PKG_FILES; do + prefix="$(rel_prefix_for "$pkg_file")" cp "$pkg_file" "${pkg_file}.bak" - - local tmp="${pkg_file}.tmp" + tmp="${pkg_file}.tmp" cp "$pkg_file" "$tmp" - while IFS='=' read -r pkg_name tarball; do [[ -z "$pkg_name" ]] && continue - node -e " - const fs = require('fs'); - const file = process.argv[1]; - const pkgName = process.argv[2]; - const replacement = process.argv[3]; - const p = JSON.parse(fs.readFileSync(file, 'utf8')); - for (const section of ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']) { - if (p[section] && Object.prototype.hasOwnProperty.call(p[section], pkgName)) { - p[section][pkgName] = replacement; - } - } - fs.writeFileSync(file, JSON.stringify(p, null, 2) + '\n'); - " "$tmp" "$pkg_name" "file:${rel_prefix}.docker-deps/${tarball}" + sed_inplace "s|\"${pkg_name}\": \"[^\"]*\"|\"${pkg_name}\": \"file:${prefix}.docker-deps/${tarball}\"|g" "$tmp" done < "$TARBALL_MAP_FILE" - mv "$tmp" "$pkg_file" - echo " Rewrote $pkg_file" -} + echo " ✓ Rewrote $pkg_file" +done -# Backend package.json -rewrite_package_json "${REPO_DIR}/backend/package.json" "../" - -# Web package.json -rewrite_package_json "${REPO_DIR}/web/package.json" "../" - -# ── Inject pnpm.overrides for transitive @bytelyst/* deps ───────── -# Tarball packages may depend on other @bytelyst/* packages (e.g. -# @bytelyst/fastify-core → @bytelyst/errors). Without overrides, pnpm -# tries to fetch them from the npm registry which fails. -inject_overrides() { - local pkg_file="$1" - local rel_prefix="$2" - - if [[ ! -f "$pkg_file" ]]; then return; fi - - local overrides="" +# Inject overrides for transitive @bytelyst/* deps +echo "" +echo "Injecting pnpm.overrides..." +for pkg_file in $PKG_FILES; do + prefix="$(rel_prefix_for "$pkg_file")" + overrides="" while IFS='=' read -r pkg_name tarball; do [[ -z "$pkg_name" ]] && continue if [[ -n "$overrides" ]]; then overrides="$overrides, "; fi - overrides="$overrides\"${pkg_name}\": \"file:${rel_prefix}.docker-deps/${tarball}\"" + overrides="$overrides\"${pkg_name}\": \"file:${prefix}.docker-deps/${tarball}\"" done < "$TARBALL_MAP_FILE" - if [[ -n "$overrides" ]]; then - node -e " - const fs = require('fs'); - const p = JSON.parse(fs.readFileSync('${pkg_file}', 'utf8')); - p.pnpm = p.pnpm || {}; - p.pnpm.overrides = { ...(p.pnpm.overrides || {}), ...JSON.parse('{${overrides}}') }; - fs.writeFileSync('${pkg_file}', JSON.stringify(p, null, 2) + '\n'); - " - echo " Injected pnpm.overrides into $pkg_file" + PKG_FILE_ARG="$pkg_file" OVERRIDES_ARG="{${overrides}}" \ + node "$SCRIPT_DIR/_docker-prep-inject.js" + echo " ✓ Injected overrides into $pkg_file" fi -} +done -inject_overrides "${REPO_DIR}/backend/package.json" "../" -inject_overrides "${REPO_DIR}/web/package.json" "../" +rm -f "$TARBALL_MAP_FILE" +trap - EXIT echo "" -echo "Done. Tarballs in $TARBALL_DIR" +echo "✅ Done. Tarballs in $TARBALL_DIR" echo "" -echo "To build Docker images:" -echo " docker compose build" -echo "" -echo "To restore after build:" -echo " ./scripts/docker-prep.sh --restore" +echo "Next:" +echo " docker compose build # build images" +echo " bash scripts/docker-prep.sh --restore # undo when done" +echo " bash scripts/docker-prep.sh --check # verify clean state"