From a418a23e568d1f7abb1083b0cd2fbc80b0c3b46b Mon Sep 17 00:00:00 2001 From: saravanakumardb1 Date: Wed, 27 May 2026 03:48:46 -0700 Subject: [PATCH] feat(scripts): canonical hardened docker-prep + sync tooling (Phase B7) Promotes docker-prep.sh to canonical home in common-plat with full Phase B hardening from the docker-build-optimization-roadmap: - B1: --dry-run mode (lists actions, no side effects) - B2: idempotency guard (refuses to run if *.bak exists, --force to bypass) - B5: trap-based auto-restore on error (--keep to disable) - B6: standardized header + usage block - B7: canonical home + sync + drift-check (mirrors npmrc.template pattern) - B8: --strip-overrides for safety-net cleanup - New: --check mode for CI-friendly state verification - New: auto-discovers package.json files with @bytelyst/* deps - New: portable sed -i (BSD on macOS, GNU on Linux) - New: preserves .docker-deps/.gitkeep on clear (fixes earlier regression) - New: 2 small JS helpers (_docker-prep-*.js) avoid bash 3.2 heredoc quirks Verified on clock + peakpulse: dry-run, pack, check, idempotency guard, restore, and post-restore git status all clean. --- scripts/_docker-prep-inject.js | 9 + scripts/_docker-prep-strip.js | 15 ++ scripts/check-docker-prep-drift.sh | 6 + scripts/docker-prep.template.sh | 293 +++++++++++++++++++++++++++++ scripts/sync-docker-prep.sh | 108 +++++++++++ 5 files changed, 431 insertions(+) create mode 100644 scripts/_docker-prep-inject.js create mode 100644 scripts/_docker-prep-strip.js create mode 100755 scripts/check-docker-prep-drift.sh create mode 100755 scripts/docker-prep.template.sh create mode 100755 scripts/sync-docker-prep.sh diff --git a/scripts/_docker-prep-inject.js b/scripts/_docker-prep-inject.js new file mode 100644 index 00000000..1e4f1105 --- /dev/null +++ b/scripts/_docker-prep-inject.js @@ -0,0 +1,9 @@ +// Helper for docker-prep.sh — inject pnpm.overrides for @bytelyst/* tarballs. +// Reads: PKG_FILE_ARG (path), OVERRIDES_ARG (JSON object string) +// Writes: $PKG_FILE_ARG in place +const fs = require('fs'); +const f = process.env.PKG_FILE_ARG; +const p = JSON.parse(fs.readFileSync(f, 'utf8')); +p.pnpm = p.pnpm || {}; +p.pnpm.overrides = Object.assign({}, p.pnpm.overrides || {}, JSON.parse(process.env.OVERRIDES_ARG)); +fs.writeFileSync(f, JSON.stringify(p, null, 2) + '\n'); diff --git a/scripts/_docker-prep-strip.js b/scripts/_docker-prep-strip.js new file mode 100644 index 00000000..838ba500 --- /dev/null +++ b/scripts/_docker-prep-strip.js @@ -0,0 +1,15 @@ +// Helper for docker-prep.sh --strip-overrides — remove @bytelyst/* keys from +// pnpm.overrides. Reads: PKG_FILE_ARG (path). Writes: $PKG_FILE_ARG in place. +const fs = require('fs'); +const f = process.env.PKG_FILE_ARG; +const p = JSON.parse(fs.readFileSync(f, 'utf8')); +if (p.pnpm && p.pnpm.overrides) { + const filtered = {}; + for (const k of Object.keys(p.pnpm.overrides)) { + if (!k.startsWith('@bytelyst/')) filtered[k] = p.pnpm.overrides[k]; + } + if (Object.keys(filtered).length === 0) delete p.pnpm.overrides; + else p.pnpm.overrides = filtered; + if (Object.keys(p.pnpm).length === 0) delete p.pnpm; + fs.writeFileSync(f, JSON.stringify(p, null, 2) + '\n'); +} diff --git a/scripts/check-docker-prep-drift.sh b/scripts/check-docker-prep-drift.sh new file mode 100755 index 00000000..9b3a4039 --- /dev/null +++ b/scripts/check-docker-prep-drift.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# CI-friendly thin wrapper that checks docker-prep.sh drift across all consumer repos. +# Mirrors scripts/check-npmrc-drift.sh. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +exec bash "$HERE/sync-docker-prep.sh" --check diff --git a/scripts/docker-prep.template.sh b/scripts/docker-prep.template.sh new file mode 100755 index 00000000..6ffa8dc7 --- /dev/null +++ b/scripts/docker-prep.template.sh @@ -0,0 +1,293 @@ +#!/usr/bin/env bash +# docker-prep — pack @bytelyst/* tarballs for hermetic Docker builds. +# +# CANONICAL TEMPLATE — do not hand-edit copies in product repos. +# Sync via: bash learning_ai_common_plat/scripts/sync-docker-prep.sh +# +# Usage: +# bash scripts/docker-prep.sh # pack + rewrite (default) +# bash scripts/docker-prep.sh --restore # undo rewrite, drop .bak/tarballs +# bash scripts/docker-prep.sh --dry-run # list packs/rewrites, no side effects +# bash scripts/docker-prep.sh --check # exit 1 if rewrites still in place +# bash scripts/docker-prep.sh --strip-overrides # remove pnpm.overrides only +# bash scripts/docker-prep.sh --force # bypass idempotency guard +# bash scripts/docker-prep.sh --keep # do NOT auto-restore on error +# +# Behavior: +# - Auto-discovers package.json files containing @bytelyst/* deps (max-depth 3). +# - Idempotent: refuses to run pack mode if any *.bak exists (use --force). +# - Safe: on error, auto-restores .bak files unless --keep is passed. +# - Portable: sed -i '' for macOS, sed -i for Linux. +# +# Phase B refs: B1, B2, B5, B6, B7, B8 of docker-build-optimization-roadmap.md + +set -uo pipefail + +# ── CLI parsing ─────────────────────────────────────────────────── +MODE=pack +FORCE=false +KEEP_ON_ERROR=false +DRY_RUN=false +while [ $# -gt 0 ]; do + case "$1" in + --restore) MODE=restore ;; + --check) MODE=check ;; + --strip-overrides) MODE=strip-overrides ;; + --dry-run) DRY_RUN=true ;; + --force) FORCE=true ;; + --keep) KEEP_ON_ERROR=true ;; + -h|--help) sed -n '2,21p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "✗ Unknown arg: $1" >&2; exit 2 ;; + esac + shift +done + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +COMMON_PLAT="${COMMON_PLAT_DIR:-${REPO_DIR}/../learning_ai_common_plat}" +TARBALL_DIR="${REPO_DIR}/.docker-deps" + +# Portable sed -i (BSD vs GNU) +sed_inplace() { + if [[ "$(uname)" == "Darwin" ]]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + +# ── Discover package.json files with @bytelyst/* refs ──────────── +discover_pkg_files() { + find "$REPO_DIR" -maxdepth 3 -name 'package.json' \ + -not -path '*/node_modules/*' \ + -not -path '*/.docker-deps/*' \ + -not -path '*/dist/*' \ + -not -path '*/.next/*' 2>/dev/null \ + | while read -r f; do + if grep -q '"@bytelyst/' "$f" 2>/dev/null; then + echo "$f" + fi + done +} + +# ── relative prefix from a package.json's dir back to repo root ── +rel_prefix_for() { + local pkg_file="$1" + local pkg_dir + pkg_dir="$(dirname "$pkg_file")" + # If package.json is at repo root, prefix is ./ + if [[ "$pkg_dir" == "$REPO_DIR" ]]; then + echo "" + return + fi + # Count dir depth from repo root → produce ../ * depth + local rel="${pkg_dir#$REPO_DIR/}" + local depth + depth=$(echo "$rel" | awk -F/ '{print NF}') + local prefix="" + for ((i=0; i/dev/null 2>&1; then + echo "✗ Tarballs still in $TARBALL_DIR" + found=1 + fi + for bak in $(find "$REPO_DIR" -name 'package.json.bak' -not -path '*/node_modules/*' 2>/dev/null); do + echo "✗ Backup still present: $bak" + found=1 + done + if [ $found -eq 0 ]; then + echo "✓ docker-prep state clean" + exit 0 + fi + exit 1 +fi + +# ── --restore mode (idempotent) ────────────────────────────────── +restore_all() { + local restored=0 + for bak in $(find "$REPO_DIR" -name 'package.json.bak' -not -path '*/node_modules/*' 2>/dev/null); do + if $DRY_RUN; then + echo " [dry-run] would restore ${bak%.bak}" + else + mv "$bak" "${bak%.bak}" + echo " ✓ Restored ${bak%.bak}" + fi + restored=$((restored+1)) + done + if [ -d "$TARBALL_DIR" ]; then + if $DRY_RUN; then + echo " [dry-run] would clear tarballs from $TARBALL_DIR (preserving .gitkeep)" + else + # Preserve .gitkeep so the dir survives for fresh clones / Dockerfile COPY + find "$TARBALL_DIR" -mindepth 1 ! -name '.gitkeep' -delete 2>/dev/null || true + echo " ✓ Cleared tarballs from $TARBALL_DIR (preserved .gitkeep)" + fi + fi + if [ $restored -eq 0 ] && [ ! -d "$TARBALL_DIR" ]; then + echo " (nothing to restore — already clean)" + fi +} + +if [[ "$MODE" == "restore" ]]; then + echo "=== docker-prep: restoring original state ===" + restore_all + echo "Done." + exit 0 +fi + +# ── --strip-overrides (B8) ────────────────────────────────────── +if [[ "$MODE" == "strip-overrides" ]]; then + echo "=== docker-prep: stripping pnpm.overrides ===" + for pkg in $(discover_pkg_files); do + if grep -q '"overrides"' "$pkg"; then + if $DRY_RUN; then + echo " [dry-run] would strip overrides from $pkg" + else + PKG_FILE_ARG="$pkg" node "$SCRIPT_DIR/_docker-prep-strip.js" + echo " ✓ Stripped @bytelyst/* overrides from $pkg" + fi + fi + done + echo "Done." + exit 0 +fi + +# ── Pack mode (default) ───────────────────────────────────────── +echo "=== docker-prep: packing @bytelyst/* tarballs ===" + +# B2: idempotency guard +EXISTING_BAKS=$(find "$REPO_DIR" -name 'package.json.bak' -not -path '*/node_modules/*' 2>/dev/null | head -5) +if [[ -n "$EXISTING_BAKS" ]] && ! $FORCE; then + echo "✗ Existing *.bak files detected — previous docker-prep run not cleaned up:" + echo "$EXISTING_BAKS" | sed 's/^/ /' + echo "" + echo "Run one of:" + echo " bash scripts/docker-prep.sh --restore # clean up" + echo " bash scripts/docker-prep.sh --force # ignore guard (DANGEROUS)" + exit 1 +fi + +# Verify common-plat sibling +if [ ! -d "$COMMON_PLAT/packages" ]; then + echo "✗ Cannot find common-plat packages at $COMMON_PLAT/packages" + echo " Clone learning_ai_common_plat as a sibling directory, or set COMMON_PLAT_DIR." + exit 1 +fi + +# B5: trap to auto-restore on error +cleanup_on_error() { + local code=$? + if [ $code -ne 0 ] && ! $KEEP_ON_ERROR && ! $DRY_RUN; then + echo "" + echo "⚠ docker-prep failed (exit $code) — auto-restoring (pass --keep to disable)..." + restore_all + fi + return $code +} +trap cleanup_on_error EXIT + +if $DRY_RUN; then + echo " [dry-run] would build @bytelyst/* packages in $COMMON_PLAT" +else + echo "Building @bytelyst/* packages..." + (cd "$COMMON_PLAT" && pnpm -r --filter './packages/*' build) >/dev/null +fi + +# Pack each package +if $DRY_RUN; then + echo " [dry-run] would pack packages to $TARBALL_DIR" +else + mkdir -p "$TARBALL_DIR" + # Clear stale tarballs but preserve .gitkeep + find "$TARBALL_DIR" -mindepth 1 ! -name '.gitkeep' -delete 2>/dev/null || true +fi + +TARBALL_MAP_FILE=$(mktemp) +# Note: not adding to existing EXIT trap; clean up explicitly at end +for pkg_dir in "$COMMON_PLAT"/packages/*/; do + pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) + [[ -z "$pkg_name" ]] && continue + if $DRY_RUN; then + echo " [dry-run] would pack $pkg_name" + continue + fi + echo " Packing $pkg_name..." + tarball=$(cd "$pkg_dir" && pnpm pack --pack-destination "$TARBALL_DIR" 2>/dev/null | tail -1) + filename=$(basename "$tarball") + echo "${pkg_name}=${filename}" >> "$TARBALL_MAP_FILE" + echo " -> $filename" +done + +if $DRY_RUN; then + echo "" + echo " [dry-run] would rewrite refs in:" + discover_pkg_files | sed 's/^/ /' + rm -f "$TARBALL_MAP_FILE" + trap - EXIT + exit 0 +fi + +# ── Rewrite + inject overrides ─────────────────────────────────── +echo "" +echo "Rewriting package.json @bytelyst/* refs to .docker-deps/ tarballs..." + +PKG_FILES=$(discover_pkg_files) +if [[ -z "$PKG_FILES" ]]; then + echo "✗ No package.json with @bytelyst/* deps found in $REPO_DIR" + rm -f "$TARBALL_MAP_FILE" + trap - EXIT + exit 1 +fi + +for pkg_file in $PKG_FILES; do + prefix="$(rel_prefix_for "$pkg_file")" + cp "$pkg_file" "${pkg_file}.bak" + tmp="${pkg_file}.tmp" + cp "$pkg_file" "$tmp" + while IFS='=' read -r pkg_name tarball; do + [[ -z "$pkg_name" ]] && continue + sed_inplace "s|\"${pkg_name}\": \"[^\"]*\"|\"${pkg_name}\": \"file:${prefix}.docker-deps/${tarball}\"|g" "$tmp" + done < "$TARBALL_MAP_FILE" + mv "$tmp" "$pkg_file" + echo " ✓ Rewrote $pkg_file" +done + +# Inject overrides for transitive @bytelyst/* deps +echo "" +echo "Injecting pnpm.overrides..." +for pkg_file in $PKG_FILES; do + prefix="$(rel_prefix_for "$pkg_file")" + overrides="" + while IFS='=' read -r pkg_name tarball; do + [[ -z "$pkg_name" ]] && continue + if [[ -n "$overrides" ]]; then overrides="$overrides, "; fi + overrides="$overrides\"${pkg_name}\": \"file:${prefix}.docker-deps/${tarball}\"" + done < "$TARBALL_MAP_FILE" + if [[ -n "$overrides" ]]; then + PKG_FILE_ARG="$pkg_file" OVERRIDES_ARG="{${overrides}}" \ + node "$SCRIPT_DIR/_docker-prep-inject.js" + echo " ✓ Injected overrides into $pkg_file" + fi +done + +rm -f "$TARBALL_MAP_FILE" +trap - EXIT + +echo "" +echo "✅ Done. Tarballs in $TARBALL_DIR" +echo "" +echo "Next:" +echo " docker compose build # build images" +echo " bash scripts/docker-prep.sh --restore # undo when done" +echo " bash scripts/docker-prep.sh --check # verify clean state" diff --git a/scripts/sync-docker-prep.sh b/scripts/sync-docker-prep.sh new file mode 100755 index 00000000..979e6c67 --- /dev/null +++ b/scripts/sync-docker-prep.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Sync canonical docker-prep.template.sh into every product repo's scripts/docker-prep.sh. +# Mirrors scripts/sync-npmrc.sh pattern. +# +# Usage: +# bash scripts/sync-docker-prep.sh # sync all known consumers +# bash scripts/sync-docker-prep.sh --check # exit 1 if any drift +# bash scripts/sync-docker-prep.sh --repo PATH # sync just one repo + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TEMPLATE="$SCRIPT_DIR/docker-prep.template.sh" +COMMON_PLAT="$(cd "$SCRIPT_DIR/.." && pwd)" +SIBLING_ROOT="$(cd "$COMMON_PLAT/.." && pwd)" + +CHECK_ONLY=false +SINGLE_REPO="" +while [ $# -gt 0 ]; do + case "$1" in + --check) CHECK_ONLY=true ;; + --repo) SINGLE_REPO="${2:-}"; shift ;; + -h|--help) sed -n '2,11p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown arg: $1" >&2; exit 2 ;; + esac + shift +done + +if [ ! -f "$TEMPLATE" ]; then + echo "✗ Template not found: $TEMPLATE" >&2; exit 1 +fi + +# Repos that consume docker-prep.sh (have scripts/docker-prep.sh today) +CONSUMERS=( + "learning_ai_clock" + "learning_ai_peakpulse" + "learning_ai_notes" + "learning_ai_fastgap" + "learning_ai_jarvis_jr" + "learning_ai_flowmonk" + "learning_ai_trails" + "learning_ai_local_memory_gpt" + "learning_ai_efforise" +) + +if [ -n "$SINGLE_REPO" ]; then + CONSUMERS=("$(basename "$SINGLE_REPO")") + SIBLING_ROOT="$(cd "$(dirname "$SINGLE_REPO")" && pwd)" +fi + +drift=0 +synced=0 +skipped=0 + +for repo in "${CONSUMERS[@]}"; do + target_dir="$SIBLING_ROOT/$repo" + target="$target_dir/scripts/docker-prep.sh" + if [ ! -d "$target_dir" ]; then + echo " ⊘ skip $repo (not present at $target_dir)" + skipped=$((skipped+1)) + continue + fi + if [ ! -f "$target" ]; then + echo " ⊘ skip $repo (no existing scripts/docker-prep.sh)" + skipped=$((skipped+1)) + continue + fi + + # Sync 3 files together: main script + 2 JS helpers + HELPER_INJECT="$SCRIPT_DIR/_docker-prep-inject.js" + HELPER_STRIP="$SCRIPT_DIR/_docker-prep-strip.js" + T_HELPER_INJECT="$(dirname "$target")/_docker-prep-inject.js" + T_HELPER_STRIP="$(dirname "$target")/_docker-prep-strip.js" + + in_sync=true + cmp -s "$TEMPLATE" "$target" || in_sync=false + cmp -s "$HELPER_INJECT" "$T_HELPER_INJECT" 2>/dev/null || in_sync=false + cmp -s "$HELPER_STRIP" "$T_HELPER_STRIP" 2>/dev/null || in_sync=false + + if $in_sync; then + echo " ✓ $repo (in sync)" + continue + fi + + if $CHECK_ONLY; then + echo " ✗ $repo DRIFT" + drift=$((drift+1)) + else + mkdir -p "$(dirname "$target")" + cp "$TEMPLATE" "$target" + cp "$HELPER_INJECT" "$T_HELPER_INJECT" + cp "$HELPER_STRIP" "$T_HELPER_STRIP" + chmod +x "$target" + echo " ↺ $repo synced (script + 2 helpers)" + synced=$((synced+1)) + fi +done + +echo "" +if $CHECK_ONLY; then + if [ $drift -gt 0 ]; then + echo "❌ $drift repo(s) drifted from canonical template" + exit 1 + fi + echo "✅ All repos in sync with canonical docker-prep template" +else + echo "✅ Synced: $synced, skipped: $skipped" +fi