diff --git a/.npmrc.docker b/.npmrc.docker index b4fabd8..1bc9e12 100644 --- a/.npmrc.docker +++ b/.npmrc.docker @@ -1,4 +1 @@ -strict-ssl=false -# @bytelyst/* packages resolved via file: tarballs from .docker-deps/ -# No registry needed when using docker-prep.sh - +@bytelyst:registry=http://gitea.bytelyst.com:3300/api/packages/bytelyst/npm/ \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile index a25cfab..dc58142 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,19 +1,14 @@ FROM node:22-alpine AS builder WORKDIR /app/backend -ARG GITEA_NPM_HOST -ENV NODE_TLS_REJECT_UNAUTHORIZED=0 -ENV NPM_CONFIG_STRICT_SSL=false -ENV GITEA_NPM_HOST=$GITEA_NPM_HOST - -RUN npm config set strict-ssl false \ - && npm install -g pnpm@10.6.5 +RUN npm install -g pnpm@10.6.5 COPY .npmrc.docker ./.npmrc -COPY .docker-deps/ /app/.docker-deps/ COPY backend/package.json ./package.json + RUN --mount=type=secret,id=gitea_npm_token \ - export GITEA_NPM_TOKEN="$(cat /run/secrets/gitea_npm_token 2>/dev/null || echo '')" && \ + TOKEN=$(cat /run/secrets/gitea_npm_token) && \ + echo "//gitea.bytelyst.com:3300/:_authToken=$TOKEN" >> .npmrc && \ pnpm install --ignore-scripts --lockfile=false COPY backend/tsconfig.json ./tsconfig.json @@ -24,7 +19,6 @@ RUN pnpm run build FROM node:22-alpine WORKDIR /app/backend ENV NODE_ENV=production -ENV NODE_TLS_REJECT_UNAUTHORIZED=0 COPY --from=builder /app/backend/node_modules ./node_modules COPY --from=builder /app/backend/package.json ./package.json @@ -32,4 +26,4 @@ COPY --from=builder /app/backend/dist ./dist COPY shared/ ../shared/ EXPOSE 4011 -CMD ["node", "dist/server.js"] +CMD ["node", "dist/server.js"] \ No newline at end of file diff --git a/scripts/docker-prep.sh b/scripts/docker-prep.sh deleted file mode 100755 index 04b8613..0000000 --- a/scripts/docker-prep.sh +++ /dev/null @@ -1,316 +0,0 @@ -#!/usr/bin/env bash -# Optimized docker-prep.sh with 6 optimizations: -# 1. Git-based incremental builds (only rebuild changed packages) -# 2. Hash-based caching (content-addressable cache) -# 3. Parallel builds (concurrent package building) -# 4. Persistent tarball cache (survives git clean) -# 5. Smart manifest tracking (track what's been built) -# 6. Docker BuildKit integration hints -# -# Usage: -# ./scripts/docker-prep-optimized.sh # pack tarballs + rewrite package.json -# ./scripts/docker-prep-optimized.sh --restore # undo rewrite -# ./scripts/docker-prep-optimized.sh --clean # clear cache -# ./scripts/docker-prep-optimized.sh --force # force full rebuild - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -COMMON_PLAT="${COMMON_PLAT:-${REPO_DIR}/../learning_ai/learning_ai_common_plat}" -if [[ ! -d "$COMMON_PLAT" && -d "${REPO_DIR}/../learning_ai_common_plat" ]]; then - COMMON_PLAT="${REPO_DIR}/../learning_ai_common_plat" -fi - -TARBALL_DIR="${REPO_DIR}/.docker-deps" -# Shared cache location - can be overridden with BYTELYST_CACHE_DIR env var -CACHE_DIR="${BYTELYST_CACHE_DIR:-${HOME}/.cache/bytelyst-packages}" -MANIFEST_FILE="${CACHE_DIR}/.manifest" - -# ── Parse arguments ─────────────────────────────────────────────────── -MODE="pack" -FORCE_REBUILD=false - -while [[ $# -gt 0 ]]; do - case "$1" in - --restore) MODE="restore"; shift ;; - --clean) MODE="clean"; shift ;; - --force) FORCE_REBUILD=true; shift ;; - *) echo "Unknown option: $1" >&2; exit 1 ;; - esac -done - -# ── Restore mode ───────────────────────────────────────────────────── -if [[ "$MODE" == "restore" ]]; then - echo "Restoring original package.json files..." - for bak in $(find "$REPO_DIR" -name "package.json.bak" -not -path "*/node_modules/*"); do - mv "$bak" "${bak%.bak}" - echo " Restored ${bak%.bak}" - done - rm -rf "$TARBALL_DIR" - echo "Done." - exit 0 -fi - -# ── Clean mode ─────────────────────────────────────────────────────── -if [[ "$MODE" == "clean" ]]; then - echo "Cleaning cache and tarball directory..." - rm -rf "$CACHE_DIR" "$TARBALL_DIR" - echo "Done." - exit 0 -fi - -# ── Validation ─────────────────────────────────────────────────────── -if [[ ! -d "$COMMON_PLAT" ]]; then - echo "Common platform checkout not found: $COMMON_PLAT" >&2 - echo "Set COMMON_PLAT=/path/to/learning_ai_common_plat or place it at ../learning_ai/learning_ai_common_plat." >&2 - exit 1 -fi - -# ── Initialize directories ─────────────────────────────────────────── -mkdir -p "$TARBALL_DIR" "$CACHE_DIR" - -# ── Load manifest if exists ─────────────────────────────────────────── -# Simple file-based manifest to avoid bash associative array issues - -# ── Determine which packages to rebuild ─────────────────────────────── -if [[ "$FORCE_REBUILD" == true ]]; then - echo "Force rebuild: building all packages" - BUILD_ALL=true -else - BUILD_ALL=false - # Git-based incremental build: check which packages changed - cd "$COMMON_PLAT" - if git rev-parse --git-dir > /dev/null 2>&1; then - # Get packages changed since last commit - CHANGED_PACKAGES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null | grep '^packages/' | cut -d'/' -f2 | sort -u || echo "") - - # If no git history or no changes, check against manifest - if [[ -z "$CHANGED_PACKAGES" ]] && [[ -f "$MANIFEST_FILE" ]]; then - echo "No git changes detected, checking cache validity..." - CHANGED_PACKAGES="" - for pkg_dir in "$COMMON_PLAT"/packages/*/; do - pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) - if [[ -z "$pkg_name" ]]; then continue; fi - - # Hash package.json + source files - HASH=$(find "$pkg_dir" -name "*.ts" -o -name "*.json" -o -name "*.js" 2>/dev/null | xargs cat 2>/dev/null | sha256sum | cut -d' ' -f1 || echo "") - - # Check manifest for existing hash - CACHED_HASH=$(grep "^${pkg_name}=" "$MANIFEST_FILE" 2>/dev/null | cut -d'=' -f2 || echo "") - if [[ -n "$HASH" ]] && [[ "$CACHED_HASH" != "$HASH" ]]; then - pkg_basename=$(basename "$pkg_dir") - CHANGED_PACKAGES="$CHANGED_PACKAGES $pkg_basename" - fi - done - fi - - # If still no changes detected but cache is empty, build all - if [[ -z "$CHANGED_PACKAGES" ]] && [[ -z "$(ls -A $CACHE_DIR 2>/dev/null)" ]]; then - echo "Cache empty, building all packages..." - BUILD_ALL=true - fi - else - echo "Not a git repo or no history, building all packages..." - BUILD_ALL=true - fi -fi - -cd "$REPO_DIR" - -# ── Build packages with caching ──────────────────────────────────────── -echo "=== docker-prep-optimized: packing @bytelyst/* tarballs ===" - -rm -rf "$TARBALL_DIR" -mkdir -p "$TARBALL_DIR" - -# Function to build a single package -build_package() { - local pkg_dir=$1 - local force_build=${2:-false} - - if [[ ! -d "$pkg_dir" ]]; then - echo " ✗ Package directory not found: $pkg_dir" >&2 - return 1 - fi - - pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) - if [[ -z "$pkg_name" ]]; then - echo " ✗ Could not read package name from: $pkg_dir" >&2 - return 1 - fi - - pkg_version=$(node -p "require('${pkg_dir}package.json').version" 2>/dev/null || echo "0.0.0") - - # Hash package.json + source files - HASH=$(find "$pkg_dir" -name "*.ts" -o -name "*.json" -o -name "*.js" 2>/dev/null | xargs cat 2>/dev/null | sha256sum | cut -d' ' -f1 || echo "unknown") - # Sanitize package name for filename (replace @ and / with -) - SAFE_PKG_NAME=$(echo "$pkg_name" | sed 's/@//g' | sed 's/\//_/g') - CACHE_FILE="$CACHE_DIR/${SAFE_PKG_NAME}-${pkg_version}.tgz" - - # Check cache - if [[ -f "$CACHE_FILE" ]] && [[ "$force_build" != true ]] && [[ "$FORCE_REBUILD" != true ]]; then - echo " ✓ Cache hit: $pkg_name" - cp "$CACHE_FILE" "$TARBALL_DIR/" - echo "${pkg_name}=${HASH}" >> "$TARBALL_DIR/.manifest.tmp" - return 0 - fi - - # Build package - echo " → Building: $pkg_name" - (cd "$pkg_dir" && pnpm build > /dev/null 2>&1) - - # Pack to cache and tarball dir - tarball=$(cd "$pkg_dir" && pnpm pack --pack-destination "$TARBALL_DIR" 2>/dev/null | tail -1) - if [[ -z "$tarball" ]]; then - echo " ✗ Failed to pack: $pkg_name" >&2 - return 1 - fi - - # Copy to cache - cp "$TARBALL_DIR/$(basename $tarball)" "$CACHE_FILE" - echo "${pkg_name}=${HASH}" >> "$TARBALL_DIR/.manifest.tmp" - - echo " → $(basename $tarball)" - return 0 -} - -# Build packages -if [[ "$BUILD_ALL" == true ]]; then - echo "Building all packages..." - for pkg_dir in "$COMMON_PLAT"/packages/*/; do - build_package "$pkg_dir" || true - done -else - echo "Building changed packages: $CHANGED_PACKAGES" - for pkg in $CHANGED_PACKAGES; do - pkg_dir="$COMMON_PLAT/packages/$pkg" - if [[ -d "$pkg_dir" ]]; then - build_package "$pkg_dir" || true - fi - done -fi - -# Copy unchanged packages from cache -if [[ "$FORCE_REBUILD" != true ]]; then - echo "" - echo "Copying unchanged packages from cache..." - for pkg_dir in "$COMMON_PLAT"/packages/*/; do - pkg_name=$(node -p "require('${pkg_dir}package.json').name" 2>/dev/null || true) - if [[ -z "$pkg_name" ]]; then continue; fi - - pkg_version=$(node -p "require('${pkg_dir}package.json').version" 2>/dev/null || echo "0.0.0") - # Sanitize package name for filename (replace @ and / with -) - SAFE_PKG_NAME=$(echo "$pkg_name" | sed 's/@//g' | sed 's/\//_/g') - CACHE_FILE="$CACHE_DIR/${SAFE_PKG_NAME}-${pkg_version}.tgz" - - if [[ -f "$CACHE_FILE" ]] && [[ ! -f "$TARBALL_DIR/$(basename $CACHE_FILE)" ]]; then - echo " ✓ Cached: $pkg_name" - cp "$CACHE_FILE" "$TARBALL_DIR/" - fi - done -fi - -# Update manifest -if [[ -f "$TARBALL_DIR/.manifest.tmp" ]]; then - mv "$TARBALL_DIR/.manifest.tmp" "$MANIFEST_FILE" -fi - -# ── Rewrite package.json files ───────────────────────────────────── -echo "" -echo "Rewriting package.json @bytelyst/* refs to .docker-deps/ tarballs..." - -# Build tarball mapping -TARBALL_MAP_FILE=$(mktemp) -trap 'rm -f "$TARBALL_MAP_FILE"' EXIT - -for tarball in "$TARBALL_DIR"/*.tgz; do - [[ -f "$tarball" ]] || continue - filename=$(basename "$tarball") - # Extract package name from tarball filename - pkg_name=$(echo "$filename" | sed 's/-[0-9].*//' | sed 's/^bytelyst-/@bytelyst\//') - echo "${pkg_name}=${filename}" >> "$TARBALL_MAP_FILE" -done - -rewrite_package_json() { - local pkg_file="$1" - local rel_prefix="$2" - - if [[ ! -f "$pkg_file" ]]; then return; fi - - # Backup - cp "$pkg_file" "${pkg_file}.bak" - - local tmp="${pkg_file}.tmp" - cp "$pkg_file" "$tmp" - - while IFS='=' read -r pkg_name tarball; do - [[ -z "$pkg_name" ]] && continue - node -e " - const fs = require('fs'); - const file = process.argv[1]; - const pkgName = process.argv[2]; - const replacement = process.argv[3]; - const p = JSON.parse(fs.readFileSync(file, 'utf8')); - for (const section of ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']) { - if (p[section] && Object.prototype.hasOwnProperty.call(p[section], pkgName)) { - p[section][pkgName] = replacement; - } - } - fs.writeFileSync(file, JSON.stringify(p, null, 2) + '\n'); - " "$tmp" "$pkg_name" "file:${rel_prefix}.docker-deps/${tarball}" - done < "$TARBALL_MAP_FILE" - - mv "$tmp" "$pkg_file" - echo " Rewrote $pkg_file" -} - -# Backend package.json -rewrite_package_json "${REPO_DIR}/backend/package.json" "../" - -# Web package.json -rewrite_package_json "${REPO_DIR}/web/package.json" "../" - -# ── Inject pnpm.overrides for transitive @bytelyst/* deps ───────── -inject_overrides() { - local pkg_file="$1" - local rel_prefix="$2" - - if [[ ! -f "$pkg_file" ]]; then return; fi - - local overrides="" - while IFS='=' read -r pkg_name tarball; do - [[ -z "$pkg_name" ]] && continue - if [[ -n "$overrides" ]]; then overrides="$overrides, "; fi - overrides="$overrides\"${pkg_name}\": \"file:${rel_prefix}.docker-deps/${tarball}\"" - done < "$TARBALL_MAP_FILE" - - if [[ -n "$overrides" ]]; then - node -e " - const fs = require('fs'); - const p = JSON.parse(fs.readFileSync('${pkg_file}', 'utf8')); - p.pnpm = p.pnpm || {}; - p.pnpm.overrides = { ...(p.pnpm.overrides || {}), ...JSON.parse('{${overrides}}') }; - fs.writeFileSync('${pkg_file}', JSON.stringify(p, null, 2) + '\n'); - " - echo " Injected pnpm.overrides into $pkg_file" - fi -} - -inject_overrides "${REPO_DIR}/backend/package.json" "../" -inject_overrides "${REPO_DIR}/web/package.json" "../" - -# ── Summary ─────────────────────────────────────────────────────────── -echo "" -echo "✓ Done. $(ls $TARBALL_DIR/*.tgz 2>/dev/null | wc -l) tarballs in $TARBALL_DIR" -echo "✓ Cache size: $(du -sh $CACHE_DIR 2>/dev/null | cut -f1)" -echo "" -echo "To build Docker images:" -echo " docker compose build" -echo "" -echo "To restore after build:" -echo " ./scripts/docker-prep-optimized.sh --restore" -echo "" -echo "To clear cache and force full rebuild:" -echo " ./scripts/docker-prep-optimized.sh --clean" -echo " ./scripts/docker-prep-optimized.sh --force" \ No newline at end of file