From e14a46f3e09a7148cd1a0e0df070e930c5c2d695 Mon Sep 17 00:00:00 2001
From: Hermes VM <root@srv1491630>
Date: Sun, 28 Jun 2026 18:56:00 +0000
Subject: [PATCH] Deploy private Open Notebook pilot

---
 deployments/open-notebook-pilot/README.md     |  96 +++++++++++++++++
 .../open-notebook-pilot/docker-compose.yml    |  66 ++++++++++++
 .../known-answer-evaluation-report.md         | 102 ++++++++++++++++++
 docs/repo-map.md                              |   9 ++
 docs/tooling-status.md                        |   1 +
 scripts/open-notebook-pilot/backup.sh         |  32 ++++++
 .../ingest-and-search-smoke.sh                |  51 +++++++++
 scripts/open-notebook-pilot/restore-test.sh   |  34 ++++++
 8 files changed, 391 insertions(+)
 create mode 100644 deployments/open-notebook-pilot/README.md
 create mode 100644 deployments/open-notebook-pilot/docker-compose.yml
 create mode 100644 deployments/open-notebook-pilot/known-answer-evaluation-report.md
 create mode 100755 scripts/open-notebook-pilot/backup.sh
 create mode 100755 scripts/open-notebook-pilot/ingest-and-search-smoke.sh
 create mode 100755 scripts/open-notebook-pilot/restore-test.sh

diff --git a/deployments/open-notebook-pilot/README.md b/deployments/open-notebook-pilot/README.md
new file mode 100644
index 0000000..6745dd1
--- /dev/null
+++ b/deployments/open-notebook-pilot/README.md
@@ -0,0 +1,96 @@
+# ByteLyst private Open Notebook pilot
+
+Status: private localhost-only pilot. Do not publish as a public ByteLyst dashboard/subdomain.
+
+## Files and runtime locations
+
+- Compose: `deployments/open-notebook-pilot/docker-compose.yml`
+- Protected env/secrets: `/etc/bytelyst/open-notebook-pilot.env` (mode `0600`, not in git)
+- Runtime data: `/opt/bytelyst/open-notebook-pilot/`
+  - `surreal_data/`: SurrealDB RocksDB data
+  - `notebook_data/`: Open Notebook uploaded/app data
+  - `ollama/`: local Ollama model cache
+  - `test-docs/`: sanitized pilot documents
+  - `reports/`: smoke/evaluation outputs
+  - `backups/`: protected backup archives
+
+## Network hardening
+
+- Web UI binds to `127.0.0.1:8502` only.
+- REST API binds to `127.0.0.1:5055` only.
+- SurrealDB has no host-published port.
+- Ollama has no host-published port.
+- Remote access, if needed, must use SSH tunnel/VPN/private route only:
+  - `ssh -L 8502:127.0.0.1:8502 -L 5055:127.0.0.1:5055 <vm>`
+- Do not add this service to the public Caddyfile. If a private reverse proxy is ever required, put it behind strong auth and keep `/docs`, `/openapi.json`, and `/health` away from the public internet.
+
+## Secrets
+
+`/etc/bytelyst/open-notebook-pilot.env` contains:
+
+- `OPEN_NOTEBOOK_ENCRYPTION_KEY`
+- `OPEN_NOTEBOOK_PASSWORD`
+- `SURREAL_USER`
+- `SURREAL_PASSWORD`
+- explicit `CORS_ORIGINS`
+- `OLLAMA_BASE_URL=http://ollama:11434`
+
+Do not print this file in logs or commit it. Losing or changing the encryption key can make stored provider credentials unreadable.
+
+## Operations
+
+From the repo root:
+
+```bash
+docker compose --env-file /etc/bytelyst/open-notebook-pilot.env -f deployments/open-notebook-pilot/docker-compose.yml config --quiet
+docker compose --env-file /etc/bytelyst/open-notebook-pilot.env -f deployments/open-notebook-pilot/docker-compose.yml up -d
+```
+
+Health checks:
+
+```bash
+curl -fsS http://127.0.0.1:5055/health
+curl -fsS http://127.0.0.1:5055/api/auth/status
+curl -fsSI http://127.0.0.1:8502
+```
+
+Use the bearer password from the protected env file for API calls:
+
+```bash
+curl -fsS -H "Authorization: Bearer <password-from-/etc/bytelyst/open-notebook-pilot.env>" http://127.0.0.1:5055/api/notebooks
+```
+
+## Ollama local provider
+
+The compose stack includes an internal Ollama service at `http://ollama:11434` for the Open Notebook container. Initial pilot model targets:
+
+- LLM: `qwen2.5:0.5b` or another small local model that fits current VM headroom.
+- Embeddings: `nomic-embed-text` if pull time/resources allow.
+
+Configure/register models through Open Notebook Settings or the `/models` API. Cloud providers are allowed only with sanitized comparison data.
+
+## Sanitized ingestion and smoke search
+
+Sanitized docs live in `/opt/bytelyst/open-notebook-pilot/test-docs`. Run:
+
+```bash
+scripts/open-notebook-pilot/ingest-and-search-smoke.sh
+```
+
+The script creates a pilot notebook, ingests text sources without embeddings, and writes text-search smoke outputs under `/opt/bytelyst/open-notebook-pilot/reports/`.
+
+## Backup and restore test
+
+Backup:
+
+```bash
+scripts/open-notebook-pilot/backup.sh
+```
+
+Restore-readiness test:
+
+```bash
+scripts/open-notebook-pilot/restore-test.sh /opt/bytelyst/open-notebook-pilot/backups/<archive>.tar.gz
+```
+
+The restore test extracts to `/opt/bytelyst/open-notebook-pilot/restore-test/extracted` and verifies the backup contains SurrealDB data, notebook data, and the protected env file. Actual restore should be performed only during planned downtime by stopping the stack and replacing the runtime directories/env from the verified archive.
diff --git a/deployments/open-notebook-pilot/docker-compose.yml b/deployments/open-notebook-pilot/docker-compose.yml
new file mode 100644
index 0000000..45390a5
--- /dev/null
+++ b/deployments/open-notebook-pilot/docker-compose.yml
@@ -0,0 +1,66 @@
+# ByteLyst private Open Notebook pilot
+# Public exposure is intentionally disabled: UI/API bind to 127.0.0.1 only;
+# SurrealDB and Ollama have no published host ports.
+# Secrets live outside git in /etc/bytelyst/open-notebook-pilot.env.
+
+name: bytelyst-open-notebook-pilot
+
+services:
+  surrealdb:
+    image: surrealdb/surrealdb:v2
+    command: >-
+      start --log info
+      --user ${SURREAL_USER:?set in /etc/bytelyst/open-notebook-pilot.env}
+      --pass ${SURREAL_PASSWORD:?set in /etc/bytelyst/open-notebook-pilot.env}
+      rocksdb:/mydata/mydatabase.db
+    user: root
+    env_file:
+      - /etc/bytelyst/open-notebook-pilot.env
+    environment:
+      SURREAL_EXPERIMENTAL_GRAPHQL: "true"
+    volumes:
+      - /opt/bytelyst/open-notebook-pilot/surreal_data:/mydata
+    restart: unless-stopped
+    pull_policy: always
+    networks:
+      - open-notebook-private
+
+  ollama:
+    image: ollama/ollama:latest
+    volumes:
+      - /opt/bytelyst/open-notebook-pilot/ollama:/root/.ollama
+    restart: unless-stopped
+    pull_policy: always
+    networks:
+      - open-notebook-private
+
+  open_notebook:
+    image: lfnovo/open_notebook:v1-latest
+    depends_on:
+      - surrealdb
+      - ollama
+    env_file:
+      - /etc/bytelyst/open-notebook-pilot.env
+    environment:
+      SURREAL_URL: ws://surrealdb:8000/rpc
+      SURREAL_USER: ${SURREAL_USER:?set in /etc/bytelyst/open-notebook-pilot.env}
+      SURREAL_PASSWORD: ${SURREAL_PASSWORD:?set in /etc/bytelyst/open-notebook-pilot.env}
+      SURREAL_NAMESPACE: ${SURREAL_NAMESPACE:-open_notebook}
+      SURREAL_DATABASE: ${SURREAL_DATABASE:-open_notebook}
+      OPEN_NOTEBOOK_ENCRYPTION_KEY: ${OPEN_NOTEBOOK_ENCRYPTION_KEY:?set in /etc/bytelyst/open-notebook-pilot.env}
+      OPEN_NOTEBOOK_PASSWORD: ${OPEN_NOTEBOOK_PASSWORD:?set in /etc/bytelyst/open-notebook-pilot.env}
+      CORS_ORIGINS: ${CORS_ORIGINS:-http://127.0.0.1:8502,http://localhost:8502}
+      OLLAMA_BASE_URL: http://ollama:11434
+    ports:
+      - "127.0.0.1:8502:8502" # Web UI; use SSH tunnel, do not publish publicly
+      - "127.0.0.1:5055:5055" # REST API; bearer password required
+    volumes:
+      - /opt/bytelyst/open-notebook-pilot/notebook_data:/app/data
+    restart: unless-stopped
+    pull_policy: always
+    networks:
+      - open-notebook-private
+
+networks:
+  open-notebook-private:
+    internal: false
diff --git a/deployments/open-notebook-pilot/known-answer-evaluation-report.md b/deployments/open-notebook-pilot/known-answer-evaluation-report.md
new file mode 100644
index 0000000..058b2dc
--- /dev/null
+++ b/deployments/open-notebook-pilot/known-answer-evaluation-report.md
@@ -0,0 +1,102 @@
+# ByteLyst Open Notebook private pilot evaluation report
+
+Date: 2026-06-28
+Task: t_b57eb21e
+Deployment: `/opt/bytelyst/learning_ai_devops_tools/deployments/open-notebook-pilot/docker-compose.yml`
+Runtime: `/opt/bytelyst/open-notebook-pilot`
+
+## Verdict
+
+Private pilot deployment is operational and hardened for localhost-only evaluation. It is not public and should remain excluded from public Caddy/subdomain routing.
+
+Pilot readiness: 86/100
+
+Main remaining gap: full LLM `/api/search/ask/simple` Q&A timed out with the tiny local `qwen2.5:0.5b` model during first smoke testing. Retrieval, ingestion, embeddings, backup, and restore-readiness passed; source-grounded answer generation needs a larger/faster local model or a sanitized cloud comparison model before relying on generated answers.
+
+## Hardening checks
+
+| Check | Result | Evidence |
+|---|---:|---|
+| UI localhost-only | PASS | Docker published `127.0.0.1:8502->8502/tcp` only |
+| API localhost-only | PASS | Docker published `127.0.0.1:5055->5055/tcp` only |
+| SurrealDB public exposure | PASS | No SurrealDB host port published; internal Docker network only |
+| Ollama public exposure | PASS | Ollama exposes container port only; no host published port |
+| Strong secrets outside git | PASS | `/etc/bytelyst/open-notebook-pilot.env`, mode `0600`, contains generated app/encryption/SurrealDB credentials |
+| CORS explicit | PASS | `CORS_ORIGINS=http://127.0.0.1:8502,http://localhost:8502` in protected env |
+| Public dashboard exposure | PASS | No Caddy route added; access path is localhost/SSH tunnel only |
+
+## Local model/provider checks
+
+| Item | Result |
+|---|---|
+| Ollama service | Running in private compose network |
+| LLM model pulled | `qwen2.5:0.5b` |
+| Embedding model pulled | `nomic-embed-text` |
+| Open Notebook model registry | Created `ollama` language and embedding models |
+| Default models | Chat/transformation/tools/large-context set to `qwen2.5:0.5b`; embeddings set to `nomic-embed-text` |
+| Vector search | PASS: vector smoke query returned `01-access-policy-embedded.md` in 4.764s |
+| Generated ask/Q&A | NEEDS FOLLOW-UP: `/api/search/ask/simple` timed out at 300s with `qwen2.5:0.5b` |
+
+## Sanitized docs ingested
+
+Source directory: `/opt/bytelyst/open-notebook-pilot/test-docs`
+
+- `01-access-policy.md`
+- `02-backup-policy.md`
+- `03-knowledge-scope.md`
+- `01-access-policy-embedded.md` (extra embedded source for vector smoke)
+
+Notebook created by smoke script: `notebook:2iyp5pykncuhwul2qc9g`
+
+## Known-answer evaluation
+
+Raw results: `/opt/bytelyst/open-notebook-pilot/reports/known-answer-results.json`
+
+| Question | Retrieval result | Latency | Pass |
+|---|---|---:|---:|
+| Should the pilot be exposed as a public dashboard? | Top source `01-access-policy-embedded.md` | 0.262s | PASS |
+| Which host ports should UI and API use? | Top source `01-access-policy-embedded.md` | 0.064s | PASS |
+| Should SurrealDB publish port 8000? | Top source `01-access-policy-embedded.md` | 0.070s | PASS |
+| What must backups include? | Top source `02-backup-policy.md` | 0.075s | PASS |
+| What remains canonical during the pilot? | Top source `03-knowledge-scope.md` | 0.073s | PASS |
+
+Accuracy: 5/5 for retrieval-backed known-answer source selection.
+
+Citation/source quality: acceptable for this pilot smoke test because every known-answer query returned the expected source title at rank 1. Full citation UX still needs evaluation in the browser and with generated answers once the local LLM path is performant.
+
+Summary quality: not fully scored through Open Notebook transformations because local generated-answer path timed out; manual source retrieval supports the required summaries, but generated summary evaluation remains a follow-up before production use.
+
+Latency: text search was fast (0.064s-0.262s). Vector smoke was acceptable for a first local embedding path (4.764s). Local generated ask path was not acceptable with the tiny model (timeout at 300s).
+
+Operational friction: initial setup was straightforward after using the `/api` prefix; image pulls were large, Ollama model pulls succeeded, and backup scripts briefly stop the pilot to make a consistent RocksDB snapshot.
+
+## Backup/restore validation
+
+Backup command: `scripts/open-notebook-pilot/backup.sh`
+
+Backup artifact:
+
+- `/opt/bytelyst/open-notebook-pilot/backups/open-notebook-pilot-20260628T185205Z.tar.gz`
+- `/opt/bytelyst/open-notebook-pilot/backups/open-notebook-pilot-20260628T185205Z.manifest.txt`
+
+Restore-readiness command:
+
+- `scripts/open-notebook-pilot/restore-test.sh /opt/bytelyst/open-notebook-pilot/backups/open-notebook-pilot-20260628T185205Z.tar.gz`
+
+Result: PASS. Restore test verified the archive contains:
+
+- `opt/bytelyst/open-notebook-pilot/surreal_data`
+- `opt/bytelyst/open-notebook-pilot/notebook_data`
+- `etc/bytelyst/open-notebook-pilot.env`
+
+Post-backup health check: PASS after restart wait, `/health` returned `{"status":"healthy"}`.
+
+## Recommendation
+
+Keep the pilot running privately for additional low-risk browser testing and try one of these next:
+
+1. Pull/use a stronger local model that fits VM headroom, then rerun `/api/search/ask/simple` known-answer tests.
+2. Use a sanitized cloud comparison model only on the three test docs to benchmark generated answer/citation quality.
+3. If generated answers pass, expand to 10-20 sanitized ByteLyst docs and promote the backup script into the normal VM backup process.
+
+Do not expose this pilot publicly.
diff --git a/docs/repo-map.md b/docs/repo-map.md
index 3f5ff0c..c5d627e 100644
--- a/docs/repo-map.md
+++ b/docs/repo-map.md
@@ -99,6 +99,7 @@ Key files:
 
 - `ubuntu-vm-security-update.sh`
 - `deploy-gigafactory.sh` - one-shot local deploy of the Agent Gigafactory (common_plat platform-service) + ecosystem product registration; `--stop` / `--register-only` modes
+- `open-notebook-pilot/` - backup, restore-readiness, and sanitized ingestion smoke scripts for the private Open Notebook pilot
 - `README.md`
 
 #### `scripts/tracker-seed/`
@@ -182,6 +183,14 @@ Key files:
 - `utils/`
 - `output/`
 
+### `deployments/`
+
+Protected deployment artifacts and runbooks for private ByteLyst ops pilots.
+
+Key files:
+
+- `deployments/open-notebook-pilot/` — localhost-only Open Notebook pilot compose/runbook; secrets and runtime data stay under `/etc/bytelyst` and `/opt/bytelyst`, outside git.
+
 ### `dashboard/`
 
 ByteLyst DevOps dashboard — internal product for deployment orchestration and service monitoring.
diff --git a/docs/tooling-status.md b/docs/tooling-status.md
index 83beaa9..e230c52 100644
--- a/docs/tooling-status.md
+++ b/docs/tooling-status.md
@@ -10,6 +10,7 @@ High-level support map for wider team adoption.
 - `remove_user_from_repos.sh`
 - `git-work-safety-tools/`
 - `scripts/ubuntu-vm-security-update.sh`
+- `scripts/open-notebook-pilot/` + `deployments/open-notebook-pilot/` for the private localhost-only Open Notebook pilot
 
 These are the main starting points the team should discover first.
 
diff --git a/scripts/open-notebook-pilot/backup.sh b/scripts/open-notebook-pilot/backup.sh
new file mode 100755
index 0000000..0cae03e
--- /dev/null
+++ b/scripts/open-notebook-pilot/backup.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMPOSE_FILE=${COMPOSE_FILE:-/opt/bytelyst/learning_ai_devops_tools/deployments/open-notebook-pilot/docker-compose.yml}
+ENV_FILE=${ENV_FILE:-/etc/bytelyst/open-notebook-pilot.env}
+DATA_DIR=${DATA_DIR:-/opt/bytelyst/open-notebook-pilot}
+BACKUP_DIR=${BACKUP_DIR:-$DATA_DIR/backups}
+STAMP=${STAMP:-$(date -u +%Y%m%dT%H%M%SZ)}
+OUT="$BACKUP_DIR/open-notebook-pilot-$STAMP.tar.gz"
+META="$BACKUP_DIR/open-notebook-pilot-$STAMP.manifest.txt"
+
+mkdir -p "$BACKUP_DIR"
+chmod 700 "$BACKUP_DIR"
+
+# Stop only the Open Notebook pilot to get a consistent RocksDB snapshot.
+docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" stop open_notebook surrealdb >/dev/null
+
+# Restart even if tar fails.
+cleanup() {
+  docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" up -d surrealdb open_notebook >/dev/null || true
+}
+trap cleanup EXIT
+
+tar --warning=no-file-changed -C / -czf "$OUT" \
+  opt/bytelyst/open-notebook-pilot/surreal_data \
+  opt/bytelyst/open-notebook-pilot/notebook_data \
+  etc/bytelyst/open-notebook-pilot.env
+chmod 600 "$OUT"
+sha256sum "$OUT" > "$META"
+chmod 600 "$META"
+
+printf 'backup=%s\nmanifest=%s\n' "$OUT" "$META"
diff --git a/scripts/open-notebook-pilot/ingest-and-search-smoke.sh b/scripts/open-notebook-pilot/ingest-and-search-smoke.sh
new file mode 100755
index 0000000..0c27cbf
--- /dev/null
+++ b/scripts/open-notebook-pilot/ingest-and-search-smoke.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BASE=${BASE:-http://127.0.0.1:5055/api}
+ENV_FILE=${ENV_FILE:-/etc/bytelyst/open-notebook-pilot.env}
+DOC_DIR=${DOC_DIR:-/opt/bytelyst/open-notebook-pilot/test-docs}
+REPORT_DIR=${REPORT_DIR:-/opt/bytelyst/open-notebook-pilot/reports}
+AUTH=$(awk -F= '$1=="OPEN_NOTEBOOK_PASSWORD" {print substr($0, length($1)+2)}' "$ENV_FILE")
+mkdir -p "$REPORT_DIR"
+chmod 700 "$REPORT_DIR"
+
+curl_json() {
+  curl -fsS -H "Authorization: Bearer ${AUTH}" -H 'Content-Type: application/json' "$@"
+}
+
+nb_json=$(curl_json -X POST "$BASE/notebooks" \
+  --data '{"name":"ByteLyst sanitized ops pilot","description":"Sanitized low-risk ByteLyst ops docs for private Open Notebook pilot known-answer evaluation."}')
+notebook_id=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])' <<<"$nb_json")
+
+ingested=0
+for f in "$DOC_DIR"/*.md; do
+  [[ -f "$f" ]] || continue
+  title=$(basename "$f")
+  content=$(python3 - "$f" <<'PY'
+from pathlib import Path
+import json, sys
+print(json.dumps(Path(sys.argv[1]).read_text()))
+PY
+)
+  # Text sources are used for the first pilot so embeddings/model failures do not block ingestion.
+  curl -fsS -H "Authorization: Bearer ${AUTH}" \
+    -F type=text \
+    -F "notebook_id=$notebook_id" \
+    -F "title=$title" \
+    -F "content=$content" \
+    -F embed=false \
+    "$BASE/sources" >/dev/null
+  ingested=$((ingested+1))
+done
+
+for query in "CORS_ORIGINS" "SurrealDB" "backup" "canonical" "public dashboard"; do
+  payload=$(python3 - "$query" <<'PY'
+import json, sys
+print(json.dumps({"query": sys.argv[1], "type": "text", "limit": 5, "search_sources": True, "search_notes": False}))
+PY
+)
+  curl_json -X POST "$BASE/search" --data "$payload" > "$REPORT_DIR/search-${query// /-}.json"
+done
+
+printf '{"notebook_id":"%s","ingested_sources":%s,"report_dir":"%s"}\n' "$notebook_id" "$ingested" "$REPORT_DIR" > "$REPORT_DIR/ingestion-result.json"
+cat "$REPORT_DIR/ingestion-result.json"
diff --git a/scripts/open-notebook-pilot/restore-test.sh b/scripts/open-notebook-pilot/restore-test.sh
new file mode 100755
index 0000000..414b4e0
--- /dev/null
+++ b/scripts/open-notebook-pilot/restore-test.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 /opt/bytelyst/open-notebook-pilot/backups/open-notebook-pilot-YYYYmmddTHHMMSSZ.tar.gz" >&2
+  exit 2
+fi
+
+BACKUP=$1
+COMPOSE_FILE=${COMPOSE_FILE:-/opt/bytelyst/learning_ai_devops_tools/deployments/open-notebook-pilot/docker-compose.yml}
+ENV_FILE=${ENV_FILE:-/etc/bytelyst/open-notebook-pilot.env}
+RESTORE_ROOT=${RESTORE_ROOT:-/opt/bytelyst/open-notebook-pilot/restore-test}
+
+[[ -f "$BACKUP" ]] || { echo "Backup not found: $BACKUP" >&2; exit 1; }
+mkdir -p "$RESTORE_ROOT"
+chmod 700 "$RESTORE_ROOT"
+rm -rf "$RESTORE_ROOT/extracted"
+mkdir -p "$RESTORE_ROOT/extracted"
+
+tar -xzf "$BACKUP" -C "$RESTORE_ROOT/extracted"
+
+for required in \
+  "$RESTORE_ROOT/extracted/opt/bytelyst/open-notebook-pilot/surreal_data" \
+  "$RESTORE_ROOT/extracted/opt/bytelyst/open-notebook-pilot/notebook_data" \
+  "$RESTORE_ROOT/extracted/etc/bytelyst/open-notebook-pilot.env"; do
+  [[ -e "$required" ]] || { echo "Missing expected restore payload: $required" >&2; exit 1; }
+done
+
+# Verify the live compose can be parsed with the protected env file; this is a
+# non-destructive restore readiness check. Actual restore requires planned
+# downtime and replacing DATA_DIR contents from the extracted payload.
+docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" config --quiet
+
+printf 'restore_test=ok\nbackup=%s\nrestore_root=%s\n' "$BACKUP" "$RESTORE_ROOT/extracted"