52 lines
1.9 KiB
Bash
Executable File
52 lines
1.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
BASE=${BASE:-http://127.0.0.1:5055/api}
|
|
ENV_FILE=${ENV_FILE:-/etc/bytelyst/open-notebook-pilot.env}
|
|
DOC_DIR=${DOC_DIR:-/opt/bytelyst/open-notebook-pilot/test-docs}
|
|
REPORT_DIR=${REPORT_DIR:-/opt/bytelyst/open-notebook-pilot/reports}
|
|
AUTH=$(awk -F= '$1=="OPEN_NOTEBOOK_PASSWORD" {print substr($0, length($1)+2)}' "$ENV_FILE")
|
|
mkdir -p "$REPORT_DIR"
|
|
chmod 700 "$REPORT_DIR"
|
|
|
|
curl_json() {
|
|
curl -fsS -H "Authorization: Bearer ${AUTH}" -H 'Content-Type: application/json' "$@"
|
|
}
|
|
|
|
nb_json=$(curl_json -X POST "$BASE/notebooks" \
|
|
--data '{"name":"ByteLyst sanitized ops pilot","description":"Sanitized low-risk ByteLyst ops docs for private Open Notebook pilot known-answer evaluation."}')
|
|
notebook_id=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])' <<<"$nb_json")
|
|
|
|
ingested=0
|
|
for f in "$DOC_DIR"/*.md; do
|
|
[[ -f "$f" ]] || continue
|
|
title=$(basename "$f")
|
|
content=$(python3 - "$f" <<'PY'
|
|
from pathlib import Path
|
|
import json, sys
|
|
print(json.dumps(Path(sys.argv[1]).read_text()))
|
|
PY
|
|
)
|
|
# Text sources are used for the first pilot so embeddings/model failures do not block ingestion.
|
|
curl -fsS -H "Authorization: Bearer ${AUTH}" \
|
|
-F type=text \
|
|
-F "notebook_id=$notebook_id" \
|
|
-F "title=$title" \
|
|
-F "content=$content" \
|
|
-F embed=false \
|
|
"$BASE/sources" >/dev/null
|
|
ingested=$((ingested+1))
|
|
done
|
|
|
|
for query in "CORS_ORIGINS" "SurrealDB" "backup" "canonical" "public dashboard"; do
|
|
payload=$(python3 - "$query" <<'PY'
|
|
import json, sys
|
|
print(json.dumps({"query": sys.argv[1], "type": "text", "limit": 5, "search_sources": True, "search_notes": False}))
|
|
PY
|
|
)
|
|
curl_json -X POST "$BASE/search" --data "$payload" > "$REPORT_DIR/search-${query// /-}.json"
|
|
done
|
|
|
|
printf '{"notebook_id":"%s","ingested_sources":%s,"report_dir":"%s"}\n' "$notebook_id" "$ingested" "$REPORT_DIR" > "$REPORT_DIR/ingestion-result.json"
|
|
cat "$REPORT_DIR/ingestion-result.json"
|