From 4f31437433dedb3d974848cedd62b81bc09f7eec Mon Sep 17 00:00:00 2001 From: Tomas Kracmar Date: Mon, 8 Sep 2025 10:07:22 +0200 Subject: [PATCH] PodX-tools introduction --- scripts/podx-tools.sh | 263 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 scripts/podx-tools.sh diff --git a/scripts/podx-tools.sh b/scripts/podx-tools.sh new file mode 100644 index 0000000..32b0f25 --- /dev/null +++ b/scripts/podx-tools.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +# podx-tools.sh — host-side helper for Meili + OpenWebUI +# Usage: ./scripts/podx-tools.sh [args...] +# Run without args to see help. + +set -euo pipefail + +# ---------- Pretty-print JSON (no jq required) ---------- +ppjson() { + if command -v python3 >/dev/null 2>&1; then + python3 -m json.tool || cat + elif command -v jq >/dev/null 2>&1; then + jq . + elif command -v node >/dev/null 2>&1; then + node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{try{console.log(JSON.stringify(JSON.parse(d),null,2))}catch{console.log(d)}})" || cat + else + cat + fi +} + +# ---------- Load .env from repo root (respects var references) ---------- +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ENV_FILE="$ROOT_DIR/.env" +if [ -f "$ENV_FILE" ]; then + set -a + # shellcheck source=/dev/null + source "$ENV_FILE" + set +a +fi + +# ---------- Defaults (can be overridden by .env) ---------- +: "${MEILI_URL:=http://localhost:7700}" +: "${MEILI_KEY:=${MEILI_MASTER_KEY:-}}" + +: "${OPENWEBUI_URL:=http://localhost:3003}" +: "${OPENWEBUI_API_KEY:=}" + +# ---------- Helpers ---------- +_require() { + local name="$1" val="${2:-}" + if [ -z "$val" ]; then + echo "Missing $name (set it in .env or env): $name" >&2 + exit 1 + fi +} + +_kb_id_by_name() { + local kb_name="$1" + _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ + "$OPENWEBUI_URL/api/v1/knowledge/list" \ + | python3 - "$kb_name" <<'PY' +import sys,json,os +name=sys.argv[1] +try: + data=json.load(sys.stdin) +except Exception: + print("", end="") + sys.exit(0) +for kb in data: + if kb.get("name")==name: + print(kb.get("id",""), end="") + break +PY +} + +_help() { + cat <" [limit] # search the 'library' index + meili-reindex # index all /transcripts/*.json via worker container + +OpenWebUI: + owui-health # check API health (200) + owui-kbs # list knowledge bases + owui-kb-id "" # print the KB UUID by exact name + owui-upload # upload a file, prints file_id + owui-attach "" # upload + attach to KB + owui-kb-files "" # list files in KB + owui-batch-attach "" # attach all files matching glob + +Examples: + ./scripts/podx-tools.sh meili-health + ./scripts/podx-tools.sh meili-search "grand canyon" 10 + ./scripts/podx-tools.sh owui-attach "Homelab Library" /mnt/.../episode.txt + ./scripts/podx-tools.sh owui-kb-files "Homelab Library" + +Environment comes from .env at repo root (MEILI_URL/KEY, OPENWEBUI_URL/API_KEY). +EOF +} + +# ---------- Commands ---------- +cmd="${1:-}" +case "$cmd" in + ""|-h|--help|help) + _help + ;; + + # ---- Meili ---- + meili-health) + _require "MEILI_URL" "$MEILI_URL" + curl -sS "$MEILI_URL/health" | ppjson + ;; + + meili-keys) + _require "MEILI_URL" "$MEILI_URL" + _require "MEILI_KEY" "$MEILI_KEY" + curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/keys" | ppjson + ;; + + meili-stats) + _require "MEILI_URL" "$MEILI_URL" + _require "MEILI_KEY" "$MEILI_KEY" + curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/indexes/library/stats" | ppjson + ;; + + meili-tasks) + _require "MEILI_URL" "$MEILI_URL" + _require "MEILI_KEY" "$MEILI_KEY" + curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/tasks?limit=20&from=0" | ppjson + ;; + + meili-init) + _require "MEILI_URL" "$MEILI_URL" + _require "MEILI_KEY" "$MEILI_KEY" + # Create index if missing (idempotent-ish) + curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \ + --data '{"uid":"library"}' \ + "$MEILI_URL/indexes" | ppjson + ;; + + meili-search) + shift || true + q="${1:-}" + lim="${2:-5}" + if [ -z "$q" ]; then echo "Usage: meili-search \"\" [limit]" >&2; exit 1; fi + _require "MEILI_URL" "$MEILI_URL" + _require "MEILI_KEY" "$MEILI_KEY" + curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \ + --data "{\"q\":\"$q\",\"limit\":$lim}" \ + "$MEILI_URL/indexes/library/search" | ppjson + ;; + + meili-reindex) + # Run inside the worker container; index all /transcripts/*.json + if ! command -v docker >/dev/null 2>&1; then + echo "docker not found on host" >&2; exit 1 + fi + docker compose exec -T podx-worker sh -lc ' + python -c "import json,glob,worker,sys; n=0 +for p in glob.glob(\"/transcripts/*.json\"): + try: + worker.index_meili(__import__(\"pathlib\").Path(p)) + print(f\"[meili] indexed {__import__(\"pathlib\").Path(p).name}\") + n+=1 + except Exception as e: + print(f\"[meili] FAILED {p}: {e}\", file=sys.stderr) +print(f\"Indexed {n} document(s).\")" + ' + ;; + + # ---- OpenWebUI ---- + owui-health) + _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + curl -sS -o /dev/null -w "%{http_code}\n" "$OPENWEBUI_URL/api/health" + ;; + + owui-kbs) + _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ + "$OPENWEBUI_URL/api/v1/knowledge/list" | ppjson + ;; + + owui-kb-id) + shift || true + name="${1:-}" + if [ -z "$name" ]; then echo "Usage: owui-kb-id \"\"" >&2; exit 1; fi + _id="$(_kb_id_by_name "$name")" + if [ -z "$_id" ]; then echo "KB '$name' not found" >&2; exit 1; fi + echo "$_id" + ;; + + owui-upload) + shift || true + file="${1:-}" + if [ -z "$file" ] || [ ! -f "$file" ]; then echo "Usage: owui-upload " >&2; exit 1; fi + _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ + -F "file=@$file" \ + "$OPENWEBUI_URL/api/v1/files/" | ppjson + ;; + + owui-attach) + shift || true + kb_name="${1:-}"; file="${2:-}" + if [ -z "$kb_name" ] || [ -z "$file" ] || [ ! -f "$file" ]; then + echo "Usage: owui-attach \"\" " >&2; exit 1 + fi + _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + FILE_JSON="$(curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" -F "file=@$file" "$OPENWEBUI_URL/api/v1/files/")" + FILE_ID="$(python3 -c 'import sys,json; d=json.loads(sys.stdin.read()); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")" + if [ -z "$FILE_ID" ]; then echo "Upload failed (no file id)"; echo "$FILE_JSON" | ppjson; exit 1; fi + KB_ID="$(_kb_id_by_name "$kb_name")" + if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found"; exit 1; fi + curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" -H "Content-Type: application/json" \ + -d "{\"file_id\":\"$FILE_ID\"}" \ + "$OPENWEBUI_URL/api/v1/knowledge/$KB_ID/file/add" | ppjson + ;; + + owui-kb-files) + shift || true + kb_name="${1:-}" + if [ -z "$kb_name" ]; then echo "Usage: owui-kb-files \"\"" >&2; exit 1; fi + _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + KB_ID="$(_kb_id_by_name "$kb_name")" + if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found"; exit 1; fi + curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ + "$OPENWEBUI_URL/api/v1/knowledge/$KB_ID/files" | ppjson + ;; + + owui-batch-attach) + shift || true + kb_name="${1:-}"; glob_pat="${2:-}" + if [ -z "$kb_name" ] || [ -z "$glob_pat" ]; then + echo "Usage: owui-batch-attach \"\" " >&2; exit 1 + fi + KB_ID="$(_kb_id_by_name "$kb_name")" + if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found"; exit 1; fi + _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + shopt -s nullglob + matched=($glob_pat) + shopt -u nullglob + if [ ${#matched[@]} -eq 0 ]; then echo "No files match: $glob_pat"; exit 1; fi + for f in "${matched[@]}"; do + echo "[owui] uploading: $f" + FILE_JSON="$(curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" -F "file=@$f" "$OPENWEBUI_URL/api/v1/files/")" + FILE_ID="$(python3 -c 'import sys,json; d=json.loads(sys.stdin.read()); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")" + if [ -z "$FILE_ID" ]; then echo " upload failed, skipping"; continue; fi + curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" -H "Content-Type: application/json" \ + -d "{\"file_id\":\"$FILE_ID\"}" \ + "$OPENWEBUI_URL/api/v1/knowledge/$KB_ID/file/add" | ppjson + done + ;; + + *) + echo "Unknown command: $cmd" >&2 + _help + exit 1 + ;; +esac \ No newline at end of file