#!/usr/bin/env bash # podx-tools.sh — host-side helper for Meili + OpenWebUI # Usage: ./scripts/podx-tools.sh [args...] # Run without args to see help. set -euo pipefail # Determine repo root early (used by temp dir and .env loader) ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # Writable temp dir (override with PODX_TMP or TMPDIR); default to repo tmp/ : "${PODX_TMP:=${TMPDIR:-$ROOT_DIR/tmp}}" mkdir -p "$PODX_TMP" # Curl timeouts used for OWUI/Meili requests (override via env if needed) : "${PODX_CURL_CONNECT_TIMEOUT:=5}" : "${PODX_CURL_MAX_TIME:=20}" CURL_TMO="--connect-timeout $PODX_CURL_CONNECT_TIMEOUT --max-time $PODX_CURL_MAX_TIME" # Portable mktemp helper that always writes inside $PODX_TMP (macOS/GNU compatible) _mktemp() { # mktemp on macOS doesn't support -p; use path template instead mktemp "$PODX_TMP/podx.XXXXXX" } # ---------- Pretty-print JSON (no jq required) ---------- ppjson() { if command -v python3 >/dev/null 2>&1; then python3 -m json.tool || cat elif command -v jq >/dev/null 2>&1; then jq . elif command -v node >/dev/null 2>&1; then node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{try{console.log(JSON.stringify(JSON.parse(d),null,2))}catch{console.log(d)}})" || cat else cat fi } # ---------- Load .env from repo root (robust, handles spaces & ${VAR}) ---------- ENV_FILE="$ROOT_DIR/.env" if [ -f "$ENV_FILE" ]; then # Minimal .env parser: # - ignores comments/blank lines # - preserves spaces in values (quotes optional) # - expands ${VAR} references using current environment / previously set keys while IFS= read -r __line || [ -n "$__line" ]; do # trim leading/trailing whitespace __line="${__line#"${__line%%[![:space:]]*}"}" __line="${__line%"${__line##*[![:space:]]}"}" # skip empty or comment [ -z "$__line" ] && continue [ "${__line:0:1}" = "#" ] && continue # split at first '=' case "$__line" in *=*) __key="${__line%%=*}" __val="${__line#*=}" ;; *) continue ;; esac # strip surrounding quotes if present if [[ "$__val" == \"*\" && "$__val" == *\" ]]; then __val="${__val:1:${#__val}-2}" elif [[ "$__val" == \'*\' && "$__val" == *\' ]]; then __val="${__val:1:${#__val}-2}" fi # expand ${VAR} references eval "__val_expanded=\"${__val}\"" export "${__key}=${__val_expanded}" done < "$ENV_FILE" unset __line __key __val __val_expanded fi # ---------- Defaults (can be overridden by .env) ---------- : "${MEILI_URL:=http://localhost:7700}" : "${MEILI_KEY:=${MEILI_MASTER_KEY:-}}" : "${OPENWEBUI_URL:=http://localhost:3003}" : "${OPENWEBUI_URL_HOST:=}" : "${OPENWEBUI_API_KEY:=}" # Resolve a working OpenWebUI base URL (fallback from host.docker.internal -> localhost) _owui_url() { # If a host-only override is provided, prefer it unconditionally local host_u="${OPENWEBUI_URL_HOST:-}" if [ -n "$host_u" ]; then echo "$host_u" return fi # Otherwise use OPENWEBUI_URL (defaulting to localhost), and rewrite # host.docker.internal -> localhost so it always resolves on the host local u="${OPENWEBUI_URL:-http://localhost:3003}" u="${u//host.docker.internal/localhost}" echo "$u" } # ---------- Helpers ---------- _require() { local name="$1" val="${2:-}" if [ -z "$val" ]; then echo "Missing $name (set it in .env or env): $name" >&2 exit 1 fi } _owui_get_kb_list() { _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" _require "OPENWEBUI_URL" "$OPENWEBUI_URL" local url; url="$(_owui_url)/api/v1/knowledge/list" curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$url" } _kb_create() { local kb_name="$1" _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" _require "OPENWEBUI_URL" "$OPENWEBUI_URL" curl -sS -X POST \ -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -H "Content-Type: application/json" \ -d "{\"name\":\"$kb_name\",\"description\":\"Created by podx-tools\"}" \ "$(_owui_url)/api/v1/knowledge/create" } _kb_id_by_name() { local kb_name="$1" local json; json="$(_owui_get_kb_list)" # Normalize to a flat list and choose the most recently updated match (robust name matching) python3 - "$kb_name" <<'PY' || true import sys, json want = (sys.argv[1] or "").strip() raw = sys.stdin.read().strip() if not raw: sys.exit(0) try: d = json.loads(raw) except Exception: sys.exit(0) # Support both list and {data:[...]} if isinstance(d, dict) and isinstance(d.get("data"), list): items = d["data"] elif isinstance(d, list): items = d else: items = [] def ts(kb): for key in ("updated_at", "created_at"): v = kb.get(key) if isinstance(v, (int, float)): return int(v) return 0 # Helper to normalize a KB name for comparison norm = lambda s: (s or "").strip().casefold() want_n = norm(want) # Try exact match (case/space-insensitive) exact = [kb for kb in items if norm(kb.get("name")) == want_n] if exact: print(sorted(exact, key=ts, reverse=True)[0].get("id", ""), end=""); sys.exit(0) # Then case-insensitive substring match subs = [kb for kb in items if want_n and want_n in norm(kb.get("name"))] if subs: print(sorted(subs, key=ts, reverse=True)[0].get("id", ""), end=""); sys.exit(0) print("", end="") PY } _help() { cat <" [limit] # search the 'library' index meili-reindex # index all /transcripts/*.json via worker container OpenWebUI: owui-health # check API health (200) owui-kbs # list knowledge bases owui-kb-id "" # print the KB UUID by exact name owui-kb-id-all "" # list all matching KB ids (if duplicates exist) owui-kb-resolve "" # debug name->id resolution with raw listing owui-upload # upload a file, prints file_id owui-attach "" # upload + attach to KB owui-attach-id # upload + attach using explicit KB id owui-kb-create "" # create a KB (prints JSON with id) owui-kbs-raw # raw JSON from /knowledge/list owui-batch-attach "" # attach all files matching glob Examples: ./scripts/podx-tools.sh meili-health ./scripts/podx-tools.sh meili-search "grand canyon" 10 ./scripts/podx-tools.sh owui-attach "Homelab Library" /mnt/.../episode.txt ./scripts/podx-tools.sh owui-kb-files "Homelab Library" Environment comes from .env at repo root (MEILI_URL/KEY, OPENWEBUI_URL/API_KEY). EOF } # ---------- Commands ---------- cmd="${1:-}" case "$cmd" in ""|-h|--help|help) _help ;; # ---- Meili ---- meili-health) _require "MEILI_URL" "$MEILI_URL" curl -sS "$MEILI_URL/health" | ppjson ;; meili-keys) _require "MEILI_URL" "$MEILI_URL" _require "MEILI_KEY" "$MEILI_KEY" curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/keys" | ppjson ;; meili-stats) _require "MEILI_URL" "$MEILI_URL" _require "MEILI_KEY" "$MEILI_KEY" curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/indexes/library/stats" | ppjson ;; meili-tasks) _require "MEILI_URL" "$MEILI_URL" _require "MEILI_KEY" "$MEILI_KEY" curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/tasks?limit=20&from=0" | ppjson ;; meili-init) _require "MEILI_URL" "$MEILI_URL" _require "MEILI_KEY" "$MEILI_KEY" # Create index if missing (idempotent-ish) curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \ --data '{"uid":"library"}' \ "$MEILI_URL/indexes" | ppjson ;; meili-search) shift || true q="${1:-}" lim="${2:-5}" if [ -z "$q" ]; then echo "Usage: meili-search \"\" [limit]" >&2; exit 1; fi _require "MEILI_URL" "$MEILI_URL" _require "MEILI_KEY" "$MEILI_KEY" curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \ --data "{\"q\":\"$q\",\"limit\":$lim}" \ "$MEILI_URL/indexes/library/search" | ppjson ;; meili-reindex) # Run inside the worker container; index all /transcripts/*.json if ! command -v docker >/dev/null 2>&1; then echo "docker not found on host" >&2; exit 1 fi docker compose exec -T podx-worker sh -lc ' python -c "import json,glob,worker,sys; n=0 for p in glob.glob(\"/transcripts/*.json\"): try: worker.index_meili(__import__(\"pathlib\").Path(p)) print(f\"[meili] indexed {__import__(\"pathlib\").Path(p).name}\") n+=1 except Exception as e: print(f\"[meili] FAILED {p}: {e}\", file=sys.stderr) print(f\"Indexed {n} document(s).\")" ' ;; # ---- OpenWebUI ---- # (debug) show resolved OpenWebUI URL for host-side calls # echo "Using OWUI URL: $(_owui_url)" >&2 owui-health) _require "OPENWEBUI_URL" "$OPENWEBUI_URL" curl -sS $CURL_TMO -o /dev/null -w "%{http_code}\n" "$(_owui_url)/api/health" ;; owui-kbs) _require "OPENWEBUI_URL" "$OPENWEBUI_URL" _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ "$(_owui_url)/api/v1/knowledge/list" | ppjson ;; owui-kbs-raw) _owui_get_kb_list | ppjson ;; owui-kb-create) shift || true name="${1:-}" if [ -z "$name" ]; then echo "Usage: owui-kb-create \"\"" >&2; exit 1; fi _kb_create "$name" | ppjson ;; owui-kb-id) shift || true name="${1:-}" if [ -z "$name" ]; then echo "Usage: owui-kb-id \"\"" >&2; exit 1; fi _id="$(_kb_id_by_name "$name")" if [ -z "$_id" ]; then echo "KB '$name' not found" >&2; exit 1; fi echo "$_id" ;; owui-kb-id-all) shift || true name="${1:-}" if [ -z "$name" ]; then echo "Usage: owui-kb-id-all \"\"" >&2; exit 1; fi _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" _require "OPENWEBUI_URL" "$OPENWEBUI_URL" _owui_get_kb_list | python3 - "$name" <<'PY' || exit 0 import sys, json want = (sys.argv[1] or "").strip() raw = sys.stdin.read().strip() try: d = json.loads(raw) except Exception: sys.exit(0) items = d.get("data") if isinstance(d, dict) and isinstance(d.get("data"), list) else (d if isinstance(d, list) else []) def norm(s): return (s or "").strip().casefold() want_n = norm(want) matches = [kb for kb in items if norm(kb.get("name")) == want_n or (want_n and want_n in norm(kb.get("name")))] for kb in matches: print(f"{kb.get('id','')}\t{kb.get('name','')}\tcreated_at={kb.get('created_at','')}\tupdated_at={kb.get('updated_at','')}") PY ;; owui-kb-resolve) shift || true name="${1:-}" if [ -z "$name" ]; then echo "Usage: owui-kb-resolve \"\"" >&2; exit 1; fi echo "[owui] base URL: $(_owui_url)" echo "[owui] KBs returned:" tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -w "%{http_code}" --output "$tmp_body" \ "$(_owui_url)/api/v1/knowledge/list" >"$tmp_code" || true http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"; rm -f "$tmp_code" bytes="$(wc -c <"$tmp_body" 2>/dev/null || echo 0)" cat "$tmp_body" | ppjson echo "[owui] http_code=$http_code bytes=$bytes" json="$(cat "$tmp_body")"; rm -f "$tmp_body" id="$(python3 - "$name" <<'PY' import sys, json want = (sys.argv[1] or "").strip() raw = sys.stdin.read() try: d = json.loads(raw) except Exception: print("", end=""); sys.exit(0) items = d.get("data") if isinstance(d, dict) and isinstance(d.get("data"), list) else (d if isinstance(d, list) else []) norm = lambda s: (s or "").strip().casefold() want_n = norm(want) def ts(kb): for k in ("updated_at","created_at"): v = kb.get(k) if isinstance(v,(int,float)): return int(v) return 0 exact = [kb for kb in items if norm(kb.get("name")) == want_n] subs = [kb for kb in items if want_n and want_n in norm(kb.get("name"))] cands = exact or subs print((sorted(cands, key=ts, reverse=True)[0].get("id","")) if cands else "", end="") PY <<<"$json")" if [ -n "$id" ]; then echo "[owui] resolved id for \"$name\": $id" else echo "[owui] could not resolve an id for \"$name\"" >&2 fi ;; owui-upload) shift || true file="${1:-}" if [ -z "$file" ] || [ ! -f "$file" ]; then echo "Usage: owui-upload " >&2; exit 1; fi _require "OPENWEBUI_URL" "$OPENWEBUI_URL" _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" # write response body to tmp_body; write HTTP code to tmp_code; capture exit code from $? curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -F "file=@$file" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true curl_exit=$? http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" cat "$tmp_body" | ppjson rm -f "$tmp_body" "$tmp_code" if [ $curl_exit -ne 0 ]; then echo "Upload failed: curl exit $curl_exit" >&2 exit $curl_exit fi if [ "$http_code" != "200" ]; then echo "Upload failed (HTTP $http_code)" >&2 exit 1 fi ;; owui-attach) shift || true kb_name="${1:-}"; file="${2:-}" if [ -z "$kb_name" ] || [ -z "$file" ] || [ ! -f "$file" ]; then echo "Usage: owui-attach \"\" " >&2; exit 1 fi _require "OPENWEBUI_URL" "$OPENWEBUI_URL" _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" # 1) Upload tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -F "file=@$file" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" FILE_JSON="$(cat "$tmp_body")" rm -f "$tmp_body" "$tmp_code" echo "$FILE_JSON" | ppjson if [ $curl_exit -ne 0 ]; then echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit fi if [ "$http_code" != "200" ]; then echo "Upload failed (HTTP $http_code)" >&2; exit 1 fi FILE_ID="$(python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")" if [ -z "$FILE_ID" ]; then echo "Upload failed (no file id)"; exit 1; fi # 2) Resolve KB and attach KB_ID="$(_kb_id_by_name "$kb_name")" echo "[owui] attaching to KB: $kb_name (id: $KB_ID)" if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found (or ambiguous)." >&2 echo "Tip: run './scripts/podx-tools.sh owui-kb-resolve \"$kb_name\"' to inspect and resolve, or create one:" >&2 echo " ./scripts/podx-tools.sh owui-kb-create \"$kb_name\"" >&2 exit 1 fi # Attach: capture headers, body, http code and curl exit tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"; tmp_hdrs="$(_mktemp)" curl -sS -X POST \ -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -H "Content-Type: application/json" \ -d "{\"file_id\":\"$FILE_ID\"}" \ -D "$tmp_hdrs" \ -w "%{http_code}" --output "$tmp_body" \ "$(_owui_url)/api/v1/knowledge/$KB_ID/file/add" >"$tmp_code" || true curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" echo "[owui] response headers:"; sed -n '1,5p' "$tmp_hdrs" || true RESP="$(cat "$tmp_body")" echo "$RESP" | ppjson rm -f "$tmp_body" "$tmp_code" "$tmp_hdrs" if [ $curl_exit -ne 0 ]; then echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit fi # Some environments report curl(23) sporadically; treat missing code as non-200 if [ -z "$http_code" ] || [ "$http_code" = "000" ]; then echo "Attach failed: no HTTP code returned" >&2; exit 1 fi case "$http_code" in 200|201|204) : # success ;; *) echo "Attach failed (HTTP $http_code)" >&2; exit 1 ;; esac ;; owui-attach-id) shift || true kb_id="${1:-}"; file="${2:-}" if [ -z "$kb_id" ] || [ -z "$file" ] || [ ! -f "$file" ]; then echo "Usage: owui-attach-id " >&2; exit 1 fi _require "OPENWEBUI_URL" "$OPENWEBUI_URL" _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" # 1) Upload tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -F "file=@$file" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" FILE_JSON="$(cat "$tmp_body")" rm -f "$tmp_body" "$tmp_code" echo "$FILE_JSON" | ppjson if [ $curl_exit -ne 0 ]; then echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit fi if [ "$http_code" != "200" ]; then echo "Upload failed (HTTP $http_code)" >&2; exit 1 fi FILE_ID="$(python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get(\"id\") or (d.get(\"data\") or {}).get(\"id\",\"\"))' <<<"$FILE_JSON")" if [ -z "$FILE_ID" ]; then echo "Upload failed (no file id)"; exit 1; fi # 2) Attach using explicit KB id tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"; tmp_hdrs="$(_mktemp)" curl -sS -X POST \ -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -H "Content-Type: application/json" \ -d "{\"file_id\":\"$FILE_ID\"}" \ -D "$tmp_hdrs" \ -w "%{http_code}" --output "$tmp_body" \ "$(_owui_url)/api/v1/knowledge/$kb_id/file/add" >"$tmp_code" || true curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" echo "[owui] response headers:"; sed -n '1,5p' "$tmp_hdrs" || true RESP="$(cat "$tmp_body")" echo "$RESP" | ppjson rm -f "$tmp_body" "$tmp_code" "$tmp_hdrs" if [ $curl_exit -ne 0 ]; then echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit fi if [ -z "$http_code" ] || [ "$http_code" = "000" ]; then echo "Attach failed: no HTTP code returned" >&2; exit 1 fi case "$http_code" in 200|201|204) : ;; *) echo "Attach failed (HTTP $http_code)" >&2; exit 1 ;; esac ;; owui-kb-files) shift || true kb_name="${1:-}" if [ -z "$kb_name" ]; then echo "Usage: owui-kb-files \"\"" >&2; exit 1; fi _require "OPENWEBUI_URL" "$OPENWEBUI_URL" _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" KB_ID="$(_kb_id_by_name "$kb_name")" if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found"; exit 1; fi curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ "$(_owui_url)/api/v1/knowledge/$KB_ID/files" | ppjson ;; owui-kb-debug) shift || true name="${1:-}" if [ -z "$name" ]; then echo "Usage: owui-kb-debug \"\"" >&2; exit 1; fi echo "[owui] base URL: $(_owui_url)" echo "[owui] KBs returned:" _owui_get_kb_list | ppjson id="$(_kb_id_by_name "$name")" if [ -n "$id" ]; then echo "[owui] resolved id for \"$name\": $id" else echo "[owui] could not resolve an id for \"$name\"" >&2 fi ;; owui-batch-attach) shift || true kb_name="${1:-}"; glob_pat="${2:-}" if [ -z "$kb_name" ] || [ -z "$glob_pat" ]; then echo "Usage: owui-batch-attach \"\" " >&2; exit 1 fi KB_ID="$(_kb_id_by_name "$kb_name")" if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found"; exit 1; fi _require "OPENWEBUI_URL" "$OPENWEBUI_URL" _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" shopt -s nullglob matched=($glob_pat) shopt -u nullglob if [ ${#matched[@]} -eq 0 ]; then echo "No files match: $glob_pat"; exit 1; fi for f in "${matched[@]}"; do echo "[owui] uploading: $f" tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -F "file=@$f" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true curl_up_exit=$?; code_up="$(cat "$tmp_code" 2>/dev/null || echo 0)" FILE_JSON="$(cat "$tmp_body")" rm -f "$tmp_body" "$tmp_code" if [ $curl_up_exit -ne 0 ] || [ "$code_up" != "200" ]; then echo " upload failed (curl=$curl_up_exit http=$code_up), skipping" echo "$FILE_JSON" | ppjson continue fi FILE_ID="$(python3 -c 'import sys,json; d=json.loads(sys.stdin.read()); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")" if [ -z "$FILE_ID" ]; then echo " upload failed (no id), skipping"; echo "$FILE_JSON" | ppjson; continue; fi tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS -X POST -H "Authorization: Bearer $OPENWEBUI_API_KEY" -H "Content-Type: application/json" \ -d "{\"file_id\":\"$FILE_ID\"}" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/knowledge/$KB_ID/file/add" >"$tmp_code" || true curl_att_exit=$?; code_att="$(cat "$tmp_code" 2>/dev/null || echo 0)" RESP="$(cat "$tmp_body")" rm -f "$tmp_body" "$tmp_code" echo "$RESP" | ppjson if [ "$code_att" != "200" ]; then echo " attach failed (HTTP $code_att)" fi done ;; *) echo "Unknown command: $cmd" >&2 _help exit 1 ;; esac