diff --git a/scripts/podx-tools.sh b/scripts/podx-tools.sh index 4dd1660..4a297c3 100755 --- a/scripts/podx-tools.sh +++ b/scripts/podx-tools.sh @@ -1,17 +1,13 @@ #!/usr/bin/env bash -# podx-tools.sh — host-side helper for Meili + OpenWebUI +# podx-tools.sh — host-side helper for Meili + OpenWebUI (+ worker switches) # Usage: ./scripts/podx-tools.sh [args...] # Run without args to see help. - set -euo pipefail -# Determine repo root early (used by temp dir and .env loader) +# ------------------------------ Paths / tmp ------------------------------ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -# Writable temp dir (override with PODX_TMP or TMPDIR); default to repo tmp/ : "${PODX_TMP:=${TMPDIR:-$ROOT_DIR/tmp}}" - mkdir -p "$PODX_TMP" # Curl timeouts used for OWUI/Meili requests (override via env if needed) @@ -20,15 +16,12 @@ mkdir -p "$PODX_TMP" CURL_TMO="--connect-timeout $PODX_CURL_CONNECT_TIMEOUT --max-time $PODX_CURL_MAX_TIME" # Portable mktemp helper that always writes inside $PODX_TMP (macOS/GNU compatible) -_mktemp() { - # mktemp on macOS doesn't support -p; use path template instead - mktemp "$PODX_TMP/podx.XXXXXX" -} +_mktemp() { mktemp "$PODX_TMP/podx.XXXXXX"; } -# ---------- Pretty-print JSON (no jq required) ---------- +# ------------------------------ Pretty JSON ------------------------------ ppjson() { if command -v python3 >/dev/null 2>&1; then - python3 -m json.tool || cat + python3 -m json.tool 2>/dev/null || cat elif command -v jq >/dev/null 2>&1; then jq . elif command -v node >/dev/null 2>&1; then @@ -38,72 +31,40 @@ ppjson() { fi } -# ---------- Load .env from repo root (robust, handles spaces & ${VAR}) ---------- +# ------------------------------ Load .env ------------------------------ ENV_FILE="$ROOT_DIR/.env" if [ -f "$ENV_FILE" ]; then - # Minimal .env parser: - # - ignores comments/blank lines - # - preserves spaces in values (quotes optional) - # - expands ${VAR} references using current environment / previously set keys while IFS= read -r __line || [ -n "$__line" ]; do - # trim leading/trailing whitespace __line="${__line#"${__line%%[![:space:]]*}"}" __line="${__line%"${__line##*[![:space:]]}"}" - # skip empty or comment [ -z "$__line" ] && continue [ "${__line:0:1}" = "#" ] && continue - # split at first '=' case "$__line" in - *=*) - __key="${__line%%=*}" - __val="${__line#*=}" - ;; - *) - continue - ;; + *=*) __key="${__line%%=*}"; __val="${__line#*=}";; + *) continue;; esac - # strip surrounding quotes if present if [[ "$__val" == \"*\" && "$__val" == *\" ]]; then __val="${__val:1:${#__val}-2}" elif [[ "$__val" == \'*\' && "$__val" == *\' ]]; then __val="${__val:1:${#__val}-2}" fi - # expand ${VAR} references eval "__val_expanded=\"${__val}\"" export "${__key}=${__val_expanded}" done < "$ENV_FILE" unset __line __key __val __val_expanded fi -# ---------- Defaults (can be overridden by .env) ---------- +# ------------------------------ Defaults ------------------------------ : "${MEILI_URL:=http://localhost:7700}" : "${MEILI_KEY:=${MEILI_MASTER_KEY:-}}" : "${OPENWEBUI_URL:=http://localhost:3003}" : "${OPENWEBUI_URL_HOST:=}" : "${OPENWEBUI_API_KEY:=}" -# Optional: explicit KB id to use for all KB operations (bypasses name resolution) : "${OPENWEBUI_KB_ID:=}" - -# Max seconds to wait for OWUI to finish extracting file content before attach : "${OPENWEBUI_WAIT_SECS:=180}" -# Resolve a working OpenWebUI base URL (fallback from host.docker.internal -> localhost) -_owui_url() { - # If a host-only override is provided, prefer it unconditionally - local host_u="${OPENWEBUI_URL_HOST:-}" - if [ -n "$host_u" ]; then - echo "$host_u" - return - fi - # Otherwise use OPENWEBUI_URL (defaulting to localhost), and rewrite - # host.docker.internal -> localhost so it always resolves on the host - local u="${OPENWEBUI_URL:-http://localhost:3003}" - u="${u//host.docker.internal/localhost}" - echo "$u" -} - -# ---------- Helpers ---------- +# ------------------------------ Helpers ------------------------------ _require() { local name="$1" val="${2:-}" if [ -z "$val" ]; then @@ -112,11 +73,20 @@ _require() { fi } +_owui_url() { + local host_u="${OPENWEBUI_URL_HOST:-}" + if [ -n "$host_u" ]; then + echo "$host_u"; return + fi + local u="${OPENWEBUI_URL:-http://localhost:3003}" + u="${u//host.docker.internal/localhost}" + echo "$u" +} + _owui_get_kb_list() { _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" _require "OPENWEBUI_URL" "$OPENWEBUI_URL" - local url; url="$(_owui_url)/api/v1/knowledge/list" - curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$url" + curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$(_owui_url)/api/v1/knowledge/list" } _kb_create() { @@ -132,105 +102,72 @@ _kb_create() { _kb_id_by_name() { local kb_name="$1" - # If an explicit KB id is provided via env, use it directly if [ -n "${OPENWEBUI_KB_ID:-}" ]; then - echo "$OPENWEBUI_KB_ID" - return 0 + echo "$OPENWEBUI_KB_ID"; return 0 fi local json; json="$(_owui_get_kb_list)" local __id="" __id=$(printf '%s' "$json" | python3 - "$kb_name" <<'PY' import sys, json, unicodedata as ud - -def norm(s: str) -> str: - s = ud.normalize('NFKC', s or '') - return s.strip().casefold() - -want = norm(sys.argv[1] if len(sys.argv) > 1 else '') +def norm(s): return ud.normalize('NFKC', (s or '')).strip().casefold() +want = norm(sys.argv[1] if len(sys.argv)>1 else '') raw = sys.stdin.read().strip() -if not raw: - print('', end=''); raise SystemExit(0) -try: - d = json.loads(raw) -except Exception: - print('', end=''); raise SystemExit(0) - +if not raw: print('', end=''); raise SystemExit(0) +try: d = json.loads(raw) +except: print('', end=''); raise SystemExit(0) items = d.get('data') if isinstance(d, dict) and isinstance(d.get('data'), list) else (d if isinstance(d, list) else []) - -# If only one KB exists, return it -if isinstance(items, list) and len(items) == 1: - print(items[0].get('id',''), end=''); raise SystemExit(0) - -# Prefer exact normalized name match +if isinstance(items, list) and len(items)==1: + print(items[0].get('id',''), end=''); raise SystemExit(0) exact = [kb for kb in items if norm(kb.get('name')) == want] if exact: - exact.sort(key=lambda kb: int(kb.get('updated_at') or kb.get('created_at') or 0), reverse=True) - print(exact[0].get('id',''), end=''); raise SystemExit(0) - -# Fallback: substring match + exact.sort(key=lambda kb: int(kb.get('updated_at') or kb.get('created_at') or 0), reverse=True) + print(exact[0].get('id',''), end=''); raise SystemExit(0) subs = [kb for kb in items if want and want in norm(kb.get('name'))] if subs: - subs.sort(key=lambda kb: int(kb.get('updated_at') or kb.get('created_at') or 0), reverse=True) - print(subs[0].get('id',''), end=''); raise SystemExit(0) - + subs.sort(key=lambda kb: int(kb.get('updated_at') or kb.get('created_at') or 0), reverse=True) + print(subs[0].get('id',''), end=''); raise SystemExit(0) print('', end='') PY ) - - # If python resolution failed, try jq (if available) - if [ -z "${__id:-}" ]; then - if command -v jq >/dev/null 2>&1; then - __id=$(printf '%s' "$json" | jq -r ' - if type=="array" and length==1 then .[0].id // empty - elif type=="array" then (map(select((.name // "")|ascii_downcase==("'"$kb_name"'"|ascii_downcase))) | .[0].id) // empty - elif has("data") and (.data|type=="array") and (.data|length==1) then .data[0].id // empty - else empty end') - fi + if [ -z "${__id:-}" ] && command -v jq >/dev/null 2>&1; then + __id=$(printf '%s' "$json" | jq -r ' + if type=="array" and length==1 then .[0].id // empty + elif type=="array" then (map(select((.name // "")|ascii_downcase==("'"$kb_name"'"|ascii_downcase))) | .[0].id) // empty + elif has("data") and (.data|type=="array") and (.data|length==1) then .data[0].id // empty + else empty end') fi - # Last resort: naive sed/grep — take the first id field if [ -z "${__id:-}" ]; then __id=$(printf '%s' "$json" | sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]\+\)".*/\1/p' | head -n1) fi printf '%s' "${__id:-}" } -# --- OWUI file helpers ------------------------------------------------------- - +# ------------------------------ OWUI file helpers ------------------------------ _owui_file_get() { - # usage: _owui_file_get local fid="$1" - curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ - "$(_owui_url)/api/v1/files/$fid" + curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$(_owui_url)/api/v1/files/$fid" } _owui_wait_file() { - # usage: _owui_wait_file [timeout_secs] - local fid="$1" - local timeout="${2:-120}" # default 2 minutes - local start now status content_len - start="$(date +%s)" + local fid="$1" timeout="${2:-120}" + local start="$(date +%s)" while :; do - now="$(date +%s)" - if [ $((now - start)) -ge "$timeout" ]; then - return 1 - fi - # Fetch status and content length + local now="$(date +%s)" + if [ $((now - start)) -ge "$timeout" ]; then return 1; fi read -r status content_len </dev/null 2>&1 && docker compose ps redis >/dev/null 2>&1; then docker compose exec -T redis redis-cli "$@" elif command -v redis-cli >/dev/null 2>&1; then @@ -240,9 +177,9 @@ _redis_cli() { return 1 fi } - _transcribe_key="podx:transcribe:paused" +# ------------------------------ Help ------------------------------ _help() { cat <" # print the KB UUID by exact name owui-kb-id-all "" # list all matching KB ids (if duplicates exist) owui-kb-resolve "" # debug name->id resolution with raw listing - # Name resolution is Unicode/space-insensitive and prefers the most recently updated match. - owui-upload # upload a file, prints file_id - owui-attach "" # upload + attach to KB + owui-upload # upload a file, prints file JSON + owui-attach "" # upload + attach to KB (waits for extraction) owui-attach-id # upload + attach using explicit KB id + owui-kb-files "" # list files for a KB (best-effort) owui-kb-create "" # create a KB (prints JSON with id) owui-kbs-raw # raw JSON from /knowledge/list owui-batch-attach "" # attach all files matching glob Transcription control: - transcribe-status # show whether transcription workers are paused - transcribe-pause # pause CPU-heavy transcription jobs - transcribe-resume # resume transcription jobs + transcribe-status # show whether transcription workers are paused + transcribe-pause # pause CPU-heavy transcription jobs + transcribe-resume # resume transcription jobs Examples: ./scripts/podx-tools.sh meili-health @@ -281,130 +218,100 @@ Examples: ./scripts/podx-tools.sh owui-attach "Homelab Library" /mnt/.../episode.txt ./scripts/podx-tools.sh owui-kb-files "Homelab Library" -Environment comes from .env at repo root (MEILI_URL/KEY, OPENWEBUI_URL/API_KEY). +Environment comes from .env at repo root (MEILI_URL/KEY, OPENWEBUI_URL/API_KEY, OPENWEBUI_KB_ID). EOF } -# ---------- Commands ---------- +# ------------------------------ Commands ------------------------------ cmd="${1:-}" case "$cmd" in - ""|-h|--help|help) - _help - ;; + ""|-h|--help|help) _help ;; - # ---- Meili ---- + # ---------- Meili ---------- meili-health) _require "MEILI_URL" "$MEILI_URL" curl -sS "$MEILI_URL/health" | ppjson ;; - meili-keys) - _require "MEILI_URL" "$MEILI_URL" - _require "MEILI_KEY" "$MEILI_KEY" + _require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY" curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/keys" | ppjson ;; - meili-stats) - _require "MEILI_URL" "$MEILI_URL" - _require "MEILI_KEY" "$MEILI_KEY" + _require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY" curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/indexes/library/stats" | ppjson ;; - meili-tasks) - _require "MEILI_URL" "$MEILI_URL" - _require "MEILI_KEY" "$MEILI_KEY" + _require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY" curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/tasks?limit=20&from=0" | ppjson ;; - meili-init) - _require "MEILI_URL" "$MEILI_URL" - _require "MEILI_KEY" "$MEILI_KEY" - # Create index if missing (idempotent-ish) + _require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY" curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \ - --data '{"uid":"library"}' \ - "$MEILI_URL/indexes" | ppjson + --data '{"uid":"library"}' "$MEILI_URL/indexes" | ppjson ;; - meili-search) - shift || true - q="${1:-}" - lim="${2:-5}" - if [ -z "$q" ]; then echo "Usage: meili-search \"\" [limit]" >&2; exit 1; fi - _require "MEILI_URL" "$MEILI_URL" - _require "MEILI_KEY" "$MEILI_KEY" + shift || true; q="${1:-}"; lim="${2:-5}" + [ -z "$q" ] && { echo "Usage: meili-search \"\" [limit]" >&2; exit 1; } + _require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY" curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \ --data "{\"q\":\"$q\",\"limit\":$lim}" \ "$MEILI_URL/indexes/library/search" | ppjson ;; - meili-reindex) - # Run inside the worker container; index all /transcripts/*.json - if ! command -v docker >/dev/null 2>&1; then - echo "docker not found on host" >&2; exit 1 - fi + if ! command -v docker >/dev/null 2>&1; then echo "docker not found on host" >&2; exit 1; fi docker compose exec -T podx-worker sh -lc ' - python -c "import json,glob,worker,sys; n=0 -for p in glob.glob(\"/transcripts/*.json\"): +python - <&2 - + # ---------- OpenWebUI ---------- owui-health) _require "OPENWEBUI_URL" "$OPENWEBUI_URL" curl -sS $CURL_TMO -o /dev/null -w "%{http_code}\n" "$(_owui_url)/api/health" ;; - owui-kbs) - _require "OPENWEBUI_URL" "$OPENWEBUI_URL" - _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" - curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ - "$(_owui_url)/api/v1/knowledge/list" | ppjson + _require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$(_owui_url)/api/v1/knowledge/list" | ppjson ;; - owui-kbs-raw) _owui_get_kb_list | ppjson ;; - owui-kb-create) - shift || true - name="${1:-}" - if [ -z "$name" ]; then echo "Usage: owui-kb-create \"\"" >&2; exit 1; fi + shift || true; name="${1:-}" + [ -z "$name" ] && { echo "Usage: owui-kb-create \"\"" >&2; exit 1; } _kb_create "$name" | ppjson ;; - owui-kb-id) - shift || true - name="${1:-}" - if [ -z "$name" ]; then echo "Usage: owui-kb-id \"\"" >&2; exit 1; fi + shift || true; name="${1:-}" + [ -z "$name" ] && { echo "Usage: owui-kb-id \"\"" >&2; exit 1; } _id="$(_kb_id_by_name "$name")" - if [ -z "$_id" ]; then echo "KB '$name' not found" >&2; exit 1; fi + [ -z "$_id" ] && { echo "KB '$name' not found" >&2; exit 1; } echo "$_id" ;; - owui-kb-id-all) - shift || true - name="${1:-}" - if [ -z "$name" ]; then echo "Usage: owui-kb-id-all \"\"" >&2; exit 1; fi - _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" - _require "OPENWEBUI_URL" "$OPENWEBUI_URL" + shift || true; name="${1:-}" + [ -z "$name" ] && { echo "Usage: owui-kb-id-all \"\"" >&2; exit 1; } _owui_get_kb_list | python3 - "$name" <<'PY' || exit 0 import sys, json want = (sys.argv[1] or "").strip() raw = sys.stdin.read().strip() -try: - d = json.loads(raw) -except Exception: - sys.exit(0) +try: d = json.loads(raw) +except: sys.exit(0) items = d.get("data") if isinstance(d, dict) and isinstance(d.get("data"), list) else (d if isinstance(d, list) else []) def norm(s): return (s or "").strip().casefold() want_n = norm(want) @@ -413,11 +320,9 @@ for kb in matches: print(f"{kb.get('id','')}\t{kb.get('name','')}\tcreated_at={kb.get('created_at','')}\tupdated_at={kb.get('updated_at','')}") PY ;; - owui-kb-resolve) - shift || true - name="${1:-}" - if [ -z "$name" ]; then echo "Usage: owui-kb-resolve \"\"" >&2; exit 1; fi + shift || true; name="${1:-}" + [ -z "$name" ] && { echo "Usage: owui-kb-resolve \"\"" >&2; exit 1; } echo "[owui] base URL: $(_owui_url)" echo "[owui] KBs returned:" tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" @@ -430,62 +335,37 @@ PY echo "[owui] http_code=$http_code bytes=$bytes" json="$(cat "$tmp_body")"; rm -f "$tmp_body" id="$(_kb_id_by_name "$name")" - if [ -n "$id" ]; then - echo "[owui] resolved id for \"$name\": $id" - else - echo "[owui] could not resolve an id for \"$name\"" >&2 + if [ -n "$id" ]; then echo "[owui] resolved id for \"$name\": $id" + else echo "[owui] could not resolve an id for \"$name\"" >&2 fi ;; - owui-upload) - shift || true - file="${1:-}" - if [ -z "$file" ] || [ ! -f "$file" ]; then echo "Usage: owui-upload " >&2; exit 1; fi - _require "OPENWEBUI_URL" "$OPENWEBUI_URL" - _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" - + shift || true; file="${1:-}" + [ -z "$file" ] || [ ! -f "$file" ] && { echo "Usage: owui-upload " >&2; exit 1; } + _require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" - # write response body to tmp_body; write HTTP code to tmp_code; capture exit code from $? curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -F "file=@$file" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true - curl_exit=$? - http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" - + curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" cat "$tmp_body" | ppjson rm -f "$tmp_body" "$tmp_code" - - if [ $curl_exit -ne 0 ]; then - echo "Upload failed: curl exit $curl_exit" >&2 - exit $curl_exit - fi - if [ "$http_code" != "200" ]; then - echo "Upload failed (HTTP $http_code)" >&2 - exit 1 - fi + [ $curl_exit -ne 0 ] && { echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit; } + [ "$http_code" != "200" ] && { echo "Upload failed (HTTP $http_code)" >&2; exit 1; } ;; - owui-attach) - shift || true - kb_name="${1:-}"; file="${2:-}" - if [ -z "$kb_name" ] || [ -z "$file" ] || [ ! -f "$file" ]; then - echo "Usage: owui-attach \"\" " >&2; exit 1 - fi - _require "OPENWEBUI_URL" "$OPENWEBUI_URL" - _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + shift || true; kb_name="${1:-}"; file="${2:-}" + [ -z "$kb_name" ] || [ -z "$file" ] || [ ! -f "$file" ] && { echo "Usage: owui-attach \"\" " >&2; exit 1; } + _require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" TMP_EXTRACT="" - # Decide how to send the file (force text/plain for .txt/.md; optionally extract JSON->text) upload_flag=("-F" "file=@$file") - ext="${file##*.}" - base="$(basename "$file")" + ext="${file##*.}"; base="$(basename "$file")" if [[ "$ext" =~ ^([Tt][Xx][Tt]|[Mm][Dd]|[Mm][Aa][Rr][Kk][Dd][Oo][Ww][Nn])$ ]]; then upload_flag=("-F" "file=@$file;type=text/plain;filename=$base") elif [[ "$ext" =~ ^([Jj][Ss][Oo][Nn])$ ]]; then - # Try to extract human text from JSON and upload as text/plain if command -v jq >/dev/null 2>&1; then tmp_txt="$(_mktemp)" - # Extract common transcript shapes: .text or .segments[].text (strings only) if jq -er ' if type=="object" and (.text|type=="string") then .text elif type=="object" and (.segments|type=="array") then @@ -495,19 +375,18 @@ PY else empty end ' "$file" >"$tmp_txt"; then if [ -s "$tmp_txt" ]; then - # Keep original stem but force .txt for OWUI indexing stem="${base%.*}" upload_flag=("-F" "file=@$tmp_txt;type=text/plain;filename=${stem}.txt") echo "[owui] extracted text from JSON -> ${stem}.txt" TMP_EXTRACT="$tmp_txt" else - echo "[owui] WARNING: JSON had no extractable text, uploading raw JSON (may be rejected)" >&2 + echo "[owui] WARNING: JSON had no extractable text, uploading raw JSON" >&2 fi else - echo "[owui] WARNING: jq failed to parse JSON, uploading raw JSON (may be rejected)" >&2 + echo "[owui] WARNING: jq failed to parse JSON, uploading raw JSON" >&2 fi else - echo "[owui] NOTE: jq not installed; uploading raw JSON (may be rejected)" >&2 + echo "[owui] NOTE: jq not installed; uploading raw JSON" >&2 fi fi @@ -519,34 +398,23 @@ PY curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" FILE_JSON="$(cat "$tmp_body")" rm -f "$tmp_body" "$tmp_code" - echo "$FILE_JSON" | ppjson - if [ $curl_exit -ne 0 ]; then - echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit - fi - if [ "$http_code" != "200" ]; then - echo "Upload failed (HTTP $http_code)" >&2; exit 1 - fi + [ $curl_exit -ne 0 ] && { echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit; } + [ "$http_code" != "200" ] && { echo "Upload failed (HTTP $http_code)" >&2; exit 1; } FILE_ID="$(python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")" - if [ -z "$FILE_ID" ]; then echo "Upload failed (no file id)"; exit 1; fi + [ -z "$FILE_ID" ] && { echo "Upload failed (no file id)"; exit 1; } - # Wait until OWUI finishes processing/extracting text for this file (prevents "content empty" 400) + # Wait for content extraction (prevents EMPTY_CONTENT) if ! _owui_wait_file "$FILE_ID" "$OPENWEBUI_WAIT_SECS"; then - echo "[owui] WARNING: timed out waiting for file content; attach may fail if OWUI hasn't extracted text yet" >&2 + echo "[owui] WARNING: timed out waiting for file extraction; attach may fail" >&2 fi # 2) Resolve KB and attach KB_ID="$(_kb_id_by_name "$kb_name")" - echo "[owui] attaching to KB: $kb_name (id: $KB_ID)" - if [ -z "$KB_ID" ]; then - echo "KB '$kb_name' not found (or ambiguous)." >&2 - echo "Tip: run './scripts/podx-tools.sh owui-kb-resolve \"$kb_name\"' to inspect and resolve, or create one:" >&2 - echo " ./scripts/podx-tools.sh owui-kb-create \"$kb_name\"" >&2 - exit 1 - fi + echo "[owui] attaching to KB: $kb_name (id: ${KB_ID:-})" + [ -z "$KB_ID" ] && { echo "KB '$kb_name' not found (or ambiguous)." >&2; exit 1; } - # Attach: capture headers, body, http code and curl exit tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"; tmp_hdrs="$(_mktemp)" curl -sS -X POST \ -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ @@ -556,56 +424,37 @@ PY -w "%{http_code}" --output "$tmp_body" \ "$(_owui_url)/api/v1/knowledge/$KB_ID/file/add" >"$tmp_code" || true curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" - echo "[owui] response headers:"; sed -n '1,5p' "$tmp_hdrs" || true RESP="$(cat "$tmp_body")" echo "$RESP" | ppjson - rm -f "$tmp_body" "$tmp_code" "$tmp_hdrs" - [ -n "${TMP_EXTRACT:-}" ] && rm -f "$TMP_EXTRACT"; unset TMP_EXTRACT + [ -n "${TMP_EXTRACT:-}" ] && rm -f "$TMP_EXTRACT" || true - if [ $curl_exit -ne 0 ]; then - echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit - fi - # Some environments report curl(23) sporadically; treat missing code as non-200 - if [ -z "$http_code" ] || [ "$http_code" = "000" ]; then - echo "Attach failed: no HTTP code returned" >&2; exit 1 - fi + [ $curl_exit -ne 0 ] && { echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit; } + [ -z "$http_code" ] || [ "$http_code" = "000" ] && { echo "Attach failed: no HTTP code returned" >&2; exit 1; } case "$http_code" in - 200|201|204) - : # success - ;; + 200|201|204) : ;; *) if printf '%s' "$RESP" | grep -qi "Duplicate content"; then - echo "[owui] duplicate content — already indexed. Treating as success." - exit 0 + echo "[owui] duplicate content — already indexed. Treating as success."; exit 0 fi echo "Attach failed (HTTP $http_code)" >&2; exit 1 ;; esac ;; - owui-attach-id) - shift || true - kb_id="${1:-}"; file="${2:-}" - if [ -z "$kb_id" ] || [ -z "$file" ] || [ ! -f "$file" ]; then - echo "Usage: owui-attach-id " >&2; exit 1 - fi - _require "OPENWEBUI_URL" "$OPENWEBUI_URL" - _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + shift || true; kb_id="${1:-}"; file="${2:-}" + [ -z "$kb_id" ] || [ -z "$file" ] || [ ! -f "$file" ] && { echo "Usage: owui-attach-id " >&2; exit 1; } + _require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" TMP_EXTRACT="" - # Decide how to send the file (force text/plain for .txt/.md; optionally extract JSON->text) upload_flag=("-F" "file=@$file") - ext="${file##*.}" - base="$(basename "$file")" + ext="${file##*.}"; base="$(basename "$file")" if [[ "$ext" =~ ^([Tt][Xx][Tt]|[Mm][Dd]|[Mm][Aa][Rr][Kk][Dd][Oo][Ww][Nn])$ ]]; then upload_flag=("-F" "file=@$file;type=text/plain;filename=$base") elif [[ "$ext" =~ ^([Jj][Ss][Oo][Nn])$ ]]; then - # Try to extract human text from JSON and upload as text/plain if command -v jq >/dev/null 2>&1; then tmp_txt="$(_mktemp)" - # Extract common transcript shapes: .text or .segments[].text (strings only) if jq -er ' if type=="object" and (.text|type=="string") then .text elif type=="object" and (.segments|type=="array") then @@ -615,48 +464,32 @@ PY else empty end ' "$file" >"$tmp_txt"; then if [ -s "$tmp_txt" ]; then - # Keep original stem but force .txt for OWUI indexing stem="${base%.*}" upload_flag=("-F" "file=@$tmp_txt;type=text/plain;filename=${stem}.txt") echo "[owui] extracted text from JSON -> ${stem}.txt" TMP_EXTRACT="$tmp_txt" - else - echo "[owui] WARNING: JSON had no extractable text, uploading raw JSON (may be rejected)" >&2 fi - else - echo "[owui] WARNING: jq failed to parse JSON, uploading raw JSON (may be rejected)" >&2 fi - else - echo "[owui] NOTE: jq not installed; uploading raw JSON (may be rejected)" >&2 fi fi - # 1) Upload tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ "${upload_flag[@]}" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" - FILE_JSON="$(cat "$tmp_body")" - rm -f "$tmp_body" "$tmp_code" - + FILE_JSON="$(cat "$tmp_body")"; rm -f "$tmp_body" "$tmp_code" echo "$FILE_JSON" | ppjson - if [ $curl_exit -ne 0 ]; then - echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit - fi - if [ "$http_code" != "200" ]; then - echo "Upload failed (HTTP $http_code)" >&2; exit 1 - fi + [ $curl_exit -ne 0 ] && { echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit; } + [ "$http_code" != "200" ] && { echo "Upload failed (HTTP $http_code)" >&2; exit 1; } FILE_ID="$(python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")" - if [ -z "$FILE_ID" ]; then echo "Upload failed (no file id)"; exit 1; fi + [ -z "$FILE_ID" ] && { echo "Upload failed (no file id)"; exit 1; } - # Wait until OWUI finishes processing/extracting text for this file (prevents "content empty" 400) if ! _owui_wait_file "$FILE_ID" "$OPENWEBUI_WAIT_SECS"; then - echo "[owui] WARNING: timed out waiting for file content; attach may fail if OWUI hasn't extracted text yet" >&2 + echo "[owui] WARNING: timed out waiting for file extraction; attach may fail" >&2 fi - # 2) Attach using explicit KB id tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"; tmp_hdrs="$(_mktemp)" curl -sS -X POST \ -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ @@ -666,118 +499,80 @@ PY -w "%{http_code}" --output "$tmp_body" \ "$(_owui_url)/api/v1/knowledge/$kb_id/file/add" >"$tmp_code" || true curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" - echo "[owui] response headers:"; sed -n '1,5p' "$tmp_hdrs" || true - RESP="$(cat "$tmp_body")" - echo "$RESP" | ppjson - + RESP="$(cat "$tmp_body")"; echo "$RESP" | ppjson rm -f "$tmp_body" "$tmp_code" "$tmp_hdrs" - [ -n "${TMP_EXTRACT:-}" ] && rm -f "$TMP_EXTRACT"; unset TMP_EXTRACT + [ -n "${TMP_EXTRACT:-}" ] && rm -f "$TMP_EXTRACT" || true - if [ $curl_exit -ne 0 ]; then - echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit - fi - if [ -z "$http_code" ] || [ "$http_code" = "000" ]; then - echo "Attach failed: no HTTP code returned" >&2; exit 1 - fi + [ $curl_exit -ne 0 ] && { echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit; } + [ -z "$http_code" ] || [ "$http_code" = "000" ] && { echo "Attach failed: no HTTP code returned" >&2; exit 1; } case "$http_code" in 200|201|204) : ;; *) if printf '%s' "$RESP" | grep -qi "Duplicate content"; then - echo "[owui] duplicate content — already indexed. Treating as success." - exit 0 + echo "[owui] duplicate content — already indexed. Treating as success."; exit 0 fi echo "Attach failed (HTTP $http_code)" >&2; exit 1 ;; esac ;; - owui-kb-files) - shift || true - kb_name="${1:-}" - if [ -z "$kb_name" ]; then echo "Usage: owui-kb-files \"\"" >&2; exit 1; fi - _require "OPENWEBUI_URL" "$OPENWEBUI_URL" - _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" - KB_ID="$(_kb_id_by_name "$kb_name")" - if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found"; exit 1; fi + shift || true; kb_name="${1:-}" + [ -z "$kb_name" ] && { echo "Usage: owui-kb-files \"\"" >&2; exit 1; } + _require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + KB_ID="$(_kb_id_by_name "$kb_name")"; [ -z "$KB_ID" ] && { echo "KB '$kb_name' not found"; exit 1; } - # First, fetch the KB object (includes files array in many OWUI versions) tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -w "%{http_code}" --output "$tmp_body" \ "$(_owui_url)/api/v1/knowledge/$KB_ID" >"$tmp_code" || true - http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)" - body="$(cat "$tmp_body" 2>/dev/null || echo '')" - rm -f "$tmp_code" + http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"; body="$(cat "$tmp_body" 2>/dev/null || echo '')"; rm -f "$tmp_code" if [ "$http_code" = "200" ] && [ -n "$body" ]; then - # Try to extract and pretty-print just the files array if present if command -v python3 >/dev/null 2>&1; then python3 - <<'PY' 2>/dev/null || echo "$body" | ppjson import sys, json d = json.loads(sys.stdin.read()) files = d.get("files") if files is None and isinstance(d.get("data"), dict): - # some versions pack file_ids under data.file_ids (UUIDs only) fids = d["data"].get("file_ids") or [] files = [{"id": fid} for fid in fids] -if files is None: - raise SystemExit(1) +if files is None: raise SystemExit(1) print(json.dumps(files, indent=2)) PY else echo "$body" | ppjson fi - rm -f "$tmp_body" - exit 0 + rm -f "$tmp_body"; exit 0 fi - # Fallback to /files endpoint if the KB object route didn't yield JSON files tmp_code2="$(_mktemp)" curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \ -w "%{http_code}" --output "$tmp_body" \ "$(_owui_url)/api/v1/knowledge/$KB_ID/files" >"$tmp_code2" || true - http_code2="$(cat "$tmp_code2" 2>/dev/null || echo 0)" - body2="$(cat "$tmp_body" 2>/dev/null || echo '')" + http_code2="$(cat "$tmp_code2" 2>/dev/null || echo 0)"; body2="$(cat "$tmp_body" 2>/dev/null || echo '')" rm -f "$tmp_body" "$tmp_code2" - - if [ "$http_code2" = "200" ] && [ -n "$body2" ]; then - echo "$body2" | ppjson - else - echo "Failed to fetch KB files (HTTP $http_code / $http_code2)" >&2 - exit 1 - fi + if [ "$http_code2" = "200" ] && [ -n "$body2" ]; then echo "$body2" | ppjson + else echo "Failed to fetch KB files (HTTP $http_code / $http_code2)" >&2; exit 1; fi ;; - owui-kb-debug) - shift || true - name="${1:-}" - if [ -z "$name" ]; then echo "Usage: owui-kb-debug \"\"" >&2; exit 1; fi + shift || true; name="${1:-}" + [ -z "$name" ] && { echo "Usage: owui-kb-debug \"\"" >&2; exit 1; } echo "[owui] base URL: $(_owui_url)" echo "[owui] KBs returned:" _owui_get_kb_list | ppjson id="$(_kb_id_by_name "$name")" - if [ -n "$id" ]; then - echo "[owui] resolved id for \"$name\": $id" - else - echo "[owui] could not resolve an id for \"$name\"" >&2 + if [ -n "$id" ]; then echo "[owui] resolved id for \"$name\": $id" + else echo "[owui] could not resolve an id for \"$name\"" >&2 fi ;; - owui-batch-attach) - shift || true - kb_name="${1:-}"; glob_pat="${2:-}" - if [ -z "$kb_name" ] || [ -z "$glob_pat" ]; then - echo "Usage: owui-batch-attach \"\" " >&2; exit 1 - fi - KB_ID="$(_kb_id_by_name "$kb_name")" - if [ -z "$KB_ID" ]; then echo "KB '$kb_name' not found"; exit 1; fi - _require "OPENWEBUI_URL" "$OPENWEBUI_URL" - _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" - shopt -s nullglob - matched=($glob_pat) - shopt -u nullglob - if [ ${#matched[@]} -eq 0 ]; then echo "No files match: $glob_pat"; exit 1; fi + shift || true; kb_name="${1:-}"; glob_pat="${2:-}" + [ -z "$kb_name" ] || [ -z "$glob_pat" ] && { echo "Usage: owui-batch-attach \"\" " >&2; exit 1; } + KB_ID="$(_kb_id_by_name "$kb_name")"; [ -z "$KB_ID" ] && { echo "KB '$kb_name' not found"; exit 1; } + _require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY" + shopt -s nullglob; matched=($glob_pat); shopt -u nullglob + [ ${#matched[@]} -eq 0 ] && { echo "No files match: $glob_pat"; exit 1; } for f in "${matched[@]}"; do echo "[owui] uploading: $f" tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" @@ -785,26 +580,18 @@ PY -F "file=@$f" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true curl_up_exit=$?; code_up="$(cat "$tmp_code" 2>/dev/null || echo 0)" - FILE_JSON="$(cat "$tmp_body")" - rm -f "$tmp_body" "$tmp_code" - + FILE_JSON="$(cat "$tmp_body")"; rm -f "$tmp_body" "$tmp_code" if [ $curl_up_exit -ne 0 ] || [ "$code_up" != "200" ]; then - echo " upload failed (curl=$curl_up_exit http=$code_up), skipping" - echo "$FILE_JSON" | ppjson - continue + echo " upload failed (curl=$curl_up_exit http=$code_up), skipping"; echo "$FILE_JSON" | ppjson; continue fi - FILE_ID="$(python3 -c 'import sys,json; d=json.loads(sys.stdin.read()); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")" if [ -z "$FILE_ID" ]; then echo " upload failed (no id), skipping"; echo "$FILE_JSON" | ppjson; continue; fi - tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)" curl -sS -X POST -H "Authorization: Bearer $OPENWEBUI_API_KEY" -H "Content-Type: application/json" \ -d "{\"file_id\":\"$FILE_ID\"}" \ -w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/knowledge/$KB_ID/file/add" >"$tmp_code" || true - curl_att_exit=$?; code_att="$(cat "$tmp_code" 2>/dev/null || echo 0)" - RESP="$(cat "$tmp_body")" + curl_att_exit=$?; code_att="$(cat "$tmp_code" 2>/dev/null || echo 0)"; RESP="$(cat "$tmp_body")" rm -f "$tmp_body" "$tmp_code" - echo "$RESP" | ppjson if [ "$code_att" != "200" ]; then echo " attach failed (HTTP $code_att)" @@ -812,31 +599,18 @@ PY done ;; + # ---------- Toggles ---------- transcribe-status) - # Prints 'paused' if the switch key is set to a truthy value, else 'running' val="$(_redis_cli GET "$_transcribe_key" 2>/dev/null || true)" - if [ -n "${val:-}" ] && [ "${val}" != "(nil)" ] && [ "${val}" != "0" ]; then - echo "paused" - exit 0 - fi - echo "running" + if [ -n "${val:-}" ] && [ "${val}" != "(nil)" ] && [ "${val}" != "0" ]; then echo "paused"; else echo "running"; fi ;; transcribe-pause) - if _redis_cli SET "$_transcribe_key" 1 >/dev/null; then - echo "Transcription: paused" - else - echo "Failed to talk to redis to set pause switch." >&2 - exit 1 - fi + if _redis_cli SET "$_transcribe_key" 1 >/dev/null; then echo "Transcription: paused"; else echo "Failed to set pause switch." >&2; exit 1; fi ;; transcribe-resume) - if _redis_cli DEL "$_transcribe_key" >/dev/null; then - echo "Transcription: resumed" - else - echo "Failed to talk to redis to clear pause switch." >&2 - exit 1 - fi + if _redis_cli DEL "$_transcribe_key" >/dev/null; then echo "Transcription: resumed"; else echo "Failed to clear pause switch." >&2; exit 1; fi ;; + *) echo "Unknown command: $cmd" >&2 _help