994 lines
40 KiB
Bash
Executable File
994 lines
40 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# podx-tools.sh — host-side helper for Meili + OpenWebUI (+ worker switches)
|
|
# Usage: ./scripts/podx-tools.sh <command> [args...]
|
|
# Run without args to see help.
|
|
|
|
set -euo pipefail
|
|
|
|
# ------------------------------ Paths / tmp ------------------------------
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
: "${PODX_TMP:=${TMPDIR:-$ROOT_DIR/tmp}}"
|
|
mkdir -p "$PODX_TMP"
|
|
|
|
# Curl timeouts used for OWUI/Meili requests (override via env if needed)
|
|
: "${PODX_CURL_CONNECT_TIMEOUT:=5}"
|
|
: "${PODX_CURL_MAX_TIME:=20}"
|
|
CURL_TMO="--connect-timeout $PODX_CURL_CONNECT_TIMEOUT --max-time $PODX_CURL_MAX_TIME"
|
|
|
|
# Portable mktemp helper that always writes inside $PODX_TMP (macOS/GNU compatible)
|
|
_mktemp() { mktemp "$PODX_TMP/podx.XXXXXX"; }
|
|
|
|
# ------------------------------ Pretty JSON ------------------------------
|
|
ppjson() {
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 -m json.tool 2>/dev/null || cat
|
|
elif command -v jq >/dev/null 2>&1; then
|
|
jq .
|
|
elif command -v node >/dev/null 2>&1; then
|
|
node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{try{console.log(JSON.stringify(JSON.parse(d),null,2))}catch{console.log(d)}})" || cat
|
|
else
|
|
cat
|
|
fi
|
|
}
|
|
|
|
# ------------------------------ Load .env ------------------------------
|
|
ENV_FILE="$ROOT_DIR/.env"
|
|
if [ -f "$ENV_FILE" ]; then
|
|
while IFS= read -r __line || [ -n "$__line" ]; do
|
|
__line="${__line#"${__line%%[![:space:]]*}"}"
|
|
__line="${__line%"${__line##*[![:space:]]}"}"
|
|
[ -z "$__line" ] && continue
|
|
[ "${__line:0:1}" = "#" ] && continue
|
|
case "$__line" in
|
|
*=*) __key="${__line%%=*}"; __val="${__line#*=}";;
|
|
*) continue;;
|
|
esac
|
|
if [[ "$__val" == \"*\" && "$__val" == *\" ]]; then
|
|
__val="${__val:1:${#__val}-2}"
|
|
elif [[ "$__val" == \'*\' && "$__val" == *\' ]]; then
|
|
__val="${__val:1:${#__val}-2}"
|
|
fi
|
|
eval "__val_expanded=\"${__val}\""
|
|
export "${__key}=${__val_expanded}"
|
|
done < "$ENV_FILE"
|
|
unset __line __key __val __val_expanded
|
|
fi
|
|
|
|
# ------------------------------ Defaults ------------------------------
|
|
: "${MEILI_URL:=http://localhost:7700}"
|
|
: "${MEILI_KEY:=${MEILI_MASTER_KEY:-}}"
|
|
|
|
: "${OPENWEBUI_URL:=http://localhost:3003}"
|
|
: "${OPENWEBUI_URL_HOST:=}"
|
|
: "${OPENWEBUI_API_KEY:=}"
|
|
: "${OPENWEBUI_KB_ID:=}"
|
|
: "${OPENWEBUI_WAIT_SECS:=180}"
|
|
: "${OPENWEBUI_AUTO_FIX_METADATA:=1}"
|
|
: "${OPENWEBUI_METADATA_TEMPLATE_JSON:=}"
|
|
|
|
__OWUI_METADATA_PATCHED=""
|
|
|
|
# ------------------------------ Helpers ------------------------------
|
|
_require() {
|
|
local name="$1" val="${2:-}"
|
|
if [ -z "$val" ]; then
|
|
echo "Missing $name (set it in .env or env): $name" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
_owui_url() {
|
|
local host_u="${OPENWEBUI_URL_HOST:-}"
|
|
if [ -n "$host_u" ]; then
|
|
echo "$host_u"; return
|
|
fi
|
|
local u="${OPENWEBUI_URL:-http://localhost:3003}"
|
|
u="${u//host.docker.internal/localhost}"
|
|
echo "$u"
|
|
}
|
|
|
|
_owui_get_kb_list() {
|
|
_require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"
|
|
curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$(_owui_url)/api/v1/knowledge/list"
|
|
}
|
|
|
|
_kb_create() {
|
|
local kb_name="$1"
|
|
_require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"
|
|
curl -sS -X POST \
|
|
-H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"name\":\"$kb_name\",\"description\":\"Created by podx-tools\"}" \
|
|
"$(_owui_url)/api/v1/knowledge/create"
|
|
}
|
|
|
|
_kb_id_by_name() {
|
|
local kb_name="$1"
|
|
if [ -n "${OPENWEBUI_KB_ID:-}" ]; then
|
|
echo "$OPENWEBUI_KB_ID"; return 0
|
|
fi
|
|
local json; json="$(_owui_get_kb_list)"
|
|
local __id=""
|
|
__id=$(printf '%s' "$json" | python3 - "$kb_name" <<'PY'
|
|
import sys, json, unicodedata as ud
|
|
def norm(s): return ud.normalize('NFKC', (s or '')).strip().casefold()
|
|
want = norm(sys.argv[1] if len(sys.argv)>1 else '')
|
|
raw = sys.stdin.read().strip()
|
|
if not raw: print('', end=''); raise SystemExit(0)
|
|
try: d = json.loads(raw)
|
|
except: print('', end=''); raise SystemExit(0)
|
|
items = d.get('data') if isinstance(d, dict) and isinstance(d.get('data'), list) else (d if isinstance(d, list) else [])
|
|
if isinstance(items, list) and len(items)==1:
|
|
print(items[0].get('id',''), end=''); raise SystemExit(0)
|
|
exact = [kb for kb in items if norm(kb.get('name')) == want]
|
|
if exact:
|
|
exact.sort(key=lambda kb: int(kb.get('updated_at') or kb.get('created_at') or 0), reverse=True)
|
|
print(exact[0].get('id',''), end=''); raise SystemExit(0)
|
|
subs = [kb for kb in items if want and want in norm(kb.get('name'))]
|
|
if subs:
|
|
subs.sort(key=lambda kb: int(kb.get('updated_at') or kb.get('created_at') or 0), reverse=True)
|
|
print(subs[0].get('id',''), end=''); raise SystemExit(0)
|
|
print('', end='')
|
|
PY
|
|
)
|
|
if [ -z "${__id:-}" ] && command -v jq >/dev/null 2>&1; then
|
|
__id=$(printf '%s' "$json" | jq -r '
|
|
if type=="array" and length==1 then .[0].id // empty
|
|
elif type=="array" then (map(select((.name // "")|ascii_downcase==("'"$kb_name"'"|ascii_downcase))) | .[0].id) // empty
|
|
elif has("data") and (.data|type=="array") and (.data|length==1) then .data[0].id // empty
|
|
else empty end')
|
|
fi
|
|
if [ -z "${__id:-}" ]; then
|
|
__id=$(printf '%s' "$json" | sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]\+\)".*/\1/p' | head -n1)
|
|
fi
|
|
printf '%s' "${__id:-}"
|
|
}
|
|
|
|
_owui_metadata_template_payload() {
|
|
python3 - "$OPENWEBUI_METADATA_TEMPLATE_JSON" <<'PY'
|
|
import sys, json
|
|
raw = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
raw = (raw or "").strip()
|
|
if not raw:
|
|
payload = {"metadata_template": {}}
|
|
else:
|
|
try:
|
|
payload = {"metadata_template": json.loads(raw)}
|
|
except Exception:
|
|
payload = {"metadata_template": raw}
|
|
print(json.dumps(payload))
|
|
PY
|
|
}
|
|
|
|
_owui_fix_metadata_template() {
|
|
local kb_id="$1" force="${2:-0}"
|
|
case "${OPENWEBUI_AUTO_FIX_METADATA,,}" in
|
|
0|false|no) return 1 ;;
|
|
esac
|
|
[ -z "$kb_id" ] && return 1
|
|
if [ "$force" != "1" ]; then
|
|
for existing in $__OWUI_METADATA_PATCHED; do
|
|
[ "$existing" = "$kb_id" ] && return 0
|
|
done
|
|
fi
|
|
|
|
local payload methods http_code tmp_body tmp_code
|
|
payload="$(_owui_metadata_template_payload)"
|
|
methods=(PATCH PUT)
|
|
for method in "${methods[@]}"; do
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS -X "$method" \
|
|
-H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" \
|
|
-w "%{http_code}" --output "$tmp_body" \
|
|
"$(_owui_url)/api/v1/knowledge/$kb_id" >"$tmp_code" || true
|
|
http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"
|
|
rm -f "$tmp_body" "$tmp_code"
|
|
case "$http_code" in
|
|
200|201|202|204)
|
|
__OWUI_METADATA_PATCHED="${__OWUI_METADATA_PATCHED} $kb_id"
|
|
echo "[owui] metadata template adjusted via $method for KB $kb_id"
|
|
return 0
|
|
;;
|
|
0|405) ;;
|
|
esac
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# ------------------------------ OWUI file helpers ------------------------------
|
|
_owui_file_get() {
|
|
local fid="$1"
|
|
curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$(_owui_url)/api/v1/files/$fid"
|
|
}
|
|
|
|
_owui_wait_file() {
|
|
local fid="$1" timeout="${2:-120}"
|
|
local start="$(date +%s)"
|
|
while :; do
|
|
local now="$(date +%s)"
|
|
if [ $((now - start)) -ge "$timeout" ]; then return 1; fi
|
|
read -r status content_len <<EOF
|
|
$(_owui_file_get "$fid" | python3 -c 'import sys,json
|
|
d=json.load(sys.stdin)
|
|
s=((d.get("data") or {}).get("status") or "")
|
|
c=len(((d.get("data") or {}).get("content") or ""))
|
|
print(s, c)')
|
|
EOF
|
|
[ "${status:-}" = "completed" ] && [ "${content_len:-0}" -gt 0 ] && return 0
|
|
sleep 2
|
|
done
|
|
}
|
|
|
|
# ------------------------------ Redis toggles ------------------------------
|
|
_redis_cli() {
|
|
# Allow explicit override with PODX_REDIS_CLI, e.g.:
|
|
# PODX_REDIS_CLI="docker exec -i myredis redis-cli"
|
|
if [ -n "${PODX_REDIS_CLI:-}" ]; then
|
|
# shellcheck disable=SC2086
|
|
${PODX_REDIS_CLI} "$@"
|
|
return $?
|
|
fi
|
|
|
|
# Defaults (overridable via .env)
|
|
: "${PODX_REDIS_SERVICE:=redis}"
|
|
: "${PODX_REDIS_HOST:=${REDIS_HOST:-127.0.0.1}}"
|
|
: "${PODX_REDIS_PORT:=${REDIS_PORT:-6379}}"
|
|
|
|
if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null 2>&1; then
|
|
# If the configured service exists, use it
|
|
if docker compose ps --services 2>/dev/null | grep -qx "$PODX_REDIS_SERVICE"; then
|
|
docker compose exec -T "$PODX_REDIS_SERVICE" redis-cli "$@"
|
|
return $?
|
|
fi
|
|
# Auto-detect a redis-like service name (redis, redis-*, *-redis, *_redis)
|
|
auto_srv="$(docker compose ps --services 2>/dev/null | grep -Ei '(^redis$)|(^redis[-_])|([-_]redis$)' | head -n1)"
|
|
if [ -n "$auto_srv" ]; then
|
|
docker compose exec -T "$auto_srv" redis-cli "$@"
|
|
return $?
|
|
fi
|
|
fi
|
|
|
|
if command -v redis-cli >/dev/null 2>&1; then
|
|
redis-cli -h "$PODX_REDIS_HOST" -p "$PODX_REDIS_PORT" "$@"
|
|
return $?
|
|
fi
|
|
|
|
echo "Could not find a way to run redis-cli.\n- Set PODX_REDIS_CLI to a full command (e.g. 'docker exec -i <container> redis-cli')\n- Or set PODX_REDIS_SERVICE to your compose service name\n- Or install redis-cli / set PODX_REDIS_HOST & PODX_REDIS_PORT" >&2
|
|
return 1
|
|
}
|
|
|
|
# --- Redis key utilities --------------------------------------------------
|
|
_redis_keys() { # usage: _redis_keys <pattern>
|
|
local pat="$1"; _redis_cli KEYS "$pat" 2>/dev/null | sed '/^(empty array)/d'
|
|
}
|
|
|
|
_redis_del_pattern() { # usage: _redis_del_pattern <pattern>
|
|
local pat="$1"; local any=0
|
|
# shellcheck disable=SC2046
|
|
mapfile -t __k < <(_redis_keys "$pat")
|
|
if [ ${#__k[@]} -gt 0 ]; then
|
|
any=1
|
|
# DEL supports multiple keys; chunk to avoid arg limits
|
|
local chunk=0 buf=()
|
|
for k in "${__k[@]}"; do
|
|
buf+=("$k"); chunk=$((chunk+1))
|
|
if [ $chunk -ge 100 ]; then _redis_cli DEL "${buf[@]}" >/dev/null; buf=(); chunk=0; fi
|
|
done
|
|
[ ${#buf[@]} -gt 0 ] && _redis_cli DEL "${buf[@]}" >/dev/null
|
|
fi
|
|
return $any
|
|
}
|
|
|
|
_redis_llen() { # usage: _redis_llen <key>
|
|
_redis_cli LLEN "$1" 2>/dev/null | tr -d '\r' || echo 0
|
|
}
|
|
|
|
_redis_type() { # usage: _redis_type <key>
|
|
_redis_cli TYPE "$1" 2>/dev/null | tr -d '\r' || echo none
|
|
}
|
|
|
|
_redis_size() { # usage: _redis_size <key>
|
|
local k="$1"
|
|
local t; t="$(_redis_type "$k")"
|
|
case "$t" in
|
|
list) _redis_cli LLEN "$k" 2>/dev/null | tr -d '\r' || echo 0 ;;
|
|
set) _redis_cli SCARD "$k" 2>/dev/null | tr -d '\r' || echo 0 ;;
|
|
zset) _redis_cli ZCARD "$k" 2>/dev/null | tr -d '\r' || echo 0 ;;
|
|
hash) _redis_cli HLEN "$k" 2>/dev/null | tr -d '\r' || echo 0 ;;
|
|
string) _redis_cli STRLEN "$k" 2>/dev/null | tr -d '\r' || echo 0 ;;
|
|
stream) _redis_cli XLEN "$k" 2>/dev/null | tr -d '\r' || echo 0 ;;
|
|
none) echo 0 ;;
|
|
*) # unknown or module-specific; try LLEN, then ZCARD, then HLEN
|
|
_redis_cli LLEN "$k" 2>/dev/null | tr -d '\r' || \
|
|
_redis_cli ZCARD "$k" 2>/dev/null | tr -d '\r' || \
|
|
_redis_cli HLEN "$k" 2>/dev/null | tr -d '\r' || echo 0 ;;
|
|
esac
|
|
}
|
|
|
|
_transcribe_key="podx:transcribe:paused"
|
|
|
|
# Queue namespace & patterns (override in .env if your deployment differs)
|
|
: "${PODX_QUEUE_NS:=podx:q}" # base namespace for our worker queues
|
|
: "${PODX_WORKER_QUEUES:=transcribe index meili enrich owui_kb}" # space-separated logical queues
|
|
: "${PODX_QUEUE_SUFFIXES:= :inflight :retry :dlq }" # related keys per queue
|
|
: "${OPENWEBUI_QUEUE_PATTERNS:=owui:kb:* owui:tasks:* open-webui:*queue* openwebui:*queue*}" # OWUI queue-ish keys
|
|
|
|
# ------------------------------ Help ------------------------------
|
|
_help() {
|
|
cat <<EOF
|
|
podx-tools — Meili & OpenWebUI helpers
|
|
|
|
Meilisearch:
|
|
meili-health # {"status":"available"} if up
|
|
meili-keys # list API keys
|
|
meili-stats # index stats for 'library'
|
|
meili-tasks # last tasks
|
|
meili-init # create 'library' index if missing
|
|
meili-search "<query>" [limit] # search the 'library' index
|
|
meili-reindex # index all /transcripts/*.json via worker container
|
|
|
|
OpenWebUI:
|
|
owui-health # check API health (200)
|
|
owui-kbs # list knowledge bases
|
|
owui-kb-id "<KB Name>" # print the KB UUID by exact name
|
|
owui-kb-id-all "<KB Name>" # list all matching KB ids (if duplicates exist)
|
|
owui-kb-resolve "<KB Name>" # debug name->id resolution with raw listing
|
|
owui-upload </abs/path/file> # upload a file, prints file JSON
|
|
owui-attach "<KB Name>" </abs/path/file> # upload + attach to KB (waits for extraction)
|
|
owui-attach-id <KB_ID> </abs/path/file> # upload + attach using explicit KB id
|
|
owui-kb-files "<KB Name>" # list files for a KB (best-effort)
|
|
owui-kb-create "<KB Name>" # create a KB (prints JSON with id)
|
|
owui-kbs-raw # raw JSON from /knowledge/list
|
|
owui-batch-attach "<KB Name>" <glob> # attach all files matching glob
|
|
|
|
Transcription control:
|
|
scan-once # run a single library scan to enqueue new files
|
|
transcribe-status # show whether transcription workers are paused
|
|
transcribe-pause # pause CPU-heavy transcription jobs
|
|
transcribe-resume # resume transcription jobs
|
|
|
|
RQ helpers:
|
|
rq-info # show RQ queue summary via worker container
|
|
rq-transcribe-requeue # requeue all failed 'transcribe' jobs back to active
|
|
rq-clear-transcribe # purge 'transcribe' queue and clean its registries (failed/started/scheduled/etc.)
|
|
rq-clear-default # purge 'default' queue and clean its registries
|
|
rq-clear-all # purge both 'default' and 'transcribe' queues and registries
|
|
|
|
Queues (Redis):
|
|
queues-workers-list # show worker queue sizes (LLEN) and related keys
|
|
queues-workers-clear # purge worker queues (ns=\${PODX_QUEUE_NS})
|
|
queues-owui-list # show OpenWebUI queue-like keys
|
|
queues-owui-clear # purge OpenWebUI queue-like keys
|
|
|
|
Examples:
|
|
./scripts/podx-tools.sh meili-health
|
|
./scripts/podx-tools.sh meili-search "grand canyon" 10
|
|
./scripts/podx-tools.sh owui-attach "Homelab Library" /mnt/.../episode.txt
|
|
./scripts/podx-tools.sh owui-kb-files "Homelab Library"
|
|
|
|
Environment comes from .env at repo root (MEILI_URL/KEY, OPENWEBUI_URL/API_KEY, OPENWEBUI_KB_ID).
|
|
EOF
|
|
}
|
|
|
|
# ------------------------------ Commands ------------------------------
|
|
cmd="${1:-}"
|
|
case "$cmd" in
|
|
""|-h|--help|help) _help ;;
|
|
|
|
# ---------- Queue inspection/cleanup ----------
|
|
queues-workers-list)
|
|
# Show lengths for configured worker queues and related keys
|
|
echo "[workers] namespace: ${PODX_QUEUE_NS}"
|
|
for q in ${PODX_WORKER_QUEUES}; do
|
|
base="${PODX_QUEUE_NS}:${q}"
|
|
printf "%-28s %6s\n" "$base" "$(_redis_size "$base")"
|
|
for suf in ${PODX_QUEUE_SUFFIXES}; do
|
|
k="${base}${suf}"
|
|
if [ -n "$(_redis_keys "$k")" ]; then
|
|
printf "%-28s %6s\n" "$k" "$(_redis_size "$k")"
|
|
fi
|
|
done
|
|
done
|
|
;;
|
|
|
|
queues-workers-clear)
|
|
# Delete worker queue keys (main lists + inflight/retry/dlq). Safe: does not touch other namespaces.
|
|
echo "[workers] clearing queues in ns=${PODX_QUEUE_NS}"
|
|
for q in ${PODX_WORKER_QUEUES}; do
|
|
base="${PODX_QUEUE_NS}:${q}"
|
|
_redis_del_pattern "$base" >/dev/null || true
|
|
for suf in ${PODX_QUEUE_SUFFIXES}; do
|
|
_redis_del_pattern "${base}${suf}" >/dev/null || true
|
|
done
|
|
done
|
|
# Also remove common locks for this namespace
|
|
_redis_del_pattern "${PODX_QUEUE_NS}:lock:*" >/dev/null || true
|
|
echo "[workers] done"
|
|
;;
|
|
|
|
queues-owui-list)
|
|
# List OpenWebUI queue-like keys and lengths
|
|
for pat in ${OPENWEBUI_QUEUE_PATTERNS}; do
|
|
mapfile -t keys < <(_redis_keys "$pat")
|
|
for k in "${keys[@]}"; do
|
|
printf "%-40s %6s\n" "$k" "$(_redis_size "$k")"
|
|
done
|
|
done
|
|
;;
|
|
|
|
queues-owui-clear)
|
|
# Clear OpenWebUI queue-like keys
|
|
echo "[owui] clearing keys matching: ${OPENWEBUI_QUEUE_PATTERNS}"
|
|
for pat in ${OPENWEBUI_QUEUE_PATTERNS}; do
|
|
_redis_del_pattern "$pat" >/dev/null || true
|
|
done
|
|
echo "[owui] done"
|
|
;;
|
|
|
|
# ---------- Meili ----------
|
|
meili-health)
|
|
_require "MEILI_URL" "$MEILI_URL"
|
|
curl -sS "$MEILI_URL/health" | ppjson
|
|
;;
|
|
meili-keys)
|
|
_require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY"
|
|
curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/keys" | ppjson
|
|
;;
|
|
meili-stats)
|
|
_require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY"
|
|
curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/indexes/library/stats" | ppjson
|
|
;;
|
|
meili-tasks)
|
|
_require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY"
|
|
curl -sS -H "Authorization: Bearer $MEILI_KEY" "$MEILI_URL/tasks?limit=20&from=0" | ppjson
|
|
;;
|
|
meili-init)
|
|
_require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY"
|
|
curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \
|
|
--data '{"uid":"library"}' "$MEILI_URL/indexes" | ppjson
|
|
;;
|
|
meili-search)
|
|
shift || true; q="${1:-}"; lim="${2:-5}"
|
|
[ -z "$q" ] && { echo "Usage: meili-search \"<query>\" [limit]" >&2; exit 1; }
|
|
_require "MEILI_URL" "$MEILI_URL"; _require "MEILI_KEY" "$MEILI_KEY"
|
|
curl -sS -X POST -H "Authorization: Bearer $MEILI_KEY" -H "Content-Type: application/json" \
|
|
--data "{\"q\":\"$q\",\"limit\":$lim}" \
|
|
"$MEILI_URL/indexes/library/search" | ppjson
|
|
;;
|
|
meili-reindex)
|
|
if ! command -v docker >/dev/null 2>&1; then echo "docker not found on host" >&2; exit 1; fi
|
|
docker compose exec -T podx-worker sh -lc '
|
|
python - <<PY
|
|
import json,glob,sys,pathlib
|
|
try:
|
|
import worker
|
|
except Exception as e:
|
|
print("[meili] import worker failed:", e, file=sys.stderr); sys.exit(1)
|
|
n=0
|
|
for p in glob.glob("/transcripts/*.json"):
|
|
try:
|
|
worker.index_meili(pathlib.Path(p))
|
|
print(f"[meili] indexed {pathlib.Path(p).name}")
|
|
n+=1
|
|
except Exception as e:
|
|
print(f"[meili] FAILED {p}: {e}", file=sys.stderr)
|
|
print(f"Indexed {n} document(s).")
|
|
PY
|
|
'
|
|
;;
|
|
|
|
# ---------- OpenWebUI ----------
|
|
owui-health)
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"
|
|
curl -sS $CURL_TMO -o /dev/null -w "%{http_code}\n" "$(_owui_url)/api/health"
|
|
;;
|
|
owui-kbs)
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" "$(_owui_url)/api/v1/knowledge/list" | ppjson
|
|
;;
|
|
owui-kbs-raw)
|
|
_owui_get_kb_list | ppjson
|
|
;;
|
|
owui-kb-create)
|
|
shift || true; name="${1:-}"
|
|
[ -z "$name" ] && { echo "Usage: owui-kb-create \"<KB Name>\"" >&2; exit 1; }
|
|
_kb_create "$name" | ppjson
|
|
;;
|
|
owui-kb-id)
|
|
shift || true; name="${1:-}"
|
|
[ -z "$name" ] && { echo "Usage: owui-kb-id \"<KB Name>\"" >&2; exit 1; }
|
|
_id="$(_kb_id_by_name "$name")"
|
|
[ -z "$_id" ] && { echo "KB '$name' not found" >&2; exit 1; }
|
|
echo "$_id"
|
|
;;
|
|
owui-kb-id-all)
|
|
shift || true; name="${1:-}"
|
|
[ -z "$name" ] && { echo "Usage: owui-kb-id-all \"<KB Name>\"" >&2; exit 1; }
|
|
_owui_get_kb_list | python3 - "$name" <<'PY' || exit 0
|
|
import sys, json
|
|
want = (sys.argv[1] or "").strip()
|
|
raw = sys.stdin.read().strip()
|
|
try: d = json.loads(raw)
|
|
except: sys.exit(0)
|
|
items = d.get("data") if isinstance(d, dict) and isinstance(d.get("data"), list) else (d if isinstance(d, list) else [])
|
|
def norm(s): return (s or "").strip().casefold()
|
|
want_n = norm(want)
|
|
matches = [kb for kb in items if norm(kb.get("name")) == want_n or (want_n and want_n in norm(kb.get("name")))]
|
|
for kb in matches:
|
|
print(f"{kb.get('id','')}\t{kb.get('name','')}\tcreated_at={kb.get('created_at','')}\tupdated_at={kb.get('updated_at','')}")
|
|
PY
|
|
;;
|
|
owui-kb-resolve)
|
|
shift || true; name="${1:-}"
|
|
[ -z "$name" ] && { echo "Usage: owui-kb-resolve \"<KB Name>\"" >&2; exit 1; }
|
|
echo "[owui] base URL: $(_owui_url)"
|
|
echo "[owui] KBs returned:"
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-w "%{http_code}" --output "$tmp_body" \
|
|
"$(_owui_url)/api/v1/knowledge/list" >"$tmp_code" || true
|
|
http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"; rm -f "$tmp_code"
|
|
bytes="$(wc -c <"$tmp_body" 2>/dev/null || echo 0)"
|
|
cat "$tmp_body" | ppjson
|
|
echo "[owui] http_code=$http_code bytes=$bytes"
|
|
json="$(cat "$tmp_body")"; rm -f "$tmp_body"
|
|
id="$(_kb_id_by_name "$name")"
|
|
if [ -n "$id" ]; then echo "[owui] resolved id for \"$name\": $id"
|
|
else echo "[owui] could not resolve an id for \"$name\"" >&2
|
|
fi
|
|
;;
|
|
owui-upload)
|
|
shift || true; file="${1:-}"
|
|
[ -z "$file" ] || [ ! -f "$file" ] && { echo "Usage: owui-upload </abs/path/file>" >&2; exit 1; }
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-F "file=@$file" \
|
|
-w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true
|
|
curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"
|
|
cat "$tmp_body" | ppjson
|
|
rm -f "$tmp_body" "$tmp_code"
|
|
[ $curl_exit -ne 0 ] && { echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit; }
|
|
[ "$http_code" != "200" ] && { echo "Upload failed (HTTP $http_code)" >&2; exit 1; }
|
|
;;
|
|
owui-attach)
|
|
shift || true; kb_name="${1:-}"; file="${2:-}"
|
|
[ -z "$kb_name" ] || [ -z "$file" ] || [ ! -f "$file" ] && { echo "Usage: owui-attach \"<KB Name>\" </abs/path/file>" >&2; exit 1; }
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
|
|
TMP_EXTRACT=""
|
|
upload_flag=("-F" "file=@$file")
|
|
ext="${file##*.}"; base="$(basename "$file")"
|
|
if [[ "$ext" =~ ^([Tt][Xx][Tt]|[Mm][Dd]|[Mm][Aa][Rr][Kk][Dd][Oo][Ww][Nn])$ ]]; then
|
|
upload_flag=("-F" "file=@$file;type=text/plain;filename=$base")
|
|
elif [[ "$ext" =~ ^([Jj][Ss][Oo][Nn])$ ]]; then
|
|
if command -v jq >/dev/null 2>&1; then
|
|
tmp_txt="$(_mktemp)"
|
|
if jq -er '
|
|
if type=="object" and (.text|type=="string") then .text
|
|
elif type=="object" and (.segments|type=="array") then
|
|
(.segments[]? | if type=="object" and (.text|type=="string") then .text
|
|
elif type=="string" then .
|
|
else empty end)
|
|
else empty end
|
|
' "$file" >"$tmp_txt"; then
|
|
if [ -s "$tmp_txt" ]; then
|
|
stem="${base%.*}"
|
|
upload_flag=("-F" "file=@$tmp_txt;type=text/plain;filename=${stem}.txt")
|
|
echo "[owui] extracted text from JSON -> ${stem}.txt"
|
|
TMP_EXTRACT="$tmp_txt"
|
|
else
|
|
echo "[owui] WARNING: JSON had no extractable text, uploading raw JSON" >&2
|
|
fi
|
|
else
|
|
echo "[owui] WARNING: jq failed to parse JSON, uploading raw JSON" >&2
|
|
fi
|
|
else
|
|
echo "[owui] NOTE: jq not installed; uploading raw JSON" >&2
|
|
fi
|
|
fi
|
|
|
|
# 1) Upload
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
"${upload_flag[@]}" \
|
|
-w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true
|
|
curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"
|
|
FILE_JSON="$(cat "$tmp_body")"
|
|
rm -f "$tmp_body" "$tmp_code"
|
|
echo "$FILE_JSON" | ppjson
|
|
[ $curl_exit -ne 0 ] && { echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit; }
|
|
[ "$http_code" != "200" ] && { echo "Upload failed (HTTP $http_code)" >&2; exit 1; }
|
|
|
|
FILE_ID="$(python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")"
|
|
[ -z "$FILE_ID" ] && { echo "Upload failed (no file id)"; exit 1; }
|
|
|
|
# Wait for content extraction (prevents EMPTY_CONTENT)
|
|
if ! _owui_wait_file "$FILE_ID" "$OPENWEBUI_WAIT_SECS"; then
|
|
echo "[owui] WARNING: timed out waiting for file extraction; attach may fail" >&2
|
|
fi
|
|
|
|
# 2) Resolve KB and attach
|
|
KB_ID="$(_kb_id_by_name "$kb_name")"
|
|
echo "[owui] attaching to KB: $kb_name (id: ${KB_ID:-<none>})"
|
|
[ -z "$KB_ID" ] && { echo "KB '$kb_name' not found (or ambiguous)." >&2; exit 1; }
|
|
_owui_fix_metadata_template "$KB_ID" || true
|
|
|
|
attach_payload="{\"file_id\":\"$FILE_ID\"}"
|
|
attempt=0
|
|
while :; do
|
|
attempt=$((attempt+1))
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"; tmp_hdrs="$(_mktemp)"
|
|
curl -sS -X POST \
|
|
-H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$attach_payload" \
|
|
-D "$tmp_hdrs" \
|
|
-w "%{http_code}" --output "$tmp_body" \
|
|
"$(_owui_url)/api/v1/knowledge/$KB_ID/file/add" >"$tmp_code" || true
|
|
curl_exit=$?
|
|
http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"
|
|
echo "[owui] response headers:"; sed -n '1,5p' "$tmp_hdrs" || true
|
|
RESP="$(cat "$tmp_body")"
|
|
echo "$RESP" | ppjson
|
|
rm -f "$tmp_body" "$tmp_code" "$tmp_hdrs"
|
|
|
|
[ $curl_exit -ne 0 ] && { echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit; }
|
|
[ -z "$http_code" ] || [ "$http_code" = "000" ] && { echo "Attach failed: no HTTP code returned" >&2; exit 1; }
|
|
|
|
if [ "$http_code" = "400" ] && printf '%s' "$RESP" | grep -qi "metadata"; then
|
|
if [ "$attempt" -lt 3 ] && _owui_fix_metadata_template "$KB_ID" 1; then
|
|
echo "[owui] retrying attach after metadata template fix"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
case "$http_code" in
|
|
200|201|204)
|
|
break
|
|
;;
|
|
*)
|
|
if printf '%s' "$RESP" | grep -qi "Duplicate content"; then
|
|
echo "[owui] duplicate content — already indexed. Treating as success."
|
|
break
|
|
fi
|
|
echo "Attach failed (HTTP $http_code)" >&2; exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
[ -n "${TMP_EXTRACT:-}" ] && rm -f "$TMP_EXTRACT" || true
|
|
;;
|
|
owui-attach-id)
|
|
shift || true; kb_id="${1:-}"; file="${2:-}"
|
|
[ -z "$kb_id" ] || [ -z "$file" ] || [ ! -f "$file" ] && { echo "Usage: owui-attach-id <KB_ID> </abs/path/file>" >&2; exit 1; }
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
|
|
TMP_EXTRACT=""
|
|
upload_flag=("-F" "file=@$file")
|
|
ext="${file##*.}"; base="$(basename "$file")"
|
|
if [[ "$ext" =~ ^([Tt][Xx][Tt]|[Mm][Dd]|[Mm][Aa][Rr][Kk][Dd][Oo][Ww][Nn])$ ]]; then
|
|
upload_flag=("-F" "file=@$file;type=text/plain;filename=$base")
|
|
elif [[ "$ext" =~ ^([Jj][Ss][Oo][Nn])$ ]]; then
|
|
if command -v jq >/dev/null 2>&1; then
|
|
tmp_txt="$(_mktemp)"
|
|
if jq -er '
|
|
if type=="object" and (.text|type=="string") then .text
|
|
elif type=="object" and (.segments|type=="array") then
|
|
(.segments[]? | if type=="object" and (.text|type=="string") then .text
|
|
elif type=="string" then .
|
|
else empty end)
|
|
else empty end
|
|
' "$file" >"$tmp_txt"; then
|
|
if [ -s "$tmp_txt" ]; then
|
|
stem="${base%.*}"
|
|
upload_flag=("-F" "file=@$tmp_txt;type=text/plain;filename=${stem}.txt")
|
|
echo "[owui] extracted text from JSON -> ${stem}.txt"
|
|
TMP_EXTRACT="$tmp_txt"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
"${upload_flag[@]}" \
|
|
-w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true
|
|
curl_exit=$?; http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"
|
|
FILE_JSON="$(cat "$tmp_body")"; rm -f "$tmp_body" "$tmp_code"
|
|
echo "$FILE_JSON" | ppjson
|
|
[ $curl_exit -ne 0 ] && { echo "Upload failed: curl exit $curl_exit" >&2; exit $curl_exit; }
|
|
[ "$http_code" != "200" ] && { echo "Upload failed (HTTP $http_code)" >&2; exit 1; }
|
|
|
|
FILE_ID="$(python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")"
|
|
[ -z "$FILE_ID" ] && { echo "Upload failed (no file id)"; exit 1; }
|
|
|
|
if ! _owui_wait_file "$FILE_ID" "$OPENWEBUI_WAIT_SECS"; then
|
|
echo "[owui] WARNING: timed out waiting for file extraction; attach may fail" >&2
|
|
fi
|
|
|
|
_owui_fix_metadata_template "$kb_id" || true
|
|
|
|
attach_payload="{\"file_id\":\"$FILE_ID\"}"
|
|
attempt=0
|
|
while :; do
|
|
attempt=$((attempt+1))
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"; tmp_hdrs="$(_mktemp)"
|
|
curl -sS -X POST \
|
|
-H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$attach_payload" \
|
|
-D "$tmp_hdrs" \
|
|
-w "%{http_code}" --output "$tmp_body" \
|
|
"$(_owui_url)/api/v1/knowledge/$kb_id/file/add" >"$tmp_code" || true
|
|
curl_exit=$?
|
|
http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"
|
|
echo "[owui] response headers:"; sed -n '1,5p' "$tmp_hdrs" || true
|
|
RESP="$(cat "$tmp_body")"; echo "$RESP" | ppjson
|
|
rm -f "$tmp_body" "$tmp_code" "$tmp_hdrs"
|
|
|
|
[ $curl_exit -ne 0 ] && { echo "Attach failed: curl exit $curl_exit" >&2; exit $curl_exit; }
|
|
[ -z "$http_code" ] || [ "$http_code" = "000" ] && { echo "Attach failed: no HTTP code returned" >&2; exit 1; }
|
|
|
|
if [ "$http_code" = "400" ] && printf '%s' "$RESP" | grep -qi "metadata"; then
|
|
if [ "$attempt" -lt 3 ] && _owui_fix_metadata_template "$kb_id" 1; then
|
|
echo "[owui] retrying attach after metadata template fix"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
case "$http_code" in
|
|
200|201|204)
|
|
break
|
|
;;
|
|
*)
|
|
if printf '%s' "$RESP" | grep -qi "Duplicate content"; then
|
|
echo "[owui] duplicate content — already indexed. Treating as success."
|
|
break
|
|
fi
|
|
echo "Attach failed (HTTP $http_code)" >&2; exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
[ -n "${TMP_EXTRACT:-}" ] && rm -f "$TMP_EXTRACT" || true
|
|
;;
|
|
owui-kb-files)
|
|
shift || true; kb_name="${1:-}"
|
|
[ -z "$kb_name" ] && { echo "Usage: owui-kb-files \"<KB Name>\"" >&2; exit 1; }
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
KB_ID="$(_kb_id_by_name "$kb_name")"; [ -z "$KB_ID" ] && { echo "KB '$kb_name' not found"; exit 1; }
|
|
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-w "%{http_code}" --output "$tmp_body" \
|
|
"$(_owui_url)/api/v1/knowledge/$KB_ID" >"$tmp_code" || true
|
|
http_code="$(cat "$tmp_code" 2>/dev/null || echo 0)"; body="$(cat "$tmp_body" 2>/dev/null || echo '')"; rm -f "$tmp_code"
|
|
|
|
if [ "$http_code" = "200" ] && [ -n "$body" ]; then
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 - <<'PY' 2>/dev/null || echo "$body" | ppjson
|
|
import sys, json
|
|
d = json.loads(sys.stdin.read())
|
|
files = d.get("files")
|
|
if files is None and isinstance(d.get("data"), dict):
|
|
fids = d["data"].get("file_ids") or []
|
|
files = [{"id": fid} for fid in fids]
|
|
if files is None: raise SystemExit(1)
|
|
print(json.dumps(files, indent=2))
|
|
PY
|
|
else
|
|
echo "$body" | ppjson
|
|
fi
|
|
rm -f "$tmp_body"; exit 0
|
|
fi
|
|
|
|
tmp_code2="$(_mktemp)"
|
|
curl -sS $CURL_TMO -H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-w "%{http_code}" --output "$tmp_body" \
|
|
"$(_owui_url)/api/v1/knowledge/$KB_ID/files" >"$tmp_code2" || true
|
|
http_code2="$(cat "$tmp_code2" 2>/dev/null || echo 0)"; body2="$(cat "$tmp_body" 2>/dev/null || echo '')"
|
|
rm -f "$tmp_body" "$tmp_code2"
|
|
if [ "$http_code2" = "200" ] && [ -n "$body2" ]; then echo "$body2" | ppjson
|
|
else echo "Failed to fetch KB files (HTTP $http_code / $http_code2)" >&2; exit 1; fi
|
|
;;
|
|
owui-kb-debug)
|
|
shift || true; name="${1:-}"
|
|
[ -z "$name" ] && { echo "Usage: owui-kb-debug \"<KB Name>\"" >&2; exit 1; }
|
|
echo "[owui] base URL: $(_owui_url)"
|
|
echo "[owui] KBs returned:"
|
|
_owui_get_kb_list | ppjson
|
|
id="$(_kb_id_by_name "$name")"
|
|
if [ -n "$id" ]; then echo "[owui] resolved id for \"$name\": $id"
|
|
else echo "[owui] could not resolve an id for \"$name\"" >&2
|
|
fi
|
|
;;
|
|
owui-batch-attach)
|
|
shift || true; kb_name="${1:-}"; glob_pat="${2:-}"
|
|
[ -z "$kb_name" ] || [ -z "$glob_pat" ] && { echo "Usage: owui-batch-attach \"<KB Name>\" <glob>" >&2; exit 1; }
|
|
KB_ID="$(_kb_id_by_name "$kb_name")"; [ -z "$KB_ID" ] && { echo "KB '$kb_name' not found"; exit 1; }
|
|
_require "OPENWEBUI_URL" "$OPENWEBUI_URL"; _require "OPENWEBUI_API_KEY" "$OPENWEBUI_API_KEY"
|
|
shopt -s nullglob; matched=($glob_pat); shopt -u nullglob
|
|
[ ${#matched[@]} -eq 0 ] && { echo "No files match: $glob_pat"; exit 1; }
|
|
for f in "${matched[@]}"; do
|
|
echo "[owui] uploading: $f"
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS -H "Authorization: Bearer $OPENWEBUI_API_KEY" \
|
|
-F "file=@$f" \
|
|
-w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/files/" >"$tmp_code" || true
|
|
curl_up_exit=$?; code_up="$(cat "$tmp_code" 2>/dev/null || echo 0)"
|
|
FILE_JSON="$(cat "$tmp_body")"; rm -f "$tmp_body" "$tmp_code"
|
|
if [ $curl_up_exit -ne 0 ] || [ "$code_up" != "200" ]; then
|
|
echo " upload failed (curl=$curl_up_exit http=$code_up), skipping"; echo "$FILE_JSON" | ppjson; continue
|
|
fi
|
|
FILE_ID="$(python3 -c 'import sys,json; d=json.loads(sys.stdin.read()); print(d.get("id") or (d.get("data") or {}).get("id",""))' <<<"$FILE_JSON")"
|
|
if [ -z "$FILE_ID" ]; then echo " upload failed (no id), skipping"; echo "$FILE_JSON" | ppjson; continue; fi
|
|
tmp_body="$(_mktemp)"; tmp_code="$(_mktemp)"
|
|
curl -sS -X POST -H "Authorization: Bearer $OPENWEBUI_API_KEY" -H "Content-Type: application/json" \
|
|
-d "{\"file_id\":\"$FILE_ID\"}" \
|
|
-w "%{http_code}" --output "$tmp_body" "$(_owui_url)/api/v1/knowledge/$KB_ID/file/add" >"$tmp_code" || true
|
|
curl_att_exit=$?; code_att="$(cat "$tmp_code" 2>/dev/null || echo 0)"; RESP="$(cat "$tmp_body")"
|
|
rm -f "$tmp_body" "$tmp_code"
|
|
echo "$RESP" | ppjson
|
|
if [ "$code_att" != "200" ]; then
|
|
echo " attach failed (HTTP $code_att)"
|
|
fi
|
|
done
|
|
;;
|
|
|
|
# ---------- Toggles ----------
|
|
transcribe-status)
|
|
val="$(_redis_cli GET "$_transcribe_key" 2>/dev/null || true)"
|
|
if [ -n "${val:-}" ] && [ "${val}" != "(nil)" ] && [ "${val}" != "0" ]; then echo "paused"; else echo "running"; fi
|
|
;;
|
|
transcribe-pause)
|
|
if _redis_cli SET "$_transcribe_key" 1 >/dev/null; then echo "Transcription: paused"; else echo "Failed to set pause switch." >&2; exit 1; fi
|
|
;;
|
|
transcribe-resume)
|
|
if _redis_cli DEL "$_transcribe_key" >/dev/null; then echo "Transcription: resumed"; else echo "Failed to clear pause switch." >&2; exit 1; fi
|
|
;;
|
|
|
|
# ---------- Scanner / RQ helpers ----------
|
|
scan-once)
|
|
# Run a single scan pass inside the scanner service to enqueue new files
|
|
if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null 2>&1; then
|
|
docker compose run --rm -e SCAN_INTERVAL=0 podx-scanner \
|
|
python -c "import scanner; n=scanner.enqueue_new_files(); print(f'Enqueued {n} new file(s)')"
|
|
else
|
|
echo "docker compose not found; run the scanner container manually." >&2; exit 1
|
|
fi
|
|
;;
|
|
rq-info)
|
|
: "${REDIS_URL:=redis://redis:6379/0}"
|
|
if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null 2>&1; then
|
|
docker compose exec -T podx-worker-transcribe rq info -u "$REDIS_URL"
|
|
else
|
|
echo "docker compose not found; cannot run rq inside container." >&2; exit 1
|
|
fi
|
|
;;
|
|
rq-transcribe-requeue)
|
|
: "${REDIS_URL:=redis://redis:6379/0}"
|
|
if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null 2>&1; then
|
|
docker compose exec -T podx-worker-transcribe rq requeue --all --queue transcribe -u "$REDIS_URL"
|
|
else
|
|
echo "docker compose not found; cannot run rq inside container." >&2; exit 1
|
|
fi
|
|
;;
|
|
|
|
rq-clear-transcribe)
|
|
: "${REDIS_URL:=redis://redis:6379/0}"
|
|
if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null 2>&1; then
|
|
docker compose exec -T podx-worker-transcribe python - "$REDIS_URL" <<'PY'
|
|
import sys
|
|
from redis import Redis
|
|
from rq import Queue
|
|
from rq.registry import FailedJobRegistry, StartedJobRegistry, DeferredJobRegistry, FinishedJobRegistry, ScheduledJobRegistry
|
|
url = sys.argv[1] if len(sys.argv)>1 else 'redis://redis:6379/0'
|
|
conn = Redis.from_url(url)
|
|
def clear(name):
|
|
q = Queue(name, connection=conn)
|
|
regs = [
|
|
FailedJobRegistry(name=q.name, connection=conn),
|
|
StartedJobRegistry(name=q.name, connection=conn),
|
|
DeferredJobRegistry(name=q.name, connection=conn),
|
|
FinishedJobRegistry(name=q.name, connection=conn),
|
|
ScheduledJobRegistry(name=q.name, connection=conn),
|
|
]
|
|
for reg in regs:
|
|
try:
|
|
for jid in reg.get_job_ids():
|
|
try: reg.remove(jid, delete_job=True)
|
|
except Exception: pass
|
|
except Exception: pass
|
|
try: q.empty()
|
|
except Exception: pass
|
|
print(f"[rq] cleared queue '{name}' and registries.")
|
|
clear('transcribe')
|
|
PY
|
|
else
|
|
echo "docker compose not found; cannot run rq inside container." >&2; exit 1
|
|
fi
|
|
;;
|
|
|
|
rq-clear-default)
|
|
: "${REDIS_URL:=redis://redis:6379/0}"
|
|
if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null 2>&1; then
|
|
docker compose exec -T podx-worker python - "$REDIS_URL" <<'PY'
|
|
import sys
|
|
from redis import Redis
|
|
from rq import Queue
|
|
from rq.registry import FailedJobRegistry, StartedJobRegistry, DeferredJobRegistry, FinishedJobRegistry, ScheduledJobRegistry
|
|
url = sys.argv[1] if len(sys.argv)>1 else 'redis://redis:6379/0'
|
|
conn = Redis.from_url(url)
|
|
def clear(name):
|
|
q = Queue(name, connection=conn)
|
|
regs = [
|
|
FailedJobRegistry(name=q.name, connection=conn),
|
|
StartedJobRegistry(name=q.name, connection=conn),
|
|
DeferredJobRegistry(name=q.name, connection=conn),
|
|
FinishedJobRegistry(name=q.name, connection=conn),
|
|
ScheduledJobRegistry(name=q.name, connection=conn),
|
|
]
|
|
for reg in regs:
|
|
try:
|
|
for jid in reg.get_job_ids():
|
|
try: reg.remove(jid, delete_job=True)
|
|
except Exception: pass
|
|
except Exception: pass
|
|
try: q.empty()
|
|
except Exception: pass
|
|
print(f"[rq] cleared queue '{name}' and registries.")
|
|
clear('default')
|
|
PY
|
|
else
|
|
echo "docker compose not found; cannot run rq inside container." >&2; exit 1
|
|
fi
|
|
;;
|
|
|
|
rq-clear-all)
|
|
: "${REDIS_URL:=redis://redis:6379/0}"
|
|
if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null 2>&1; then
|
|
docker compose exec -T podx-worker-transcribe python - "$REDIS_URL" <<'PY'
|
|
import sys
|
|
from redis import Redis
|
|
from rq import Queue
|
|
from rq.registry import FailedJobRegistry, StartedJobRegistry, DeferredJobRegistry, FinishedJobRegistry, ScheduledJobRegistry
|
|
url = sys.argv[1] if len(sys.argv)>1 else 'redis://redis:6379/0'
|
|
conn = Redis.from_url(url)
|
|
def clear(name):
|
|
q = Queue(name, connection=conn)
|
|
regs = [
|
|
FailedJobRegistry(name=q.name, connection=conn),
|
|
StartedJobRegistry(name=q.name, connection=conn),
|
|
DeferredJobRegistry(name=q.name, connection=conn),
|
|
FinishedJobRegistry(name=q.name, connection=conn),
|
|
ScheduledJobRegistry(name=q.name, connection=conn),
|
|
]
|
|
for reg in regs:
|
|
try:
|
|
for jid in reg.get_job_ids():
|
|
try: reg.remove(jid, delete_job=True)
|
|
except Exception: pass
|
|
except Exception: pass
|
|
try: q.empty()
|
|
except Exception: pass
|
|
print(f"[rq] cleared queue '{name}' and registries.")
|
|
for name in ('default','transcribe'):
|
|
clear(name)
|
|
PY
|
|
else
|
|
echo "docker compose not found; cannot run rq inside container." >&2; exit 1
|
|
fi
|
|
;;
|
|
|
|
*)
|
|
echo "Unknown command: $cmd" >&2
|
|
_help
|
|
exit 1
|
|
;;
|
|
esac
|