Added queue management tools to podx-tools.sh

This commit is contained in:
2025-09-11 16:49:30 +02:00
parent 5f311d7d30
commit 2daccb1c69

View File

@@ -177,8 +177,41 @@ _redis_cli() {
return 1
fi
}
# --- Redis key utilities --------------------------------------------------
_redis_keys() { # usage: _redis_keys <pattern>
local pat="$1"; _redis_cli KEYS "$pat" 2>/dev/null | sed '/^(empty array)/d'
}
_redis_del_pattern() { # usage: _redis_del_pattern <pattern>
local pat="$1"; local any=0
# shellcheck disable=SC2046
mapfile -t __k < <(_redis_keys "$pat")
if [ ${#__k[@]} -gt 0 ]; then
any=1
# DEL supports multiple keys; chunk to avoid arg limits
local chunk=0 buf=()
for k in "${__k[@]}"; do
buf+=("$k"); chunk=$((chunk+1))
if [ $chunk -ge 100 ]; then _redis_cli DEL "${buf[@]}" >/dev/null; buf=(); chunk=0; fi
done
[ ${#buf[@]} -gt 0 ] && _redis_cli DEL "${buf[@]}" >/dev/null
fi
return $any
}
_redis_llen() { # usage: _redis_llen <key>
_redis_cli LLEN "$1" 2>/dev/null | tr -d '\r' || echo 0
}
_transcribe_key="podx:transcribe:paused"
# Queue namespace & patterns (override in .env if your deployment differs)
: "${PODX_QUEUE_NS:=podx:q}" # base namespace for our worker queues
: "${PODX_WORKER_QUEUES:=transcribe index meili enrich owui_kb}" # space-separated logical queues
: "${PODX_QUEUE_SUFFIXES:= :inflight :retry :dlq }" # related keys per queue
: "${OPENWEBUI_QUEUE_PATTERNS:=owui:kb:* owui:tasks:* open-webui:*queue* openwebui:*queue*}" # OWUI queue-ish keys
# ------------------------------ Help ------------------------------
_help() {
cat <<EOF
@@ -212,6 +245,12 @@ Transcription control:
transcribe-pause # pause CPU-heavy transcription jobs
transcribe-resume # resume transcription jobs
Queues (Redis):
queues-workers-list # show worker queue sizes (LLEN) and related keys
queues-workers-clear # purge worker queues (ns=\${PODX_QUEUE_NS})
queues-owui-list # show OpenWebUI queue-like keys
queues-owui-clear # purge OpenWebUI queue-like keys
Examples:
./scripts/podx-tools.sh meili-health
./scripts/podx-tools.sh meili-search "grand canyon" 10
@@ -227,6 +266,56 @@ cmd="${1:-}"
case "$cmd" in
""|-h|--help|help) _help ;;
# ---------- Queue inspection/cleanup ----------
queues-workers-list)
# Show lengths for configured worker queues and related keys
echo "[workers] namespace: ${PODX_QUEUE_NS}"
for q in ${PODX_WORKER_QUEUES}; do
base="${PODX_QUEUE_NS}:${q}"
printf "%-28s %6s\n" "$base" "$(_redis_llen "$base")"
for suf in ${PODX_QUEUE_SUFFIXES}; do
k="${base}${suf}"
if [ -n "$(_redis_keys "$k")" ]; then
printf "%-28s %6s\n" "$k" "$(_redis_llen "$k")"
fi
done
done
;;
queues-workers-clear)
# Delete worker queue keys (main lists + inflight/retry/dlq). Safe: does not touch other namespaces.
echo "[workers] clearing queues in ns=${PODX_QUEUE_NS}"
for q in ${PODX_WORKER_QUEUES}; do
base="${PODX_QUEUE_NS}:${q}"
_redis_del_pattern "$base" >/dev/null || true
for suf in ${PODX_QUEUE_SUFFIXES}; do
_redis_del_pattern "${base}${suf}" >/dev/null || true
done
done
# Also remove common locks for this namespace
_redis_del_pattern "${PODX_QUEUE_NS}:lock:*" >/dev/null || true
echo "[workers] done"
;;
queues-owui-list)
# List OpenWebUI queue-like keys and lengths
for pat in ${OPENWEBUI_QUEUE_PATTERNS}; do
mapfile -t keys < <(_redis_keys "$pat")
for k in "${keys[@]}"; do
printf "%-40s %6s\n" "$k" "$(_redis_llen "$k")"
done
done
;;
queues-owui-clear)
# Clear OpenWebUI queue-like keys
echo "[owui] clearing keys matching: ${OPENWEBUI_QUEUE_PATTERNS}"
for pat in ${OPENWEBUI_QUEUE_PATTERNS}; do
_redis_del_pattern "$pat" >/dev/null || true
done
echo "[owui] done"
;;
# ---------- Meili ----------
meili-health)
_require "MEILI_URL" "$MEILI_URL"