#!/usr/bin/env bash # Regression: Docker full-model hot-swap failures must restore the last # known-good model config and recreate llama-server from that config. set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TARGET="$ROOT_DIR/scripts/bootstrap-upgrade.sh" fail() { echo "[FAIL] $*" >&2 if [[ -n "${tmp:-}" ]]; then [[ -f "$tmp/bootstrap.log" ]] && sed 's/^/[bootstrap] /' "$tmp/bootstrap.log" >&2 [[ -f "${install_dir:-}/.env" ]] && sed 's/^/[env] /' "$install_dir/.env" >&2 [[ -f "${install_dir:-}/config/llama-server/models.ini" ]] && sed 's/^/[models.ini] /' "$install_dir/config/llama-server/models.ini" >&2 [[ -f "${docker_calls:-}" ]] && sed 's/^/[docker] /' "$docker_calls" >&2 fi exit 1 } pass() { echo "[PASS] $*" } tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT fakebin="$tmp/bin" install_dir="$tmp/install" docker_calls="$tmp/docker-calls.log" mkdir -p "$fakebin" "$install_dir/data/models" "$install_dir/config/llama-server" cat > "$fakebin/curl" <<'EOF' #!/usr/bin/env bash case " $* " in *" -sI "*) printf 'HTTP/2 200\r\ncontent-length: 10\r\n\r\n' exit 0 ;; esac printf 'health:%s\n' "$*" >> "${ODS_FAKE_CURL_LOG:?}" if [[ " $* " == *"11434/health"* ]] && grep -q '^GGUF_FILE=Bootstrap.gguf$' .env 2>/dev/null; then printf '{"status":"ok"}\n' exit 0 fi exit 7 EOF chmod +x "$fakebin/curl" cat > "$fakebin/sleep" <<'EOF' #!/usr/bin/env bash exit 0 EOF chmod +x "$fakebin/sleep" cat > "$fakebin/uname" <<'EOF' #!/usr/bin/env bash printf 'Linux\n' EOF chmod +x "$fakebin/uname" cat > "$fakebin/docker" <<'EOF' #!/usr/bin/env bash set -euo pipefail env_value() { local key="$1" [[ -f .env ]] || return 0 grep -E "^${key}=" .env 2>/dev/null | head -1 | cut -d= -f2- } case "${1:-}" in info) exit 0 ;; compose) if [[ "${2:-}" == "version" ]]; then exit 0 fi if [[ " $* " == *" up -d --force-recreate --no-deps llama-server "* ]]; then printf 'compose-up:%s\n' "$(env_value GGUF_FILE)" >> "${ODS_FAKE_DOCKER_LOG:?}" exit 0 fi exit 0 ;; ps) if [[ " $* " == *"name=ods-llama-server"* ]]; then printf 'ods-llama-server\n' fi exit 0 ;; inspect) if [[ "${2:-}" == "ods-llama-server" ]]; then if [[ " $* " == *".State.Status"* ]]; then printf 'restarting true 1\n' exit 0 fi printf '/app/llama-server --model /models/%s --ctx-size %s\n' \ "$(env_value GGUF_FILE)" "$(env_value CTX_SIZE)" fi exit 0 ;; esac exit 0 EOF chmod +x "$fakebin/docker" cat > "$install_dir/.env" <<'EOF' GGUF_FILE=Bootstrap.gguf LLM_MODEL=bootstrap-model MAX_CONTEXT=8192 CTX_SIZE=8192 GPU_BACKEND=nvidia OLLAMA_PORT=11434 EOF cat > "$install_dir/config/llama-server/models.ini" <<'EOF' [bootstrap-model] filename = Bootstrap.gguf load-on-startup = true n-ctx = 8192 EOF cat > "$install_dir/.compose-flags" <<'EOF' -f docker-compose.base.yml -f docker-compose.nvidia.yml EOF printf 'bootstrap\n' > "$install_dir/data/models/Bootstrap.gguf" printf 'full-model\n' > "$install_dir/data/models/Full.gguf" curl_calls="$tmp/curl-calls.log" set +e PATH="$fakebin:$PATH" ODS_FAKE_DOCKER_LOG="$docker_calls" ODS_FAKE_CURL_LOG="$curl_calls" bash "$TARGET" \ "$install_dir" \ "Full.gguf" \ "https://example.invalid/Full.gguf" \ "" \ "full-model" \ "32768" \ "Bootstrap.gguf" \ > "$tmp/bootstrap.log" 2>&1 rc=$? set -e [[ $rc -ne 0 ]] || fail "bootstrap-upgrade must fail when Docker llama-server never becomes healthy" grep -q '^GGUF_FILE=Bootstrap.gguf$' "$install_dir/.env" \ || fail "Docker hot-swap failure must restore previous GGUF_FILE" grep -q '^LLM_MODEL=bootstrap-model$' "$install_dir/.env" \ || fail "Docker hot-swap failure must restore previous LLM_MODEL" grep -q '^CTX_SIZE=8192$' "$install_dir/.env" \ || fail "Docker hot-swap failure must restore previous CTX_SIZE" grep -q 'filename = Bootstrap.gguf' "$install_dir/config/llama-server/models.ini" \ || fail "Docker hot-swap failure must restore previous models.ini" grep -q 'compose-up:Full.gguf' "$docker_calls" \ || fail "test did not exercise the full-model compose recreate" grep -q 'compose-up:Bootstrap.gguf' "$docker_calls" \ || fail "rollback must recreate llama-server from the restored bootstrap config" grep -q 'Restoring previous active model config after Docker llama-server swap failure' "$tmp/bootstrap.log" \ || fail "bootstrap-upgrade should log the Docker rollback" grep -q 'llama-server container exited or is restarting while loading the full model' "$tmp/bootstrap.log" \ || fail "bootstrap-upgrade should detect a failed llama-server container before waiting for the full health timeout" health_attempts=$(grep -c '^health:' "$curl_calls" 2>/dev/null || true) [[ "$health_attempts" -lt 60 ]] \ || fail "failed Docker hot-swap should not wait for all health attempts after the container is restarting" grep -q '"status": "failed"' "$install_dir/data/bootstrap-status.json" \ || fail "failed Docker hot-swap must mark bootstrap-status failed" pass "Docker hot-swap failure restores previous active model config"