From 7504e2bf73b870cc3588eca78e6831b55b29ebc9 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 4 Aug 2026 11:19:46 +0100 Subject: [PATCH 1/3] fix(litellm): pin litellm[proxy]==1.95.0 + fastapi==0.140.0 for proxy startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unpinned `uvx --from 'litellm[proxy]'` resolves the newest fastapi, but a 0.140.x patch removed `get_flat_dependant`, which litellm 1.95.0's proxy imports — so startup died with a masked `ModuleNotFoundError: proxy_server`. Pin both deps (overridable via LITELLM_SPEC / LITELLM_FASTAPI) and update the config's manual-run comment to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- litellm/litellm-config.yaml | 6 ++++-- litellm/start-litellm.sh | 11 ++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/litellm/litellm-config.yaml b/litellm/litellm-config.yaml index 5ec10b8e..a86e1d45 100644 --- a/litellm/litellm-config.yaml +++ b/litellm/litellm-config.yaml @@ -18,8 +18,10 @@ # AWS_REGION=eu-north-1 — EU residency (Stockholm). # LITELLM_MASTER_KEY — the virtual key clients present as LITELLM_AUTH_TOKEN. # -# Run manually: -# uvx --from 'litellm[proxy]' litellm --config litellm/litellm-config.yaml --port 4000 +# Run manually (pin fastapi==0.140.0 — a later 0.140.x patch removed a symbol +# litellm's proxy imports; start-litellm.sh does this for you): +# uvx --from 'litellm[proxy]==1.95.0' --with 'fastapi==0.140.0' \ +# litellm --config litellm/litellm-config.yaml --port 4000 model_list: # DeepSeek V3.2 — cost lead ($0.74 / $2.22 per Mtok). diff --git a/litellm/start-litellm.sh b/litellm/start-litellm.sh index 169ece47..984a6c0d 100755 --- a/litellm/start-litellm.sh +++ b/litellm/start-litellm.sh @@ -96,4 +96,13 @@ Set these in coder_eval's .env (or shell) to use it: EOF -exec uvx --from 'litellm[proxy]' litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" +# Pin the proxy deps. `uvx --from 'litellm[proxy]'` unpinned drifts: litellm 1.95.0 +# declares `fastapi>=0.136.3,<1.0`, so uvx grabs the newest fastapi — but fastapi +# dropped `get_flat_dependant` (which litellm's proxy still imports) in a 0.140.x +# PATCH (0.140.0 has it, 0.140.13 doesn't), so a range cap isn't enough and startup +# dies with a (masked) `ModuleNotFoundError: proxy_server`. Pin fastapi to an exact +# verified-good version and pin litellm so the sidecar can't silently re-break. +# Override for an upgrade: LITELLM_SPEC='litellm[proxy]==' LITELLM_FASTAPI='fastapi=='. +LITELLM_SPEC="${LITELLM_SPEC:-litellm[proxy]==1.95.0}" +LITELLM_FASTAPI="${LITELLM_FASTAPI:-fastapi==0.140.0}" +exec uvx --from "$LITELLM_SPEC" --with "$LITELLM_FASTAPI" litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" From c03dbcfc0c87e108b59762ccffd68d9e655513c5 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 4 Aug 2026 11:27:54 +0100 Subject: [PATCH 2/3] docs(litellm): surface the proxy dep-pin override vars in start script Address code-review notes: list LITELLM_SPEC / LITELLM_FASTAPI in the header Usage block and echo the resolved pins in the startup banner so an operator debugging a startup failure can see which versions actually ran. Co-Authored-By: Claude Opus 4.8 (1M context) --- litellm/start-litellm.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/start-litellm.sh b/litellm/start-litellm.sh index 984a6c0d..ad641097 100755 --- a/litellm/start-litellm.sh +++ b/litellm/start-litellm.sh @@ -12,7 +12,8 @@ # Usage: # litellm/start-litellm.sh # foreground; Ctrl-C to stop # Overridable via env: -# LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY +# LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY, +# LITELLM_SPEC / LITELLM_FASTAPI (proxy dep pins — see the launch comment below) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" @@ -105,4 +106,5 @@ EOF # Override for an upgrade: LITELLM_SPEC='litellm[proxy]==' LITELLM_FASTAPI='fastapi=='. LITELLM_SPEC="${LITELLM_SPEC:-litellm[proxy]==1.95.0}" LITELLM_FASTAPI="${LITELLM_FASTAPI:-fastapi==0.140.0}" +echo "proxy deps : $LITELLM_SPEC + $LITELLM_FASTAPI" exec uvx --from "$LITELLM_SPEC" --with "$LITELLM_FASTAPI" litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" From b7b4a9804c07b42b297fc4798dc58d1c34f00dad Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 4 Aug 2026 12:25:11 +0100 Subject: [PATCH 3/3] =?UTF-8?q?refactor(litellm):=20address=20PR=20review?= =?UTF-8?q?=20=E2=80=94=20pin=20SSOT=20guard,=20rename,=20doc=20ripple?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply code-review feedback on the proxy dep-pin fix: - Rename LITELLM_FASTAPI -> LITELLM_FASTAPI_SPEC for symmetry with LITELLM_SPEC and add a fail-loud guard rejecting a bare version (the `=0.140.13` footgun). - Move the pin defs + `proxy deps` echo up into the aligned preflight status block; leave only `exec` at the bottom. - Ripple the new knobs into litellm/README.md (env-override list + a Troubleshooting row for the masked ModuleNotFoundError) and refresh the stale unpinned invocation in cost_logger.py's docstring. - Add tests/test_litellm_config.py::TestLitellmSidecarPins: start-litellm.sh is the SSOT for the pins; assert both exact pins are restated in the config comment and cost_logger docstring, that defaults are full pip specifiers, and that the launcher passes `bash -n`. Co-Authored-By: Claude Opus 4.8 (1M context) --- litellm/README.md | 5 +++- litellm/cost_logger.py | 2 +- litellm/start-litellm.sh | 36 +++++++++++++++-------- tests/test_litellm_config.py | 56 ++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 14 deletions(-) diff --git a/litellm/README.md b/litellm/README.md index 6c3afdc1..6ca8e6b6 100644 --- a/litellm/README.md +++ b/litellm/README.md @@ -55,7 +55,9 @@ bash litellm/start-litellm.sh # foreground on :4000, Ctrl-C to stop ``` Overridable via env: `LITELLM_PORT` (default 4000), `LITELLM_CONFIG`, `ENV_FILE`, -`LITELLM_MASTER_KEY`. +`LITELLM_MASTER_KEY`, and the proxy dep pins `LITELLM_SPEC` / `LITELLM_FASTAPI_SPEC` +(full pip specifiers, e.g. `litellm[proxy]==1.95.0` / `fastapi==0.140.0` — set both +together when bumping). ### Point coder_eval at it @@ -190,4 +192,5 @@ subject to this. Bedrock models are single-provider and not affected. | `LiteLLM proxy not reachable at ...` (coder_eval startup) | Proxy not running — start it, or unset `LITELLM_BASE_URL`. | | `Invalid model name passed in model=...` | Model added to yaml but proxy not restarted — restart it. | | HTTP 401 / "Unable to locate credentials" | Missing `AWS_BEARER_TOKEN_BEDROCK` / `OPENROUTER_API_KEY` in `.env`, or key mismatch between `LITELLM_AUTH_TOKEN` (client) and the proxy's master key. | +| `ModuleNotFoundError: No module named 'proxy_server'` (masked startup death) | fastapi drifted past 0.140.0 (`get_flat_dependant` removed). Use `start-litellm.sh` (it pins the deps), or run with `--with 'fastapi==0.140.0'`. If overriding `LITELLM_SPEC`, bump `LITELLM_FASTAPI_SPEC` to match. | | evalboard cost column blank for a model | Model missing from `evalboard/lib/pricing.ts`. | diff --git a/litellm/cost_logger.py b/litellm/cost_logger.py index 03dc86f3..4a3eaef4 100644 --- a/litellm/cost_logger.py +++ b/litellm/cost_logger.py @@ -26,7 +26,7 @@ This module is intentionally **self-contained** (no ``coder_eval`` import): the proxy may run in its own ephemeral environment -(``uvx --from 'litellm[proxy]' litellm``). +(``uvx --from 'litellm[proxy]==1.95.0' --with 'fastapi==0.140.0' litellm``). """ from __future__ import annotations diff --git a/litellm/start-litellm.sh b/litellm/start-litellm.sh index ad641097..517c0721 100755 --- a/litellm/start-litellm.sh +++ b/litellm/start-litellm.sh @@ -13,7 +13,8 @@ # litellm/start-litellm.sh # foreground; Ctrl-C to stop # Overridable via env: # LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY, -# LITELLM_SPEC / LITELLM_FASTAPI (proxy dep pins — see the launch comment below) +# LITELLM_SPEC / LITELLM_FASTAPI_SPEC (proxy dep pins — FULL pip specifiers, +# e.g. 'litellm[proxy]==1.95.0' / 'fastapi==0.140.0'; see the pin comment below) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" @@ -70,11 +71,31 @@ if [ -z "$AWS_BEARER_TOKEN_BEDROCK" ]; then echo " Set it in .env or 'export AWS_BEARER_TOKEN_BEDROCK=...' before running." >&2 exit 1 fi +# Pin the proxy deps. `uvx --from 'litellm[proxy]'` unpinned drifts: litellm 1.95.0 +# declares `fastapi>=0.136.3,<1.0`, so uvx grabs the newest fastapi — but fastapi +# dropped `get_flat_dependant` (which litellm's proxy still imports) in a 0.140.x +# PATCH (0.140.0 has it, 0.140.13 doesn't), so a range cap isn't enough and startup +# dies with a (masked) `ModuleNotFoundError: proxy_server`. Pin fastapi to an exact +# verified-good version and pin litellm so the sidecar can't silently re-break. +# Both are FULL pip specifiers; override for an upgrade (set BOTH together): +# LITELLM_SPEC='litellm[proxy]==' LITELLM_FASTAPI_SPEC='fastapi=='. +LITELLM_SPEC="${LITELLM_SPEC:-litellm[proxy]==1.95.0}" +LITELLM_FASTAPI_SPEC="${LITELLM_FASTAPI_SPEC:-fastapi==0.140.0}" +# Fail loud on a bare version (e.g. '0.140.0'): `uvx --with 0.140.0` would die with +# an opaque 'package not found' instead of a pin error. +for _spec in "$LITELLM_SPEC" "$LITELLM_FASTAPI_SPEC"; do + case "$_spec" in + *[=\<\>~]*) ;; + *) echo "ERROR: '$_spec' is not a pip specifier (expected e.g. fastapi==0.140.0)." >&2; exit 1 ;; + esac +done + echo "config : $CONFIG" echo "region : $AWS_REGION" echo "bedrock tok: set (${#AWS_BEARER_TOKEN_BEDROCK} chars)" echo "master key : $LITELLM_MASTER_KEY" echo "cost log : $LITELLM_COST_LOG" +echo "proxy deps : $LITELLM_SPEC + $LITELLM_FASTAPI_SPEC" # --- stop any stale proxy on the port (the classic 'creds-less running proxy') --- existing=$(lsof -tiTCP:"$PORT" -sTCP:LISTEN 2>/dev/null || true) @@ -97,14 +118,5 @@ Set these in coder_eval's .env (or shell) to use it: EOF -# Pin the proxy deps. `uvx --from 'litellm[proxy]'` unpinned drifts: litellm 1.95.0 -# declares `fastapi>=0.136.3,<1.0`, so uvx grabs the newest fastapi — but fastapi -# dropped `get_flat_dependant` (which litellm's proxy still imports) in a 0.140.x -# PATCH (0.140.0 has it, 0.140.13 doesn't), so a range cap isn't enough and startup -# dies with a (masked) `ModuleNotFoundError: proxy_server`. Pin fastapi to an exact -# verified-good version and pin litellm so the sidecar can't silently re-break. -# Override for an upgrade: LITELLM_SPEC='litellm[proxy]==' LITELLM_FASTAPI='fastapi=='. -LITELLM_SPEC="${LITELLM_SPEC:-litellm[proxy]==1.95.0}" -LITELLM_FASTAPI="${LITELLM_FASTAPI:-fastapi==0.140.0}" -echo "proxy deps : $LITELLM_SPEC + $LITELLM_FASTAPI" -exec uvx --from "$LITELLM_SPEC" --with "$LITELLM_FASTAPI" litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" +# Launch with the pinned deps resolved above (rationale in the pin comment). +exec uvx --from "$LITELLM_SPEC" --with "$LITELLM_FASTAPI_SPEC" litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" diff --git a/tests/test_litellm_config.py b/tests/test_litellm_config.py index 97b10d17..31e6dd22 100644 --- a/tests/test_litellm_config.py +++ b/tests/test_litellm_config.py @@ -4,19 +4,32 @@ proxy startup rather than in CI, so pin the structure the open-weight cost feature relies on: `usage.include` + the vetted provider pins per model, and the callback registration matching the real symbol in cost_logger.py. + +Same rationale covers the sidecar dependency pins: `start-litellm.sh` is the SSOT +for `LITELLM_SPEC` / `LITELLM_FASTAPI_SPEC`, and the pins are restated in the +config's `# Run manually` comment and the cost_logger docstring. Nothing else +mechanically checks the shell launcher (no shellcheck hook, no CI job touches +`litellm/`), so a future pin bump could silently desync those copies and only fail +at proxy startup — exactly the failure mode the pins exist to fix. These guards +keep the copies in lockstep and syntax-check the launcher. """ from __future__ import annotations import importlib.util +import re +import shutil +import subprocess from pathlib import Path +import pytest import yaml _REPO_ROOT = Path(__file__).resolve().parent.parent _CONFIG = _REPO_ROOT / "litellm" / "litellm-config.yaml" _COST_LOGGER = _REPO_ROOT / "litellm" / "cost_logger.py" +_START_SCRIPT = _REPO_ROOT / "litellm" / "start-litellm.sh" _CALLBACK = "cost_logger.proxy_handler_instance" @@ -25,6 +38,17 @@ def _load() -> dict: return yaml.safe_load(_CONFIG.read_text(encoding="utf-8")) +def _pin_defaults() -> dict[str, str]: + """Extract the `${VAR:-}` pin defaults from start-litellm.sh (the SSOT).""" + text = _START_SCRIPT.read_text(encoding="utf-8") + pins: dict[str, str] = {} + for var in ("LITELLM_SPEC", "LITELLM_FASTAPI_SPEC"): + m = re.search(rf'{var}="\$\{{{var}:-(?P[^}}]+)\}}"', text) + assert m, f"{var} default not found in {_START_SCRIPT.name}" + pins[var] = m.group("val") + return pins + + class TestLitellmConfigShape: def test_cost_callback_registered(self): assert _load()["litellm_settings"]["callbacks"] == _CALLBACK @@ -55,3 +79,35 @@ def test_callback_symbol_exists_in_cost_logger(self): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) assert hasattr(module, attr) + + +class TestLitellmSidecarPins: + def test_defaults_are_full_pip_specifiers(self): + # A bare version (e.g. LITELLM_FASTAPI_SPEC=0.140.13) would become + # `uvx --with 0.140.13` and fail with an opaque 'package not found' rather + # than a pin error, so every default must carry a version comparator. + for var, spec in _pin_defaults().items(): + assert var.endswith("_SPEC"), f"{var} holds a specifier; name it *_SPEC" + assert re.search(r"[=<>~]=?", spec), f"{var}={spec!r} is not a pip specifier" + + def test_pins_are_restated_in_every_committed_copy(self): + # start-litellm.sh is the SSOT; the config's `# Run manually` comment and the + # cost_logger docstring each restate the invocation. Assert both exact pins + # appear verbatim in each, so a future bump can't silently desync them. + pins = _pin_defaults().values() + config_text = _CONFIG.read_text(encoding="utf-8") + cost_logger_text = _COST_LOGGER.read_text(encoding="utf-8") + for pin in pins: + assert pin in config_text, f"{pin!r} missing from litellm-config.yaml comment" + assert pin in cost_logger_text, f"{pin!r} missing from cost_logger.py docstring" + + def test_start_script_has_valid_bash_syntax(self): + bash = shutil.which("bash") + if bash is None: + pytest.skip("bash not available") + result = subprocess.run( + [bash, "-n", str(_START_SCRIPT)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr