fix(litellm): pin proxy deps (litellm 1.95.0 + fastapi 0.140.0) to fix startup crash

litellm/README.md

@@ -55,7 +55,9 @@ bash litellm/start-litellm.sh          # foreground on :4000, Ctrl-C to stop
 ```
 
 Overridable via env: `LITELLM_PORT` (default 4000), `LITELLM_CONFIG`, `ENV_FILE`,
-`LITELLM_MASTER_KEY`.
+`LITELLM_MASTER_KEY`, and the proxy dep pins `LITELLM_SPEC` / `LITELLM_FASTAPI_SPEC`
+(full pip specifiers, e.g. `litellm[proxy]==1.95.0` / `fastapi==0.140.0` — set both
+together when bumping).
 
 ### Point coder_eval at it
 
@@ -190,4 +192,5 @@ subject to this. Bedrock models are single-provider and not affected.
 | `LiteLLM proxy not reachable at ...` (coder_eval startup) | Proxy not running — start it, or unset `LITELLM_BASE_URL`. |
 | `Invalid model name passed in model=...` | Model added to yaml but proxy not restarted — restart it. |
 | HTTP 401 / "Unable to locate credentials" | Missing `AWS_BEARER_TOKEN_BEDROCK` / `OPENROUTER_API_KEY` in `.env`, or key mismatch between `LITELLM_AUTH_TOKEN` (client) and the proxy's master key. |
+| `ModuleNotFoundError: No module named 'proxy_server'` (masked startup death) | fastapi drifted past 0.140.0 (`get_flat_dependant` removed). Use `start-litellm.sh` (it pins the deps), or run with `--with 'fastapi==0.140.0'`. If overriding `LITELLM_SPEC`, bump `LITELLM_FASTAPI_SPEC` to match. |
 | evalboard cost column blank for a model | Model missing from `evalboard/lib/pricing.ts`. |

litellm/cost_logger.py

@@ -26,7 +26,7 @@
 
 This module is intentionally **self-contained** (no ``coder_eval`` import): the
 proxy may run in its own ephemeral environment
-(``uvx --from 'litellm[proxy]' litellm``).
+(``uvx --from 'litellm[proxy]==1.95.0' --with 'fastapi==0.140.0' litellm``).
 """
 
 from __future__ import annotations

litellm/litellm-config.yaml

@@ -18,8 +18,10 @@
 #   AWS_REGION=eu-north-1      — EU residency (Stockholm).
 #   LITELLM_MASTER_KEY         — the virtual key clients present as LITELLM_AUTH_TOKEN.
 #
-# Run manually:
-#   uvx --from 'litellm[proxy]' litellm --config litellm/litellm-config.yaml --port 4000
+# Run manually (pin fastapi==0.140.0 — a later 0.140.x patch removed a symbol
+# litellm's proxy imports; start-litellm.sh does this for you):
+#   uvx --from 'litellm[proxy]==1.95.0' --with 'fastapi==0.140.0' \
+#     litellm --config litellm/litellm-config.yaml --port 4000
 
 model_list:
   # DeepSeek V3.2 — cost lead ($0.74 / $2.22 per Mtok).

litellm/start-litellm.sh

@@ -12,7 +12,9 @@
 # Usage:
 #   litellm/start-litellm.sh                 # foreground; Ctrl-C to stop
 # Overridable via env:
-#   LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY
+#   LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY,
+#   LITELLM_SPEC / LITELLM_FASTAPI_SPEC (proxy dep pins — FULL pip specifiers,
+#   e.g. 'litellm[proxy]==1.95.0' / 'fastapi==0.140.0'; see the pin comment below)
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
@@ -69,11 +71,31 @@ if [ -z "$AWS_BEARER_TOKEN_BEDROCK" ]; then
   echo "       Set it in .env or 'export AWS_BEARER_TOKEN_BEDROCK=...' before running." >&2
   exit 1
 fi
+# Pin the proxy deps. `uvx --from 'litellm[proxy]'` unpinned drifts: litellm 1.95.0
+# declares `fastapi>=0.136.3,<1.0`, so uvx grabs the newest fastapi — but fastapi
+# dropped `get_flat_dependant` (which litellm's proxy still imports) in a 0.140.x
+# PATCH (0.140.0 has it, 0.140.13 doesn't), so a range cap isn't enough and startup
+# dies with a (masked) `ModuleNotFoundError: proxy_server`. Pin fastapi to an exact
+# verified-good version and pin litellm so the sidecar can't silently re-break.
+# Both are FULL pip specifiers; override for an upgrade (set BOTH together):
+#   LITELLM_SPEC='litellm[proxy]==<ver>' LITELLM_FASTAPI_SPEC='fastapi==<ver>'.
+LITELLM_SPEC="${LITELLM_SPEC:-litellm[proxy]==1.95.0}"
+LITELLM_FASTAPI_SPEC="${LITELLM_FASTAPI_SPEC:-fastapi==0.140.0}"
+# Fail loud on a bare version (e.g. '0.140.0'): `uvx --with 0.140.0` would die with
+# an opaque 'package not found' instead of a pin error.
+for _spec in "$LITELLM_SPEC" "$LITELLM_FASTAPI_SPEC"; do
+  case "$_spec" in
+    *[=\<\>~]*) ;;
+    *) echo "ERROR: '$_spec' is not a pip specifier (expected e.g. fastapi==0.140.0)." >&2; exit 1 ;;
+  esac
+done
+
 echo "config     : $CONFIG"
 echo "region     : $AWS_REGION"
 echo "bedrock tok: set (${#AWS_BEARER_TOKEN_BEDROCK} chars)"
 echo "master key : $LITELLM_MASTER_KEY"
 echo "cost log   : $LITELLM_COST_LOG"
+echo "proxy deps : $LITELLM_SPEC + $LITELLM_FASTAPI_SPEC"
 
 # --- stop any stale proxy on the port (the classic 'creds-less running proxy') ---
 existing=$(lsof -tiTCP:"$PORT" -sTCP:LISTEN 2>/dev/null || true)
@@ -96,4 +118,5 @@ Set these in coder_eval's .env (or shell) to use it:
 
 EOF
 
-exec uvx --from 'litellm[proxy]' litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT"
+# Launch with the pinned deps resolved above (rationale in the pin comment).
+exec uvx --from "$LITELLM_SPEC" --with "$LITELLM_FASTAPI_SPEC" litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT"

tests/test_litellm_config.py

@@ -4,19 +4,32 @@
 proxy startup rather than in CI, so pin the structure the open-weight cost feature
 relies on: `usage.include` + the vetted provider pins per model, and the callback
 registration matching the real symbol in cost_logger.py.
+
+Same rationale covers the sidecar dependency pins: `start-litellm.sh` is the SSOT
+for `LITELLM_SPEC` / `LITELLM_FASTAPI_SPEC`, and the pins are restated in the
+config's `# Run manually` comment and the cost_logger docstring. Nothing else
+mechanically checks the shell launcher (no shellcheck hook, no CI job touches
+`litellm/`), so a future pin bump could silently desync those copies and only fail
+at proxy startup — exactly the failure mode the pins exist to fix. These guards
+keep the copies in lockstep and syntax-check the launcher.
 """
 
 from __future__ import annotations
 
 import importlib.util
+import re
+import shutil
+import subprocess
 from pathlib import Path
 
+import pytest
 import yaml
 
 
 _REPO_ROOT = Path(__file__).resolve().parent.parent
 _CONFIG = _REPO_ROOT / "litellm" / "litellm-config.yaml"
 _COST_LOGGER = _REPO_ROOT / "litellm" / "cost_logger.py"
+_START_SCRIPT = _REPO_ROOT / "litellm" / "start-litellm.sh"
 
 _CALLBACK = "cost_logger.proxy_handler_instance"
 
@@ -25,6 +38,17 @@ def _load() -> dict:
     return yaml.safe_load(_CONFIG.read_text(encoding="utf-8"))
 
 
+def _pin_defaults() -> dict[str, str]:
+    """Extract the `${VAR:-<default>}` pin defaults from start-litellm.sh (the SSOT)."""
+    text = _START_SCRIPT.read_text(encoding="utf-8")
+    pins: dict[str, str] = {}
+    for var in ("LITELLM_SPEC", "LITELLM_FASTAPI_SPEC"):
+        m = re.search(rf'{var}="\$\{{{var}:-(?P<val>[^}}]+)\}}"', text)
+        assert m, f"{var} default not found in {_START_SCRIPT.name}"
+        pins[var] = m.group("val")
+    return pins
+
+
 class TestLitellmConfigShape:
     def test_cost_callback_registered(self):
         assert _load()["litellm_settings"]["callbacks"] == _CALLBACK
@@ -55,3 +79,35 @@ def test_callback_symbol_exists_in_cost_logger(self):
         module = importlib.util.module_from_spec(spec)
         spec.loader.exec_module(module)
         assert hasattr(module, attr)
+
+
+class TestLitellmSidecarPins:
+    def test_defaults_are_full_pip_specifiers(self):
+        # A bare version (e.g. LITELLM_FASTAPI_SPEC=0.140.13) would become
+        # `uvx --with 0.140.13` and fail with an opaque 'package not found' rather
+        # than a pin error, so every default must carry a version comparator.
+        for var, spec in _pin_defaults().items():
+            assert var.endswith("_SPEC"), f"{var} holds a specifier; name it *_SPEC"
+            assert re.search(r"[=<>~]=?", spec), f"{var}={spec!r} is not a pip specifier"
+
+    def test_pins_are_restated_in_every_committed_copy(self):
+        # start-litellm.sh is the SSOT; the config's `# Run manually` comment and the
+        # cost_logger docstring each restate the invocation. Assert both exact pins
+        # appear verbatim in each, so a future bump can't silently desync them.
+        pins = _pin_defaults().values()
+        config_text = _CONFIG.read_text(encoding="utf-8")
+        cost_logger_text = _COST_LOGGER.read_text(encoding="utf-8")
+        for pin in pins:
+            assert pin in config_text, f"{pin!r} missing from litellm-config.yaml comment"
+            assert pin in cost_logger_text, f"{pin!r} missing from cost_logger.py docstring"
+
+    def test_start_script_has_valid_bash_syntax(self):
+        bash = shutil.which("bash")
+        if bash is None:
+            pytest.skip("bash not available")
+        result = subprocess.run(
+            [bash, "-n", str(_START_SCRIPT)],
+            capture_output=True,
+            text=True,
+        )
+        assert result.returncode == 0, result.stderr