diff --git a/litellm/README.md b/litellm/README.md index 6c3afdc1..6ca8e6b6 100644 --- a/litellm/README.md +++ b/litellm/README.md @@ -55,7 +55,9 @@ bash litellm/start-litellm.sh # foreground on :4000, Ctrl-C to stop ``` Overridable via env: `LITELLM_PORT` (default 4000), `LITELLM_CONFIG`, `ENV_FILE`, -`LITELLM_MASTER_KEY`. +`LITELLM_MASTER_KEY`, and the proxy dep pins `LITELLM_SPEC` / `LITELLM_FASTAPI_SPEC` +(full pip specifiers, e.g. `litellm[proxy]==1.95.0` / `fastapi==0.140.0` — set both +together when bumping). ### Point coder_eval at it @@ -190,4 +192,5 @@ subject to this. Bedrock models are single-provider and not affected. | `LiteLLM proxy not reachable at ...` (coder_eval startup) | Proxy not running — start it, or unset `LITELLM_BASE_URL`. | | `Invalid model name passed in model=...` | Model added to yaml but proxy not restarted — restart it. | | HTTP 401 / "Unable to locate credentials" | Missing `AWS_BEARER_TOKEN_BEDROCK` / `OPENROUTER_API_KEY` in `.env`, or key mismatch between `LITELLM_AUTH_TOKEN` (client) and the proxy's master key. | +| `ModuleNotFoundError: No module named 'proxy_server'` (masked startup death) | fastapi drifted past 0.140.0 (`get_flat_dependant` removed). Use `start-litellm.sh` (it pins the deps), or run with `--with 'fastapi==0.140.0'`. If overriding `LITELLM_SPEC`, bump `LITELLM_FASTAPI_SPEC` to match. | | evalboard cost column blank for a model | Model missing from `evalboard/lib/pricing.ts`. | diff --git a/litellm/cost_logger.py b/litellm/cost_logger.py index 03dc86f3..4a3eaef4 100644 --- a/litellm/cost_logger.py +++ b/litellm/cost_logger.py @@ -26,7 +26,7 @@ This module is intentionally **self-contained** (no ``coder_eval`` import): the proxy may run in its own ephemeral environment -(``uvx --from 'litellm[proxy]' litellm``). +(``uvx --from 'litellm[proxy]==1.95.0' --with 'fastapi==0.140.0' litellm``). """ from __future__ import annotations diff --git a/litellm/litellm-config.yaml b/litellm/litellm-config.yaml index 5ec10b8e..a86e1d45 100644 --- a/litellm/litellm-config.yaml +++ b/litellm/litellm-config.yaml @@ -18,8 +18,10 @@ # AWS_REGION=eu-north-1 — EU residency (Stockholm). # LITELLM_MASTER_KEY — the virtual key clients present as LITELLM_AUTH_TOKEN. # -# Run manually: -# uvx --from 'litellm[proxy]' litellm --config litellm/litellm-config.yaml --port 4000 +# Run manually (pin fastapi==0.140.0 — a later 0.140.x patch removed a symbol +# litellm's proxy imports; start-litellm.sh does this for you): +# uvx --from 'litellm[proxy]==1.95.0' --with 'fastapi==0.140.0' \ +# litellm --config litellm/litellm-config.yaml --port 4000 model_list: # DeepSeek V3.2 — cost lead ($0.74 / $2.22 per Mtok). diff --git a/litellm/start-litellm.sh b/litellm/start-litellm.sh index 169ece47..517c0721 100755 --- a/litellm/start-litellm.sh +++ b/litellm/start-litellm.sh @@ -12,7 +12,9 @@ # Usage: # litellm/start-litellm.sh # foreground; Ctrl-C to stop # Overridable via env: -# LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY +# LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY, +# LITELLM_SPEC / LITELLM_FASTAPI_SPEC (proxy dep pins — FULL pip specifiers, +# e.g. 'litellm[proxy]==1.95.0' / 'fastapi==0.140.0'; see the pin comment below) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" @@ -69,11 +71,31 @@ if [ -z "$AWS_BEARER_TOKEN_BEDROCK" ]; then echo " Set it in .env or 'export AWS_BEARER_TOKEN_BEDROCK=...' before running." >&2 exit 1 fi +# Pin the proxy deps. `uvx --from 'litellm[proxy]'` unpinned drifts: litellm 1.95.0 +# declares `fastapi>=0.136.3,<1.0`, so uvx grabs the newest fastapi — but fastapi +# dropped `get_flat_dependant` (which litellm's proxy still imports) in a 0.140.x +# PATCH (0.140.0 has it, 0.140.13 doesn't), so a range cap isn't enough and startup +# dies with a (masked) `ModuleNotFoundError: proxy_server`. Pin fastapi to an exact +# verified-good version and pin litellm so the sidecar can't silently re-break. +# Both are FULL pip specifiers; override for an upgrade (set BOTH together): +# LITELLM_SPEC='litellm[proxy]==' LITELLM_FASTAPI_SPEC='fastapi=='. +LITELLM_SPEC="${LITELLM_SPEC:-litellm[proxy]==1.95.0}" +LITELLM_FASTAPI_SPEC="${LITELLM_FASTAPI_SPEC:-fastapi==0.140.0}" +# Fail loud on a bare version (e.g. '0.140.0'): `uvx --with 0.140.0` would die with +# an opaque 'package not found' instead of a pin error. +for _spec in "$LITELLM_SPEC" "$LITELLM_FASTAPI_SPEC"; do + case "$_spec" in + *[=\<\>~]*) ;; + *) echo "ERROR: '$_spec' is not a pip specifier (expected e.g. fastapi==0.140.0)." >&2; exit 1 ;; + esac +done + echo "config : $CONFIG" echo "region : $AWS_REGION" echo "bedrock tok: set (${#AWS_BEARER_TOKEN_BEDROCK} chars)" echo "master key : $LITELLM_MASTER_KEY" echo "cost log : $LITELLM_COST_LOG" +echo "proxy deps : $LITELLM_SPEC + $LITELLM_FASTAPI_SPEC" # --- stop any stale proxy on the port (the classic 'creds-less running proxy') --- existing=$(lsof -tiTCP:"$PORT" -sTCP:LISTEN 2>/dev/null || true) @@ -96,4 +118,5 @@ Set these in coder_eval's .env (or shell) to use it: EOF -exec uvx --from 'litellm[proxy]' litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" +# Launch with the pinned deps resolved above (rationale in the pin comment). +exec uvx --from "$LITELLM_SPEC" --with "$LITELLM_FASTAPI_SPEC" litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" diff --git a/tests/test_litellm_config.py b/tests/test_litellm_config.py index 97b10d17..31e6dd22 100644 --- a/tests/test_litellm_config.py +++ b/tests/test_litellm_config.py @@ -4,19 +4,32 @@ proxy startup rather than in CI, so pin the structure the open-weight cost feature relies on: `usage.include` + the vetted provider pins per model, and the callback registration matching the real symbol in cost_logger.py. + +Same rationale covers the sidecar dependency pins: `start-litellm.sh` is the SSOT +for `LITELLM_SPEC` / `LITELLM_FASTAPI_SPEC`, and the pins are restated in the +config's `# Run manually` comment and the cost_logger docstring. Nothing else +mechanically checks the shell launcher (no shellcheck hook, no CI job touches +`litellm/`), so a future pin bump could silently desync those copies and only fail +at proxy startup — exactly the failure mode the pins exist to fix. These guards +keep the copies in lockstep and syntax-check the launcher. """ from __future__ import annotations import importlib.util +import re +import shutil +import subprocess from pathlib import Path +import pytest import yaml _REPO_ROOT = Path(__file__).resolve().parent.parent _CONFIG = _REPO_ROOT / "litellm" / "litellm-config.yaml" _COST_LOGGER = _REPO_ROOT / "litellm" / "cost_logger.py" +_START_SCRIPT = _REPO_ROOT / "litellm" / "start-litellm.sh" _CALLBACK = "cost_logger.proxy_handler_instance" @@ -25,6 +38,17 @@ def _load() -> dict: return yaml.safe_load(_CONFIG.read_text(encoding="utf-8")) +def _pin_defaults() -> dict[str, str]: + """Extract the `${VAR:-}` pin defaults from start-litellm.sh (the SSOT).""" + text = _START_SCRIPT.read_text(encoding="utf-8") + pins: dict[str, str] = {} + for var in ("LITELLM_SPEC", "LITELLM_FASTAPI_SPEC"): + m = re.search(rf'{var}="\$\{{{var}:-(?P[^}}]+)\}}"', text) + assert m, f"{var} default not found in {_START_SCRIPT.name}" + pins[var] = m.group("val") + return pins + + class TestLitellmConfigShape: def test_cost_callback_registered(self): assert _load()["litellm_settings"]["callbacks"] == _CALLBACK @@ -55,3 +79,35 @@ def test_callback_symbol_exists_in_cost_logger(self): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) assert hasattr(module, attr) + + +class TestLitellmSidecarPins: + def test_defaults_are_full_pip_specifiers(self): + # A bare version (e.g. LITELLM_FASTAPI_SPEC=0.140.13) would become + # `uvx --with 0.140.13` and fail with an opaque 'package not found' rather + # than a pin error, so every default must carry a version comparator. + for var, spec in _pin_defaults().items(): + assert var.endswith("_SPEC"), f"{var} holds a specifier; name it *_SPEC" + assert re.search(r"[=<>~]=?", spec), f"{var}={spec!r} is not a pip specifier" + + def test_pins_are_restated_in_every_committed_copy(self): + # start-litellm.sh is the SSOT; the config's `# Run manually` comment and the + # cost_logger docstring each restate the invocation. Assert both exact pins + # appear verbatim in each, so a future bump can't silently desync them. + pins = _pin_defaults().values() + config_text = _CONFIG.read_text(encoding="utf-8") + cost_logger_text = _COST_LOGGER.read_text(encoding="utf-8") + for pin in pins: + assert pin in config_text, f"{pin!r} missing from litellm-config.yaml comment" + assert pin in cost_logger_text, f"{pin!r} missing from cost_logger.py docstring" + + def test_start_script_has_valid_bash_syntax(self): + bash = shutil.which("bash") + if bash is None: + pytest.skip("bash not available") + result = subprocess.run( + [bash, "-n", str(_START_SCRIPT)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr