diff --git a/.gitignore b/.gitignore index cb6a633..63a2eac 100644 --- a/.gitignore +++ b/.gitignore @@ -222,3 +222,4 @@ __marimo__/ # Personal pycodeloop provider config (may end up holding local overrides) pycodeloop.config.json +.DS_Store diff --git a/pycodeloop/cli/commands/chat.py b/pycodeloop/cli/commands/chat.py index e2d4987..9b63fe4 100644 --- a/pycodeloop/cli/commands/chat.py +++ b/pycodeloop/cli/commands/chat.py @@ -17,6 +17,16 @@ def chat( url: str = typer.Option( None, help="Endpoint URL, required for --provider generic." ), + temperature: float | None = typer.Option( + None, + "--temperature", + help="Sampling temperature.", + ), + max_tokens: int | None = typer.Option( + None, + "--max-tokens", + help="Maximum number of tokens to generate.", + ), mcp: list[str] = typer.Option( None, help="MCP server as 'command arg1 arg2'; repeatable." ), @@ -72,6 +82,8 @@ def chat( base_url, url, mcp, + temperature=temperature, + max_tokens=max_tokens, auto_approve=yes, skills=skills, skills_refresh=skills_refresh, diff --git a/pycodeloop/cli/commands/run.py b/pycodeloop/cli/commands/run.py index 1a93d4b..afab186 100644 --- a/pycodeloop/cli/commands/run.py +++ b/pycodeloop/cli/commands/run.py @@ -64,6 +64,14 @@ def run( "arbitrary shell commands. On by default." ), ), + temperature: float | None = typer.Option( + None, + help="Sampling temperature.", + ), + max_tokens: int | None = typer.Option( + None, + help="Maximum number of tokens to generate", + ), ) -> None: """Run a single prompt to completion, non-interactively.""" flow, _provider_name, _model = build_flow( @@ -72,6 +80,8 @@ def run( base_url, url, mcp, + temperature=temperature, + max_tokens=max_tokens, auto_approve=yes, skills=skills, skills_refresh=skills_refresh, diff --git a/pycodeloop/cli/flow.py b/pycodeloop/cli/flow.py index 665c194..02ee9bc 100644 --- a/pycodeloop/cli/flow.py +++ b/pycodeloop/cli/flow.py @@ -101,6 +101,8 @@ def resolve_provider( model: str | None, base_url: str | None = None, url: str | None = None, + temperature: float | None = None, + max_tokens: int | None = None, ) -> tuple[Provider, str]: """Build a `Provider` from CLI-style args, returning it alongside the resolved provider name — shared by the interactive CLI and `serve`.""" @@ -125,6 +127,17 @@ def resolve_provider( else: provider_kwargs = {"model": model or Settings.MODEL} + inference_params = {} + + if temperature is not None: + inference_params["temperature"] = temperature + + if max_tokens is not None: + inference_params["max_tokens"] = max_tokens + + if inference_params: + provider_kwargs["inference_params"] = inference_params + return get_provider(provider_name, **provider_kwargs), provider_name @@ -134,6 +147,8 @@ def build_flow( base_url: str | None = None, url: str | None = None, mcp: list[str] | None = None, + temperature: float | None = None, + max_tokens: int | None = None, auto_approve: bool = False, skills: bool = False, skills_refresh: bool = False, @@ -142,7 +157,7 @@ def build_flow( workspace: bool = True, ) -> tuple[CodeLoop, str, str]: provider, provider_name = resolve_provider( - provider_name, model, base_url, url + provider_name, model, base_url, url, temperature, max_tokens ) buffer = TurnBuffer(console) diff --git a/pycodeloop/providers/__init__.py b/pycodeloop/providers/__init__.py index 49aa5a1..95a49b5 100644 --- a/pycodeloop/providers/__init__.py +++ b/pycodeloop/providers/__init__.py @@ -63,6 +63,12 @@ def _from_json_path(path: str, kwargs: dict) -> GenericProvider: if kwargs.get("api_key"): provider.api_key = kwargs["api_key"] + if kwargs.get("inference_params"): + provider.inference_params = { + **provider.inference_params, + **kwargs["inference_params"], + } + return provider diff --git a/pycodeloop/providers/generic.py b/pycodeloop/providers/generic.py index d2b3531..aeada17 100644 --- a/pycodeloop/providers/generic.py +++ b/pycodeloop/providers/generic.py @@ -122,6 +122,7 @@ class _ConnectionSnapshot: response_parser: ResponseParser supports_openai_sse: bool include_usage_in_stream: bool + inference_params: dict class GenericProvider(Provider): @@ -198,6 +199,7 @@ def __init__( context_window: int | None = None, supports_openai_sse: bool = True, include_usage_in_stream: bool = True, + inference_params: dict | None = None, **kwargs, ) -> None: super().__init__(model=model, api_key=api_key, **kwargs) @@ -214,6 +216,7 @@ def __init__( self.context_window = context_window self._supports_openai_sse = supports_openai_sse self._include_usage_in_stream = include_usage_in_stream + self.inference_params = inference_params or {} self._config_path: Path | None = None self._lock = threading.Lock() @@ -249,6 +252,16 @@ def _build_from_json(cls, path: str | Path) -> GenericProvider: if "request" in data: request_builder = request_builder_from_config(data["request"]) + # Static `request.params` are already baked into the body by + # `request_builder_from_config` when a custom builder is in play — + # only fall back to `inference_params` (used by `_default_request`) + # when there's no custom builder to double-apply them. + request_params = ( + {} + if request_builder + else (data.get("request") or {}).get("params") or {} + ) + return cls( url=data["url"], model=data.get("model", ""), @@ -262,6 +275,7 @@ def _build_from_json(cls, path: str | Path) -> GenericProvider: context_window=data.get("context_window"), supports_openai_sse=response_shape != "anthropic", include_usage_in_stream=data.get("include_usage_in_stream", True), + inference_params=request_params or None, ) def reload(self) -> None: @@ -286,6 +300,7 @@ def reload(self) -> None: self.context_window = fresh.context_window self._supports_openai_sse = fresh._supports_openai_sse self._include_usage_in_stream = fresh._include_usage_in_stream + self.inference_params = fresh.inference_params @staticmethod def _default_request( @@ -314,6 +329,7 @@ def _snapshot_locked(self) -> _ConnectionSnapshot: response_parser=self.response_parser, supports_openai_sse=self._supports_openai_sse, include_usage_in_stream=self._include_usage_in_stream, + inference_params=dict(self.inference_params), ) def _headers(self, config: _ConnectionSnapshot) -> dict[str, str]: @@ -357,6 +373,8 @@ def complete( body = config.request_builder( system_prompt, messages, tools, config.model ) + if config.inference_params: + body = {**body, **config.inference_params} known_tools = {tool["name"] for tool in tools} if on_delta is not None and config.supports_openai_sse: diff --git a/tests/providers/test_generic.py b/tests/providers/test_generic.py index 4fbd9dc..7f3f189 100644 --- a/tests/providers/test_generic.py +++ b/tests/providers/test_generic.py @@ -34,6 +34,76 @@ def _write_config(self, config: dict) -> Path: class TestLoadProviderFromJson(GenericProviderTestCase): + def test_inference_params_are_merged_into_the_request_body(self): + provider = GenericProvider( + url="http://fake/v1/chat/completions", + model="my-model", + inference_params={ + "temperature": 0, + "max_tokens": 500, + }, + ) + + captured_requests = [] + + def fake_urlopen(request, timeout=None): + captured_requests.append(json.loads(request.data)) + payload = json.dumps( + { + "choices": [ + {"message": {"content": "ok"}, "finish_reason": "stop"} + ], + "usage": {}, + } + ).encode() + return _FakeResponse(payload) + + with mock.patch( + "pycodeloop.providers.generic.urllib.request.urlopen", + side_effect=fake_urlopen, + ): + provider.complete("sys", [], []) + + self.assertEqual(captured_requests[0]["temperature"], 0) + self.assertEqual(captured_requests[0]["max_tokens"], 500) + + def test_inference_params_apply_on_top_of_a_custom_request_builder(self): + """Config-declared `request.params` are baked into the body once by + the custom builder — `inference_params` (CLI overrides forwarded via + `get_provider`) must still reach the wire without doubling them up.""" + path = self._write_config( + { + "url": "http://fake/v1/chat/completions", + "model": "my-model", + "request": {"params": {"temperature": 0.7}}, + } + ) + provider = get_provider( + str(path), inference_params={"temperature": 0.1} + ) + + captured_requests = [] + + def fake_urlopen(request, timeout=None): + captured_requests.append(json.loads(request.data)) + payload = json.dumps( + { + "choices": [ + {"message": {"content": "ok"}, "finish_reason": "stop"} + ], + "usage": {}, + } + ).encode() + return _FakeResponse(payload) + + with mock.patch( + "pycodeloop.providers.generic.urllib.request.urlopen", + side_effect=fake_urlopen, + ): + provider.complete("sys", [], []) + + self.assertEqual(captured_requests[0]["temperature"], 0.1) + def test_builds_generic_provider_from_config(self): path = self._write_config( {