From dbededa8dfd0096ac507df7c6c0c31aa1ec1b457 Mon Sep 17 00:00:00 2001 From: Anai Guo Date: Fri, 17 Jul 2026 03:11:21 -0700 Subject: [PATCH] fix(llama): release a vision chat handler's mtmd context on close (#2342) `Llama.close()` tore down the model and its context but left the chat handler's `_exit_stack` untouched, so the mtmd/clip context it built from that model was never freed. Handlers routinely outlive the `Llama` that initialized them -- callers construct one handler and reuse it across loads -- and `_init_mtmd_context()` returns early while `mtmd_ctx` is set. After the first `close()` the handler therefore kept a context bound to an already-freed model and handed it back on the next load, where it surfaces as a null `mtmd_ctx` on the C++ side. Register the handler's exit stack on the Llama's `_stack`. It unwinds LIFO, so the mtmd context is released ahead of the model teardown registered earlier in `__init__`, and `mtmd_free` resets `mtmd_ctx` to `None` so the next load re-initializes cleanly. Duck-typed on `_exit_stack`, which both `Llava15ChatHandler` and `MTMDChatHandler` (and every subclass) expose. --- llama_cpp/llama.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 75c74b41fc..4d55da97d4 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -456,6 +456,16 @@ def free_lora_adapter(): self.chat_format = chat_format self.chat_handler = chat_handler + # A vision chat handler builds its mtmd/clip context from this model, so + # that context must be released before the model itself is freed. The + # handler object can outlive the Llama (callers commonly reuse a single + # handler across loads), and it skips re-initialization while mtmd_ctx + # is set -- leaving a context bound to an already-freed model behind. + # `_stack` unwinds LIFO, so this runs before the model teardown that was + # registered earlier in __init__. + handler_stack = getattr(chat_handler, "_exit_stack", None) + if handler_stack is not None: + self._stack.callback(handler_stack.close) self._chat_handlers: Dict[ str, llama_chat_format.LlamaChatCompletionHandler ] = {}