feat(*): send user attachments to the model, gated on vision support

raven/agent/context/builder.py

@@ -1,7 +1,5 @@
 """Context builder for assembling agent prompts."""
 
-import base64
-import mimetypes
 import platform
 import time
 from datetime import datetime
@@ -12,7 +10,7 @@
 from raven.memory_engine.skill_forge import LocalSkillCatalog
 from raven.memory_engine.skill_local.types import SkillMeta
 from raven.security.trust import wrap_untrusted, wrap_untrusted_blocks
-from raven.utils.helpers import build_assistant_message, detect_image_mime, image_block
+from raven.utils.helpers import build_assistant_message
 
 if TYPE_CHECKING:
     from raven.providers.base import LLMProvider
@@ -280,35 +278,26 @@ def build_messages(
             {"role": "user", "content": merged},
         ]
 
-    def _build_user_content(self, text: str, media: list[str] | None) -> str | list[dict[str, Any]]:
-        """Build user message content with optional base64-encoded images.
-
-        Every image's path is named in the text as well: the base64 survives only
-        this turn (the session stores a placeholder), so the path is what lets a
-        later turn re-read the picture instead of only learning one existed.
+    def _build_user_content(
+        self,
+        text: str,
+        media: list[str] | None,
+        *,
+        can_see_images: bool = True,
+        describe_tool: str | None = None,
+    ) -> str | list[dict[str, Any]]:
+        """Build user message content with optional attachments.
+
+        Delegates to the one implementation rather than keeping a second: this
+        builder only feeds MemoryConsolidator's token estimation today, so a
+        divergence here would be invisible until someone routed a real turn
+        through it, and by then the two would have drifted. The vision-aware
+        arguments are carried for that day rather than used now -- the estimator
+        passes no media at all, so nothing reaches the attachment path yet.
         """
-        if not media:
-            return text
-
-        images = []
-        notes = []
-        for path in media:
-            p = Path(path)
-            if not p.is_file():
-                continue
-            raw = p.read_bytes()
-            # Detect real MIME type from magic bytes; fallback to filename guess
-            mime = detect_image_mime(raw) or mimetypes.guess_type(path)[0]
-            if not mime or not mime.startswith("image/"):
-                continue
-            b64 = base64.b64encode(raw).decode()
-            images.append(image_block(f"data:{mime};base64,{b64}"))
-            notes.append(f"[Image: {p.name} (path: {p}) — re-read it with read_file if you need another look]")
-
-        if not images:
-            return text
-        body = (f"{text}\n\n" if text else "") + "\n".join(notes)
-        return images + [{"type": "text", "text": body}]
+        from raven.context_engine.segments import render
+
+        return render.build_user_content(text, media, can_see_images=can_see_images, describe_tool=describe_tool)
 
     def add_tool_result(
         self,

raven/agent/loop/main.py

@@ -44,7 +44,7 @@
 from raven.memory_engine.base import TokenBudget
 from raven.memory_engine.consolidate.consolidator import MemoryConsolidator, MemoryStore
 from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest
-from raven.providers.capabilities import image_placeholder_text, supports_image_tool_result
+from raven.providers.capabilities import image_placeholder_text, supports_image_tool_result, vision_verdict
 from raven.sandbox import SandboxConfig, SandboxExecutor, SandboxInitError, build_executor
 from raven.session.manager import Session, SessionManager
 from raven.spine.turn import Origin
@@ -373,6 +373,7 @@ def __init__(
         # takes a per-call model (strategies rewrite it, and the model chain
         # falls back), so one model's verdict must not answer for another's.
         self._image_tool_result_ok: dict[str, bool] = {}
+        self._vision_ok: dict[str, bool] = {}
         self.max_iterations = max_iterations
         # Empty-response recovery budgets. None → enabled defaults.
         self._recovery_limits = empty_recovery if empty_recovery is not None else RecoveryLimits()
@@ -799,6 +800,94 @@ def _supports_image_tool_result(self, model: str | None = None) -> bool:
             )
         return self._image_tool_result_ok[key]
 
+    # The tool that can read an attachment for a model that cannot see it.
+    # Contributed by the EverOS plugin, so absent on a default install.
+    _DESCRIBE_TOOL = "understand_media"
+
+    def _describe_tool_name(self) -> str | None:
+        """The description tool's name if it is registered, else ``None``.
+
+        Checked rather than assumed: the note that replaces a picture points at
+        this tool, and pointing at one the model was never given is an
+        instruction it cannot follow.
+        """
+        return self._DESCRIBE_TOOL if self.tools.get(self._DESCRIBE_TOOL) else None
+
+    def _route_result_images(
+        self,
+        model_text: str,
+        blocks: list[dict[str, Any]] | None,
+        model: str,
+    ) -> tuple[str, list[dict[str, Any]] | None, list[dict[str, Any]] | None]:
+        """Decide how a tool result's pictures reach ``model``.
+
+        Returns the text the tool result carries, the blocks to put *in* it, and
+        the blocks to attach to a following user message. Exactly one of the last
+        two is ever populated.
+
+        Three outcomes, and the wording differs because the model's next move
+        differs. No vision at all: nothing follows, so the note must not promise
+        an attachment, and it names the description tool when one is registered.
+        Vision and a transport that carries images in a ``role="tool"`` message:
+        the blocks ride along untouched. Vision but a transport that cannot (every
+        OpenAI-style Chat Completions endpoint -- image is excluded from the tool
+        role at the schema level): the result carries text and the picture follows
+        in a user message, the shape OpenClaw uses.
+
+        A method rather than a branch inside the loop so it can be tested at all:
+        the loop reaches this point only through a live provider and a real tool
+        call, and the wrong choice here is silent -- the model answers about a
+        picture it never received.
+        """
+        if not blocks:
+            return model_text, blocks, None
+        if not self._supports_vision(model):
+            return image_placeholder_text(blocks, blind=True, describe_tool=self._describe_tool_name()), None, None
+        if self._supports_image_tool_result(model):
+            return model_text, blocks, None
+        attach = [b for b in blocks if b.get("type") == "image_url"]
+        return image_placeholder_text(blocks), None, attach
+
+    def _supports_vision(self, model: str | None = None) -> bool:
+        """Cached per model: whether this model can see a picture at all.
+
+        Asked once per turn and once per tool result that returns an image, and
+        the lookup joins the model string against the gateway catalogue -- same
+        reason the sibling probe above is cached.
+
+        Only a real verdict is cached. ``vision_verdict`` returns ``None`` while
+        the catalog has no answer -- a cold install, before the background warm
+        lands -- and that is optimism rather than knowledge: caching it would
+        freeze the guess for the life of this loop, which is the life of the
+        process, and the warm would then fill a table nothing re-reads. An
+        unknown model is re-asked each turn, which costs a dict lookup.
+
+        The verdict is the routed primary's. A fallback further down the chain is
+        sent the same message list, so a vision-capable primary with a blind
+        fallback hands the blind endpoint an image block it will refuse; that
+        refusal classifies as fatal and stops the chain rather than answering
+        blind. Pre-existing in the sibling probe too, and it needs the fallback
+        chain to be assembled per candidate to fix properly.
+        """
+        key = model or self.model
+        cached = self._vision_ok.get(key)
+        if cached is not None:
+            return cached
+
+        spec = None
+        try:
+            from raven.providers.registry import find_by_model
+
+            spec = find_by_model(key)
+        except Exception:
+            pass
+        verdict = vision_verdict(key, spec, self.provider)
+        logger.debug("vision support for {}: {}", key, verdict)
+        if verdict is None:
+            return True
+        self._vision_ok[key] = verdict
+        return verdict
+
     # ── Context engine helpers ──────────────────────────────────────────
 
     def _context_messages_for_session(self, session: Session) -> list[dict[str, Any]]:
@@ -865,8 +954,15 @@ async def _assemble_context_messages(
         channel: str | None = None,
         chat_id: str | None = None,
         selected_skills: list[Any] | None = None,
+        model: str | None = None,
     ) -> list[dict[str, Any]]:
-        """Ask the active context engine for the main-agent message window."""
+        """Ask the active context engine for the main-agent message window.
+
+        ``model`` is the id the request will actually reach (the router's pick,
+        when there is one). It decides whether an attachment is inlined as a
+        picture, so defaulting it to ``self.model`` would let the configured
+        model answer for a routed one.
+        """
         from raven.context_engine import TurnContext  # deferred — see module note
 
         # Phase A / Phase C tidy: reset the metadata stash BEFORE calling
@@ -883,6 +979,8 @@ async def _assemble_context_messages(
             turn=TurnContext(
                 current_message=current_message,
                 media=media,
+                can_see_images=self._supports_vision(model),
+                describe_tool=self._describe_tool_name(),
                 channel=channel,
                 chat_id=chat_id,
                 selected_skills=selected_skills,
@@ -1863,19 +1961,9 @@ async def _run_agent_loop(
                                 "truncated": len(display_src) > 200,
                             },
                         )
-                    blocks = getattr(result, "blocks", None)
-                    attach_blocks: list[dict[str, Any]] | None = None
-                    if blocks:
-                        if self._supports_image_tool_result(call_model or effective_model):
-                            pass  # blocks ride in the tool result itself
-                        else:
-                            # This transport cannot put an image in a tool result,
-                            # so the tool result carries text naming the image and
-                            # the picture follows in a user message. Same shape
-                            # OpenClaw uses against Chat Completions endpoints.
-                            model_text = image_placeholder_text(blocks)
-                            attach_blocks = [b for b in blocks if b.get("type") == "image_url"]
-                            blocks = None
+                    model_text, blocks, attach_blocks = self._route_result_images(
+                        model_text, getattr(result, "blocks", None), call_model or effective_model
+                    )
                     if blocks:
                         messages = self.context.add_tool_result(
                             messages, tool_call.id, tool_call.name, model_text, blocks
@@ -2380,17 +2468,12 @@ async def _extract():
             content,
             context_messages,
         )
-        initial_messages = await self._assemble_context_messages(
-            session=session,
-            session_key=key,
-            current_message=content,
-            media=media_paths if media_paths else None,
-            channel=channel,
-            chat_id=chat_id,
-            selected_skills=selected_skills or None,
-        )
-
         # ── Model routing (EcoClaw-style) ────────────────────────────────────
+        # Ahead of assembly, not after it: assembly decides whether an
+        # attachment is inlined as a picture or described in text, and that
+        # question is about the model the request will actually reach. Routing
+        # needs only ``content``, so asking first costs nothing and stops one
+        # model's verdict from shaping a message another model receives.
         routed_model: str | None = None
         fallback_models: list[str] = []
         if self.router is not None:
@@ -2400,6 +2483,17 @@ async def _extract():
             if fallback_models:
                 logger.info("Router fallback chain: {}", fallback_models)
 
+        initial_messages = await self._assemble_context_messages(
+            session=session,
+            session_key=key,
+            current_message=content,
+            media=media_paths if media_paths else None,
+            channel=channel,
+            chat_id=chat_id,
+            selected_skills=selected_skills or None,
+            model=routed_model,
+        )
+
         extraction_sid = None  # Phase B-1: embedded extraction removed; always None now.
         turn_start_idx = len(initial_messages) - 1
         final_content, _, all_msgs, outcome = await self._run_agent_loop(

raven/agent/tools/filesystem.py

@@ -54,7 +54,7 @@ def description(self) -> str:
         return (
             "Read the contents of a file. Text files return numbered lines — use offset and limit to "
             "paginate through large ones. Image files (PNG, JPEG, GIF, WebP, and other common formats) "
-            "return the picture itself when the active model can see images, downscaled if needed."
+            "return the picture itself, downscaled if needed."
         )
 
     @property

raven/context_engine/assembler.py

@@ -76,6 +76,8 @@ async def assemble(
             session_key=session_key,
             current_message=turn.current_message,
             media=turn.media,
+            can_see_images=turn.can_see_images,
+            describe_tool=turn.describe_tool,
             channel=turn.channel,
             chat_id=turn.chat_id,
             session_messages=session_messages,
@@ -144,7 +146,12 @@ async def after_turn(
     def _build_user(self, ctx: AssemblyContext) -> dict[str, Any]:
         """The single structural user message: runtime context + content."""
         runtime_ctx = render.build_runtime_context(self._now_fn, ctx.channel, ctx.chat_id)
-        user_content = render.build_user_content(ctx.current_message, ctx.media)
+        user_content = render.build_user_content(
+            ctx.current_message,
+            ctx.media,
+            can_see_images=ctx.can_see_images,
+            describe_tool=ctx.describe_tool,
+        )
         if isinstance(user_content, str):
             merged: Any = f"{runtime_ctx}\n\n{user_content}"
         else:

raven/context_engine/base.py

@@ -79,6 +79,8 @@ class AssemblyContext:
     session_messages: list[dict[str, Any]]
     budget: TokenBudget
     prefix: AssembledPrefix | None = None
+    can_see_images: bool = True
+    describe_tool: str | None = None
 
 
 @dataclass

raven/context_engine/curator.py

@@ -40,6 +40,15 @@ class TurnContext:
     channel: str | None = None
     chat_id: str | None = None
     selected_skills: list[Any] | None = None
+    # Whether this turn's model can see a picture. Decided by the loop (it owns
+    # the provider and the model id) and carried here because the message is
+    # built down in render, which knows neither. Defaults True so a caller that
+    # does not set it keeps the old inline-everything behavior.
+    can_see_images: bool = True
+    # Name of a registered tool that can read an attachment the model cannot,
+    # or None when none is (it comes from an optional plugin). Naming a tool the
+    # model does not have reads as an instruction it cannot follow.
+    describe_tool: str | None = None
 
 
 @dataclass

raven/context_engine/segments/curator.py

@@ -95,6 +95,8 @@ async def build(self, ctx: AssemblyContext) -> Segment | None:
         turn = TurnContext(
             current_message=ctx.current_message,
             media=ctx.media,
+            can_see_images=ctx.can_see_images,
+            describe_tool=ctx.describe_tool,
             channel=ctx.channel,
             chat_id=ctx.chat_id,
         )