diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py index 61e237b8bd..41026843fb 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py @@ -7,6 +7,7 @@ from astrbot.core.platform.astr_message_event import AstrMessageEvent from astrbot.core.provider.entities import ProviderRequest from astrbot.core.utils.media_utils import ( + MODEL_IMAGE_MAX_BYTES, MediaResolver, is_recoverable_image_error, prepare_model_image, @@ -25,6 +26,7 @@ async def prepare_request_images( prepared: dict[str, str | None], quote_image_ref: str | None = None, montage_max_size: int | None = None, + max_bytes: int | None = MODEL_IMAGE_MAX_BYTES, ) -> None: """Replace current images on a working request and track their owned files. @@ -38,6 +40,7 @@ async def prepare_request_images( prepared: Per-request mapping reused after the request hook. quote_image_ref: Optional input for the dedicated quote caption branch. montage_max_size: Optional montage-specific limit; defaults to ``max_size``. + max_bytes: Optional byte budget per still; ``None`` keeps them byte-exact. """ req.image_urls = normalize_and_dedupe_strings(req.image_urls) refs = list(req.image_urls) @@ -59,6 +62,7 @@ async def prepare_request_images( output_dir=output_dir, quality=quality, montage_max_size=montage_max_size, + max_bytes=max_bytes, ) if path: event.track_temporary_local_file(path) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index 322506e3a8..d115660db2 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -45,6 +45,7 @@ from astrbot.core.utils.astrbot_path import get_astrbot_temp_path from astrbot.core.utils.media_utils import ( IMAGE_COMPRESS_DEFAULT_QUALITY, + MODEL_IMAGE_MAX_BYTES, normalize_model_image_max_size, ) from astrbot.core.utils.metrics import Metric @@ -264,6 +265,9 @@ async def process( # configured cap, which bounds the 3x3 canvas. Oversized # passthrough images warn below. max_size = 1_000_000 + # CUA stills stay byte-exact to avoid JPEG color shifts, so the + # byte budget applies only elsewhere; that path warns instead. + max_bytes = None if cua_pixel_mode else MODEL_IMAGE_MAX_BYTES quality = ( options.get("quality") if isinstance(options, dict) else None ) @@ -294,6 +298,7 @@ async def process( prepared=prepared, quote_image_ref=quote_image_ref, montage_max_size=montage_max_size, + max_bytes=max_bytes, ) await _process_quote_message( event, @@ -361,6 +366,7 @@ async def process( output_dir=output_dir, prepared=prepared, montage_max_size=montage_max_size, + max_bytes=max_bytes, ) if cua_pixel_mode: oversized = [] diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index ff552293d5..559872fd0a 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -1014,13 +1014,28 @@ def _inspect_image(image_bytes: bytes) -> int: return frame_count -_IMAGE_CONVERT_CACHE_VERSION = "v9-icc" +_IMAGE_CONVERT_CACHE_VERSION = "v10-bytes" """Bump when conversion output semantics change (modes, transparency, sizing) so stale cache entries produced by older code are never served.""" MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES = 1024 * 1024 """PNG outputs larger than this are flattened onto white and re-encoded as JPEG.""" +MODEL_IMAGE_MAX_BYTES = 1024 * 1024 +"""Byte budget for one prepared still image. + +The longest-edge cap bounds pixels, not compressed size: a compliant 1280x1280 +noisy PNG already exceeds 6 MiB, which providers reject before the model ever +sees it. ``None`` disables the budget and keeps oversized stills byte-exact. +""" + +MODEL_IMAGE_SHRINK_ATTEMPTS = 5 +"""Extra encodes allowed while fitting a still into its byte budget. + +Embedded metadata survives every quality step and a large canvas survives every +budget, so a bounded number of retries is needed rather than one quality step. +""" + def normalize_model_image_max_size(value: object) -> int: """Normalize the model image longest-edge cap. @@ -1067,6 +1082,7 @@ def _encode_image_frame_bytes( image: PILImage.Image, max_size: int | None = None, quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY, + keep_metadata: bool = True, ) -> bytes: """Encode a display-oriented frame as JPEG, or PNG when it carries transparency. @@ -1074,6 +1090,9 @@ def _encode_image_frame_bytes( image: Opened source frame, which is never mutated. max_size: Optional longest-edge limit; smaller images are not enlarged. quality: JPEG output quality in the range 1-100. + keep_metadata: Whether to carry the source ICC profile into the output. + Callers fitting a byte budget drop it, because the profile survives + every quality step and can keep a payload over budget on its own. Returns: Encoded single-frame JPEG or PNG bytes. A PNG larger than @@ -1106,7 +1125,7 @@ def _encode_image_frame_bytes( # JPEG saving does not auto-embed the source ICC profile like PNG does; # attach it explicitly, but only when the color space survived intact # (conversions like CMYK -> RGB invalidate the source profile). - icc_profile = image.info.get("icc_profile") + icc_profile = image.info.get("icc_profile") if keep_metadata else None if image.mode not in {"RGB", "RGBA", "L", "LA", "P", "1", "I", "I;16"}: icc_profile = None save_kwargs = {"icc_profile": icc_profile} if icc_profile else {} @@ -1224,7 +1243,10 @@ def _publish_image_cache_atomic( def _convert_image_bytes_sync( - source_bytes: bytes, max_size: int, quality: int + source_bytes: bytes, + max_size: int, + quality: int, + max_bytes: int | None = MODEL_IMAGE_MAX_BYTES, ) -> bytes: """Normalize a validated still image with an optional derived cache. @@ -1232,27 +1254,46 @@ def _convert_image_bytes_sync( source_bytes: Encoded source bytes already checked by _inspect_image. max_size: Longest-edge limit in pixels. quality: JPEG output quality in the range 1-100. + max_bytes: Optional byte budget; ``None`` keeps oversized compliant + stills byte-exact instead of bounding their payload. Returns: - Single-frame JPEG or PNG bytes. An oriented JPEG or PNG within the size - limit is reused unchanged; anything else is re-encoded. + Single-frame JPEG or PNG bytes. An oriented JPEG or PNG inside both the + pixel and the byte limit is reused unchanged; anything else is + re-encoded and shrunk until it fits the byte budget, dropping embedded + metadata and finally the canvas when quality alone is not enough. """ with PILImage.open(io.BytesIO(source_bytes)) as image: if ( image.format in {"PNG", "JPEG"} and image.getexif().get(274, 1) == 1 and max(image.size) <= max_size + and (max_bytes is None or len(source_bytes) <= max_bytes) ): return source_bytes cache_key = _image_convert_cache_key( source_bytes, - f"{_IMAGE_CONVERT_CACHE_VERSION}|convert|s={max_size}|q={quality}", + f"{_IMAGE_CONVERT_CACHE_VERSION}|convert|s={max_size}" + f"|q={quality}|b={max_bytes}", ) output_path = _image_convert_cache_dir() / (cache_key + ".img") cached = _read_valid_cached_image_bytes(output_path) if cached is not None: return cached encoded = _encode_image_frame_bytes(image, max_size=max_size, quality=quality) + for _ in range(MODEL_IMAGE_SHRINK_ATTEMPTS): + if max_bytes is None or len(encoded) <= max_bytes: + break + # The pixel cap bounds geometry and the quality bounds artefacts, but + # neither bounds the payload alone: embedded metadata survives any + # quality, and a large canvas survives any budget. Drop the metadata + # with the first retry, then derate and finally shrink the canvas. + quality = max(1, quality * max_bytes // len(encoded)) + if quality <= 1: + max_size = max(max_size // 2, ANIMATED_MONTAGE_GRID) + encoded = _encode_image_frame_bytes( + image, max_size=max_size, quality=quality, keep_metadata=False + ) _publish_image_cache_atomic(output_path, encoded) return encoded @@ -1367,6 +1408,7 @@ async def prepare_model_image( output_dir: Path, quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY, montage_max_size: int | None = None, + max_bytes: int | None = MODEL_IMAGE_MAX_BYTES, ) -> str | None: """Prepare a single local model-ready image for the caller to own until consumption. @@ -1380,6 +1422,8 @@ async def prepare_model_image( but montages are never used for coordinates, so callers pass the configured limit here to keep the 3x3 canvas bounded. Defaults to ``max_size``. + max_bytes: Optional byte budget for stills. CUA sessions pass ``None`` + because their compliant stills must stay byte-exact. Returns: An existing JPEG or PNG path, or None for a recoverable input or write @@ -1399,7 +1443,7 @@ async def prepare_model_image( ) else: converted_bytes = await asyncio.to_thread( - _convert_image_bytes_sync, image_bytes, max_size, quality + _convert_image_bytes_sync, image_bytes, max_size, quality, max_bytes ) # Publish the working file synchronously after encoding, so cancellation # cannot leave an untracked background write alive after this call. diff --git a/docs/en/providers/image-formats.md b/docs/en/providers/image-formats.md index 0c9fb96eca..109d8604fb 100644 --- a/docs/en/providers/image-formats.md +++ b/docs/en/providers/image-formats.md @@ -4,7 +4,8 @@ ## Preparation rules -- Compliant still images are sent unchanged: already JPEG or PNG, correctly oriented, and within the size limit. +- Compliant still images are sent unchanged: already JPEG or PNG, correctly oriented, inside the pixel limit and inside the 1 MB byte budget. +- Compliant stills above the byte budget are re-encoded to bound the request payload: pixel dimensions are preserved first, oversized metadata is dropped, and the JPEG quality is derated; only when the lowest quality is still oversized is the canvas scaled down. The pixel limit bounds dimensions, not compressed size, so without a byte budget such a still can reach tens of megabytes and be rejected by the provider before the model sees it. - Other still images are orientation-corrected, resized and re-encoded as needed. Opaque images become JPEG, controlled by `image_compress_options.quality` (default 95), with high-bit-depth samples normalized to 8-bit. Images with transparency become PNG; a PNG larger than 1 MB is flattened onto a white background and re-encoded as JPEG to bound the payload size. - Animations are detected from their frames, including GIF, animated WebP and APNG. Up to nine evenly sampled frames, including the first and last, become one white 3×3 grid. Unused cells stay white; an APNG independent cover is excluded. - Stills and montages share `image_compress_options.max_size` (default 1280). Small images are not enlarged. @@ -12,7 +13,7 @@ > [!TIP] -> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Compliant images pass through byte-exact (no lossy re-encoding, avoiding JPEG color shifts); format conversion for other formats and animation montages still apply. Images above roughly 5 MB may exceed provider image upload limits and trigger a warning in the logs. +> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Compliant images pass through byte-exact (no lossy re-encoding, avoiding JPEG color shifts, and no byte budget); format conversion for other formats and animation montages still apply. Images above roughly 5 MB may exceed provider image upload limits and trigger a warning in the logs. The Agent receives readable local paths. Original image files and event components keep their original content, and attachment text continues to reference the source image. Providers only read/encode references and assemble their protocols. ## Errors and lifetime diff --git a/docs/zh/providers/image-formats.md b/docs/zh/providers/image-formats.md index 4fe3997318..75697cc21a 100644 --- a/docs/zh/providers/image-formats.md +++ b/docs/zh/providers/image-formats.md @@ -10,10 +10,11 @@ > [!TIP] -> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时，为避免像素坐标漂移，输入图片不按最大边长缩放，合规图片原样发送（不重新编码，避免 JPEG 变色）；格式转换与动图拼图等其余处理仍然生效。超过约 5 MB 的大图可能超出服务商的图片体积限制，此时日志会给出警告。 +> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时，为避免像素坐标漂移，输入图片不按最大边长缩放，合规图片原样发送（不重新编码，避免 JPEG 变色，也不受 1 MB 体积预算约束）；格式转换与动图拼图等其余处理仍然生效。超过约 5 MB 的大图可能超出服务商的图片体积限制，此时日志会给出警告。 ## 处理方式 -- **合规静图原样发送**：已是 JPEG/PNG、方向正常且未超过最大边长的图片不做任何改动。 +- **合规静图原样发送**：已是 JPEG/PNG、方向正常、未超过最大边长且不超过 1 MB 的图片不做任何改动。 +- **超出体积预算的静图**：其余条件合规但超过 1 MB 的静图会重新编码以控制请求体积：优先保持像素尺寸、丢弃过大的元数据并逐步降低 JPEG 质量；只有质量降到最低仍超标时才按比例缩小尺寸。最大边长限制的是像素而不是压缩后体积，不设体积预算时合规静图可以原样发送数十 MB，被服务商直接拒绝。 - **其余静图按需转换**：方向异常、尺寸超限或格式不通用的图片会修正方向、缩放后重新编码。无透明度的图片输出 JPEG（16 位等高位深图片会归一化为 8 位）；含透明度的图片输出 PNG，若 PNG 超过 1 MB 则扁平化为白底 JPEG，控制发送体积。 - **动图**：从 GIF、动画 WebP、APNG 等图片中均匀选取最多 9 帧，包含首尾动画帧，生成白底 3×3 拼图；不足 9 帧的格子留白。单帧 GIF 和静态 WebP 按静图处理。 - **无法读取或解码的图片**：跳过该图片，其他文字和正常图片继续处理。 diff --git a/tests/test_model_image_preparation.py b/tests/test_model_image_preparation.py index 24488b88de..84e949c4eb 100644 --- a/tests/test_model_image_preparation.py +++ b/tests/test_model_image_preparation.py @@ -2,6 +2,7 @@ import asyncio import errno +import os from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock @@ -51,6 +52,48 @@ async def test_compliant_stills_pass_through_byte_identical(tmp_path, fmt): assert source.read_bytes() == original +@pytest.mark.asyncio +async def test_compliant_still_over_byte_budget_is_reencoded(tmp_path): + source = tmp_path / "noisy.png" + Image.frombytes("RGBA", (1280, 1280), os.urandom(1280 * 1280 * 4)).save(source) + original = source.read_bytes() + assert len(original) > media.MODEL_IMAGE_MAX_BYTES + path = await media.prepare_model_image( + str(source), max_size=1280, output_dir=tmp_path + ) + assert path and Path(path).read_bytes() != original + assert Path(path).stat().st_size <= media.MODEL_IMAGE_MAX_BYTES + assert source.read_bytes() == original + + +@pytest.mark.asyncio +async def test_oversized_metadata_does_not_keep_still_over_budget(tmp_path): + source = tmp_path / "profiled.jpg" + Image.new("RGB", (200, 100), "red").save( + source, icc_profile=b"\x00" * (2 * 1024 * 1024) + ) + original = source.read_bytes() + assert len(original) > media.MODEL_IMAGE_MAX_BYTES + path = await media.prepare_model_image( + str(source), max_size=1280, output_dir=tmp_path + ) + assert path and Path(path).stat().st_size <= media.MODEL_IMAGE_MAX_BYTES + assert source.read_bytes() == original + + +@pytest.mark.asyncio +async def test_disabled_byte_budget_keeps_oversized_still_byte_exact(tmp_path): + source = tmp_path / "noisy.png" + Image.frombytes("RGBA", (1280, 1280), os.urandom(1280 * 1280 * 4)).save(source) + original = source.read_bytes() + assert len(original) > media.MODEL_IMAGE_MAX_BYTES + path = await media.prepare_model_image( + str(source), max_size=1280, output_dir=tmp_path, max_bytes=None + ) + assert path and Path(path).read_bytes() == original + assert source.read_bytes() == original + + @pytest.mark.asyncio async def test_rotated_jpeg_is_normalized_to_upright_jpeg(tmp_path): source = tmp_path / "rotated.jpg"