Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
477 changes: 477 additions & 0 deletions docs/rfcs/draft/RFC-0059-image-attachment-normalization.md

Large diffs are not rendered by default.

59 changes: 56 additions & 3 deletions src/wolfharness/functional/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,26 @@
if TYPE_CHECKING:
from wolfharness.agents.native_agent import AgentKwargs
from wolfharness.common_types import PromptCompatible
from wolfharness.images.normalizer import ImageNormalizer


def _make_image_normalizer(
attachment_image: Any | None,
) -> ImageNormalizer | None:
"""Build an ``ImageNormalizer`` from an explicit config (RFC-0059)."""
if attachment_image is None:
return None
from wolfharness.images.normalizer import ImageNormalizer

return ImageNormalizer(attachment_image)


def _normalize_image_url(url: str, normalizer: ImageNormalizer | None) -> str:
"""Normalize a data-URI image URL if a normalizer is available."""
if normalizer is None:
return url
normalized, _mime = normalizer.normalize(url, "image/*")
return normalized


@overload
Expand All @@ -21,6 +41,7 @@ async def run_agent[TResult](
image_url: str | None = None,
*,
output_type: type[TResult],
attachment_image: Any | None = None,
**kwargs: Unpack[AgentKwargs],
) -> TResult: ...

Expand All @@ -29,6 +50,9 @@ async def run_agent[TResult](
async def run_agent(
prompt: PromptCompatible,
image_url: str | None = None,
*,
output_type: None = None,
attachment_image: Any | None = None,
**kwargs: Unpack[AgentKwargs],
) -> str: ...

Expand All @@ -38,15 +62,27 @@ async def run_agent(
image_url: str | None = None,
*,
output_type: type[Any] | None = None,
attachment_image: Any | None = None,
**kwargs: Unpack[AgentKwargs],
) -> Any:
"""Run prompt through agent and return result."""
"""Run prompt through agent and return result.

Args:
prompt: The user prompt.
image_url: Optional image URL or ``data:`` URI. ``data:`` URIs are
normalized when they exceed configured limits (RFC-0059).
output_type: Optional structured output type.
attachment_image: Optional ``AttachmentImageConfig`` for image
normalization. When omitted, defaults apply.
**kwargs: Additional agent constructor kwargs.
"""
async with Agent[Any, str](**kwargs) as agent:
# Convert to structured output agent if output_type specified
final = agent.to_structured(output_type) if output_type is not None else agent

if image_url:
image = ImageUrl(url=image_url)
normalized = _normalize_image_url(image_url, _make_image_normalizer(attachment_image))
image = ImageUrl(url=normalized)
result = await final.run(prompt, image)
else:
result = await final.run(prompt)
Expand All @@ -59,6 +95,7 @@ def run_agent_sync[TResult](
image_url: str | None = None,
*,
output_type: type[TResult],
attachment_image: Any | None = None,
**kwargs: Unpack[AgentKwargs],
) -> TResult: ...

Expand All @@ -67,6 +104,8 @@ def run_agent_sync[TResult](
def run_agent_sync(
prompt: PromptCompatible,
image_url: str | None = None,
*,
attachment_image: Any | None = None,
**kwargs: Unpack[AgentKwargs],
) -> str: ...

Expand All @@ -76,11 +115,25 @@ def run_agent_sync(
image_url: str | None = None,
*,
output_type: type[Any] | None = None,
attachment_image: Any | None = None,
**kwargs: Unpack[AgentKwargs],
) -> Any:
"""Sync wrapper for run_agent."""

async def _run() -> Any:
return await run_agent(prompt, image_url, output_type=output_type, **kwargs) # type: ignore[arg-type]
if output_type is None:
return await run_agent(
prompt,
image_url,
attachment_image=attachment_image,
**kwargs,
)
return await run_agent(
prompt,
image_url,
output_type=output_type,
attachment_image=attachment_image,
**kwargs,
)

return run_sync(_run())
7 changes: 7 additions & 0 deletions src/wolfharness/images/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
"""Image attachment normalization (RFC-0059)."""

from __future__ import annotations

from wolfharness.images.normalizer import ImageNormalizer, ImageSizeError

__all__ = ["ImageNormalizer", "ImageSizeError"]
211 changes: 211 additions & 0 deletions src/wolfharness/images/normalizer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
"""Image attachment normalization service (RFC-0059).

Normalizes oversized image attachments on the protocol user-upload path
before they are forwarded to the model. Reuses the existing
``resize_image_if_needed()`` from the fsspec toolset so behavior stays
consistent with the tool-read path.

Failure semantics:

- ``auto_resize: true`` (default): images exceeding dimension or byte
limits are resized and re-encoded. If normalization itself fails
(e.g. Pillow unavailable), the original is passed through unchanged —
the session is never interrupted.
- ``auto_resize: false``: the byte budget is still enforced and an
over-limit image raises :class:`ImageSizeError`.
- Only ``data:`` base64 URIs are processed. Remote ``http(s)://`` and
``file://`` URLs are left untouched (avoids SSRF / filesystem access).
"""

from __future__ import annotations

import base64
import binascii
import io
import logging
from typing import Any

from wolfharness_config.attachment import AttachmentImageConfig


logger = logging.getLogger(__name__)


class ImageSizeError(Exception):
"""Raised when an image attachment cannot be brought within limits.

Used for the ``auto_resize: false`` path: an over-limit user attachment
fails with this error instead of silently passing an oversized image.
"""


class ImageNormalizer:
"""Normalize oversized image attachments.

Args:
config: Image attachment normalization configuration. When omitted,
falls back to ``AttachmentImageConfig()`` defaults.
"""

def __init__(self, config: AttachmentImageConfig | None = None) -> None:
self._config = config if config is not None else AttachmentImageConfig()

def normalize(self, url: str, mime: str) -> tuple[str, str]:
"""Normalize an image attachment URL.

Args:
url: Image URL. Only ``data:`` base64 URIs are processed.
mime: MIME type of the image (e.g. ``image/png``).

Returns:
Tuple of ``(possibly_normalized_url, mime)``. Passes the inputs
through unchanged for non-``data:`` URLs and for images already
within limits.

Raises:
ImageSizeError: When ``auto_resize`` is disabled and the image
exceeds configured limits.
"""
if not url.startswith("data:"):
return url, mime

payload = _data_uri_payload(url)
if payload is None:
return url, mime

try:
data = base64.b64decode(payload)
except (binascii.Error, ValueError):
logger.warning("Invalid base64 in image data URI; passing through unchanged")
return url, mime

normalized_data, new_mime = self.normalize_bytes(data, mime)
if normalized_data is data:
return url, mime
return _make_data_uri(new_mime, _b64encode(normalized_data)), new_mime

def normalize_bytes(self, data: bytes, mime: str) -> tuple[bytes, str]:
"""Normalize raw image bytes.

Args:
data: Raw image bytes.
mime: MIME type of the image (e.g. ``image/png``).

Returns:
Tuple of ``(possibly_normalized_bytes, mime)``. Returns the
original input unchanged when already within limits or not
normalizable.

Raises:
ImageSizeError: When ``auto_resize`` is disabled and the image
exceeds configured limits.
"""
target_bytes = self._config.max_base64_bytes * 3 // 4
max_size = min(self._config.max_width, self._config.max_height)

if len(data) <= target_bytes and not self._config.auto_resize:
return data, mime
if len(data) <= target_bytes and _fits_dimensions(data, max_size):
return data, mime

if not self._config.auto_resize:
return self._normalize_disabled(data, mime, max_size, target_bytes)

return self._normalize_enabled(data, mime, max_size, target_bytes)

def _normalize_enabled(
self,
data: bytes,
mime: str,
max_size: int,
target_bytes: int,
) -> tuple[bytes, str]:
"""Normalize an image with ``auto_resize: true``."""
from wolfharness_toolsets.fsspec_toolset.image_utils import (
resize_image_if_needed,
)

try:
resized, new_mime, note = resize_image_if_needed(
data,
mime,
max_size=max_size,
max_bytes=target_bytes,
)
except Exception:
logger.warning("Image normalization failed; passing through unchanged", exc_info=True)
return data, mime

if note is None:
return data, mime

if len(resized) > target_bytes * 4 // 3:
logger.warning(
"Re-encoded image still exceeds max_base64_bytes; passing original through"
)
return data, mime

return resized, new_mime

def _normalize_disabled(
self,
data: bytes,
mime: str,
max_size: int,
target_bytes: int,
) -> tuple[bytes, str]:
"""Raise :class:`ImageSizeError` for over-limit images."""
try:
with _open_image(data) as img:
width, height = img.size
except Exception: # noqa: BLE001
logger.warning("Image decode failed with auto_resize disabled; passing through")
return data, mime

if width > max_size or height > max_size or len(data) > target_bytes:
raise ImageSizeError(
f"Image attachment {width}x{height} exceeds configured limits "
f"({max_size}x{max_size} px, {target_bytes} bytes) and auto_resize is disabled"
)
return data, mime

@property
def config(self) -> AttachmentImageConfig:
"""The underlying normalization configuration."""
return self._config


def _open_image(data: bytes) -> Any:
"""Open an image from bytes as a context manager."""
from PIL import Image

return Image.open(io.BytesIO(data))


def _fits_dimensions(data: bytes, max_size: int) -> bool:
"""Return whether an image's dimensions are within ``max_size``."""
try:
with _open_image(data) as img:
width, height = img.size
except Exception: # noqa: BLE001
return True
return int(width) <= max_size and int(height) <= max_size


def _data_uri_payload(url: str) -> str | None:
"""Extract the base64 payload from a data URI, or None if not base64."""
marker = ";base64,"
index = url.find(marker)
if index < 0:
return None
return url[index + len(marker) :]


def _make_data_uri(mime: str, encoded: str) -> str:
"""Build a ``data:`` URI from a MIME type and base64 payload."""
return f"data:{mime};base64,{encoded}"


def _b64encode(data: bytes) -> str:
"""Base64-encode bytes to ASCII (no newlines)."""
return base64.b64encode(data).decode("ascii")
18 changes: 18 additions & 0 deletions src/wolfharness/models/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from wolfharness.models.agents import NativeAgentConfig
from wolfharness.models.file_agents import FileAgentConfig
from wolfharness.models.model_configs import AnyModelConfig, StringModelConfig
from wolfharness_config.attachment import AttachmentImageConfig
from wolfharness_config.commands import CommandConfig, StaticCommandConfig
from wolfharness_config.compaction import CompactionConfig
from wolfharness_config.context import ConfigContextManager
Expand Down Expand Up @@ -372,6 +373,23 @@ class AgentsManifest(Schema):
```
"""

attachment: AttachmentImageConfig = Field(default_factory=AttachmentImageConfig)
"""Image attachment normalization configuration (RFC-0059).

Controls automatic resizing/re-encoding of oversized image attachments
on the protocol user-upload path. Defaults mirror opencode's limits.

Example:
```yaml
attachment:
image:
auto_resize: true
max_width: 2000
max_height: 2000
max_base64_bytes: 5242880
```
"""

session_pool: SessionPoolConfig = Field(default_factory=SessionPoolConfig)
"""Session pool configuration for session lifecycle management.

Expand Down
2 changes: 2 additions & 0 deletions src/wolfharness_config/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from wolfharness_config.wolfharness_tools import AgentpoolToolConfig
from wolfharness_config.builtin_tools import BuiltinToolConfig

from wolfharness_config.attachment import AttachmentImageConfig
from wolfharness_config.capabilities import CapabilityConfig
from wolfharness_config.forward_targets import ForwardingTarget
from wolfharness_config.session import SessionQuery
Expand Down Expand Up @@ -85,6 +86,7 @@
"DEFAULT_SKILLS_PATHS",
"ACPConfig",
"AnyToolConfig",
"AttachmentImageConfig",
"BaseEventHandlerConfig",
"BaseHookConfig",
"BaseMCPServerConfig",
Expand Down
Loading
Loading