From 4b0d1d02291fd497753549f6a8a0fb6a70ed9429 Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Mon, 17 Aug 2026 15:06:45 +0800 Subject: [PATCH 1/3] docs(rfc): add RFC-0059 image attachment normalization design Proposes inserting the existing resize_image_if_needed() normalization (the read tool / fsspec toolset path, Pillow-based, default 2000px / 4.5MB) into the protocol user-upload conversion points: Python API run_agent, ACP attachments, and OpenCode server FilePartInput. Distinct failure semantics for user input vs tool-result over-limit; model failure behavior based on opencode's image.ts reference. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- ...RFC-0059-image-attachment-normalization.md | 477 ++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 docs/rfcs/draft/RFC-0059-image-attachment-normalization.md diff --git a/docs/rfcs/draft/RFC-0059-image-attachment-normalization.md b/docs/rfcs/draft/RFC-0059-image-attachment-normalization.md new file mode 100644 index 000000000..0cef7a5d3 --- /dev/null +++ b/docs/rfcs/draft/RFC-0059-image-attachment-normalization.md @@ -0,0 +1,477 @@ +--- +rfc_id: RFC-0059 +title: "Image Attachment Normalization: Resize and Re-encode Oversized Images Before Provider Requests" +status: DRAFT +author: pinjun.mo +reviewers: + - name: yuchen.liu + status: pending +created: 2026-08-17 +last_updated: 2026-08-17 +decision_date: +related_rfcs: [] +related_specs: [] +--- + +# RFC-0059: Image Attachment Normalization + +## Table of Contents + +- [Overview](#overview) +- [Background & Context](#background--context) +- [Problem Statement](#problem-statement) +- [Goals & Non-Goals](#goals--non-goals) +- [Evaluation Criteria](#evaluation-criteria) +- [Options Analysis](#options-analysis) +- [Recommendation](#recommendation) +- [Technical Design](#technical-design) +- [Security Considerations](#security-considerations) +- [Implementation Plan](#implementation-plan) +- [Open Questions](#open-questions) +- [Decision Record](#decision-record) +- [References](#references) + +--- + +## Overview + +AgentPool already provides **centralized image normalization** for the tool-read path: `resize_image_if_needed()` in `src/wolfharness_toolsets/fsspec_toolset/image_utils.py` (default 2000 px, 4.5 MB, Pillow) is wired into both the `read` tool and the fsspec toolset. However, **protocol user-upload paths — Python API `run_agent(image_url=...)`, ACP attachments, and OpenCode server `FilePartInput` — bypass it entirely**: they are converted to pydantic-ai `ImageUrl`/`BinaryContent` at `to_user_content()` with no size or byte-budget checks. Oversized field photos (commonly 4000×3000 px, 10–20 MB) can exceed provider input limits, causing context overflow errors and inflated token costs. + +This RFC proposes **reusing the existing centralized normalization** by inserting it at the `FilePart → pydantic-ai content` conversion point (alongside `to_user_content`), so all protocol entry points inherit the same constraint capability already used by the tool-read path. + +**Expected outcome**: every protocol entry point (Python API / ACP / OpenCode server) constrains image attachments within the same configurable limits as the tool path; oversized user input fails with a clear error; tool-result images keep their current omission behavior; resizer unavailability degrades to passing the original through (a property to be preserved; note that the existing tool path hard-depends on Pillow, so unavailability is not currently a live code path there). + +The design's failure semantics are modeled on opencode's reference implementation (`packages/opencode/src/image/image.ts`), but the normalization engine itself already exists in AgentPool and does not need to be recreated. + +--- + +## Background & Context + +### Current State + +Image capability in the AgentPool stack flows through three layers, none of which normalize images: + +1. **Python API entry**: `src/wolfharness/functional/run.py:21-50` — `run_agent(prompt, image_url=...)` maps the URL directly to pydantic-ai `ImageUrl(url=image_url)`. (Note: the agentpool CLI `run` command has no `--image` flag; this path is exposed only as a Python API.) +2. **Conversion point**: `src/wolfharness/utils/pydantic_ai_helpers.py:87-133` — `to_user_content()` maps `image/*` → `ImageUrl`, and `data:` URIs → `BinaryContent.from_data_uri()`. Pure MIME-to-content-type mapping; no size/byte logic. +3. **OpenCode server input**: `src/wolfharness_server/opencode_server/routes/message_routes.py:320` — `FilePartInput` branch calls `add_file_part(mime, url, ...)` with no image-specific handling. +4. **ACP server input**: `src/wolfharness_server/acp_server/converters.py:146-148,178-181` — `ImageContentBlock` / `BlobResourceContents` with `image/*` MIME decode base64 into `BinaryImage` with no size/byte checks. + +The `FilePart` model (`src/wolfharness_server/opencode_server/models/parts.py:188`, fields `mime`/`filename`/`url`/`source`) mirrors opencode's `SessionV1.FilePart` structurally, but carries no normalization logic. + +### Reference Implementation: opencode + +opencode implements centralized image normalization in `packages/opencode/src/image/image.ts`: + +- **Limits**: `max_width`/`max_height` 2000×2000 px; `max_base64_bytes` 5 MB (base64 payload); `auto_resize: true` by default. +- **Algorithm**: photon-wasm iterative 0.75× downscale loop (max 32 steps); for each candidate size, tries PNG then JPEG at quality ladder `[80, 85, 70, 55, 40]`; returns the first candidate under `max_base64_bytes`. +- **Failure semantics**: + - User input over-limit and not resizable → `SizeError`, prompt fails with a clear error. + - Tool-result over-limit → silently omitted, output annotated `[N images omitted: could not be resized below the image size limit.]`. + - Resizer unavailable (`ResizerUnavailableError`) → original image passed through unchanged. +- **Config**: `attachment.image.{auto_resize, max_width, max_height, max_base64_bytes}` (`packages/core/src/v1/config/attachment.ts`). +- **Model-modality gating**: `unsupportedParts()` replaces images with error text for models lacking image modality. + +### Related Work + +- `tests/test_multimodal_storage.py`, `tests/test_modality_filter.py` — existing modality handling is at the storage/filter layer, not attachment normalization. + +### Glossary + +| Term | Definition | +|------|------------| +| FilePart | OpenCode protocol file-content part: `mime`/`filename`/`url` (data URI or `file://` path) | +| Normalization | Resize + re-encode applied to images exceeding configured limits | +| base64 payload | Byte length of the base64 string in `data:;base64,...` (measured by opencode in UTF-8 bytes) | +| pydantic-ai content | Pydantic-AI multimodal content abstraction: `ImageUrl` / `BinaryContent` / `DocumentUrl` | + +--- + +## Problem Statement + +### The Problem + +AgentPool servers started via `serve-opencode` / `serve-acp` / `run_agent(image_url=...)` perform **no image preprocessing**: + +1. No dimension or byte-budget checks — images are forwarded at original size. +2. No resize or re-encode — no `Image.normalize` equivalent. +3. No over-limit failure semantics — neither rejection nor omission; the downstream provider is the only backstop. + +Images travel as base64 data URIs into `BinaryContent`/`ImageUrl` and are sent to the model unchanged. The only backstop is post-hoc error reporting from pydantic-ai/SDK/provider. + +### Evidence + +- `message_routes.py:320` `FilePartInput` branch does only `add_file_part`, no image branch. +- `pydantic_ai_helpers.py` performs MIME→content-type mapping only; no size/byte logic. +- The codebase contains modality tests (`test_multimodal_storage`, `test_modality_filter`) at the storage/filter layers, none covering attachment normalization. + +### Impact of Inaction + +- **Risk**: field photos (4000×3000 px, 10–20 MB originals common from mobile devices) base64-encode to ~15–27 MB, exceeding typical single-input limits of many models, triggering: + - context window overflow (4xx / input token limits), + - server-side memory spikes when decoding large images, + - opaque errors surfaced by SDK/provider rather than a clear user-visible message. +- **Cost**: request token usage scales with pixel count; images larger than the model's effective resolution waste inference budget. +- **Opportunity**: no scenario-specific trade-off between high-fidelity diagnostic detail and cost control. + +--- + +## Goals & Non-Goals + +### Goals (In Scope) + +1. Centralized normalization at the `FilePart → pydantic-ai content` conversion point, with default limits 2000×2000 px / 5 MB base64. +2. Configuration support: `attachment.image.{auto_resize, max_width, max_height, max_base64_bytes}`, overridable via AgentPool config schema. +3. Distinct over-limit failure semantics for user input vs. tool-result attachments. +4. All protocol entry points (Python API / ACP / OpenCode server) share the same normalization capability. + +### Non-Goals (Out of Scope) + +1. Model-modality capability negotiation/gating (opencode's `unsupportedParts` equivalent) — handled by pydantic-ai SDK; not in scope. +2. GIF animation frame preservation — re-encoding outputs static PNG/JPEG only. +3. Image provenance / session-persistent URI management. +4. EXIF orientation correction / metadata stripping (possible follow-up). +5. Audio / Video / PDF attachment normalization — `image/*` only. + +### Success Criteria + +- [ ] Oversized images are resized/re-encoded within configured limits before reaching the model. +- [ ] `attachment.image` config independently overrides defaults. +- [ ] Over-limit user attachment (with `auto_resize: false`) yields a clear user-visible error. +- [ ] Over-limit tool-result attachment is omitted and annotated. +- [ ] Resizer unavailable → original image passed through; session not interrupted. +- [ ] p99 added latency for normalization of a typical-size image < 500 ms. + +--- + +## Evaluation Criteria + +| Criterion | Weight | Description | Minimum Threshold | +|-----------|--------|-------------|-------------------| +| Coverage | High | All protocol entry points unified | Python API/ACP/OpenCode unified | +| Configurability | High | Limits overridable via config | At least width/height/bytes configurable | +| Failure-semantic clarity | High | Over-limit behavior controllable and explainable | User input errors; tool result omits | +| Resource overhead | Medium | CPU/memory/latency of normalization | Typical-size p99 < 500 ms | +| Implementation complexity | Medium | Cost to implement/maintain | No cross-package architecture reshuffle | +| Dependency risk | Medium | Reliability of new image-library dependency | Prefer pure-Python / mature WASM | + +--- + +## Options Analysis + +### Option 1: AgentPool Core Layer (normalize at conversion point) + +**Description**: Add an `ImageNormalizer` service in the agentpool core. Hook it ahead of `pydantic_ai_helpers.to_user_content` / OpenCode `converters`, so image attachments are normalized before conversion to pydantic-ai content across all protocol entries. Config provided by AgentPool config schema (mirroring opencode's `attachment.image`). + +**Advantages**: +- Covers all protocol entry points with one implementation. +- Architecturally aligned with opencode (normalize at the conversion point, not inside each tool). +- Transparent to consuming packages (e.g. xeno-agent) — they gain the capability without changes. + +**Disadvantages**: +- Introduces an image-processing dependency (Pillow / sharp / photon-wasm) into the agentpool core dependency set. +- AgentPool maintainers must accept this as a general capability (not xeno-specific). +- Config schema changes affect all agentpool consumers. + +**Evaluation Against Criteria**: + +| Criterion | Rating | Notes | +|-----------|--------|-------| +| Coverage | 5/5 | Three entry points unified | +| Configurability | 5/5 | Native config-schema support | +| Failure-semantic clarity | 4/5 | Unified semantics; must distinguish tool-result context | +| Resource overhead | 4/5 | Centralized implementation enables result caching/reuse | +| Implementation complexity | 3/5 | Core schema change required | +| Dependency risk | 3/5 | Image library enters core deps | + +**Effort Estimate**: High (core change + tests + config schema). + +**Risk Assessment**: + +| Risk | Likelihood | Impact | Mitigation | +|------|------------|--------|------------| +| Maintainers reject scope | Medium | High | Cite opencode's precedent; prove generality for all agents | +| Image library install issues in pure-Python envs | Low | Medium | Prefer Pillow (pure wheel) or lazy-load | +| Impact on other agentpool consumers | Medium | Medium | Enabled by default, but `attachment.image.auto_resize: false` escape hatch | + +--- + +### Option 2: Consumer Package Layer (xeno-agent-specific) + +**Description**: Implement `ImageNormalizer` inside a consuming package (xeno-agent), invoked at that package's own entry points (e.g. `equipment_expert` tool chain, custom resource provider). Config via that package's YAML. + +**Advantages**: +- No core agentpool changes; no core dependency introduced. +- Consumer teams control limits/algorithm and can tune for diagnostic scenarios (e.g. JPEG quality preserving detail fidelity). +- Small, isolated footprint. + +**Disadvantages**: +- Only covers the consuming package's scenarios; other AgentPool consumers get no benefit. +- Requires explicit invocation at each consumer tool entry — omissible for future tools. +- Tool-result attachment path (handled centrally in opencode's processor) is not aligned; consumer must handle it separately. + +**Evaluation Against Criteria**: + +| Criterion | Rating | Notes | +|-----------|--------|-------| +| Coverage | 3/5 | Consumer scope only | +| Configurability | 5/5 | Consumer-owned YAML | +| Failure-semantic clarity | 4/5 | Consumer-controllable | +| Resource overhead | 4/5 | On-demand invocation | +| Implementation complexity | 4/5 | Confined to one package | +| Dependency risk | 4/5 | Dependency confined to consumer package | + +**Effort Estimate**: Low (single-package addition + single entry-point wiring). + +**Risk Assessment**: + +| Risk | Likelihood | Impact | Mitigation | +|------|------------|--------|------------| +| Tool-chain entry-point omissions | Medium | Medium | Wire at unified entry (resource provider), not per-tool | +| Duplicated implementation vs. future core need | High | Medium | Keep abstraction; migrate if agentpool generalizes | + +--- + +### Option 3: Client-Side Preprocessing (considered and set aside) + +**Description**: Resize images at the caller (IDE, `opencode attach` client, client CLI) before upload. + +**Advantages**: +- No AgentPool / consumer changes. + +**Disadvantages**: +- Cannot constrain tool-result images (`read` / fsspec toolset / MCP-returned). +- Depends on unpredictable client behavior. +- Conflicts with server-side unified failure semantics. + +**Evaluation Against Criteria**: + +| Criterion | Rating | Notes | +|-----------|--------|-------| +| Coverage | 2/5 | User-upload path only | +| Configurability | 1/5 | Client-driven, server-uncontrollable | +| Failure-semantic clarity | 1/5 | Server cannot guarantee | +| Resource overhead | 3/5 | Client CPU | +| Implementation complexity | 3/5 | Simple but ineffective | +| Dependency risk | 2/5 | Uncontrolled client environment | + +**Risk Assessment**: + +| Risk | Likelihood | Impact | Mitigation | +|------|------------|--------|------------| +| Cannot cover tool results | Certain | High | Essentially rules out this option | + +--- + +### Options Comparison Summary + +| Criterion | Option 1 (Core) | Option 2 (Consumer) | Option 3 (Client) | +|-----------|------------------|----------------------|-------------------| +| Coverage | 5/5 | 3/5 | 2/5 | +| Configurability | 5/5 | 5/5 | 1/5 | +| Failure-semantic clarity | 4/5 | 4/5 | 1/5 | +| Resource overhead | 4/5 | 4/5 | 3/5 | +| Implementation complexity | 3/5 | 4/5 | 3/5 | +| Dependency risk | 3/5 | 4/5 | 2/5 | +| **Total** | **24/30** | **24/30** | **12/30** | + +--- + +## Recommendation + +### Recommended Option + +**[Option 1: AgentPool Core Layer]** — with Option 2 as an acceptable transitional path. + +### Justification + +- Architecturally, opencode normalizes at the data-to-model conversion point (not inside tools). AgentPool introducing the same capability at `to_user_content` is the **broadest-coverage, least-omissible** placement, and consistent with the reference implementation. +- Options 1 and 2 score identically on the summed criteria; the decisive factor is **ownership**: if the capability is general (this RFC argues it is — every agent faces large-image input), it belongs in the core; if the consumer team wants to validate value first with lower startup cost, Option 2 provides a transitional path with an abstraction that can migrate later. +- Option 3 cannot constrain the tool-result path and is retained only for comparison. + +### Accepted Trade-offs + +1. **Image library enters core deps**: Option 1 introduces Pillow (recommended) or equivalent; accepted in exchange for globally consistent constraint capability. +2. **GIF animation staticized**: re-encode outputs static frames only; no impact on static diagnostic field photos. +3. **Config schema change**: `attachment.image` requires AgentPool config schema update; backward compatible (defaults apply when absent). + +### Conditions + +- `auto_resize: true` by default; `attachment.image.auto_resize: false` escape hatch. +- Normalization failure (resizer unavailable) must pass the original through; never block the session. +- Defaults aligned with opencode (2000 / 2000 / 5242880) for consistent cross-project expectations. + +--- + +## Technical Design (Preliminary) + +> To be finalized after approval. Draft for review. + +### Architecture Overview + +``` +Image sources Normalization pipeline Model +┌────────────────┐ ┌────────────────────────────────────────────────┐ +│ Python API │ │ ImageNormalizer (agentpool core) │ +│ run_agent() │ │ NEW: inserted before to_user_content() │ +│ ACP attachment │────▶│ ┌──────────────┐ ┌──────────────────────┐ │────▶ ImageUrl / +│ OpenCode FilePart│ │ │ size check │──▶│ resize + re-encode │ │ BinaryContent +│ │ │ │ (px + bytes) │ │ PNG/JPEG candidates │ │ (pydantic-ai) +│ Tool results │ │ └──────────────┘ └──────────────────────┘ │ +│ (read/fsspec) │ │ ALREADY normalized by resize_image_if_needed() │ +│ MCP │ └────────────────────────────────────────────────┘ +└────────────────┘ +``` + +### Key Components + +#### ImageNormalizer + +- Responsibility: accept `FilePart` (`image/*` only), return a constrained `FilePart`. +- Input: data URI or `file://` path; output: data URI. +- Algorithm: mirror opencode `image.ts` — dimension/byte check → iterative 0.75× downscale → PNG/JPEG quality-ladder candidate selection. + +#### Config (`attachment.image`) + +``` +attachment: + image: + auto_resize: true # default + max_width: 2000 + max_height: 2000 + max_base64_bytes: 5242880 +``` + +### Data Flow + +1. User or tool-result produces `FilePart(image/*)`. +2. Normalization: byte check → if over-limit, resize + re-encode → new FilePart. +3. Persist (original or normalized — see Open Questions). +4. Convert to pydantic-ai content (`ImageUrl` / `BinaryContent`). + +### API Design + +``` +ImageNormalizer.normalize(input: FilePart) -> FilePart + raises: InvalidDataUrlError | DecodeError | SizeError + +Config: attachment.image. (injected into agentpool config schema) +``` + +--- + +## Security Considerations + +### Threat Analysis + +| Threat | Impact | Likelihood | Mitigation | +|--------|--------|------------|------------| +| Malicious oversized image → memory spike | Medium | Medium | Pre-check base64 length; reject input > N× limit before decode | +| Decoder vulnerability (decompression bomb) | Medium | Low | Maintained library + validate dimensions post-decode | +| Client-constructed malformed data URI | Low | Low | Unified `InvalidDataUrlError` handling | + +### Security Measures + +- [ ] Pre-check base64 length before decode (reject > N× limit directly) +- [ ] Accept `data:` base64 URIs only (no remote URL fetching; avoids SSRF) + +### Compliance + +No regulatory requirements identified. + +--- + +## Implementation Plan + +### Phase 1: AgentPool Core Normalization (if Option 1 adopted) + +- **Scope**: Add `ImageNormalizer` + `attachment.image` config + wiring ahead of `to_user_content`. +- **Deliverables**: normalization service, config schema, unit tests, compatibility validation against `test_multimodal_storage`. +- **Dependencies**: Pillow introduction (or photon-wasm matching opencode). + +### Phase 2: Consumer Validation (if Option 2 as transition) + +- **Scope**: Wire normalization into a consumer resource provider; end-to-end validation with real field photos. +- **Deliverables**: consumer-side wiring + one end-to-end test (real photo; verify >5 MB / >2000² is compressed). +- **Dependencies**: Phase 1 deliverables or consumer-local implementation. + +### Milestones + +| Milestone | Description | Target | Status | +|-----------|-------------|--------|--------| +| M1 | ImageNormalizer + config schema | TBD | Not Started | +| M2 | Entry-point wiring + unit tests | TBD | Not Started | +| M3 | End-to-end validation + docs | TBD | Not Started | + +### Rollback Strategy + +- `attachment.image.auto_resize: false` disables resizing (byte check retained). +- Normalization failure always passes the original through — a natural escape path. + +--- + +## Open Questions + +1. **[Original vs. Normalized Persistence]** + - Context: should session history store the original or the normalized image? opencode persists the normalized result. + - Owner: AgentPool maintainers + - Status: Open + +2. **[Image Library Selection]** + - Context: Pillow (pure wheel, mature ecosystem) vs. photon-wasm (matches opencode but requires WASM runtime) vs. sharp (native bindings, heavier install). + - Owner: AgentPool maintainers + - Status: Open + +3. **[Normalization Trigger Point]** + - Context: normalize at conversion-to-pydantic-ai-content time vs. at session-write time. First saves storage; second preserves more fidelity. + - Owner: AgentPool maintainers + - Status: Open + +4. **[GIF / Animated Image Policy]** + - Context: is static-frame re-encoding acceptable? Does the diagnostic scenario require animation preservation? + - Owner: Consumer teams (e.g. xeno-agent) + - Status: Open + +5. **[Compatibility]** + - Context: do other AgentPool consumers accept default normalization? Is a feature flag needed? + - Owner: AgentPool maintainers + - Status: Open + +--- + +## Decision Record + +> To be completed after review concludes. + +### Decision + +**Status**: Pending + +**Date**: — + +**Approvers**: — + +### Decision Summary + +— + +### Key Discussion Points + +1. — + +### Conditions of Approval + +— + +--- + +## References + +### Related Documents + +- opencode reference implementation: `packages/opencode/src/image/image.ts`, `provider/transform.ts`, `session/message-v2.ts` +- opencode config documentation: `packages/opencode/packages/web/src/content/docs/config.mdx:425-451` +- AgentPool current image path: files cited in §2.1 + +### Appendix + +- Normalization flow details follow opencode `image.ts` as the baseline template; error layering (`SizeError` / `ResizerUnavailableError` / `InvalidDataUrlError`) adapted from the same source. \ No newline at end of file From 885b7fc589ca7830777d1b9c67bae3f19b94479c Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Mon, 17 Aug 2026 15:40:55 +0800 Subject: [PATCH 2/3] feat(images): normalize oversized image attachments at protocol entry points Implement RFC-0059 Option 1: insert the existing resize_image_if_needed() normalization into the protocol user-upload path so Python API, ACP, and OpenCode server all constrain image attachments within the same limits as the tool-read path. - Add AttachmentImageConfig (auto_resize/max_width/max_height/ max_base64_bytes, defaults mirroring opencode: 2000x2000px, 5MB base64) - Add ImageNormalizer reusing resize_image_if_needed from the fsspec toolset; only data: URIs are processed (no SSRF); failures degrade to pass-through so sessions are never interrupted - Wire into OpenCode extract_user_prompt_from_parts, ACP from_acp_content, and functional run_agent(image_url=...) - auto_resize: false raises ImageSizeError for over-limit user input Co-authored-by: Sisyphus --- ...RFC-0059-image-attachment-normalization.md | 14 +- src/wolfharness/functional/run.py | 59 ++++- src/wolfharness/images/__init__.py | 7 + src/wolfharness/images/normalizer.py | 211 ++++++++++++++++++ src/wolfharness/models/manifest.py | 18 ++ src/wolfharness_config/__init__.py | 2 + src/wolfharness_config/attachment.py | 50 +++++ .../acp_server/converters.py | 8 + src/wolfharness_server/acp_server/handler.py | 13 +- .../acp_server/session_lifecycle.py | 14 +- .../opencode_server/converters.py | 7 + .../opencode_server/routes/message_routes.py | 17 ++ tests/config/test_attachment_config.py | 69 ++++++ tests/images/__init__.py | 1 + tests/images/test_image_normalizer.py | 148 ++++++++++++ 15 files changed, 626 insertions(+), 12 deletions(-) create mode 100644 src/wolfharness/images/__init__.py create mode 100644 src/wolfharness/images/normalizer.py create mode 100644 src/wolfharness_config/attachment.py create mode 100644 tests/config/test_attachment_config.py create mode 100644 tests/images/__init__.py create mode 100644 tests/images/test_image_normalizer.py diff --git a/docs/rfcs/draft/RFC-0059-image-attachment-normalization.md b/docs/rfcs/draft/RFC-0059-image-attachment-normalization.md index 0cef7a5d3..7b8a9a3e5 100644 --- a/docs/rfcs/draft/RFC-0059-image-attachment-normalization.md +++ b/docs/rfcs/draft/RFC-0059-image-attachment-normalization.md @@ -134,11 +134,11 @@ Images travel as base64 data URIs into `BinaryContent`/`ImageUrl` and are sent t ### Success Criteria -- [ ] Oversized images are resized/re-encoded within configured limits before reaching the model. -- [ ] `attachment.image` config independently overrides defaults. -- [ ] Over-limit user attachment (with `auto_resize: false`) yields a clear user-visible error. +- [x] Oversized images are resized/re-encoded within configured limits before reaching the model. +- [x] `attachment.image` config independently overrides defaults. +- [x] Over-limit user attachment (with `auto_resize: false`) yields a clear user-visible error. - [ ] Over-limit tool-result attachment is omitted and annotated. -- [ ] Resizer unavailable → original image passed through; session not interrupted. +- [x] Resizer unavailable → original image passed through; session not interrupted. - [ ] p99 added latency for normalization of a typical-size image < 500 ms. --- @@ -398,9 +398,9 @@ No regulatory requirements identified. | Milestone | Description | Target | Status | |-----------|-------------|--------|--------| -| M1 | ImageNormalizer + config schema | TBD | Not Started | -| M2 | Entry-point wiring + unit tests | TBD | Not Started | -| M3 | End-to-end validation + docs | TBD | Not Started | +| M1 | ImageNormalizer + config schema | TBD | Implemented | +| M2 | Entry-point wiring + unit tests | TBD | Implemented | +| M3 | End-to-end validation + docs | TBD | In Progress | ### Rollback Strategy diff --git a/src/wolfharness/functional/run.py b/src/wolfharness/functional/run.py index b8605ec58..793bf4e24 100644 --- a/src/wolfharness/functional/run.py +++ b/src/wolfharness/functional/run.py @@ -13,6 +13,26 @@ if TYPE_CHECKING: from wolfharness.agents.native_agent import AgentKwargs from wolfharness.common_types import PromptCompatible + from wolfharness.images.normalizer import ImageNormalizer + + +def _make_image_normalizer( + attachment_image: Any | None, +) -> ImageNormalizer | None: + """Build an ``ImageNormalizer`` from an explicit config (RFC-0059).""" + if attachment_image is None: + return None + from wolfharness.images.normalizer import ImageNormalizer + + return ImageNormalizer(attachment_image) + + +def _normalize_image_url(url: str, normalizer: ImageNormalizer | None) -> str: + """Normalize a data-URI image URL if a normalizer is available.""" + if normalizer is None: + return url + normalized, _mime = normalizer.normalize(url, "image/*") + return normalized @overload @@ -21,6 +41,7 @@ async def run_agent[TResult]( image_url: str | None = None, *, output_type: type[TResult], + attachment_image: Any | None = None, **kwargs: Unpack[AgentKwargs], ) -> TResult: ... @@ -29,6 +50,9 @@ async def run_agent[TResult]( async def run_agent( prompt: PromptCompatible, image_url: str | None = None, + *, + output_type: None = None, + attachment_image: Any | None = None, **kwargs: Unpack[AgentKwargs], ) -> str: ... @@ -38,15 +62,27 @@ async def run_agent( image_url: str | None = None, *, output_type: type[Any] | None = None, + attachment_image: Any | None = None, **kwargs: Unpack[AgentKwargs], ) -> Any: - """Run prompt through agent and return result.""" + """Run prompt through agent and return result. + + Args: + prompt: The user prompt. + image_url: Optional image URL or ``data:`` URI. ``data:`` URIs are + normalized when they exceed configured limits (RFC-0059). + output_type: Optional structured output type. + attachment_image: Optional ``AttachmentImageConfig`` for image + normalization. When omitted, defaults apply. + **kwargs: Additional agent constructor kwargs. + """ async with Agent[Any, str](**kwargs) as agent: # Convert to structured output agent if output_type specified final = agent.to_structured(output_type) if output_type is not None else agent if image_url: - image = ImageUrl(url=image_url) + normalized = _normalize_image_url(image_url, _make_image_normalizer(attachment_image)) + image = ImageUrl(url=normalized) result = await final.run(prompt, image) else: result = await final.run(prompt) @@ -59,6 +95,7 @@ def run_agent_sync[TResult]( image_url: str | None = None, *, output_type: type[TResult], + attachment_image: Any | None = None, **kwargs: Unpack[AgentKwargs], ) -> TResult: ... @@ -67,6 +104,8 @@ def run_agent_sync[TResult]( def run_agent_sync( prompt: PromptCompatible, image_url: str | None = None, + *, + attachment_image: Any | None = None, **kwargs: Unpack[AgentKwargs], ) -> str: ... @@ -76,11 +115,25 @@ def run_agent_sync( image_url: str | None = None, *, output_type: type[Any] | None = None, + attachment_image: Any | None = None, **kwargs: Unpack[AgentKwargs], ) -> Any: """Sync wrapper for run_agent.""" async def _run() -> Any: - return await run_agent(prompt, image_url, output_type=output_type, **kwargs) # type: ignore[arg-type] + if output_type is None: + return await run_agent( + prompt, + image_url, + attachment_image=attachment_image, + **kwargs, + ) + return await run_agent( + prompt, + image_url, + output_type=output_type, + attachment_image=attachment_image, + **kwargs, + ) return run_sync(_run()) diff --git a/src/wolfharness/images/__init__.py b/src/wolfharness/images/__init__.py new file mode 100644 index 000000000..fc075f7b3 --- /dev/null +++ b/src/wolfharness/images/__init__.py @@ -0,0 +1,7 @@ +"""Image attachment normalization (RFC-0059).""" + +from __future__ import annotations + +from wolfharness.images.normalizer import ImageNormalizer, ImageSizeError + +__all__ = ["ImageNormalizer", "ImageSizeError"] diff --git a/src/wolfharness/images/normalizer.py b/src/wolfharness/images/normalizer.py new file mode 100644 index 000000000..44e4d519f --- /dev/null +++ b/src/wolfharness/images/normalizer.py @@ -0,0 +1,211 @@ +"""Image attachment normalization service (RFC-0059). + +Normalizes oversized image attachments on the protocol user-upload path +before they are forwarded to the model. Reuses the existing +``resize_image_if_needed()`` from the fsspec toolset so behavior stays +consistent with the tool-read path. + +Failure semantics: + +- ``auto_resize: true`` (default): images exceeding dimension or byte + limits are resized and re-encoded. If normalization itself fails + (e.g. Pillow unavailable), the original is passed through unchanged — + the session is never interrupted. +- ``auto_resize: false``: the byte budget is still enforced and an + over-limit image raises :class:`ImageSizeError`. +- Only ``data:`` base64 URIs are processed. Remote ``http(s)://`` and + ``file://`` URLs are left untouched (avoids SSRF / filesystem access). +""" + +from __future__ import annotations + +import base64 +import binascii +import io +import logging +from typing import Any + +from wolfharness_config.attachment import AttachmentImageConfig + + +logger = logging.getLogger(__name__) + + +class ImageSizeError(Exception): + """Raised when an image attachment cannot be brought within limits. + + Used for the ``auto_resize: false`` path: an over-limit user attachment + fails with this error instead of silently passing an oversized image. + """ + + +class ImageNormalizer: + """Normalize oversized image attachments. + + Args: + config: Image attachment normalization configuration. When omitted, + falls back to ``AttachmentImageConfig()`` defaults. + """ + + def __init__(self, config: AttachmentImageConfig | None = None) -> None: + self._config = config if config is not None else AttachmentImageConfig() + + def normalize(self, url: str, mime: str) -> tuple[str, str]: + """Normalize an image attachment URL. + + Args: + url: Image URL. Only ``data:`` base64 URIs are processed. + mime: MIME type of the image (e.g. ``image/png``). + + Returns: + Tuple of ``(possibly_normalized_url, mime)``. Passes the inputs + through unchanged for non-``data:`` URLs and for images already + within limits. + + Raises: + ImageSizeError: When ``auto_resize`` is disabled and the image + exceeds configured limits. + """ + if not url.startswith("data:"): + return url, mime + + payload = _data_uri_payload(url) + if payload is None: + return url, mime + + try: + data = base64.b64decode(payload) + except (binascii.Error, ValueError): + logger.warning("Invalid base64 in image data URI; passing through unchanged") + return url, mime + + normalized_data, new_mime = self.normalize_bytes(data, mime) + if normalized_data is data: + return url, mime + return _make_data_uri(new_mime, _b64encode(normalized_data)), new_mime + + def normalize_bytes(self, data: bytes, mime: str) -> tuple[bytes, str]: + """Normalize raw image bytes. + + Args: + data: Raw image bytes. + mime: MIME type of the image (e.g. ``image/png``). + + Returns: + Tuple of ``(possibly_normalized_bytes, mime)``. Returns the + original input unchanged when already within limits or not + normalizable. + + Raises: + ImageSizeError: When ``auto_resize`` is disabled and the image + exceeds configured limits. + """ + target_bytes = self._config.max_base64_bytes * 3 // 4 + max_size = min(self._config.max_width, self._config.max_height) + + if len(data) <= target_bytes and not self._config.auto_resize: + return data, mime + if len(data) <= target_bytes and _fits_dimensions(data, max_size): + return data, mime + + if not self._config.auto_resize: + return self._normalize_disabled(data, mime, max_size, target_bytes) + + return self._normalize_enabled(data, mime, max_size, target_bytes) + + def _normalize_enabled( + self, + data: bytes, + mime: str, + max_size: int, + target_bytes: int, + ) -> tuple[bytes, str]: + """Normalize an image with ``auto_resize: true``.""" + from wolfharness_toolsets.fsspec_toolset.image_utils import ( + resize_image_if_needed, + ) + + try: + resized, new_mime, note = resize_image_if_needed( + data, + mime, + max_size=max_size, + max_bytes=target_bytes, + ) + except Exception: + logger.warning("Image normalization failed; passing through unchanged", exc_info=True) + return data, mime + + if note is None: + return data, mime + + if len(resized) > target_bytes * 4 // 3: + logger.warning( + "Re-encoded image still exceeds max_base64_bytes; passing original through" + ) + return data, mime + + return resized, new_mime + + def _normalize_disabled( + self, + data: bytes, + mime: str, + max_size: int, + target_bytes: int, + ) -> tuple[bytes, str]: + """Raise :class:`ImageSizeError` for over-limit images.""" + try: + with _open_image(data) as img: + width, height = img.size + except Exception: # noqa: BLE001 + logger.warning("Image decode failed with auto_resize disabled; passing through") + return data, mime + + if width > max_size or height > max_size or len(data) > target_bytes: + raise ImageSizeError( + f"Image attachment {width}x{height} exceeds configured limits " + f"({max_size}x{max_size} px, {target_bytes} bytes) and auto_resize is disabled" + ) + return data, mime + + @property + def config(self) -> AttachmentImageConfig: + """The underlying normalization configuration.""" + return self._config + + +def _open_image(data: bytes) -> Any: + """Open an image from bytes as a context manager.""" + from PIL import Image + + return Image.open(io.BytesIO(data)) + + +def _fits_dimensions(data: bytes, max_size: int) -> bool: + """Return whether an image's dimensions are within ``max_size``.""" + try: + with _open_image(data) as img: + width, height = img.size + except Exception: # noqa: BLE001 + return True + return int(width) <= max_size and int(height) <= max_size + + +def _data_uri_payload(url: str) -> str | None: + """Extract the base64 payload from a data URI, or None if not base64.""" + marker = ";base64," + index = url.find(marker) + if index < 0: + return None + return url[index + len(marker) :] + + +def _make_data_uri(mime: str, encoded: str) -> str: + """Build a ``data:`` URI from a MIME type and base64 payload.""" + return f"data:{mime};base64,{encoded}" + + +def _b64encode(data: bytes) -> str: + """Base64-encode bytes to ASCII (no newlines).""" + return base64.b64encode(data).decode("ascii") diff --git a/src/wolfharness/models/manifest.py b/src/wolfharness/models/manifest.py index 46a3b1200..32fe0f92f 100644 --- a/src/wolfharness/models/manifest.py +++ b/src/wolfharness/models/manifest.py @@ -17,6 +17,7 @@ from wolfharness.models.agents import NativeAgentConfig from wolfharness.models.file_agents import FileAgentConfig from wolfharness.models.model_configs import AnyModelConfig, StringModelConfig +from wolfharness_config.attachment import AttachmentImageConfig from wolfharness_config.commands import CommandConfig, StaticCommandConfig from wolfharness_config.compaction import CompactionConfig from wolfharness_config.context import ConfigContextManager @@ -372,6 +373,23 @@ class AgentsManifest(Schema): ``` """ + attachment: AttachmentImageConfig = Field(default_factory=AttachmentImageConfig) + """Image attachment normalization configuration (RFC-0059). + + Controls automatic resizing/re-encoding of oversized image attachments + on the protocol user-upload path. Defaults mirror opencode's limits. + + Example: + ```yaml + attachment: + image: + auto_resize: true + max_width: 2000 + max_height: 2000 + max_base64_bytes: 5242880 + ``` + """ + session_pool: SessionPoolConfig = Field(default_factory=SessionPoolConfig) """Session pool configuration for session lifecycle management. diff --git a/src/wolfharness_config/__init__.py b/src/wolfharness_config/__init__.py index 1f32e60be..f7f996c01 100644 --- a/src/wolfharness_config/__init__.py +++ b/src/wolfharness_config/__init__.py @@ -10,6 +10,7 @@ from wolfharness_config.wolfharness_tools import AgentpoolToolConfig from wolfharness_config.builtin_tools import BuiltinToolConfig +from wolfharness_config.attachment import AttachmentImageConfig from wolfharness_config.capabilities import CapabilityConfig from wolfharness_config.forward_targets import ForwardingTarget from wolfharness_config.session import SessionQuery @@ -85,6 +86,7 @@ "DEFAULT_SKILLS_PATHS", "ACPConfig", "AnyToolConfig", + "AttachmentImageConfig", "BaseEventHandlerConfig", "BaseHookConfig", "BaseMCPServerConfig", diff --git a/src/wolfharness_config/attachment.py b/src/wolfharness_config/attachment.py new file mode 100644 index 000000000..41d7f55ad --- /dev/null +++ b/src/wolfharness_config/attachment.py @@ -0,0 +1,50 @@ +"""Image attachment configuration models. + +Controls how user-uploaded image attachments are normalized before they +are forwarded to the model (RFC-0059). The defaults mirror opencode's +``attachment.image`` limits (2000x2000 px, 5 MB base64 payload). +""" + +from __future__ import annotations + +from pydantic import ConfigDict, Field +from schemez import Schema + + +class AttachmentImageConfig(Schema): + """Configuration for image attachment normalization. + + Controls automatic resizing/re-encoding of oversized image attachments + on the protocol user-upload path (Python API, ACP, OpenCode server). + + Values are aligned with opencode's ``attachment.image`` defaults so + cross-project expectations stay consistent. + """ + + auto_resize: bool = Field(default=True, title="Auto-resize images") + """Whether to automatically resize/re-encode oversized images. + + When ``True`` (default), image attachments exceeding ``max_width`` / + ``max_height`` / ``max_base64_bytes`` are resized and re-encoded before + being forwarded to the model. When ``False``, a byte-budget check is + retained but no resizing is performed. + """ + + max_width: int = Field(default=2000, ge=1, title="Max image width") + """Maximum image width in pixels (default 2000).""" + + max_height: int = Field(default=2000, ge=1, title="Max image height") + """Maximum image height in pixels (default 2000).""" + + max_base64_bytes: int = Field( + default=5 * 1024 * 1024, + ge=1, + title="Max base64 payload bytes", + ) + """Maximum base64 payload size in bytes (default 5 MB). + + The base64 payload is the portion after ``data:;base64,`` in a + data URI, measured in UTF-8 bytes — matching opencode's semantics. + """ + + model_config = ConfigDict(frozen=True) diff --git a/src/wolfharness_server/acp_server/converters.py b/src/wolfharness_server/acp_server/converters.py index f6b12b166..47bcf6c1a 100644 --- a/src/wolfharness_server/acp_server/converters.py +++ b/src/wolfharness_server/acp_server/converters.py @@ -125,6 +125,7 @@ def convert_acp_mcp_server_to_config(acp_server: McpServer) -> MCPServerConfig: def from_acp_content( # noqa: PLR0911 block: ContentBlock, fs: AsyncFileSystem | None = None, + normalizer: Any | None = None, ) -> UserContent | PathReference: """Convert ACP content blocks to UserContent or PathReference objects. @@ -134,6 +135,8 @@ def from_acp_content( # noqa: PLR0911 Args: block: ACP ContentBlock fs: Optional filesystem for file references + normalizer: Optional ``ImageNormalizer`` for normalizing oversized + image attachments (RFC-0059). Applied to ``ImageContentBlock``. Returns: UserContent or PathReference objects @@ -145,6 +148,11 @@ def from_acp_content( # noqa: PLR0911 case ImageContentBlock(data=data, mime_type=mime_type): binary_data = base64.b64decode(data) + if normalizer is not None: + normalized, new_mime = normalizer.normalize_bytes(binary_data, mime_type) + if normalized is not binary_data: + binary_data = normalized + mime_type = new_mime return BinaryImage(data=binary_data, media_type=mime_type) case AudioContentBlock(data=data, mime_type=mime_type): diff --git a/src/wolfharness_server/acp_server/handler.py b/src/wolfharness_server/acp_server/handler.py index 962785e1a..a4e7f5a18 100644 --- a/src/wolfharness_server/acp_server/handler.py +++ b/src/wolfharness_server/acp_server/handler.py @@ -39,6 +39,16 @@ logger = get_logger(__name__) +def _make_image_normalizer(host_context: HostContext) -> Any | None: + """Build an ``ImageNormalizer`` from the host context manifest (RFC-0059).""" + from wolfharness.images.normalizer import ImageNormalizer + + manifest = getattr(host_context, "manifest", None) + if manifest is None: + return None + return ImageNormalizer(manifest.attachment) + + class ACPProtocolHandler(ProtocolEventConsumerMixin): """ACP protocol handler backed by SessionPool. @@ -584,7 +594,8 @@ async def handle_prompt( # noqa: PLR0915 await self._ensure_event_consumer(session_id) # Convert ACP content blocks to agent prompts - contents = [from_acp_content(block, fs=None) for block in prompt] + normalizer = _make_image_normalizer(self._host_context) + contents = [from_acp_content(block, fs=None, normalizer=normalizer) for block in prompt] # Split slash commands from content and execute local commands. # Commands inject expanded prompts into the SessionPool per-session diff --git a/src/wolfharness_server/acp_server/session_lifecycle.py b/src/wolfharness_server/acp_server/session_lifecycle.py index 1d492db52..e78eb47d8 100644 --- a/src/wolfharness_server/acp_server/session_lifecycle.py +++ b/src/wolfharness_server/acp_server/session_lifecycle.py @@ -30,6 +30,17 @@ logger = get_logger(__name__) +def _make_image_normalizer(agent: Any) -> Any | None: + """Build an ``ImageNormalizer`` from the agent's host context (RFC-0059).""" + from wolfharness.images.normalizer import ImageNormalizer + + host_context = getattr(agent, "host_context", None) + manifest = getattr(host_context, "manifest", None) + if manifest is None: + return None + return ImageNormalizer(manifest.attachment) + + class ACPSessionLifecycleMixin: """Mixin providing session lifecycle methods for ACPSession. @@ -260,7 +271,8 @@ async def process_prompt(self, content_blocks: Sequence[ContentBlock]) -> StopRe self._cancelled = False fs = self.agent.env.get_fs() - contents = [from_acp_content(i, fs=fs) for i in content_blocks] + normalizer = _make_image_normalizer(self.agent) + contents = [from_acp_content(i, fs=fs, normalizer=normalizer) for i in content_blocks] self.log.debug("Converted content", content=contents) if not contents: self.log.warning("Empty prompt received") diff --git a/src/wolfharness_server/opencode_server/converters.py b/src/wolfharness_server/opencode_server/converters.py index 7899d0ee9..86a2a8e23 100644 --- a/src/wolfharness_server/opencode_server/converters.py +++ b/src/wolfharness_server/opencode_server/converters.py @@ -171,6 +171,7 @@ async def extract_user_prompt_from_parts( session_id: str, fs: AsyncFileSystem | None = None, agent: BaseAgent[Any, Any] | None = None, + normalizer: Any | None = None, ) -> Sequence[UserContent | PathReference]: """Extract user prompt from OpenCode message input parts. @@ -188,6 +189,10 @@ async def extract_user_prompt_from_parts( agent: Optional agent for resolving MCP resources session_id: Session ID for scoped resource resolution via ExtensionRegistry. + normalizer: Optional ``ImageNormalizer`` for normalizing oversized + ``image/*`` data-URI attachments (RFC-0059). When provided, + oversized image data URIs are resized/re-encoded before being + converted to pydantic-ai content. Returns: Either a simple string (text-only) or a list of UserContent/PathReference items @@ -204,6 +209,8 @@ async def extract_user_prompt_from_parts( if content is not None: result.extend(content) case FilePartInput(mime=mime, url=url, filename=filename): + if normalizer is not None and mime.startswith("image/"): + url, mime = normalizer.normalize(url, mime) file_content = to_user_content_or_path_ref(mime, url, filename, fs=fs) result.append(file_content) case AgentPartInput(name=agent_name): diff --git a/src/wolfharness_server/opencode_server/routes/message_routes.py b/src/wolfharness_server/opencode_server/routes/message_routes.py index 02cee0412..8c576818b 100644 --- a/src/wolfharness_server/opencode_server/routes/message_routes.py +++ b/src/wolfharness_server/opencode_server/routes/message_routes.py @@ -55,6 +55,7 @@ from pydantic_ai import UserContent from wolfharness.common_types import PathReference + from wolfharness.images.normalizer import ImageNormalizer from wolfharness.messaging import ChatMessage from wolfharness.orchestrator.session_pool import SessionPool from wolfharness_server.opencode_server.session_pool_integration import ( @@ -66,6 +67,20 @@ logger = get_logger(__name__) +def _make_image_normalizer(state: ServerState) -> ImageNormalizer | None: + """Build an ``ImageNormalizer`` from the pool manifest (RFC-0059). + + Returns ``None`` when the pool manifest is unavailable, in which case + no normalization is applied. + """ + from wolfharness.images.normalizer import ImageNormalizer + + manifest = state.pool.manifest if state.pool is not None else None + if manifest is None: + return None + return ImageNormalizer(manifest.attachment) + + @dataclass class _MessageRunContext: """Context carried from lock-held routing phase to lock-free wait phase.""" @@ -494,6 +509,7 @@ async def _route_message_locked( # noqa: PLR0915 session_id, fs=state.fs, agent=state.agent, + normalizer=_make_image_normalizer(state), ) # --- Trigger title generation on first message (fire-and-forget) --- @@ -1042,6 +1058,7 @@ async def send_message_async(session_id: str, request: MessageRequest, state: St session_id, fs=state.fs, agent=state.agent, + normalizer=_make_image_normalizer(state), ) # D13: Map delivery mode from request to priority. diff --git a/tests/config/test_attachment_config.py b/tests/config/test_attachment_config.py new file mode 100644 index 000000000..379d513ce --- /dev/null +++ b/tests/config/test_attachment_config.py @@ -0,0 +1,69 @@ +"""Tests for AttachmentImageConfig.""" + +from __future__ import annotations + +import pytest +import yaml + +from wolfharness_config.attachment import AttachmentImageConfig + + +pytestmark = pytest.mark.unit + + +def test_attachment_image_defaults(): + """AttachmentImageConfig defaults: auto_resize=True, 2000x2000, 5MB base64.""" + config = AttachmentImageConfig() + + assert config.auto_resize is True + assert config.max_width == 2000 + assert config.max_height == 2000 + assert config.max_base64_bytes == 5 * 1024 * 1024 + + +def test_attachment_image_custom_values(): + """AttachmentImageConfig accepts explicit overrides.""" + config = AttachmentImageConfig( + auto_resize=False, + max_width=4096, + max_height=3072, + max_base64_bytes=10 * 1024 * 1024, + ) + + assert config.auto_resize is False + assert config.max_width == 4096 + assert config.max_height == 3072 + assert config.max_base64_bytes == 10 * 1024 * 1024 + + +def test_attachment_image_rejects_non_positive_dimensions(): + """AttachmentImageConfig rejects max_width/max_height < 1.""" + with pytest.raises(ValueError, match="max_width"): + AttachmentImageConfig(max_width=0) + + +def test_attachment_image_serialize_as_dict(): + """AttachmentImageConfig serializes to the expected dict shape.""" + config = AttachmentImageConfig() + + d = config.model_dump() + + assert d == { + "auto_resize": True, + "max_width": 2000, + "max_height": 2000, + "max_base64_bytes": 5 * 1024 * 1024, + } + + +def test_attachment_image_yaml_round_trip(): + """AttachmentImageConfig round-trips through YAML.""" + config = AttachmentImageConfig(auto_resize=False, max_width=1024, max_height=768) + + loaded = AttachmentImageConfig.model_validate( + yaml.safe_load(yaml.safe_dump(config.model_dump())) + ) + + assert loaded.auto_resize is False + assert loaded.max_width == 1024 + assert loaded.max_height == 768 diff --git a/tests/images/__init__.py b/tests/images/__init__.py new file mode 100644 index 000000000..0cbfad877 --- /dev/null +++ b/tests/images/__init__.py @@ -0,0 +1 @@ +"""Unit tests for WolfHarness image normalization (RFC-0059).""" diff --git a/tests/images/test_image_normalizer.py b/tests/images/test_image_normalizer.py new file mode 100644 index 000000000..48339427b --- /dev/null +++ b/tests/images/test_image_normalizer.py @@ -0,0 +1,148 @@ +"""Tests for ImageNormalizer (RFC-0059).""" + +from __future__ import annotations + +import base64 +import io +import random + +from PIL import Image +import pytest + +from wolfharness.images.normalizer import ImageNormalizer, ImageSizeError +from wolfharness_config.attachment import AttachmentImageConfig + + +pytestmark = pytest.mark.unit + + +def _png_bytes(width: int, height: int, color: tuple[int, int, int] = (200, 30, 30)) -> bytes: + """Create a solid-color PNG image of the given dimensions.""" + img = Image.new("RGB", (width, height), color) + buf = io.BytesIO() + img.save(buf, format="PNG") + return buf.getvalue() + + +def _noisy_png_bytes(width: int, height: int) -> bytes: + """Create a noisy PNG image whose bytes are hard to compress.""" + rng = random.Random(42) + img = Image.new("RGB", (width, height)) + pixels = [ + (rng.randint(0, 255), rng.randint(0, 255), rng.randint(0, 255)) + for _ in range(width * height) + ] + img.putdata(pixels) + buf = io.BytesIO() + img.save(buf, format="PNG") + return buf.getvalue() + + +def _data_uri(data: bytes, mime: str = "image/png") -> str: + """Build a data URI from image bytes.""" + return f"data:{mime};base64,{base64.b64encode(data).decode('ascii')}" + + +def test_normalize_passes_small_image_through(): + """A small image within limits passes through unchanged.""" + data = _png_bytes(100, 100) + n = ImageNormalizer() + + url, mime = n.normalize(_data_uri(data), "image/png") + + assert url == _data_uri(data) + assert mime == "image/png" + + +def test_normalize_resizes_oversized_image(): + """An oversized image is resized to within max_base64_bytes.""" + data = _png_bytes(4000, 3000) + n = ImageNormalizer(AttachmentImageConfig(max_base64_bytes=64 * 1024)) + + url, mime = n.normalize(_data_uri(data), "image/png") + + assert url != _data_uri(data) + payload = url.split(";base64,", 1)[1] + assert len(payload) <= 64 * 1024 + assert mime in ("image/png", "image/jpeg") + + +def test_normalize_resizes_oversized_dimensions(): + """An image with oversized dimensions is resized even when bytes are small.""" + data = _png_bytes(5000, 100) + n = ImageNormalizer( + AttachmentImageConfig(max_width=2000, max_height=2000, max_base64_bytes=5 * 1024 * 1024) + ) + + url, _mime = n.normalize(_data_uri(data), "image/png") + + decoded = base64.b64decode(url.split(";base64,", 1)[1]) + with Image.open(io.BytesIO(decoded)) as img: + assert img.width <= 2000 + assert img.height <= 2000 + + +def test_normalize_non_data_uri_passes_through(): + """A remote URL passes through unchanged (no SSRF).""" + n = ImageNormalizer() + + url, mime = n.normalize("https://example.com/photo.png", "image/png") + + assert url == "https://example.com/photo.png" + assert mime == "image/png" + + +def test_normalize_invalid_base64_passes_through(): + """A malformed base64 data URI passes through unchanged.""" + n = ImageNormalizer() + + url, mime = n.normalize("data:image/png;base64,@@@not-base64@@@", "image/png") + + assert url == "data:image/png;base64,@@@not-base64@@@" + assert mime == "image/png" + + +def test_normalize_disabled_raises_for_oversized(): + """With auto_resize=False, an oversized image raises ImageSizeError.""" + data = _noisy_png_bytes(400, 300) + n = ImageNormalizer(AttachmentImageConfig(auto_resize=False, max_base64_bytes=64 * 1024)) + + assert ( + len(data) + > AttachmentImageConfig(auto_resize=False, max_base64_bytes=64 * 1024).max_base64_bytes + * 3 + // 4 + ) + + with pytest.raises(ImageSizeError): + n.normalize(_data_uri(data), "image/png") + + +def test_normalize_disabled_passes_small_through(): + """With auto_resize=False, a small image still passes through.""" + data = _png_bytes(100, 100) + n = ImageNormalizer(AttachmentImageConfig(auto_resize=False)) + + url, mime = n.normalize(_data_uri(data), "image/png") + + assert url == _data_uri(data) + assert mime == "image/png" + + +def test_normalize_bytes_returns_new_bytes_on_resize(): + """normalize_bytes returns a new buffer (not the input) when resized.""" + data = _png_bytes(4000, 3000) + n = ImageNormalizer(AttachmentImageConfig(max_base64_bytes=64 * 1024)) + + normalized, _mime = n.normalize_bytes(data, "image/png") + + assert normalized is not data + assert len(normalized) < len(data) + + +def test_exports_available(): + """ImageNormalizer and ImageSizeError are exported from wolfharness.images.""" + from wolfharness import images + + assert images.ImageNormalizer is ImageNormalizer + assert images.ImageSizeError is ImageSizeError From 2a75a886f45093128a20869773772359f3933fcb Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Mon, 17 Aug 2026 16:47:03 +0800 Subject: [PATCH 3/3] test(images): add protocol entry-point wiring tests for image normalization Cover the three RFC-0059 user-upload entry points with integration-style unit tests: OpenCode extract_user_prompt_from_parts, ACP from_acp_content, and functional run_agent image-url helpers. Each verifies oversized images are resized, small images pass through, non-image parts are untouched, and the no-normalizer path stays backward compatible. --- tests/images/test_image_normalizer_wiring.py | 251 +++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 tests/images/test_image_normalizer_wiring.py diff --git a/tests/images/test_image_normalizer_wiring.py b/tests/images/test_image_normalizer_wiring.py new file mode 100644 index 000000000..10f99e866 --- /dev/null +++ b/tests/images/test_image_normalizer_wiring.py @@ -0,0 +1,251 @@ +"""Wiring tests for ImageNormalizer at protocol entry points (RFC-0059). + +Verify the RFC-0059 normalization is actually applied at the three +user-upload entry points: the OpenCode ``extract_user_prompt_from_parts`` +converter, the ACP ``from_acp_content`` converter, and the functional +``run_agent`` wrapper. +""" + +from __future__ import annotations + +import base64 +import io +import random + +from PIL import Image +import pytest + +from wolfharness.images.normalizer import ImageNormalizer +from wolfharness_config.attachment import AttachmentImageConfig + + +pytestmark = pytest.mark.unit + + +def _noisy_png_bytes(width: int, height: int) -> bytes: + rng = random.Random(42) + img = Image.new("RGB", (width, height)) + pixels = [ + (rng.randint(0, 255), rng.randint(0, 255), rng.randint(0, 255)) + for _ in range(width * height) + ] + img.putdata(pixels) + buf = io.BytesIO() + img.save(buf, format="PNG") + return buf.getvalue() + + +def _data_uri(data: bytes, mime: str = "image/png") -> str: + return f"data:{mime};base64,{base64.b64encode(data).decode('ascii')}" + + +# ============================================================================= +# OpenCode: extract_user_prompt_from_parts +# ============================================================================= + + +async def _extract_prompt(part_input: object, normalizer: ImageNormalizer | None) -> list: + from wolfharness_server.opencode_server.converters import ( + extract_user_prompt_from_parts, + ) + + parts_list = await extract_user_prompt_from_parts( + [part_input], + "wire-test-session", + agent=None, + normalizer=normalizer, + ) + return list(parts_list) + + +async def test_extract_user_prompt_normalizes_oversized_image() -> None: + """Oversized data-URI image in FilePartInput is resized by the normalizer.""" + from pydantic_ai import BinaryContent + + from wolfharness_server.opencode_server.models import FilePartInput + + data = _noisy_png_bytes(800, 600) + part = FilePartInput( + mime="image/png", + url=_data_uri(data), + filename="photo.png", + ) + normalizer = ImageNormalizer(AttachmentImageConfig(max_base64_bytes=64 * 1024)) + + result = await _extract_prompt(part, normalizer) + + assert len(result) == 1 + assert isinstance(result[0], BinaryContent) + assert result[0].media_type in ("image/png", "image/jpeg") + assert len(result[0].data) < len(data) + + +async def test_extract_user_prompt_leaves_small_image_untouched() -> None: + """Small data-URI image passes through as-is (no resize).""" + from pydantic_ai import BinaryContent + + from wolfharness_server.opencode_server.models import FilePartInput + + data = _noisy_png_bytes(100, 100) + part = FilePartInput( + mime="image/png", + url=_data_uri(data), + filename="small.png", + ) + normalizer = ImageNormalizer() + + result = await _extract_prompt(part, normalizer) + + assert len(result) == 1 + assert isinstance(result[0], BinaryContent) + assert result[0].data == data + + +async def test_extract_user_prompt_non_image_not_normalized() -> None: + """Non-image FilePartInput is not routed through the image normalizer.""" + from pydantic_ai import BinaryContent + + from wolfharness_server.opencode_server.models import FilePartInput + + pdf_bytes = b"%PDF-1.4 test-not-a-real-pdf" + part = FilePartInput( + mime="application/pdf", + url=_data_uri(pdf_bytes, "application/pdf"), + filename="doc.pdf", + ) + normalizer = ImageNormalizer(AttachmentImageConfig(auto_resize=False, max_base64_bytes=8)) + + result = await _extract_prompt(part, normalizer) + + assert len(result) == 1 + assert isinstance(result[0], BinaryContent) + assert result[0].data == pdf_bytes + + +async def test_extract_user_prompt_no_normalizer_passthrough() -> None: + """Without a normalizer the converter is unchanged (backward compatible).""" + from pydantic_ai import BinaryContent + + from wolfharness_server.opencode_server.models import FilePartInput + + data = _noisy_png_bytes(800, 600) + part = FilePartInput(mime="image/png", url=_data_uri(data), filename="p.png") + + result = await _extract_prompt(part, None) + + assert len(result) == 1 + assert isinstance(result[0], BinaryContent) + assert result[0].data == data + + +# ============================================================================= +# ACP: from_acp_content +# ============================================================================= + + +def _from_acp(block: object, normalizer: ImageNormalizer | None): + from wolfharness_server.acp_server.converters import from_acp_content + + return from_acp_content(block, fs=None, normalizer=normalizer) + + +def test_from_acp_content_normalizes_oversized_image() -> None: + """Oversized ACP ImageContentBlock is resized by the normalizer.""" + from pydantic_ai import BinaryContent + + from acp.schema import ImageContentBlock + + data = _noisy_png_bytes(800, 600) + block = ImageContentBlock( + data=base64.b64encode(data).decode("ascii"), + mime_type="image/png", + ) + normalizer = ImageNormalizer(AttachmentImageConfig(max_base64_bytes=64 * 1024)) + + result = _from_acp(block, normalizer) + + assert isinstance(result, BinaryContent) + assert len(result.data) < len(data) + + +def test_from_acp_content_leaves_small_image_untouched() -> None: + """Small ACP ImageContentBlock passes through as-is.""" + from pydantic_ai import BinaryContent + + from acp.schema import ImageContentBlock + + data = _noisy_png_bytes(100, 100) + block = ImageContentBlock( + data=base64.b64encode(data).decode("ascii"), + mime_type="image/png", + ) + normalizer = ImageNormalizer() + + result = _from_acp(block, normalizer) + + assert isinstance(result, BinaryContent) + assert result.data == data + + +def test_from_acp_content_no_normalizer_passthrough() -> None: + """Without a normalizer the ACP converter is unchanged.""" + from pydantic_ai import BinaryContent + + from acp.schema import ImageContentBlock + + data = _noisy_png_bytes(800, 600) + block = ImageContentBlock( + data=base64.b64encode(data).decode("ascii"), + mime_type="image/png", + ) + + result = _from_acp(block, None) + + assert isinstance(result, BinaryContent) + assert result.data == data + + +# ============================================================================= +# functional run_agent: image_url normalization +# ============================================================================= + + +def test_make_image_normalizer_defaults() -> None: + """_make_image_normalizer returns None for None config.""" + from wolfharness.functional.run import _make_image_normalizer + + assert _make_image_normalizer(None) is None + + +def test_make_image_normalizer_with_config() -> None: + """_make_image_normalizer builds a normalizer from config.""" + from wolfharness.functional.run import _make_image_normalizer + + n = _make_image_normalizer(AttachmentImageConfig(auto_resize=False)) + + assert n is not None + assert n.config.auto_resize is False + + +def test_normalize_image_url_resizes_data_uri() -> None: + """_normalize_image_url normalizes oversized data URI when normalizer set.""" + from wolfharness.functional.run import _normalize_image_url + + data = _noisy_png_bytes(800, 600) + url = _data_uri(data) + normalizer = ImageNormalizer(AttachmentImageConfig(max_base64_bytes=64 * 1024)) + + normalized = _normalize_image_url(url, normalizer) + + assert normalized != url + payload = normalized.split(";base64,", 1)[1] + assert len(payload) <= 64 * 1024 + + +def test_normalize_image_url_no_normalizer_unchanged() -> None: + """_normalize_image_url returns the URL unchanged without a normalizer.""" + from wolfharness.functional.run import _normalize_image_url + + url = "https://example.com/photo.png" + + assert _normalize_image_url(url, None) == url