From ca20e9aa827e6a735d194c13a9bada747bbf105d Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Tue, 18 Aug 2026 11:29:57 +0800 Subject: [PATCH 1/5] docs(rfc): add RFC-0061 information-preserving degradation for modality filter * docs(rfc): design discussion for modality_filter degradation Proposes replacing the bare MIME placeholder ([image/png]) that ModalityFilterCapability emits when degrading unsupported multimodal content with an information-preserving placeholder carrying retrievable metadata (filename/mime/source), and optionally a session-scoped persisted reference so a text-only model can delegate the image to a vision-capable subagent or file tool. The problem is grounded in opencode ecosystem findings (HEAD 040b856): bare placeholders were rejected in PR #29279 on hallucination risk, are reported as a defect in issue #42758 (no way to access actual content), and the community consensus (#29216) is to preserve the reference rather than destroy it. Auto model-switching is deliberately out of scope (NOT_PLANNED upstream). Surveys 4 options (status quo / metadata-only / persist+reference / hybrid) with an evaluation matrix and recommends the hybrid: upgrade describe() to honest metadata now, add an opt-in reference strategy for manifests that declare vision-capable consumers. --- ...lter-information-preserving-degradation.md | 445 ++++++++++++++++++ 1 file changed, 445 insertions(+) create mode 100644 docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md diff --git a/docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md b/docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md new file mode 100644 index 000000000..3a383946c --- /dev/null +++ b/docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md @@ -0,0 +1,445 @@ +--- +rfc_id: RFC-0061 +title: "Information-Preserving Degradation for ModalityFilter: From Bare Placeholders to Retrievable References" +status: DRAFT +author: pinjun.mo +reviewers: [] +created: 2026-08-18 +last_updated: 2026-08-18 +decision_date: +related_rfcs: + - RFC-0059 (Image Attachment Normalization: Resize and Re-encode Oversized Images Before Provider Requests) +related_specs: [] +--- + +# RFC-0061: Information-Preserving Degradation for ModalityFilter + +## Table of Contents + +- [Overview](#overview) +- [Background & Context](#background--context) +- [Problem Statement](#problem-statement) +- [Goals & Non-Goals](#goals--non-goals) +- [Evaluation Criteria](#evaluation-criteria) +- [Options Analysis](#options-analysis) +- [Recommendation](#recommendation) +- [Technical Design](#technical-design) +- [Security Considerations](#security-considerations) +- [Implementation Plan](#implementation-plan) +- [Open Questions](#open-questions) +- [Decision Record](#decision-record) +- [References](#references) + +--- + +## Overview + +AgentPool's `ModalityFilterCapability` degrades multimodal content that the active model does not support. Its default `describe` strategy replaces an unsupported image with a **bare MIME placeholder** — e.g. `[image/png]` — before the provider request is sent. This RFC argues that the placeholder destroys the **retrievability** of the content: the text-only model receives a token with no filename, no location, and no way to hand the image to a vision-capable tool or subagent, so the original user intent (analyze *this* image) is silently lost. + +Unlike RFC-0059 (which constrains image *size* for vision-capable providers), this RFC is about what happens when the provider **cannot read the image at all**. It proposes replacing the bare placeholder with an **information-preserving degradation** that keeps retrievable metadata on the image, and discusses whether AgentPool should persist unsupported media so that other agents — vision-capable subagents, file tools, or MCP resources — can actually access it. + +The design question posed here is deliberately left open for discussion. The RFC surveys the candidate strategies and presents the trade-offs, mirroring how the opencode community converged on "don't destroy the image reference in the first place" without yet landing an implementation. + +--- + +## Background & Context + +### Current State + +`ModalityFilterCapability` (in `src/wolfharness/capabilities/modality_filter.py`) is an opt-in capability that is **not auto-injected**. It is enabled by declaring `type: modality_filter` in a manifest's capabilities and configuring a per-category strategy (`describe` / `drop` / `pass`): + +- `describe` (default): replace unsupported content with a text placeholder. +- `drop`: remove the content entirely. +- `pass`: forward the content unchanged. + +The degradation is applied in two places: + +1. **`before_model_request`** — scans `ModelRequest` / `ModelResponse` messages and rewrites unsupported content in `UserPromptPart` and `ToolReturnPart` via `dataclasses.replace()`. +2. **`wrap_tool_execute`** — filters multimodal content in tool results. + +The placeholder itself comes from `describe_multimodal_content()` in `src/wolfharness/capabilities/modality_utils.py`: + +```python +case BinaryImage(media_type=media) | BinaryContent(media_type=media): + return f"[{media}]" # e.g. "[image/png]" +``` + +This function has 12 call sites, most of which are genuinely for **logging / display / persistence** (`helpers.py:_summarize_content_block` documents itself as "for logging/display"). The problem addressed by this RFC is that **the same placeholder is reused as the degradation payload delivered to the model**, where a bare MIME token is neither visible content nor a usable reference. + +### Reference Implementation: opencode + +The opencode ecosystem has explored the exact same problem and produced a documented consensus. Findings from the `anomalyco/opencode` codebase (HEAD `040b856`): + +**Current behavior — error text, not a bare placeholder.** `unsupportedParts()` in `packages/opencode/src/provider/transform.ts:410-442` rewrites the unsupported part to: + +``` +ERROR: Cannot read "photo.png" (this model does not support image input). Inform the user. +``` + +**The bare-placeholder form was explicitly rejected.** PR [#29279](https://github.com/anomalyco/opencode/pull/29279) attempted to replace the error text with `[Attached image: "photo.png" (image/png)]`. Reviewers rejected it on hallucination risk: + +> "Calling something 'Attached' when the model has no direct access to its content is misleading and invites the LLM to invent details." + +Its counter-proposal (still unmerged) carried three signals — a factual claim, an anti-hallucination guardrail, and an escape hatch: + +``` +[User provided image: "photo.png" (image/png). + Direct processing unsupported — available via tools and filesystem.] +``` + +**The bare placeholder is a known bug complaint.** The `stripMedia` path in `packages/opencode/src/session/message-v2.ts:213-218` produces `[Attached image/png: file]`. Issue [#42758](https://github.com/anomalyco/opencode/issues/42758) (author environment: `deepseek-v4-flash`, a text-only model — the same downstream scenario motivating this RFC) reports that this leaves the agent with **no way to access the actual image content**, since the placeholder carries no reference to how the image can be retrieved. + +**Community consensus: don't destroy the reference.** Issue [#29216](https://github.com/anomalyco/opencode/issues/29216) articulates the design philosophy that two further open issues (`#36006`, `#40495`) and three open PRs (`#32680`, `#26164`, `#21633` — persist unsupported images to temp files for vision MCP tools) all converge on: + +> "don't destroy the image reference in the first place" — the model doesn't need native vision; it needs an undestroyed reference (file path / MIME / filename) so it can dispatch to an MCP tool. + +**Automatic model switching is explicitly not planned.** Issues `#32601` (NOT_PLANNED) and `#31936` (CLOSED) confirm opencode will not auto-switch to a vision model when an image arrives. + +In short: the opencode community identifies bare-placeholder degradation as a defect, agrees the fix should preserve retrievability, and has not shipped it. AgentPool has an opportunity to design this properly rather than inheriting the same defect. + +--- + +## Problem Statement + +When `ModalityFilterCapability` degrades an image to `[image/png]` inside the provider request: + +1. **The content is unrecoverable by the model.** The placeholder is a MIME-only token. It carries no filename, no path, no identifier, and no hint of how the image could be opened. +2. **Delegation is impossible.** AgentPool's native agents can delegate to subagents and call tools. A vision-capable subagent or a file-reading tool would be perfectly able to consume a *reference*, but a bare placeholder gives the delegating model nothing to pass along. +3. **The degradation is misleading.** It reads like a description of content that is actually absent, which — exactly as opencode's reviewer warned — invites the model to invent details about an image it has never seen. +4. **The failure is silent.** The user attached an image with an intent; the session proceeds as if that intent were preserved. There is no surfaced signal that the image was not understood. + +The same defect affects `describe` degradation in every modality category (image/audio/video/document) and every content type (`BinaryContent`, `BinaryImage`, `ImageUrl`, `AudioUrl`, `VideoUrl`, `DocumentUrl`, `UploadedFile`). + +--- + +## Goals & Non-Goals + +### Goals + +- Preserve **retrievable metadata** (filename, MIME, storage location where applicable) in the degradation output so the model and downstream agents can act on it. +- Support the realistic working pattern: **text-only primary model + vision-capable subagent or tool** consuming the degraded content via AgentPool's existing delegation / tool infrastructure. +- Make the degradation **explicit and honest**: signal "this content exists but was not directly read" rather than pretending it was. +- Keep the change contained to the `modality_filter` capability layer; no protocol server rewrites. + +### Non-Goals + +- Auto-switch models when unsupported content arrives (opencode's NOT_PLANNED position; aligned with AgentPool's explicit model-pinning design). +- Preserve the *bytes* of unsupported media when no consumer exists — persistence should be scoped to retrievability, not archival. +- Change what vision-capable models receive. When the model supports a modality, content passes through unchanged and RFC-0059 normalization applies. +- Resolve the storage backend question in this RFC (see [Open Questions](#open-questions)); this RFC defines the degradation surface, storage is the implementation detail. + +--- + +## Evaluation Criteria + +| Criterion | Question it answers | +|-----------|---------------------| +| Retrievability | Can a downstream agent (subagent/tool) actually obtain the content from the degradation output? | +| Honesty | Does the output truthfully represent "content exists, not directly read" vs. pretending content was parsed? | +| Hallucination resistance | Does the wording discourage the model from inventing details about unseen content? | +| Boundary safety | Can the output be safely surfaced in logs, prompts, and (if persisted) an HTTP server without leaking or breaking? | +| Backward compatibility | Does the change keep working for existing manifests that rely on `describe` today? | +| Implementation complexity | How much of the capability layer / storage layer must change? | + +--- + +## Options Analysis + +### Option 1: Status Quo — Bare MIME Placeholder + +**Description**: Keep `describe` producing `[image/png]` (and `[image: ]` for URL types). + +**Advantages**: +- Zero change; current tests pass. +- Byte-lean prompt footprint. + +**Disadvantages**: +- Unrecoverable content, no delegation path, silent failure, hallucination-prone (see [Problem Statement](#problem-statement)). +- Directly contradicted by opencode PR #29279 review and issue #42758 — this is the documented defect. + +**Evaluation Against Criteria**: + +| Criterion | Rating | Notes | +|-----------|--------|-------| +| Retrievability | 1/5 | Nothing to retrieve | +| Honesty | 1/5 | Reads like content, content absent | +| Hallucination resistance | 1/5 | Invites invention | +| Boundary safety | 4/5 | No new attack surface | +| Backward compatibility | 5/5 | Unchanged | +| Implementation complexity | 5/5 | None | + +**Effort Estimate**: None. + +--- + +### Option 2: Informational Metadata Placeholder (no persistence) + +**Description**: Keep the degradation as a *text* replacement, but enrich the placeholder with retrievable metadata and explicit signals. Modeled on opencode PR #29279's counter-proposal: + +``` +[User supplied image: "photo.png" (image/png). + Direct model processing is unsupported by the active model. + The file is NOT inlined into this context — a vision-capable subagent or file tool may open the original.] +``` + +For URL types: `[image: https://...]` already carries a reference and could stay as-is. For `BinaryContent`/`BinaryImage` (which have no filename), the placeholder would degrade to `[User supplied image (image/png), source not preserved]` — honest about what is *not* available. + +**Advantages**: +- Cheap: confined to `describe_multimodal_content()` / the `describe` branch in `modality_filter.py`. +- Honest and hallucination-resistant wording; aligns with opencode's accepted direction. +- No storage, no security surface. + +**Disadvantages**: +- **Still not retrievable in the binary case**: `BinaryContent`/`BinaryImage` have no on-disk origin, so the metadata placeholder still can't hand anything to a tool. +- Requires a vision-capable *tool* that accepts a filename — AgentPool today has `read` (fsspec) which can open local paths, so the pattern is realistic only for content that originated on disk. + +**Evaluation Against Criteria**: + +| Criterion | Rating | Notes | +|-----------|--------|-------| +| Retrievability | 2/5 | Filename present, bytes not persisted for binary input | +| Honesty | 5/5 | Explicit "unsupported / not inlined" signal | +| Hallucination resistance | 5/5 | Active discouragement wording | +| Boundary safety | 4/5 | Metadata only; filename is the only new surface | +| Backward compatibility | 4/5 | New string shape is a drop-in for downstream parsers; tests updated | +| Implementation complexity | 4/5 | Single function + tests | + +**Effort Estimate**: Low (one function + unit tests + one behavior test). + +--- + +### Option 3: Session-Scoped Persistence + Retrievable Reference + +**Description**: When `describe` (or a new `reference` strategy) degrades binary multimodal content, persist the bytes into the session storage under a deterministic key, and produce a reference the model can hand to tools: + +``` +[User supplied image: "photo.png" (image/png). + Direct processing unsupported — the file is persisted for this session. + A vision-capable subagent can access it via ] +``` + +The persisted reference is designed to be consumable by AgentPool's existing delegation model: the primary agent can spawn a vision-capable subagent and pass the reference in the prompt, and the subagent's file/`read` tools open it. This is exactly the pattern opencode PRs #32680 / #21633 / #26164 tried to ship, restated in AgentPool's session-storage terms. + +**Advantages**: +- **Truly retrievable** — the delegation loop closes: text model → reference → vision subagent. +- Solves the binary-content case (no on-disk origin) by creating one. +- Consistent with RFC-0059's storage-aware direction and with AgentPool's existing `StorageManager` / session persistence. + +**Disadvantages**: +- **Storage backend coupling**: requires deciding where bytes live (in-memory per session? SQLite? filesystem?), TTL/cleanup, and multi-process visibility. This is a real design fork this RFC does not close. +- **When does persistence run, if ever?** Degradation happens in `before_model_request`, which is synchronous and hot-path. Writing bytes there couples the capability to storage I/O. +- **Boundary risk**: exposing a reference means exposing bytes; must scope to the session and consider authz on any HTTP-served reference. +- If no vision-capable consumer exists in the manifest, persistence is wasted I/O. + +**Evaluation Against Criteria**: + +| Criterion | Rating | Notes | +|-----------|--------|-------| +| Retrievability | 5/5 | Bytes persisted, subagent can consume | +| Honesty | 5/5 | Explicit signal | +| Hallucination resistance | 5/5 | Active discouragement wording | +| Boundary safety | 3/5 | Storage-cleanness + potential HTTP exposure | +| Backward compatibility | 4/5 | New value-add; existing `describe` behavior superseded by configurable strategy | +| Implementation complexity | 2/5 | Capability + storage + lifecycle | + +**Effort Estimate**: High (new strategy or storage hook + lifecycle + tests + storage decision). + +--- + +### Option 4: Hybrid — Metadata First, Persistence Optional + +**Description**: Ship Option 2 (metadata placeholder) as the default behavior now, and add Option 3's persistence behind a *new explicit strategy* (e.g. `reference`) that manifests opt into when they actually have vision-capable consumers. A manifest that wants the delegation loop declares: + +```yaml +capabilities: + - type: modality_filter + image_strategy: reference # persist + emit retrievable reference +``` + +**Advantages**: +- Immediate defect fix (metadata honesty) with zero storage commitment. +- Persistence only engaged when a consumer exists — no wasted I/O, no new security surface for manifests that don't need it. +- Preserves `describe` semantics for existing manifests (backward compatible), adds `reference` for opt-in. +- Matches opencode's trajectory exactly: metadata placeholder as the agreed direction; persistence as the still-open PR. + +**Disadvantages**: +- Two strategies to maintain. +- `reference` still has all of Option 3's open storage questions. + +**Evaluation Against Criteria**: + +| Criterion | Rating | Notes | +|-----------|--------|-------| +| Retrievability | 4/5 | Metadata now; full retrievability when `reference` used | +| Honesty | 5/5 | Explicit signal in both modes | +| Hallucination resistance | 5/5 | Same wording | +| Boundary safety | 4/5 | Persistence opt-in → less default surface | +| Backward compatibility | 5/5 | `describe` unchanged, new strategy added | +| Implementation complexity | 3/5 | Two pieces, but second is opt-in | + +**Effort Estimate**: Medium (default metadata change + opt-in reference strategy + storage decision for reference). + +--- + +### Options Comparison Summary + +| Criterion | 1: Bare | 2: Metadata | 3: Persist+Ref | 4: Hybrid | +|-----------|---------|--------------|----------------|-----------| +| Retrievability | 1/5 | 2/5 | 5/5 | 4/5 | +| Honesty | 1/5 | 5/5 | 5/5 | 5/5 | +| Hallucination resistance | 1/5 | 5/5 | 5/5 | 5/5 | +| Boundary safety | 4/5 | 4/5 | 3/5 | 4/5 | +| Backward compatibility | 5/5 | 4/5 | 4/5 | 5/5 | +| Implementation complexity | 5/5 | 4/5 | 2/5 | 3/5 | +| **Total** | **17/30** | **24/30** | **24/30** | **26/30** | + +--- + +## Recommendation + +### Recommended Option + +**[Option 4: Hybrid — Metadata First, Persistence Optional]** — with Option 2 as the minimum acceptable landing scope. + +### Justification + +- Option 1 is the documented defect (opencode PR #29279 rejection, issue #42758) and does not meet the goals. +- Options 2 and 3 score equally overall, but Option 3 commits to a storage design this RFC deliberately leaves open. Shipping storage-backed degradation speculatively, for manifests that may have no vision-capable consumer, is over-engineering. +- Option 4 captures the immediate, cheap, broadly-shared win (honest, retrievable-by-filename metadata — the opencode-consensus direction) and defers the costly, uncertain part (persistence) behind an explicit opt-in strategy that only engages when the user actually wants the delegaton loop. +- It is backward compatible: existing `describe` manifests keep their current behavior (upgraded to the informative string shape); the new `reference` strategy is opt-in. + +### Accepted Trade-offs + +1. **Filename-preserving degradation still needs a consumer.** For binary input with no on-disk origin, Option 4's metadata form (like Option 2) cannot hand a tool anything; full retrievability requires `reference`, which is out of scope for the initial landing unless reviewers prefer Option 3. +2. **`reference` inherits storage questions.** Persistence backend, TTL, cleanup, and multi-process visibility remain open (see [Open Questions](#open-questions)); they bubble to whichever option includes persistence. + +### Conditions + +- The metadata placeholder wording must include the anti-hallucination guardrail "direct model processing unsupported" (not just a filename). +- The `reference` strategy, if added, must be documented as requiring a vision-capable subagent or file tool in the manifest. +- Any persistence must be scoped to the session and cleaned up with the session (consistent with AgentPool's session-scoped storage). + +--- + +## Technical Design (Preliminary) + +> To be finalized after approval. Draft for review. + +### Architecture Overview + +``` +Unsupported image input + │ + ▼ +ModalityFilterCapability._filter_single_content() + │ strategy = describe (default) + ▼ +describe_multimodal_content() ──NEW──▶ information-preserving placeholder + │ (filename · mime · "unsupported" signal) + │ + ├── default: text replacement in request + │ + └── strategy = reference (opt-in, if approved) + ▼ + session-scoped persist ──▶ retrievable reference in prompt + │ + ▼ + text-only model delegates to + vision-capable subagent / file tool +``` + +### Key Components + +#### `describe_multimodal_content()` (information-preserving variant) + +- Signature unchanged: `(content: MultiModalContent) -> str`. +- Output string now depends on which metadata is available: + - `ImageUrl` / `AudioUrl` / `VideoUrl` / `DocumentUrl` (URL types): keep `[image: ]` — the URL *is* a retrievable reference. + - `BinaryImage` / `BinaryContent`: emit `[User supplied — direct model processing is unsupported by the active model (not inlined); a vision-capable subagent or file tool may open the original if it is on disk]`. + - `UploadedFile`: emit `[User supplied uploaded file (file_id: )]` — the id is a reference if the upload store is queryable. + +#### New strategy: `reference` (opt-in, pending discussion) + +- Persist degraded binary bytes to session storage under a deterministic key. +- Emit the reference into the replacement text with the same honesty guardrails. +- Never runs unless the manifest declares `image_strategy: reference`. + +### Data Flow + +1. Manifest declares `modality_filter` capability (existing) — defaults unchanged. +2. Degradation path in `before_model_request` now produces honest metadata text (Option 2 behavior, default). +3. If `strategy: reference`, bytes are persisted first, then the reference is emitted. +4. The model may delegate the reference to a vision-capable subagent (existing delegation infra) which opens it via file tools (existing `read` / fsspec). + +### API Design + +``` +# Existing (unchanged) +describe_multimodal_content(content) -> str # now information-preserving + +# New (if Option 3/4's reference strategy is approved) +enum ModalityStrategy += "reference" +ModalityFilterCapability.reference_strategy(...) # persists + emits reference +``` + +--- + +## Security Considerations + +- **Placeholder text is user-influenced**: filenames are provided by the caller/OS. They are already interpolated today (in logs); emitting them into the model prompt requires care with control characters and prompt-injection-ish filenames. Recommend rendering filenames via a safe repr/escape before interpolation. +- **Persistence is a new write path**: any `reference`/Option-3 persistence must scope bytes to the session, bound total size (reuse RFC-0059's byte limits), and guarantee cleanup on session teardown. Persisted references must never be world-readable by default. +- **No new secrets surface**: unsupported media can contain sensitive pixels (screenshots). Emitting "persisted at " only makes sense inside an already-trusted runtime; do not echo full absolute paths into prompts unless the deployment is trusted. + +--- + +## Implementation Plan + +### Phase 1 — Honest metadata placeholder (Option 2 scope, in-core) + +1. Rewrite `describe_multimodal_content()` to produce the information-preserving strings above. +2. Update unit tests in `tests/test_modality_utils.py` and the modality-filter behavior tests (`tests/test_modality_filter.py`, `tests/test_agent_factory_modality.py`). +3. Keep URL-type placeholders unchanged (already references). +4. Add a changelog entry under `changelog/unreleased/`. + +### Phase 2 — Opt-in `reference` strategy (only if reviewers approve the storage direction) + +1. Storage decision first (see [Open Questions](#open-questions)). +2. Add `reference` to `ModalityStrategy`; wire persistence in the degradation path. +3. VCR/E2E test: text-only primary + vision-capable subagent consuming the persisted reference. + +### Phase 3 — Docs + +- Update `docs/rfcs/draft/` status, `docs/reference` for the capability, and the configuration reference (`docs/how-to/` modality-filter page if present). + +--- + +## Open Questions + +1. **Is the bare placeholder genuinely a defect, or intended lossy behavior?** The RFC assumes the former; reviewers may decide the loss is acceptable for byte-budget reasons. +2. **Does AgentPool want a storage-backed `reference` strategy at all?** opencode's community wants it but hasn't shipped it; AgentPool can differentiate. (Opencode status: PRs #32680 / #21633 / #26164 open, unmerged.) +3. **Where do persisted bytes live?** In-memory per-run, session SQLite, filesystem scratch, or a host-served reference? Affects multi-process and restart visibility. +4. **Should degradation emit a surfaced signal to the caller** (e.g. an event) so the user knows their image was not read — beyond the honest placeholder? +5. **Filename safety**: should the replacement text escape/redact filenames before interpolation? + +--- + +## Decision Record + +- **2026-08-18**: RFC opened as DRAFT by pinjun.mo. Problem identified while debugging a real deployment (`glm52`/kimi-k2, a text-only model, failing on pasted images via the opencode server). Three opencode-codebase investigations (HEAD `040b856`) support the problem framing. No decision made yet. + +--- + +## References + +- `src/wolfharness/capabilities/modality_filter.py` — current degradation implementation +- `src/wolfharness/capabilities/modality_utils.py` — `describe_multimodal_content()` (placeholder source) +- `src/wolfharness/agents/native_agent/helpers.py` — `_summarize_content_block` (logging/display call site; documents the dual-use tension) +- RFC-0059: Image Attachment Normalization (Resize and Re-encode Oversized Images) +- opencode `anomalyco/opencode` HEAD `040b856`: + - `packages/opencode/src/provider/transform.ts:410-442` — `unsupportedParts()` (error-text degradation) + - `packages/opencode/src/session/message-v2.ts:213-218` — `stripMedia` bare-placeholder path + - PR [#29279](https://github.com/anomalyco/opencode/pull/29279) — bare placeholder rejected (hallucination risk) + - Issue [#42758](https://github.com/anomalyco/opencode/issues/42758) — "agent has no way to access the actual image content" + - Issue [#29216](https://github.com/anomalyco/opencode/issues/29216) — "don't destroy the image reference" + - Issues [#32601](https://github.com/anomalyco/opencode/issues/32601) / [#31936](https://github.com/anomalyco/opencode/issues/31936) — auto-switch NOT_PLANNED + - PRs [#32680](https://github.com/anomalyco/opencode/pull/32680) / [#21633](https://github.com/anomalyco/opencode/pull/21633) / [#26164](https://github.com/anomalyco/opencode/pull/26164) — persist-to-temp-file attempts (open, unmerged) \ No newline at end of file From d065144645e3309ea5205968e20a59107851b740 Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Wed, 19 Aug 2026 15:40:34 +0800 Subject: [PATCH 2/5] feat(modality): information-preserving degradation + reference strategy Phase 1 (RFC-0061): describe_multimodal_content() no longer emits a bare [image/png] placeholder. Binary content now states its media type, that direct model processing is unsupported, and whether a file identifier is available. Control characters in caller-supplied identifiers are escaped to prevent prompt injection via malformed filenames. URL content keeps its [image: url] / [audio: url] form since the URL is already retrievable. Phase 2: new opt-in 'reference' strategy persists binary content to a per-session scratch directory and replaces it with a [file: ] reference that a vision-capable subagent or the agent's read tool can open. Scratch dirs live under tempfile.gettempdir()/wolfharness-modality/ {session_id}/; after_node_run() removes them. URL and UploadedFile content has no local bytes and falls back to describe. The strategy is wired through the ModalityFilterCapabilityConfig schema. --- ...19-modality-info-preserving-degradation.md | 25 +++ .../capabilities/modality_filter.py | 126 +++++++++++++--- .../capabilities/modality_utils.py | 61 +++++++- src/wolfharness_config/capabilities.py | 8 +- tests/orchestrator/test_multimodal_storage.py | 19 ++- tests/test_modality_e2e.py | 15 +- tests/test_modality_filter.py | 142 ++++++++++++++++-- tests/test_modality_utils.py | 51 ++++++- 8 files changed, 387 insertions(+), 60 deletions(-) create mode 100644 changelog/unreleased/2026-08-19-modality-info-preserving-degradation.md diff --git a/changelog/unreleased/2026-08-19-modality-info-preserving-degradation.md b/changelog/unreleased/2026-08-19-modality-info-preserving-degradation.md new file mode 100644 index 000000000..4f2e5d40a --- /dev/null +++ b/changelog/unreleased/2026-08-19-modality-info-preserving-degradation.md @@ -0,0 +1,25 @@ +# Information-preserving degradation for modality filter + +The `ModalityFilterCapability` `describe` strategy previously replaced +unsupported multimodal content with a bare MIME placeholder such as +`[image/png]`. That token carried no filename, path, or identifier, so the +model could never retrieve the underlying content — a vision-capable +subagent or file tool had nothing to open, and the placeholder misled the +model into inventing content it never saw. + +The placeholder is now **information-preserving** (RFC-0061): binary content +states its media type, that direct model processing is unsupported, and +whether a file identifier is available; control characters in caller-supplied +identifiers are escaped to prevent prompt injection via malformed filenames. +URL-type content keeps its `[image: url]` / `[audio: url]` form since the URL +is already retrievable. + +A new opt-in `reference` strategy is added for each modality: instead of a +placeholder, the content bytes are persisted to a per-session scratch +directory under `tempfile.gettempdir()/wolfharness-modality/{session_id}/` and +replaced with a `[file: ]` reference a vision-capable subagent or the +agent's `read` tool can open. The directory is removed by +`after_node_run()`. URL and `UploadedFile` content has no local bytes and +falls back to `describe`. + +Resolves wolf1069b/wolfharness#377. \ No newline at end of file diff --git a/src/wolfharness/capabilities/modality_filter.py b/src/wolfharness/capabilities/modality_filter.py index ac1bbabc6..8f663ba36 100644 --- a/src/wolfharness/capabilities/modality_filter.py +++ b/src/wolfharness/capabilities/modality_filter.py @@ -7,6 +7,11 @@ - ``describe``: replace the content with a text placeholder via ``describe_multimodal_content``. +- ``reference``: persist the binary content to a per-run scratch + directory and replace it with a ``[file: ]`` reference that a + vision-capable subagent or file tool can open (RFC-0061). URL and + ``UploadedFile`` content has no local bytes, so it falls back to + ``describe``. - ``drop``: remove the content entirely. - ``pass``: leave the content unchanged (no filtering). @@ -25,7 +30,12 @@ from __future__ import annotations import dataclasses +import hashlib import logging +import mimetypes +from pathlib import Path +import shutil +import tempfile from typing import TYPE_CHECKING, Any, Literal, assert_never from pydantic_ai import BinaryContent, BinaryImage @@ -44,6 +54,7 @@ VideoUrl, ) +from wolfharness.agents.context import AgentRunContext from wolfharness.capabilities.modality_utils import ( BinaryCategory, classify_binary_content, @@ -53,7 +64,11 @@ if TYPE_CHECKING: from pydantic_ai import RunContext - from pydantic_ai.capabilities import WrapToolExecuteHandler + from pydantic_ai.capabilities import ( + AgentNode, + NodeResult, + WrapToolExecuteHandler, + ) from pydantic_ai.messages import ToolCallPart from pydantic_ai.models import ModelRequestContext from pydantic_ai.tools import ToolDefinition @@ -64,7 +79,7 @@ logger = logging.getLogger(__name__) -type ModalityStrategy = Literal["describe", "drop", "pass"] +type ModalityStrategy = Literal["describe", "reference", "drop", "pass"] _FALLBACK_DROP_TEXT = "[Tool returned only unsupported multimodal content]" @@ -105,10 +120,76 @@ class ModalityFilterCapability(AbstractCapability[Any]): video_strategy: ModalityStrategy = "describe" document_strategy: ModalityStrategy = "describe" + _scratch_dirs: set[Path] = dataclasses.field(default_factory=set, init=False, repr=False) + @property def has_wrap_node_run(self) -> bool: return False + # ---- Scratch root (RFC-0061 `reference` strategy) ---- + + def _scratch_dir(self, ctx: RunContext[Any]) -> Path: + """Return the per-run scratch directory for ``reference`` writes. + + Rooted at ``tempfile.gettempdir()/wolfharness-modality/{session_id}`` + so different runs never collide and the OS can reclaim stale + directories. Falls back to a shared run-id-scoped directory when + no session id is available (standalone ``agent.run()``). + """ + base = Path(tempfile.gettempdir()) / "wolfharness-modality" + session_id = self._session_id(ctx) + scratch = base / session_id + scratch.mkdir(parents=True, exist_ok=True) + self._scratch_dirs.add(scratch) + return scratch + + def _session_id(self, ctx: RunContext[Any]) -> str: + """Get the current run's session id, or a safe fallback.""" + run_ctx = ctx.deps.run_ctx if hasattr(ctx.deps, "run_ctx") else None + if isinstance(run_ctx, AgentRunContext): + return run_ctx.session_id or "default" + return str(getattr(run_ctx, "run_id", None) or "default") + + def _reference_content(self, content: MultiModalContent, ctx: RunContext[Any]) -> str: + """Persist binary content and return a ``[file: ]`` reference. + + URL types and ``UploadedFile`` have no local bytes to persist, so + they fall back to ``describe_multimodal_content``. + """ + if not isinstance(content, (BinaryContent, BinaryImage)): + return describe_multimodal_content(content) + + scratch = self._scratch_dir(ctx) + ext = mimetypes.guess_extension(content.media_type) or ".bin" + # Hash-based name so identical payloads dedupe and filenames stay + # filesystem-safe regardless of any caller-supplied identifier. + digest = hashlib.sha256(content.data, usedforsecurity=False).hexdigest()[:16] + path = scratch / f"content-{digest}{ext}" + path.write_bytes(content.data) + return f"[file: {path}]" + + async def after_node_run( + self, + ctx: RunContext[Any], + *, + node: AgentNode[Any], + result: NodeResult[Any], + ) -> NodeResult[Any]: + """Remove scratch directories written by this run. + + ``reference`` strategy persists degraded content to disk; the + session can be resumed later via restart, so cleanup happens only + when the run itself ends. Per-instance tracking avoids deleting + directories another concurrent run may still need. + """ + for scratch in self._scratch_dirs: + try: + shutil.rmtree(scratch, ignore_errors=True) + except OSError: + logger.warning("Failed to remove modality scratch dir: %s", scratch) + self._scratch_dirs.clear() + return result + # ---- Ordering ---- def get_ordering(self) -> CapabilityOrdering | None: @@ -154,7 +235,7 @@ async def wrap_tool_execute( handler: WrapToolExecuteHandler, ) -> Any: result = await handler(args) - return self._filter_tool_result(result) + return self._filter_tool_result(ctx, result) # ---- Pre-request message filtering ---- @@ -184,12 +265,12 @@ async def before_model_request( for msg in messages: match msg: case ModelRequest(): - filtered_req = self._filter_model_request(msg) + filtered_req = self._filter_model_request(ctx, msg) if filtered_req is not msg: changed = True new_messages.append(filtered_req) case ModelResponse(): - filtered_resp = self._filter_model_response(msg) + filtered_resp = self._filter_model_response(ctx, msg) if filtered_resp is not msg: changed = True new_messages.append(filtered_resp) @@ -203,7 +284,7 @@ async def before_model_request( # ---- Internal: tool result filtering ---- - def _filter_tool_result(self, result: Any) -> Any: + def _filter_tool_result(self, ctx: RunContext[Any], result: Any) -> Any: """Degrade multimodal content in a tool result. Handles ``str``, ``list``, and direct ``MultiModalContent``. @@ -212,10 +293,10 @@ def _filter_tool_result(self, result: Any) -> Any: case str(): return result case list(): - return self._filter_content_list(result) + return self._filter_content_list(ctx, result) case _: if isinstance(result, _MULTIMODAL_TYPES): - filtered = self._filter_single_content(result) # type: ignore[arg-type] + filtered = self._filter_single_content(ctx, result) # type: ignore[arg-type] match filtered: case str(): return filtered @@ -225,7 +306,7 @@ def _filter_tool_result(self, result: Any) -> Any: return filtered return result - def _filter_content_list(self, items: list[Any]) -> Any: + def _filter_content_list(self, ctx: RunContext[Any], items: list[Any]) -> Any: """Filter a list of content items. Returns the original list if nothing changed, a new list if @@ -236,7 +317,7 @@ def _filter_content_list(self, items: list[Any]) -> Any: changed = False for item in items: if isinstance(item, _MULTIMODAL_TYPES): - filtered = self._filter_single_content(item) # type: ignore[arg-type] + filtered = self._filter_single_content(ctx, item) # type: ignore[arg-type] match filtered: case None: changed = True @@ -259,6 +340,7 @@ def _filter_content_list(self, items: list[Any]) -> Any: def _filter_single_content( self, + ctx: RunContext[Any], content: ( BinaryContent | BinaryImage @@ -284,6 +366,8 @@ def _filter_single_content( match strategy: case "describe": return describe_multimodal_content(content) + case "reference": + return self._reference_content(content, ctx) case "drop": return None case "pass": @@ -295,7 +379,7 @@ def _filter_single_content( # ---- Internal: message filtering ---- - def _filter_model_request(self, msg: ModelRequest) -> ModelRequest: + def _filter_model_request(self, ctx: RunContext[Any], msg: ModelRequest) -> ModelRequest: """Filter multimodal content in a ``ModelRequest``. Returns the original message if nothing changed, or a new @@ -306,7 +390,7 @@ def _filter_model_request(self, msg: ModelRequest) -> ModelRequest: for part in msg.parts: match part: case UserPromptPart(): - new_part = self._filter_user_prompt_part(part) + new_part = self._filter_user_prompt_part(ctx, part) if new_part is not part: changed = True new_parts.append(new_part) @@ -317,7 +401,7 @@ def _filter_model_request(self, msg: ModelRequest) -> ModelRequest: return msg return dataclasses.replace(msg, parts=new_parts) - def _filter_model_response(self, msg: ModelResponse) -> ModelResponse: + def _filter_model_response(self, ctx: RunContext[Any], msg: ModelResponse) -> ModelResponse: """Filter multimodal content in a ``ModelResponse``. Returns the original message if nothing changed, or a new @@ -328,7 +412,7 @@ def _filter_model_response(self, msg: ModelResponse) -> ModelResponse: for part in msg.parts: match part: case ToolReturnPart(): - new_part = self._filter_tool_return_part(part) + new_part = self._filter_tool_return_part(ctx, part) if new_part is not part: changed = True new_parts.append(new_part) @@ -339,7 +423,9 @@ def _filter_model_response(self, msg: ModelResponse) -> ModelResponse: return msg return dataclasses.replace(msg, parts=new_parts) - def _filter_user_prompt_part(self, part: UserPromptPart) -> UserPromptPart: + def _filter_user_prompt_part( + self, ctx: RunContext[Any], part: UserPromptPart + ) -> UserPromptPart: """Filter multimodal content in a ``UserPromptPart``. ``UserPromptPart.content`` can be ``str`` or @@ -350,7 +436,7 @@ def _filter_user_prompt_part(self, part: UserPromptPart) -> UserPromptPart: case str(): return part case list(): - new_items = self._filter_content_list(content) + new_items = self._filter_content_list(ctx, content) match new_items: case list(): return dataclasses.replace(part, content=new_items) @@ -362,7 +448,9 @@ def _filter_user_prompt_part(self, part: UserPromptPart) -> UserPromptPart: case _: return part - def _filter_tool_return_part(self, part: ToolReturnPart) -> ToolReturnPart: + def _filter_tool_return_part( + self, ctx: RunContext[Any], part: ToolReturnPart + ) -> ToolReturnPart: """Filter multimodal content in a ``ToolReturnPart``. ``ToolReturnPart.content`` can be ``str``, ``MultiModalContent``, @@ -374,7 +462,7 @@ def _filter_tool_return_part(self, part: ToolReturnPart) -> ToolReturnPart: case str(): pass case list(): - new_items = self._filter_content_list(content) + new_items = self._filter_content_list(ctx, content) match new_items: case list() | str(): new_content = new_items @@ -382,7 +470,7 @@ def _filter_tool_return_part(self, part: ToolReturnPart) -> ToolReturnPart: pass case _: if isinstance(content, _MULTIMODAL_TYPES): - filtered = self._filter_single_content(content) # type: ignore[arg-type] + filtered = self._filter_single_content(ctx, content) # type: ignore[arg-type] match filtered: case str(): new_content = filtered diff --git a/src/wolfharness/capabilities/modality_utils.py b/src/wolfharness/capabilities/modality_utils.py index 8b4776c71..26aeae7ea 100644 --- a/src/wolfharness/capabilities/modality_utils.py +++ b/src/wolfharness/capabilities/modality_utils.py @@ -7,6 +7,7 @@ from __future__ import annotations +import re from typing import Literal, assert_never from pydantic_ai import BinaryContent, BinaryImage @@ -23,24 +24,76 @@ type BinaryCategory = Literal["image", "audio", "video", "document", "unknown"] +# Control characters that could break prompt formatting if a caller-supplied +# filename were interpolated verbatim (RFC-0061 Security considerations). +_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") + + +def _safe_ref(value: str) -> str: + r"""Sanitize a caller-influenced identifier (filename/path) for interpolation. + + Replaces control characters (newlines, tabs, NUL, ...) with a visible + ``\xNN`` escape so a malicious or malformed filename cannot inject + prompt structure or break log lines. Non-control characters pass through. + """ + + def _escape(match: re.Match[str]) -> str: + return f"\\x{ord(match.group()):02x}" + + return _CONTROL_CHARS.sub(_escape, value) + + +def _describe_binary(binary: BinaryImage | BinaryContent) -> str: + """Build an information-preserving placeholder for binary content. + + Uses the underlying ``_identifier`` field, which is only set when a + caller explicitly provided one (e.g. the tool that produced the bytes + knew the source path). The public ``.identifier`` property is avoided + because it falls back to a content hash that is not a retrievable + reference. + """ + media = binary.media_type + identifier = binary._identifier + if identifier: + safe_id = _safe_ref(identifier) + return ( + f"[User supplied {media} — direct model processing is " + f"unsupported by the active model. File: {safe_id} " + f"(may be opened by a vision-capable subagent or file tool)]" + ) + return ( + f"[User supplied {media} — direct model processing is " + f"unsupported by the active model (content not inlined; " + f"no file reference available)]" + ) + + def describe_multimodal_content(content: MultiModalContent) -> str: """Produce a short text placeholder for a pydantic-ai multimodal content item. Handles all ``MultiModalContent`` variant types, producing meaningful placeholders instead of raw ``repr()`` output for binary/URL types. + For URL types the URL itself is returned (it is already a retrievable + reference). For ``UploadedFile`` the file id is returned. For binary + content (``BinaryImage`` / ``BinaryContent``) the placeholder is + *information-preserving* (RFC-0061): it states the media type, whether an + ``identifier`` is available, and that direct model processing is + unsupported, rather than emitting a bare ``[image/png]`` token that + misleads the model into inventing content it cannot see. + Args: content: A ``MultiModalContent`` variant — ``BinaryImage``, ``BinaryContent``, ``ImageUrl``, ``AudioUrl``, ``VideoUrl``, ``DocumentUrl``, or ``UploadedFile``. Returns: - A lowercase placeholder string suitable for logs, ``ChatMessage.content``, - and other text-only contexts. + An information-preserving placeholder string suitable for logs, + ``ChatMessage.content``, and other text-only contexts. """ match content: - case BinaryImage(media_type=media) | BinaryContent(media_type=media): - return f"[{media}]" + case BinaryImage() | BinaryContent() as binary: + return _describe_binary(binary) case ImageUrl(url=url): return f"[image: {url}]" case AudioUrl(url=url): diff --git a/src/wolfharness_config/capabilities.py b/src/wolfharness_config/capabilities.py index 193890dc3..3317d0b4c 100644 --- a/src/wolfharness_config/capabilities.py +++ b/src/wolfharness_config/capabilities.py @@ -151,13 +151,13 @@ class ModalityFilterCapabilityConfig(BaseModel): """ type: Literal["modality_filter"] = "modality_filter" - image_strategy: Literal["describe", "drop", "pass"] = "describe" + image_strategy: Literal["describe", "reference", "drop", "pass"] = "describe" """Degradation strategy for unsupported image content.""" - audio_strategy: Literal["describe", "drop", "pass"] = "describe" + audio_strategy: Literal["describe", "reference", "drop", "pass"] = "describe" """Degradation strategy for unsupported audio content.""" - video_strategy: Literal["describe", "drop", "pass"] = "describe" + video_strategy: Literal["describe", "reference", "drop", "pass"] = "describe" """Degradation strategy for unsupported video content.""" - document_strategy: Literal["describe", "drop", "pass"] = "describe" + document_strategy: Literal["describe", "reference", "drop", "pass"] = "describe" """Degradation strategy for unsupported document content.""" diff --git a/tests/orchestrator/test_multimodal_storage.py b/tests/orchestrator/test_multimodal_storage.py index efaab099e..0bc4666fe 100644 --- a/tests/orchestrator/test_multimodal_storage.py +++ b/tests/orchestrator/test_multimodal_storage.py @@ -95,18 +95,20 @@ def test_extract_text_from_messages_empty() -> None: @pytest.mark.unit def test_summarize_binary_image() -> None: - """BinaryImage produces '[image/png]' placeholder.""" + """BinaryImage produces an information-preserving placeholder.""" img = BinaryImage(data=b"\x89PNG\r\n\x1a\n", media_type="image/png") result = _summarize_content_block(img) - assert result == "[image/png]" + assert "image/png" in result + assert "unsupported" in result @pytest.mark.unit def test_summarize_binary_content() -> None: - """BinaryContent produces '[audio/wav]' placeholder.""" + """BinaryContent produces an information-preserving placeholder.""" audio = BinaryContent(data=b"RIFF....", media_type="audio/wav") result = _summarize_content_block(audio) - assert result == "[audio/wav]" + assert "audio/wav" in result + assert "unsupported" in result @pytest.mark.unit @@ -285,7 +287,8 @@ def test_prompt_text_uses_summarize_content_block() -> None: assert "b'\\x89PNG" not in prompt_text assert "\\x89PNG" not in prompt_text # Should contain the meaningful placeholder - assert "[image/png]" in prompt_text + assert "image/png" in prompt_text + assert "unsupported" in prompt_text assert "hello" in prompt_text assert "describe this" in prompt_text @@ -306,7 +309,8 @@ def test_opencode_converter_uses_summarize_content_block() -> None: assert "text before image" in text assert "text after image" in text - assert "[image/png]" in text + assert "image/png" in text + assert "unsupported" in text assert "BinaryContent" not in text assert "b'\\x89PNG" not in text @@ -330,7 +334,8 @@ def test_compaction_extract_text_content_uses_summarize() -> None: result = _extract_text_content(msg) assert "describe this" in result - assert "[image/png]" in result + assert "image/png" in result + assert "unsupported" in result assert "BinaryContent" not in result diff --git a/tests/test_modality_e2e.py b/tests/test_modality_e2e.py index 0c0e90773..97a530dc1 100644 --- a/tests/test_modality_e2e.py +++ b/tests/test_modality_e2e.py @@ -126,7 +126,8 @@ async def test_tool_image_text_only_model_receives_placeholder() -> None: ) # The tool result should be a text placeholder, not a BinaryImage. assert isinstance(result, str) - assert result == "[image/png]" + assert "image/png" in result + assert "unsupported" in result # --------------------------------------------------------------------------- @@ -201,8 +202,10 @@ async def test_fallback_model_intersection_injects_filter() -> None: # Verify image content is degraded by this capability. # None is treated as unsupported via ``is True`` check → text-only behavior. img = _binary_image() - degraded = filter_caps[0]._filter_tool_result(img) - assert degraded == "[image/png]" + degraded = filter_caps[0]._filter_tool_result(None, img) # type: ignore[arg-type] + assert isinstance(degraded, str) + assert "image/png" in degraded + assert "unsupported" in degraded # --------------------------------------------------------------------------- @@ -240,7 +243,8 @@ async def test_history_tool_return_image_degraded_for_text_only() -> None: assert isinstance(new_msg, ModelResponse) new_part = new_msg.parts[0] assert isinstance(new_part, ToolReturnPart) - assert new_part.content == "[image/png]" + assert "image/png" in new_part.content + assert "unsupported" in new_part.content # --------------------------------------------------------------------------- @@ -367,7 +371,8 @@ async def test_before_model_request_does_not_mutate_original() -> None: assert result.messages is not ctx.messages new_part = result.messages[0].parts[0] # type: ignore[union-attr] assert isinstance(new_part, ToolReturnPart) - assert new_part.content == "[image/png]" + assert "image/png" in new_part.content + assert "unsupported" in new_part.content assert new_part.content is not original_content diff --git a/tests/test_modality_filter.py b/tests/test_modality_filter.py index de052647e..43113bf44 100644 --- a/tests/test_modality_filter.py +++ b/tests/test_modality_filter.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path +from types import SimpleNamespace from typing import Any from pydantic_ai import BinaryContent, BinaryImage @@ -19,6 +21,7 @@ from pydantic_ai.models.test import TestModel import pytest +from wolfharness.agents.context import AgentRunContext from wolfharness.capabilities.modality_filter import ModalityFilterCapability from wolfharness_config.model_capabilities import ModelCapabilities @@ -119,7 +122,8 @@ async def test_wrap_tool_image_describe_strategy() -> None: args={}, handler=_handler(img), ) - assert result == "[image/png]" + assert "image/png" in result + assert "unsupported" in result @pytest.mark.unit @@ -164,7 +168,8 @@ async def test_wrap_tool_audio_describe() -> None: args={}, handler=_handler(audio), ) - assert result == "[audio/wav]" + assert "audio/wav" in result + assert "unsupported" in result @pytest.mark.unit @@ -179,7 +184,8 @@ async def test_wrap_tool_video_describe() -> None: args={}, handler=_handler(video), ) - assert result == "[video/mp4]" + assert "video/mp4" in result + assert "unsupported" in result @pytest.mark.unit @@ -194,7 +200,8 @@ async def test_wrap_tool_document_describe() -> None: args={}, handler=_handler(pdf), ) - assert result == "[application/pdf]" + assert "application/pdf" in result + assert "unsupported" in result @pytest.mark.unit @@ -218,7 +225,8 @@ async def test_wrap_tool_mixed_content_list() -> None: assert isinstance(result, list) assert result[0] is img # image is supported, passes through assert result[1] == text - assert result[2] == "[audio/wav]" # audio is unsupported, described + assert "audio/wav" in result[2] # audio is unsupported, described + assert "unsupported" in result[2] @pytest.mark.unit @@ -268,7 +276,8 @@ async def test_wrap_tool_binary_content_audio_wav() -> None: args={}, handler=_handler(audio), ) - assert result == "[audio/wav]" + assert "audio/wav" in result + assert "unsupported" in result @pytest.mark.unit @@ -283,7 +292,8 @@ async def test_wrap_tool_binary_content_video_mp4() -> None: args={}, handler=_handler(video), ) - assert result == "[video/mp4]" + assert "video/mp4" in result + assert "unsupported" in result @pytest.mark.unit @@ -298,7 +308,8 @@ async def test_wrap_tool_binary_content_pdf() -> None: args={}, handler=_handler(pdf), ) - assert result == "[application/pdf]" + assert "application/pdf" in result + assert "unsupported" in result @pytest.mark.unit @@ -319,7 +330,8 @@ async def test_wrap_tool_binary_image_always_image() -> None: handler=_handler(img), ) # Should use image strategy, not audio strategy. - assert result == "[image/png]" + assert "image/png" in result + assert "unsupported" in result @pytest.mark.unit @@ -405,7 +417,8 @@ async def test_before_model_request_tool_return_image_degraded() -> None: assert isinstance(new_msg, ModelResponse) new_part = new_msg.parts[0] assert isinstance(new_part, ToolReturnPart) - assert new_part.content == "[image/png]" + assert "image/png" in new_part.content + assert "unsupported" in new_part.content @pytest.mark.unit @@ -425,7 +438,8 @@ async def test_before_model_request_user_prompt_image_degraded() -> None: assert isinstance(new_part, UserPromptPart) assert isinstance(new_part.content, list) assert new_part.content[0] == "Describe this" - assert new_part.content[1] == "[image/png]" + assert "image/png" in new_part.content[1] + assert "unsupported" in new_part.content[1] @pytest.mark.unit @@ -492,13 +506,15 @@ async def test_before_model_request_mixed_message_types() -> None: new_user_part = new_req.parts[0] assert isinstance(new_user_part, UserPromptPart) assert isinstance(new_user_part.content, list) - assert new_user_part.content[1] == "[image/png]" + assert "image/png" in new_user_part.content[1] + assert "unsupported" in new_user_part.content[1] new_resp = result.messages[1] assert isinstance(new_resp, ModelResponse) new_tool_part = new_resp.parts[0] assert isinstance(new_tool_part, ToolReturnPart) - assert new_tool_part.content == "[audio/wav]" + assert "audio/wav" in new_tool_part.content + assert "unsupported" in new_tool_part.content @pytest.mark.unit @@ -560,3 +576,103 @@ async def test_for_run_returns_new_instance() -> None: assert new_cap.image_strategy == "drop" assert new_cap.audio_strategy == "pass" assert new_cap.capabilities == cap.capabilities + + +# --------------------------------------------------------------------------- +# 4.8 — reference strategy (RFC-0061 Phase 2) +# --------------------------------------------------------------------------- + + +def _run_ctx(session_id: str = "test-session") -> Any: + """Build a minimal RunContext-like object with a session id.""" + return SimpleNamespace( + deps=SimpleNamespace(run_ctx=AgentRunContext(session_id=session_id, run_id="run-1")) + ) + + +@pytest.mark.unit +async def test_wrap_tool_image_reference_strategy() -> None: + """Image with 'reference' strategy persists bytes and returns a file path.""" + cap = ModalityFilterCapability(capabilities=_text_only_caps(), image_strategy="reference") + img = _binary_image() + ctx = _run_ctx() + result = await cap.wrap_tool_execute( + ctx=ctx, + call=None, # type: ignore[arg-type] + tool_def=None, # type: ignore[arg-type] + args={}, + handler=_handler(img), + ) + assert isinstance(result, str) + assert result.startswith("[file: ") + path = Path(result.removeprefix("[file: ").removesuffix("]")) + assert path.exists() + assert path.read_bytes() == img.data + + +@pytest.mark.unit +async def test_reference_url_falls_back_to_describe() -> None: + """URL content with 'reference' strategy falls back to describe (no bytes).""" + cap = ModalityFilterCapability(capabilities=_text_only_caps(), image_strategy="reference") + url_img = ImageUrl(url="https://example.com/img.png") + result = await cap.wrap_tool_execute( + ctx=_run_ctx(), + call=None, # type: ignore[arg-type] + tool_def=None, # type: ignore[arg-type] + args={}, + handler=_handler(url_img), + ) + assert result == "[image: https://example.com/img.png]" + + +@pytest.mark.unit +async def test_reference_after_node_run_cleans_scratch() -> None: + """after_node_run removes scratch directories written this run.""" + cap = ModalityFilterCapability(capabilities=_text_only_caps(), image_strategy="reference") + img = _binary_image() + ctx = _run_ctx() + result = await cap.wrap_tool_execute( + ctx=ctx, + call=None, # type: ignore[arg-type] + tool_def=None, # type: ignore[arg-type] + args={}, + handler=_handler(img), + ) + path = Path(result.removeprefix("[file: ").removesuffix("]")) # type: ignore[arg-type] + assert path.exists() + + await cap.after_node_run( + ctx=ctx, # type: ignore[arg-type] + node=None, # type: ignore[arg-type] + result=None, # type: ignore[arg-type] + ) + assert not path.parent.exists() + + +@pytest.mark.unit +async def test_reference_session_id_scoped_paths() -> None: + """Scratch paths differ per session id.""" + cap = ModalityFilterCapability(capabilities=_text_only_caps(), image_strategy="reference") + img = _binary_image() + r1 = await cap.wrap_tool_execute( + ctx=_run_ctx("sess-a"), # type: ignore[arg-type] + call=None, # type: ignore[arg-type] + tool_def=None, # type: ignore[arg-type] + args={}, + handler=_handler(img), + ) + r2 = await cap.wrap_tool_execute( + ctx=_run_ctx("sess-b"), + call=None, # type: ignore[arg-type] + tool_def=None, # type: ignore[arg-type] + args={}, + handler=_handler(img), + ) + assert isinstance(r1, str) + assert isinstance(r2, str) + assert r1 != r2 + await cap.after_node_run( + ctx=_run_ctx(), + node=None, # type: ignore[arg-type] + result=None, # type: ignore[arg-type] + ) diff --git a/tests/test_modality_utils.py b/tests/test_modality_utils.py index 9abee3495..c0392321d 100644 --- a/tests/test_modality_utils.py +++ b/tests/test_modality_utils.py @@ -28,30 +28,65 @@ @pytest.mark.unit def test_describe_binary_image() -> None: - """BinaryImage produces '[image/png]' placeholder.""" + """BinaryImage produces an information-preserving placeholder.""" img = BinaryImage(data=b"\x89PNG\r\n\x1a\n", media_type="image/png") - assert describe_multimodal_content(img) == "[image/png]" + result = describe_multimodal_content(img) + assert "image/png" in result + assert "unsupported" in result + assert "not inlined" in result @pytest.mark.unit def test_describe_binary_content_audio() -> None: - """BinaryContent with audio media type produces '[audio/wav]'.""" + """BinaryContent with audio media type produces an information-preserving placeholder.""" audio = BinaryContent(data=b"RIFF....", media_type="audio/wav") - assert describe_multimodal_content(audio) == "[audio/wav]" + result = describe_multimodal_content(audio) + assert "audio/wav" in result + assert "unsupported" in result @pytest.mark.unit def test_describe_binary_content_video() -> None: - """BinaryContent with video media type produces '[video/mp4]'.""" + """BinaryContent with video media type produces an information-preserving placeholder.""" video = BinaryContent(data=b"\x00\x00\x00\x20ftyp", media_type="video/mp4") - assert describe_multimodal_content(video) == "[video/mp4]" + result = describe_multimodal_content(video) + assert "video/mp4" in result + assert "unsupported" in result @pytest.mark.unit def test_describe_binary_content_document() -> None: - """BinaryContent with document media type produces '[application/pdf]'.""" + """BinaryContent with document media type produces an information-preserving placeholder.""" doc = BinaryContent(data=b"%PDF-1.4", media_type="application/pdf") - assert describe_multimodal_content(doc) == "[application/pdf]" + result = describe_multimodal_content(doc) + assert "application/pdf" in result + assert "unsupported" in result + + +@pytest.mark.unit +def test_describe_binary_content_identifier_included() -> None: + """BinaryContent with an identifier emits the file reference (RFC-0061).""" + img = BinaryImage( + data=b"\x89PNG\r\n\x1a\n", + media_type="image/png", + identifier="/tmp/screenshot.png", + ) + result = describe_multimodal_content(img) + assert "/tmp/screenshot.png" in result + assert "vision-capable subagent or file tool" in result + + +@pytest.mark.unit +def test_describe_binary_content_identifier_control_chars_escaped() -> None: + """Control characters in a caller-supplied identifier are escaped (RFC-0061 security).""" + img = BinaryImage( + data=b"\x89PNG\r\n\x1a\n", + media_type="image/png", + identifier="evil\n.png", + ) + result = describe_multimodal_content(img) + assert "\n" not in result + assert "\\x0a" in result @pytest.mark.unit From 115ca7784c7720ce5a8e52fdf506d0dc440de58f Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Wed, 19 Aug 2026 19:46:07 +0800 Subject: [PATCH 3/5] fix(mime): preserve blob mime_type in MCP and ACP binary content conversions - mcp_server/conversions.py: pass BlobResourceContents.mimeType through to BinaryContent instead of hardcoding application/octet-stream - acp_server/converters.py: drop _DOCUMENT_FORMATS whitelist gate in resource_to_content so arbitrary-mime embedded blobs reach the model, with octet-stream fallback; modality filtering stays downstream in ModalityFilterCapability - add unit tests for both conversions (mime passthrough, fallback, image binary preservation) --- src/wolfharness/mcp_server/conversions.py | 4 +- .../acp_server/converters.py | 25 ++----- tests/images/test_image_normalizer_wiring.py | 72 +++++++++++++++++++ tests/mcp_server/test_conversions_mime.py | 38 ++++++++++ 4 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 tests/mcp_server/test_conversions_mime.py diff --git a/src/wolfharness/mcp_server/conversions.py b/src/wolfharness/mcp_server/conversions.py index d5cceabb3..c73b8187d 100644 --- a/src/wolfharness/mcp_server/conversions.py +++ b/src/wolfharness/mcp_server/conversions.py @@ -136,9 +136,9 @@ async def from_mcp_content( decoded_data = base64.b64decode(data) content = BinaryContent(data=decoded_data, media_type=mime_type) contents.append(content) - case BlobResourceContents(blob=blob): + case BlobResourceContents(blob=blob, mimeType=blob_mime_type): decoded_data = base64.b64decode(blob) - mime = "application/octet-stream" + mime = blob_mime_type or "application/octet-stream" content = BinaryContent(data=decoded_data, media_type=mime) contents.append(content) case ResourceLink(uri=uri, mimeType=mime_type) if client: diff --git a/src/wolfharness_server/acp_server/converters.py b/src/wolfharness_server/acp_server/converters.py index 47bcf6c1a..febf9b48b 100644 --- a/src/wolfharness_server/acp_server/converters.py +++ b/src/wolfharness_server/acp_server/converters.py @@ -56,18 +56,6 @@ logger = get_logger(__name__) -_DOCUMENT_FORMATS: dict[str, str] = { - "application/pdf": "pdf", - "text/plain": "txt", - "text/csv": "csv", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", - "text/html": "html", - "text/markdown": "md", - "application/msword": "doc", - "application/vnd.ms-excel": "xls", -} - @overload def convert_acp_mcp_server_to_config( @@ -185,12 +173,13 @@ def resource_to_content(resource: ResourceContents) -> str | BinaryImage | Binar return format_uri_as_link(uri) + f'\n\n{text}\n' case BlobResourceContents(blob=blob, mime_type=mime_type): binary_data = base64.b64decode(blob) - if mime_type and mime_type.startswith("image/"): - return BinaryImage(data=binary_data, media_type=mime_type) - if (mime_type and mime_type.startswith("audio/")) or mime_type in _DOCUMENT_FORMATS: - return BinaryContent(data=binary_data, media_type=mime_type) - formatted_uri = format_uri_as_link(resource.uri) - return f"Binary Resource: {formatted_uri}" + mime = mime_type or "application/octet-stream" + # Image resources stay BinaryImage; everything else passes as + # BinaryContent. Unsupported modalities are filtered downstream by + # ModalityFilterCapability, not dropped here. + if mime.startswith("image/"): + return BinaryImage(data=binary_data, media_type=mime) + return BinaryContent(data=binary_data, media_type=mime) case _ as unreachable: assert_never(unreachable) diff --git a/tests/images/test_image_normalizer_wiring.py b/tests/images/test_image_normalizer_wiring.py index 10f99e866..1decf2084 100644 --- a/tests/images/test_image_normalizer_wiring.py +++ b/tests/images/test_image_normalizer_wiring.py @@ -205,6 +205,78 @@ def test_from_acp_content_no_normalizer_passthrough() -> None: assert result.data == data +def test_from_acp_content_embedded_blob_unknown_mime_passes_through() -> None: + """Embedded blob with a mime outside the old whitelist reaches the model. + + A generic binary (e.g. application/json, text/yaml) must become + ``BinaryContent`` with its mime preserved instead of a text placeholder. + """ + from pydantic_ai import BinaryContent + + from acp.schema import ( + BlobResourceContents, + EmbeddedResourceContentBlock, + ) + + data = b'{"key": "value"}' + block = EmbeddedResourceContentBlock( + resource=BlobResourceContents( + uri="acp://inner/config.json", + blob=base64.b64encode(data).decode("ascii"), + mime_type="application/json", + ) + ) + + result = _from_acp(block, None) + + assert isinstance(result, BinaryContent) + assert result.data == data + assert result.media_type == "application/json" + + +def test_from_acp_content_embedded_blob_missing_mime_uses_octet_stream() -> None: + """Embedded blob without mime falls back to application/octet-stream.""" + from pydantic_ai import BinaryContent + + from acp.schema import BlobResourceContents, EmbeddedResourceContentBlock + + data = b"\x00\x01\x02" + block = EmbeddedResourceContentBlock( + resource=BlobResourceContents( + uri="acp://inner/raw.bin", + blob=base64.b64encode(data).decode("ascii"), + ) + ) + + result = _from_acp(block, None) + + assert isinstance(result, BinaryContent) + assert result.data == data + assert result.media_type == "application/octet-stream" + + +def test_from_acp_content_embedded_blob_image_is_binary_image() -> None: + """Embedded image blob stays BinaryImage, not generic BinaryContent.""" + from pydantic_ai import BinaryImage + + from acp.schema import BlobResourceContents, EmbeddedResourceContentBlock + + data = _noisy_png_bytes(64, 64) + block = EmbeddedResourceContentBlock( + resource=BlobResourceContents( + uri="acp://inner/photo.png", + blob=base64.b64encode(data).decode("ascii"), + mime_type="image/png", + ) + ) + + result = _from_acp(block, None) + + assert isinstance(result, BinaryImage) + assert result.data == data + assert result.media_type == "image/png" + + # ============================================================================= # functional run_agent: image_url normalization # ============================================================================= diff --git a/tests/mcp_server/test_conversions_mime.py b/tests/mcp_server/test_conversions_mime.py new file mode 100644 index 000000000..7a395dedb --- /dev/null +++ b/tests/mcp_server/test_conversions_mime.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import base64 + +from mcp.types import BlobResourceContents +from pydantic_ai import BinaryContent +import pytest + +from wolfharness.mcp_server.conversions import from_mcp_content + + +pytestmark = pytest.mark.unit + + +async def test_from_mcp_content_blob_preserves_mime_type() -> None: + """BlobResourceContents mimeType is propagated, not hardcoded octet-stream.""" + raw = b"\x89PNG\r\n\x1a\n" + blob = base64.b64encode(raw).decode("ascii") + result = await from_mcp_content([ + BlobResourceContents(uri="mcp://s/img.png", blob=blob, mimeType="image/png") + ]) + assert len(result) == 1 + content = result[0] + assert isinstance(content, BinaryContent) + assert content.data == raw + assert content.media_type == "image/png" + + +async def test_from_mcp_content_blob_falls_back_to_octet_stream() -> None: + """BlobResourceContents without mimeType still returns bytes.""" + raw = b"\x00\x01\x02" + blob = base64.b64encode(raw).decode("ascii") + result = await from_mcp_content([BlobResourceContents(uri="mcp://s/data.bin", blob=blob)]) + assert len(result) == 1 + content = result[0] + assert isinstance(content, BinaryContent) + assert content.data == raw + assert content.media_type == "application/octet-stream" From 460720ad29540f7af9bf2bd23261b1a050f24394 Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Wed, 19 Aug 2026 19:49:30 +0800 Subject: [PATCH 4/5] docs(rfc): sync RFC-0061 with landed implementation - Current State: note original bare placeholder vs info-preserving impl landed - Goals: mark 'no protocol server rewrites' superseded; document converter-layer mime passthrough rationale - Decision Record: add 2026-08-19 entry for describe+reference+mime fix commits --- ...lter-information-preserving-degradation.md | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md b/docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md index 3a383946c..f11a42af0 100644 --- a/docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md +++ b/docs/rfcs/draft/RFC-0061-modality-filter-information-preserving-degradation.md @@ -5,7 +5,7 @@ status: DRAFT author: pinjun.mo reviewers: [] created: 2026-08-18 -last_updated: 2026-08-18 +last_updated: 2026-08-19 decision_date: related_rfcs: - RFC-0059 (Image Attachment Normalization: Resize and Re-encode Oversized Images Before Provider Requests) @@ -57,7 +57,7 @@ The degradation is applied in two places: 1. **`before_model_request`** — scans `ModelRequest` / `ModelResponse` messages and rewrites unsupported content in `UserPromptPart` and `ToolReturnPart` via `dataclasses.replace()`. 2. **`wrap_tool_execute`** — filters multimodal content in tool results. -The placeholder itself comes from `describe_multimodal_content()` in `src/wolfharness/capabilities/modality_utils.py`: +The placeholder comes from `describe_multimodal_content()` in `src/wolfharness/capabilities/modality_utils.py`. As this RFC was being authored, the original bare placeholder was: ```python case BinaryImage(media_type=media) | BinaryContent(media_type=media): @@ -121,6 +121,18 @@ The same defect affects `describe` degradation in every modality category (image - Make the degradation **explicit and honest**: signal "this content exists but was not directly read" rather than pretending it was. - Keep the change contained to the `modality_filter` capability layer; no protocol server rewrites. + > **Status (2026-08-19): superseded.** Landing `reference` + mime integrity required minimal + > converter-layer changes so binary content actually reaches the model intact. Specifically: + > - `src/wolfharness/mcp_server/conversions.py`: pass `BlobResourceContents.mimeType` through + > to `BinaryContent` instead of hardcoding `application/octet-stream`. + > - `src/wolfharness_server/acp_server/converters.py`: drop the `_DOCUMENT_FORMATS` whitelist + > gate in `resource_to_content` so arbitrary-mime embedded blobs surface as `BinaryContent` + > (with octet-stream fallback), deferring modality decisions to `ModalityFilterCapability`. + > This is consistent with the filter treating `"unknown"` mime as pass-through. + > + > The principle still holds: capability-layer degradation is the decision point; converters + > only stop destroying the data before it gets there. + ### Non-Goals - Auto-switch models when unsupported content arrives (opencode's NOT_PLANNED position; aligned with AgentPool's explicit model-pinning design). @@ -426,6 +438,10 @@ ModalityFilterCapability.reference_strategy(...) # persists + emits referen ## Decision Record - **2026-08-18**: RFC opened as DRAFT by pinjun.mo. Problem identified while debugging a real deployment (`glm52`/kimi-k2, a text-only model, failing on pasted images via the opencode server). Three opencode-codebase investigations (HEAD `040b856`) support the problem framing. No decision made yet. +- **2026-08-19**: Option 4 (hybrid) landed as the working direction. Implementation commits on this branch: + - `d06514464` — information-preserving `describe` (honest metadata placeholder) + opt-in `reference` strategy (session-scoped scratch persistence under `tempfile`). + - `115ca7784` — converter-layer mime integrity: `BlobResourceContents.mimeType` passthrough in MCP conversions, removal of the `_DOCUMENT_FORMATS` whitelist in ACP `resource_to_content` (see Goals & Non-Goals status note). Both changes were necessary so binary content reaches `ModalityFilterCapability` intact; the filter remains the single decision point for unsupported modalities. + - Remaining open items carried forward unchanged: storage backend/TTL for `reference` (see [Open Questions](#open-questions)), surfaced degradation signal to the caller. --- From f0cbee33707e8157356f1fe3d97877069e55a557 Mon Sep 17 00:00:00 2001 From: Million <15158090088@163.com> Date: Wed, 19 Aug 2026 20:10:46 +0800 Subject: [PATCH 5/5] fix(modality): suppress hash identifiers in describe placeholder pydantic-ai falls back to a short content hash (sha1[:6]) when no explicit identifier is set. A bare hash is not a retrievable reference, so presenting it as 'File: ' misled text-only models into thinking they could open it. Hash-shaped identifiers now degrade to the 'no file reference available' branch; real paths from fsspec-style tools still surface. --- .../capabilities/modality_utils.py | 10 +++++-- tests/test_modality_utils.py | 28 ++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/wolfharness/capabilities/modality_utils.py b/src/wolfharness/capabilities/modality_utils.py index 26aeae7ea..beb74c1e2 100644 --- a/src/wolfharness/capabilities/modality_utils.py +++ b/src/wolfharness/capabilities/modality_utils.py @@ -28,6 +28,11 @@ # filename were interpolated verbatim (RFC-0061 Security considerations). _CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") +# pydantic-ai falls back to a short content hash (sha1[:6]) when no explicit +# identifier is available. A bare hash is not a retrievable reference, so it +# must not be presented to the model as a "File:" it could open. +_HASH_IDENTIFIER = re.compile(r"^[0-9a-fA-F]{4,64}$") + def _safe_ref(value: str) -> str: r"""Sanitize a caller-influenced identifier (filename/path) for interpolation. @@ -50,11 +55,12 @@ def _describe_binary(binary: BinaryImage | BinaryContent) -> str: caller explicitly provided one (e.g. the tool that produced the bytes knew the source path). The public ``.identifier`` property is avoided because it falls back to a content hash that is not a retrievable - reference. + reference. Hash-shaped identifiers are likewise suppressed — a bare + digest is not a path the model could open. """ media = binary.media_type identifier = binary._identifier - if identifier: + if identifier and not _HASH_IDENTIFIER.fullmatch(identifier): safe_id = _safe_ref(identifier) return ( f"[User supplied {media} — direct model processing is " diff --git a/tests/test_modality_utils.py b/tests/test_modality_utils.py index c0392321d..297454090 100644 --- a/tests/test_modality_utils.py +++ b/tests/test_modality_utils.py @@ -65,7 +65,7 @@ def test_describe_binary_content_document() -> None: @pytest.mark.unit def test_describe_binary_content_identifier_included() -> None: - """BinaryContent with an identifier emits the file reference (RFC-0061).""" + """BinaryContent with a path identifier emits the file reference (RFC-0061).""" img = BinaryImage( data=b"\x89PNG\r\n\x1a\n", media_type="image/png", @@ -76,6 +76,32 @@ def test_describe_binary_content_identifier_included() -> None: assert "vision-capable subagent or file tool" in result +@pytest.mark.unit +def test_describe_binary_content_hash_identifier_suppressed() -> None: + """A hash-shaped identifier is not a retrievable reference and is suppressed.""" + img = BinaryImage( + data=b"\x89PNG\r\n\x1a\n", + media_type="image/png", + identifier="a1b2c3d4e5f6", + ) + result = describe_multimodal_content(img) + assert "a1b2c3d4e5f6" not in result + assert "no file reference available" in result + + +@pytest.mark.unit +def test_describe_binary_content_sha1_hash_identifier_suppressed() -> None: + """pydantic-ai's sha1[:6] fallback identifier shape is suppressed.""" + img = BinaryImage( + data=b"\x89PNG\r\n\x1a\n", + media_type="image/png", + identifier="1a2b3c", + ) + result = describe_multimodal_content(img) + assert "1a2b3c" not in result + assert "no file reference available" in result + + @pytest.mark.unit def test_describe_binary_content_identifier_control_chars_escaped() -> None: """Control characters in a caller-supplied identifier are escaped (RFC-0061 security)."""