(Openinference Migration: Langchain): Capture multimodal image content (OpenAI image_url and Anthropic image blocks) as Blob/Uri message parts.

instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added

@@ -0,0 +1 @@
+(Openinference Migration: Langchain) - Capture multimodal image content (OpenAI ``image_url`` and Anthropic ``image`` blocks) as ``Blob``/``Uri`` message parts.

instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py

@@ -73,7 +73,7 @@ def on_chain_start(
         operation = classify_chain_run(
             serialized, metadata, kwargs, parent_run_id
         )
-
+        capture_content = self._telemetry_handler.should_capture_content()
         if operation == OperationName.INVOKE_WORKFLOW:
             workflow_name = kwargs.get("name") or serialized.get("name")
             workflow_name_override = (
@@ -82,7 +82,9 @@ def on_chain_start(
             workflow = self._telemetry_handler.workflow(
                 name=workflow_name_override or workflow_name
             )
-            workflow.input_messages = make_input_message(inputs)
+            workflow.input_messages = make_input_message(
+                inputs, capture_content
+            )
             self._invocation_manager.add_invocation_state(
                 run_id, parent_run_id, workflow
             )
@@ -107,7 +109,9 @@ def on_chain_start(
                     agent = self._telemetry_handler.invoke_local_agent(
                         agent_name=suggested_agent_name,
                     )
-                    agent.input_messages = make_input_message(inputs)
+                    agent.input_messages = make_input_message(
+                        inputs, capture_content
+                    )
 
                     if metadata:
                         agent.agent_id = metadata.get("agent_id")
@@ -162,7 +166,10 @@ def on_chain_end(
             self._invocation_manager.delete_invocation_state(run_id)
             return
 
-        invocation.output_messages = make_last_output_message(outputs)
+        capture_content = self._telemetry_handler.should_capture_content()
+        invocation.output_messages = make_last_output_message(
+            outputs, capture_content
+        )
 
         invocation.stop()
 
@@ -270,7 +277,8 @@ def on_chat_model_start(
         # :func:`to_input_messages` produce spec-conformant ``InputMessage`` s
         # with proper roles, tool-call requests, tool results, and reasoning.
         flattened: list[BaseMessage] = [msg for sub in messages for msg in sub]
-        input_messages = to_input_messages(flattened)
+        capture_content = self._telemetry_handler.should_capture_content()
+        input_messages = to_input_messages(flattened, capture_content)
 
         llm_invocation = self._telemetry_handler.inference(
             provider,

instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py

@@ -19,6 +19,7 @@
     gen_ai_attributes as GenAIAttributes,
 )
 from opentelemetry.util.genai.types import (
+    Blob,
     FunctionToolDefinition,
     InputMessage,
     MessagePart,
@@ -29,6 +30,7 @@
     ToolCallResponse,
     ToolDefinition,
 )
+from opentelemetry.util.genai.utils import decode_base64, image_from_url
 
 # Mapping from LangChain ``ls_provider`` metadata values to the well-known
 # ``gen_ai.provider.name`` values defined by the GenAI semantic conventions.
@@ -75,8 +77,66 @@ def _normalize_role(message: BaseMessage) -> str:
     return _ROLE_MAP.get(message.type, message.type)
 
 
+def _media_part(
+    item: dict[str, Any], capture_content: bool = False
+) -> MessagePart | None:
+    """Convert a LangChain multimodal image content block into a media part.
+
+    Handles the two shapes LangChain chat models accept:
+
+    - OpenAI style ``{"type": "image_url", "image_url": {"url": ...}}`` (or a
+      bare ``"image_url": "..."`` string). A ``data:<mime>;base64,<payload>``
+      URL becomes a :class:`Blob`; any other URL becomes a :class:`Uri`.
+    - Anthropic style ``{"type": "image", "source": {...}}`` where ``source``
+      is either ``{"type": "base64", "media_type": ..., "data": ...}`` (→
+      :class:`Blob`) or ``{"type": "url", "url": ...}`` (→ :class:`Uri`).
+    """
+    block_type = item.get("type")
+    if block_type == "image_url":
+        image_url = item.get("image_url")
+        url: str | None = None
+        if isinstance(image_url, str):
+            url = image_url
+        elif isinstance(image_url, dict):
+            image_url_dict = cast(dict[str, Any], image_url)
+            raw_url = image_url_dict.get("url")
+            url = raw_url if isinstance(raw_url, str) else None
+        if not url:
+            return None
+        return image_from_url(url, capture_content=capture_content)
+    if block_type == "image":
+        source = item.get("source")
+        if not isinstance(source, dict):
+            return None
+        source_dict = cast(dict[str, Any], source)
+        source_type = source_dict.get("type")
+        if source_type == "base64":
+            data = source_dict.get("data")
+            if not isinstance(data, str):
+                return None
+            decoded = decode_base64(data, capture_content)
+            if decoded is None:
+                return None
+            media_type = source_dict.get("media_type")
+            return Blob(
+                mime_type=(
+                    media_type if isinstance(media_type, str) else None
+                ),
+                modality="image",
+                content=decoded,
+            )
+        if source_type == "url":
+            source_url = source_dict.get("url")
+            if isinstance(source_url, str) and source_url:
+                return image_from_url(
+                    source_url, capture_content=capture_content
+                )
+    return None
+
+
 def _content_to_parts(
     content: str | list[str | dict[str, Any]],
+    capture_content: bool = False,
 ) -> list[MessagePart]:
     """Convert a LangChain message ``content`` payload into ``MessagePart`` s.
 
@@ -109,6 +169,10 @@ def _content_to_parts(
             )
             if isinstance(reasoning_value, str) and reasoning_value:
                 parts.append(Reasoning(content=reasoning_value))
+        elif block_type in ("image_url", "image"):
+            media = _media_part(item, capture_content)
+            if media is not None:
+                parts.append(media)
     return parts
 
 
@@ -139,14 +203,18 @@ def _legacy_function_call_request(
     return ToolCallRequest(arguments=arguments, name=name, id=None)
 
 
-def _ai_message_parts(message: AIMessage) -> list[MessagePart]:
+def _ai_message_parts(
+    message: AIMessage, capture_content: bool = False
+) -> list[MessagePart]:
     """Build :class:`MessagePart` s for an :class:`AIMessage`.
 
     Includes any text/reasoning content followed by a
     :class:`ToolCallRequest` for each entry in ``message.tool_calls``, plus a
     legacy ``additional_kwargs['function_call']`` when present.
     """
-    parts: list[MessagePart] = _content_to_parts(message.content)
+    parts: list[MessagePart] = _content_to_parts(
+        message.content, capture_content
+    )
     for call in message.tool_calls:
         name = call["name"]
         if not name:
@@ -176,16 +244,19 @@ def _tool_message_parts(message: ToolMessage) -> list[MessagePart]:
     ]
 
 
-def _message_parts(message: BaseMessage) -> list[MessagePart]:
+def _message_parts(
+    message: BaseMessage, capture_content: bool = False
+) -> list[MessagePart]:
     if isinstance(message, ToolMessage):
         return _tool_message_parts(message)
     if isinstance(message, AIMessage):
-        return _ai_message_parts(message)
-    return _content_to_parts(message.content)
+        return _ai_message_parts(message, capture_content)
+    return _content_to_parts(message.content, capture_content)
 
 
 def to_input_messages(
     messages: Iterable[Any],
+    capture_content: bool = False,
 ) -> list[InputMessage]:
     """Convert LangChain messages into spec-conformant ``InputMessage`` s."""
     try:
@@ -198,7 +269,7 @@ def to_input_messages(
         ]
     result: list[InputMessage] = []
     for message in normalized_messages:
-        parts = _message_parts(message)
+        parts = _message_parts(message, capture_content)
         if not parts:
             continue
         result.append(InputMessage(role=_normalize_role(message), parts=parts))
@@ -209,6 +280,7 @@ def to_output_messages(
     messages: Iterable[BaseMessage],
     *,
     finish_reason: str = "",
+    capture_content: bool = False,
 ) -> list[OutputMessage]:
     """Convert LangChain ``AIMessage`` instances into ``OutputMessage`` s.
 
@@ -221,7 +293,7 @@ def to_output_messages(
     for message in messages:
         if not isinstance(message, AIMessage):
             continue
-        parts = _ai_message_parts(message)
+        parts = _ai_message_parts(message, capture_content)
         if not parts:
             continue
         result.append(
@@ -283,7 +355,9 @@ def prepare_tool_definitions(tools: list[Any]) -> list[ToolDefinition] | None:
     return definitions or None
 
 
-def make_input_message(data: Any) -> list[InputMessage]:
+def make_input_message(
+    data: Any, capture_content: bool = False
+) -> list[InputMessage]:
     """Build ``InputMessage`` s from a workflow/agent input mapping.
 
     When ``data['messages']`` is present, every LangChain ``BaseMessage`` in it
@@ -305,7 +379,10 @@ def make_input_message(data: Any) -> list[InputMessage]:
             messages, Iterable
         ):
             return []
-        return to_input_messages(cast(Iterable[BaseMessage], messages))
+        return to_input_messages(
+            cast(Iterable[BaseMessage], messages),
+            capture_content,
+        )
     # Fallback: serialize non-message state fields as input.
     # Common in LangGraph where nodes use structured state fields
     # (e.g., user_query) rather than a message list.
@@ -322,7 +399,9 @@ def make_input_message(data: Any) -> list[InputMessage]:
     return []
 
 
-def make_output_message(data: Any) -> list[OutputMessage]:
+def make_output_message(
+    data: Any, capture_content: bool = False
+) -> list[OutputMessage]:
     """Build ``OutputMessage`` s from a workflow/agent output mapping.
 
     Only ``AIMessage`` entries become outputs. ``finish_reason`` is left
@@ -340,17 +419,22 @@ def make_output_message(data: Any) -> list[OutputMessage]:
         or not isinstance(messages, Iterable)
     ):
         return []
-    return to_output_messages(cast(Iterable[BaseMessage], messages))
+    return to_output_messages(
+        cast(Iterable[BaseMessage], messages),
+        capture_content=capture_content,
+    )
 
 
-def make_last_output_message(data: Any) -> list[OutputMessage]:
+def make_last_output_message(
+    data: Any, capture_content: bool = False
+) -> list[OutputMessage]:
     """Extract only the last AI message as the output.
 
     For Workflow and AgentInvocation spans, the final AI message best represents
     the actual output. Intermediate AI messages (e.g., tool-call decisions) are
     already captured in child LLM invocation spans.
     """
-    all_messages = make_output_message(data)
+    all_messages = make_output_message(data, capture_content)
     if all_messages:
         return [all_messages[-1]]
     return []

instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml

@@ -0,0 +1,48 @@
+# TODO: this is generated by AI, re-record
+# against the live Anthropic API once an ANTHROPIC_API_KEY is available.
+interactions:
+- request:
+    body: |-
+      {"model": "claude-sonnet-4-5", "max_tokens": 1024, "messages": [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg=="}}]}], "temperature": 0.1}
+    headers:
+      Content-Type:
+      - application/json
+      User-Agent:
+      - !!binary |
+        QW50aHJvcGljL1B5dGhvbiAxLjAuMA==
+      x-api-key:
+      - test_key
+      anthropic-version:
+      - '2023-06-01'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: |-
+        {
+          "id": "msg_01MultimodalImagePlaceholder",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-sonnet-4-5",
+          "content": [
+            {
+              "type": "text",
+              "text": "This is a tiny 1x1 pixel PNG image."
+            }
+          ],
+          "stop_reason": "end_turn",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 16,
+            "output_tokens": 12
+          }
+        }
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 04 Sep 2025 20:00:58 GMT
+    status:
+      code: 200
+      message: OK
+version: 1