diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 4edec05770..6ab1b495c1 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,10 +1,47 @@ +import json from typing import TYPE_CHECKING import sentry_sdk from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations import DidNotEnable from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing_utils import has_span_streaming_enabled -from sentry_sdk.utils import safe_serialize + +try: + from agents import ( + CodeInterpreterTool, + ComputerTool, + FileSearchTool, + FunctionTool, + HostedMCPTool, + ImageGenerationTool, + LocalShellTool, + WebSearchTool, + ) +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + +try: + from agents import ApplyPatchTool, ShellTool +except ImportError: + ShellTool = None + ApplyPatchTool = None + +try: + from agents import ToolSearchTool +except ImportError: + ToolSearchTool = None + +try: + from agents import CustomTool +except ImportError: + CustomTool = None + +try: + from agents import ProgrammaticToolCallingTool +except ImportError: + ProgrammaticToolCallingTool = None + from ..consts import SPAN_ORIGIN from ..utils import ( @@ -17,7 +54,146 @@ if TYPE_CHECKING: from typing import Any, Optional, Union - from agents import Agent + from agents import Agent, Tool + + from sentry_sdk._types import ToolDefinition + + +def _transform_tool_definitions(tools: "list[Tool]") -> "list[ToolDefinition]": + """ + Transform tool definitions to the schema used by the "gen_ai.tool.definitions" attribute. + Includes special handling for tools where the type includes a name, description or parameters. + + Note: These are generated from the same OpenAPI spec as Responses API tools. + """ + if not isinstance(tools, list): + return [] + + tool_definitions: "list[ToolDefinition]" = [] + for tool in tools: + if isinstance(tool, FunctionTool): + tool_definitions.append( + { + "type": "function", + "name": tool.name, + "description": tool.description, + "parameters": tool.params_json_schema, + } + ) + continue + + if isinstance(tool, WebSearchTool): + tool_definitions.append( + { + "type": "web_search", + "name": tool.name, + } + ) + continue + + if isinstance(tool, FileSearchTool): + tool_definitions.append( + { + "type": "file_search", + "name": tool.name, + } + ) + continue + + if isinstance(tool, ComputerTool): + tool_definitions.append( + { + "type": "computer", + "name": tool.name, + } + ) + continue + + if CustomTool is not None and isinstance(tool, CustomTool): + tool_definitions.append( + { + "type": "custom", + "name": tool.name, + "description": tool.description, + } + ) + continue + + if isinstance(tool, HostedMCPTool): + tool_definitions.append( + { + "type": "mcp", + "name": tool.name, + } + ) + continue + + if ApplyPatchTool is not None and isinstance(tool, ApplyPatchTool): + tool_definitions.append( + { + "type": "apply_patch", + "name": tool.name, + } + ) + continue + + if ShellTool is not None and isinstance(tool, ShellTool): + tool_definitions.append( + { + "type": "shell", + "name": tool.name, + } + ) + continue + + if isinstance(tool, ImageGenerationTool): + tool_definitions.append( + { + "type": "image_generation", + "name": tool.name, + } + ) + continue + + if isinstance(tool, CodeInterpreterTool): + tool_definitions.append( + { + "type": "code_interpreter", + "name": tool.name, + } + ) + continue + + if isinstance(tool, LocalShellTool): + tool_definitions.append( + { + "type": "local_shell", + "name": tool.name, + } + ) + continue + + if ToolSearchTool is not None and isinstance(tool, ToolSearchTool): + tool_definitions.append( + { + "type": "tool_search", + "name": tool.name, + } + ) + continue + + if ProgrammaticToolCallingTool is not None and isinstance( + tool, ProgrammaticToolCallingTool + ): + tool_definitions.append( + { + "type": "programmatic_tool_calling", + "name": tool.name, + } + ) + continue + + return tool_definitions def ai_client_span( @@ -56,11 +232,10 @@ def ai_client_span( _set_agent_data(span, agent) - if len(agent.tools) > 0: - set_on_span( - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, - safe_serialize([vars(tool) for tool in agent.tools]), - ) + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions(agent.tools)), + ) _set_input_data(span, get_response_kwargs) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index ba40aa4e5f..7277d12bf9 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -14,6 +14,7 @@ ModelSettings, Usage, ) +from agents.computer import Computer from agents.exceptions import MaxTurnsExceeded, ModelBehaviorError from agents.items import ( ResponseFunctionToolCall, @@ -24,6 +25,46 @@ from agents.tool import HostedMCPTool from agents.version import __version__ as OPENAI_AGENTS_VERSION from openai import AsyncOpenAI, InternalServerError +from openai.types.responses.tool_param import CodeInterpreter, ImageGeneration + +from sentry_sdk.integrations import DidNotEnable + +try: + from agents import ( + CodeInterpreterTool, + ComputerTool, + FileSearchTool, + HostedMCPTool, + ImageGenerationTool, + LocalShellTool, + WebSearchTool, + ) +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + +try: + from agents import ApplyPatchTool, ShellTool +except ImportError: + ShellTool = None + ApplyPatchTool = None + +try: + from agents import ToolSearchTool +except ImportError: + ToolSearchTool = None + +try: + from agents import CustomTool +except ImportError: + CustomTool = None + +try: + from agents import ProgrammaticToolCallingTool +except ImportError: + ProgrammaticToolCallingTool = None + + +from typing import Any, cast import sentry_sdk from sentry_sdk import start_span @@ -173,6 +214,286 @@ def test_agent_custom_model(): ) +class DummyEditor: + def create_file(self, operation): + return None + + def update_file(self, operation): + return None + + def delete_file(self, operation): + return None + + +class TrackingComputer(Computer): + def __init__(self): + self.calls = [] + + @property + def environment(self): + return "mac" + + @property + def dimensions(self): + return (1, 1) + + def screenshot(self): + self.calls.append("screenshot") + return "img" + + def click(self, _x, _y, _button): + self.calls.append("click") + + def double_click(self, _x, _y): + self.calls.append("double_click") + + def scroll(self, _x, _y, _scroll_x, _scroll_y): + self.calls.append("scroll") + + def type(self, _text): + self.calls.append("type") + + def wait(self): + self.calls.append("wait") + + def move(self, _x, _y): + self.calls.append("move") + + def keypress(self, _keys): + self.calls.append("keypress") + + def drag(self, _path): + self.calls.append("drag") + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.asyncio +async def test_tool_definitions( + sentry_init, + capture_events, + capture_items, + test_agent, + nonstreaming_responses_model_response, + get_model_response, + stream_gen_ai_spans, + span_streaming, +): + """ + Verifies that the `gen_ai.tool.definitions` attribute is set. + Provides one instance of each tool type to an agent. + """ + client = AsyncOpenAI(api_key="test-key") + model = OpenAIResponsesModel(model="gpt-4", openai_client=client) + + def some_function(a: str, b: list[int]) -> str: + return "hello" + + tools = [ + WebSearchTool(), + FileSearchTool(vector_store_ids=[]), + ComputerTool(computer=TrackingComputer()), + HostedMCPTool( + tool_config=cast( + Any, + { + "type": "mcp", + "server_label": "docs_server", + "server_url": "https://example.com/mcp", + }, + ) + ), + ImageGenerationTool( + tool_config=cast( + ImageGeneration, {"type": "image_generation", "model": "gpt-image-1"} + ) + ), + CodeInterpreterTool( + tool_config=cast( + CodeInterpreter, {"type": "code_interpreter", "container": "python"} + ) + ), + LocalShellTool(executor=lambda req: "ok"), + ] + + expected_available_tools = [ + {"type": "web_search", "name": "web_search_preview"} + if parse_version(OPENAI_AGENTS_VERSION) < (0, 3, 0) + else {"type": "web_search", "name": "web_search"}, + {"type": "file_search", "name": "file_search"}, + {"type": "computer", "name": "computer_use_preview"}, + {"type": "mcp", "name": "hosted_mcp"}, + {"type": "image_generation", "name": "image_generation"}, + {"type": "code_interpreter", "name": "code_interpreter"}, + {"type": "local_shell", "name": "local_shell"}, + ] + + if CustomTool is not None: + tools.append( + CustomTool( + name="custom", + description="Custom tool", + on_invoke_tool=lambda _context, _input: "ok", + ) + ) + expected_available_tools.append( + {"type": "custom", "name": "custom", "description": "Custom tool"}, + ) + + if ApplyPatchTool is not None: + tools.append(ApplyPatchTool(editor=DummyEditor())) + expected_available_tools.append( + {"type": "apply_patch", "name": "apply_patch"}, + ) + + if ShellTool is not None: + tools.append(ShellTool(executor=lambda req: "ok")) + expected_available_tools.append( + {"type": "shell", "name": "shell"}, + ) + + if ToolSearchTool is not None: + tools.append(agents.function_tool(some_function, defer_loading=True)) + tools.append(ToolSearchTool()) + expected_available_tools.append( + { + "type": "function", + "name": "some_function", + "description": "", + "parameters": { + "properties": { + "a": {"title": "A", "type": "string"}, + "b": { + "items": {"type": "integer"}, + "title": "B", + "type": "array", + }, + }, + "required": ["a", "b"], + "title": "some_function_args", + "type": "object", + "additionalProperties": False, + }, + }, + ) + expected_available_tools.append( + {"type": "tool_search", "name": "tool_search"}, + ) + + if ProgrammaticToolCallingTool is not None: + tools.append(ProgrammaticToolCallingTool()) + expected_available_tools.append( + {"type": "programmatic_tool_calling", "name": "programmatic_tool_calling"}, + ) + + agent = test_agent.clone(model=model, tools=tools) + + response = get_model_response( + nonstreaming_responses_model_response, serialize_pydantic=True + ) + + if span_streaming: + with patch.object( + agent.model._client._client, + "send", + return_value=response, + ) as _: + sentry_init( + integrations=[OpenAIAgentsIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=False, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream", + ) + + items = capture_items("span") + + result = await agents.Runner.run( + agent, + "Test input", + run_config=test_run_config, + ) + + assert result is not None + assert result.final_output == "Hello, how can I help you?" + + sentry_sdk.flush() + spans = [item.payload for item in items] + ai_client_span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) + + assert ( + json.loads(ai_client_span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) + == expected_available_tools + ) + elif stream_gen_ai_spans: + with patch.object( + agent.model._client._client, + "send", + return_value=response, + ) as _: + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + send_default_pii=False, + stream_gen_ai_spans=stream_gen_ai_spans, + ) + + items = capture_items("span", "transaction") + + result = await agents.Runner.run( + agent, + "Test input", + run_config=test_run_config, + ) + + assert result is not None + assert result.final_output == "Hello, how can I help you?" + + spans = [item.payload for item in items if item.type == "span"] + ai_client_span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) + + assert ( + json.loads(ai_client_span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) + == expected_available_tools + ) + else: + with patch.object( + agent.model._client._client, + "send", + return_value=response, + ) as _: + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + send_default_pii=False, + stream_gen_ai_spans=stream_gen_ai_spans, + ) + events = capture_events() + + result = await agents.Runner.run( + agent, + "Test input", + run_config=test_run_config, + ) + + assert result is not None + assert result.final_output == "Hello, how can I help you?" + + (transaction,) = events + spans = transaction["spans"] + ai_client_span = next(span for span in spans if span["op"] == OP.GEN_AI_CHAT) + + assert ( + json.loads(ai_client_span["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) + == expected_available_tools + ) + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @@ -1958,41 +2279,15 @@ def simple_test_tool(message: str) -> str: available_tool = { "name": "simple_test_tool", "description": "A simple tool", - "params_json_schema": { + "parameters": { "properties": {"message": {"title": "Message", "type": "string"}}, "required": ["message"], "title": "simple_test_tool_args", "type": "object", "additionalProperties": False, }, - "on_invoke_tool": mock.ANY, - "strict_json_schema": True, - "is_enabled": True, } - if parse_version(OPENAI_AGENTS_VERSION) >= (0, 3, 3): - available_tool.update( - {"tool_input_guardrails": None, "tool_output_guardrails": None} - ) - - if parse_version(OPENAI_AGENTS_VERSION) >= ( - 0, - 8, - ): - available_tool["needs_approval"] = False - if parse_version(OPENAI_AGENTS_VERSION) >= ( - 0, - 9, - 0, - ): - available_tool.update( - { - "timeout_seconds": None, - "timeout_behavior": "error_as_result", - "timeout_error_function": None, - } - ) - assert agent_span["name"] == "invoke_agent test_agent" assert agent_span["attributes"]["sentry.origin"] == "auto.ai.openai_agents" assert agent_span["attributes"]["gen_ai.agent.name"] == "test_agent" @@ -2010,7 +2305,7 @@ def simple_test_tool(message: str) -> str: assert ai_client_span1["attributes"]["gen_ai.agent.name"] == "test_agent" ai_client_span1_available_tool = json.loads( - ai_client_span1["attributes"]["gen_ai.request.available_tools"] + ai_client_span1["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] )[0] assert all( @@ -2077,7 +2372,7 @@ def simple_test_tool(message: str) -> str: assert ai_client_span2["attributes"]["gen_ai.operation.name"] == "chat" ai_client_span2_available_tool = json.loads( - ai_client_span2["attributes"]["gen_ai.request.available_tools"] + ai_client_span2["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] )[0] assert all( @@ -2178,41 +2473,15 @@ def simple_test_tool(message: str) -> str: available_tool = { "name": "simple_test_tool", "description": "A simple tool", - "params_json_schema": { + "parameters": { "properties": {"message": {"title": "Message", "type": "string"}}, "required": ["message"], "title": "simple_test_tool_args", "type": "object", "additionalProperties": False, }, - "on_invoke_tool": mock.ANY, - "strict_json_schema": True, - "is_enabled": True, } - if parse_version(OPENAI_AGENTS_VERSION) >= (0, 3, 3): - available_tool.update( - {"tool_input_guardrails": None, "tool_output_guardrails": None} - ) - - if parse_version(OPENAI_AGENTS_VERSION) >= ( - 0, - 8, - ): - available_tool["needs_approval"] = False - if parse_version(OPENAI_AGENTS_VERSION) >= ( - 0, - 9, - 0, - ): - available_tool.update( - { - "timeout_seconds": None, - "timeout_behavior": "error_as_result", - "timeout_error_function": None, - } - ) - assert agent_span["name"] == "invoke_agent test_agent" assert agent_span["attributes"]["sentry.origin"] == "auto.ai.openai_agents" assert agent_span["attributes"]["gen_ai.agent.name"] == "test_agent" @@ -2230,7 +2499,7 @@ def simple_test_tool(message: str) -> str: assert ai_client_span1["attributes"]["gen_ai.agent.name"] == "test_agent" ai_client_span1_available_tool = json.loads( - ai_client_span1["attributes"]["gen_ai.request.available_tools"] + ai_client_span1["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] )[0] assert all( @@ -2297,7 +2566,7 @@ def simple_test_tool(message: str) -> str: assert ai_client_span2["attributes"]["gen_ai.operation.name"] == "chat" ai_client_span2_available_tool = json.loads( - ai_client_span2["attributes"]["gen_ai.request.available_tools"] + ai_client_span2["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] )[0] assert all( @@ -2389,41 +2658,15 @@ def simple_test_tool(message: str) -> str: available_tool = { "name": "simple_test_tool", "description": "A simple tool", - "params_json_schema": { + "parameters": { "properties": {"message": {"title": "Message", "type": "string"}}, "required": ["message"], "title": "simple_test_tool_args", "type": "object", "additionalProperties": False, }, - "on_invoke_tool": mock.ANY, - "strict_json_schema": True, - "is_enabled": True, } - if parse_version(OPENAI_AGENTS_VERSION) >= (0, 3, 3): - available_tool.update( - {"tool_input_guardrails": None, "tool_output_guardrails": None} - ) - - if parse_version(OPENAI_AGENTS_VERSION) >= ( - 0, - 8, - ): - available_tool["needs_approval"] = False - if parse_version(OPENAI_AGENTS_VERSION) >= ( - 0, - 9, - 0, - ): - available_tool.update( - { - "timeout_seconds": None, - "timeout_behavior": "error_as_result", - "timeout_error_function": None, - } - ) - assert transaction["transaction"] == "test_agent workflow" assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" @@ -2444,7 +2687,7 @@ def simple_test_tool(message: str) -> str: assert ai_client_span1["data"]["gen_ai.agent.name"] == "test_agent" ai_client_span1_available_tool = json.loads( - ai_client_span1["data"]["gen_ai.request.available_tools"] + ai_client_span1["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] )[0] assert all( ai_client_span1_available_tool[k] == v for k, v in available_tool.items() @@ -2504,7 +2747,7 @@ def simple_test_tool(message: str) -> str: assert ai_client_span2["data"]["gen_ai.operation.name"] == "chat" ai_client_span2_available_tool = json.loads( - ai_client_span2["data"]["gen_ai.request.available_tools"] + ai_client_span2["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] )[0] assert all( ai_client_span2_available_tool[k] == v for k, v in available_tool.items()