From 1afe5d6e8abcf9aa79a2475fed4a319eb7b473bc Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 11:09:42 +0200 Subject: [PATCH 01/12] feat(openai): Set gen_ai.tool.definitions attribute --- sentry_sdk/ai/_openai_completions_api.py | 55 +++++- sentry_sdk/integrations/openai.py | 16 +- tests/integrations/openai/test_openai.py | 233 ++++++++++++++++------- 3 files changed, 225 insertions(+), 79 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index b5eb8c55ef..57e35397bd 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -8,9 +8,10 @@ ChatCompletionContentPartParam, ChatCompletionMessageParam, ChatCompletionSystemMessageParam, + ChatCompletionToolUnionParam, ) - from sentry_sdk._types import TextPart + from sentry_sdk._types import TextPart, ToolDefinition def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: @@ -64,3 +65,55 @@ def _transform_system_instructions( instruction_text_parts += text_parts return instruction_text_parts + + +def _transform_tool_definitions( + tools: "Iterable[ChatCompletionToolUnionParam]", +) -> "list[ToolDefinition]": + """ + Transform tool definitions to the schema used by the "gen_ai.tool.definitions" attribute. + """ + if not isinstance(tools, Iterable): + return [] + + tool_definitions = [] + for tool in tools: + if not isinstance(tool, dict): + continue + + if tool["type"] == "function": + tool_definition: ToolDefinition = { + "type": "function", + } + + if "function" not in tool: + tool_definitions.append(tool_definition) + continue + + if "name" in tool["function"]: + tool_definition["name"] = tool["function"]["name"] + + if "description" in tool["function"]: + tool_definition["description"] = tool["function"]["description"] + + if "parameters" in tool["function"]: + tool_definition["parameters"] = tool["function"]["parameters"] + + tool_definitions.append(tool_definition) + continue + + if tool["type"] == "custom": + tool_definition = { + "type": "custom", + } + + if "name" in tool["custom"]: + tool_definition["name"] = tool["custom"]["name"] + + if "description" in tool["custom"]: + tool_definition["description"] = tool["custom"]["description"] + + tool_definitions.append(tool_definition) + continue + + return tool_definitions diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 8a77668329..1b25b6b962 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -17,6 +17,9 @@ from sentry_sdk.ai._openai_completions_api import ( _is_system_instruction as _is_system_instruction_completions, ) +from sentry_sdk.ai._openai_completions_api import ( + _transform_tool_definitions as _transform_tool_definitions_completions, +) from sentry_sdk.ai._openai_responses_api import ( _get_system_instructions as _get_system_instructions_responses, ) @@ -463,15 +466,16 @@ def _set_completions_api_input_data( "messages" ) - tools = kwargs.get("tools") - if tools is not None and _is_given(tools) and len(tools) > 0: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) - ) - set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data ) + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_completions(tools)), + ) + model = kwargs.get("model") if model is not None: set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 7d243657f7..36e37f7fbd 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -107,6 +107,167 @@ async def __call__(self, *args, **kwargs): ) +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +def test_chat_completion_tool_definitions( + sentry_init, + capture_events, + capture_items, + nonstreaming_chat_completions_model_response, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[OpenAIIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response( + response_id="chat-id", + response_model="gpt-3.5-turbo", + message_content="the model response", + created=10000000, + usage=CompletionUsage( + prompt_tokens=20, + completion_tokens=10, + total_tokens=30, + ), + ) + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + tools=[ + { + "type": "function", + "function": { + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + "strict": True, + }, + }, + { + "type": "custom", + "custom": { + "name": "name", + "description": "description", + }, + }, + ], + ) + + sentry_sdk.flush() + span = next(item.payload for item in items) + + assert json.loads(span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + ] + else: + events = capture_events() + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + tools=[ + { + "type": "function", + "function": { + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + "strict": True, + }, + }, + { + "type": "custom", + "custom": { + "name": "name", + "description": "description", + }, + }, + ], + ) + + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + print(tx["spans"]) + + assert json.loads(span["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + ] + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( @@ -5617,78 +5778,6 @@ async def test_streaming_responses_api_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -@pytest.mark.skipif( - OPENAI_VERSION <= (1, 1, 0), - reason="OpenAI versions <=1.1.0 do not support the tools parameter.", -) -@pytest.mark.parametrize( - "tools", - [[], None, NOT_GIVEN, omit], -) -def test_empty_tools_in_chat_completion( - sentry_init, - capture_events, - capture_items, - tools, - nonstreaming_chat_completions_model_response, - stream_gen_ai_spans, - span_streaming, -): - sentry_init( - integrations=[OpenAIIntegration()], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - stream_gen_ai_spans=stream_gen_ai_spans, - trace_lifecycle="stream" if span_streaming else "static", - ) - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - return_value=nonstreaming_chat_completions_model_response( - response_id="chat-id", - response_model="gpt-3.5-turbo", - message_content="the model response", - created=10000000, - usage=CompletionUsage( - prompt_tokens=20, - completion_tokens=10, - total_tokens=30, - ), - ) - ) - - if span_streaming or stream_gen_ai_spans: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - tools=tools, - ) - - sentry_sdk.flush() - span = next(item.payload for item in items) - - assert "gen_ai.request.available_tools" not in span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - tools=tools, - ) - - (event,) = events - span = event["spans"][0] - - assert "gen_ai.request.available_tools" not in span["data"] - - @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) # Feature added in https://github.com/openai/openai-python/pull/1952 From 33faef1bb98568041d4fddc88b7d0326ca69b3ca Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 11:11:29 +0200 Subject: [PATCH 02/12] add type --- sentry_sdk/_types.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index e91fed097c..d3c70ddfa4 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -472,6 +472,12 @@ class TextPart(TypedDict): type: Literal["text"] content: str + class ToolDefinition(TypedDict): + type: str + name: NotRequired[str] + description: NotRequired[str] + parameters: NotRequired[dict[str, object]] + IgnoreSpansName = Union[str, Pattern[str]] IgnoreSpansContext = TypedDict( "IgnoreSpansContext", From 7398e26bc2a1a42e1484fc76652fd5fccb442b6f Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 11:49:46 +0200 Subject: [PATCH 03/12] make iteration safer --- sentry_sdk/ai/_openai_completions_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 57e35397bd..a61a3e7814 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -78,7 +78,7 @@ def _transform_tool_definitions( tool_definitions = [] for tool in tools: - if not isinstance(tool, dict): + if not isinstance(tool, dict) or "type" not in tool: continue if tool["type"] == "function": From 0cd572b407edacca658f17a1869990710f35d23d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 11:52:56 +0200 Subject: [PATCH 04/12] clean up test --- tests/integrations/openai/test_openai.py | 49 ++++++++++++++---------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 36e37f7fbd..2510ca970a 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -17,11 +17,18 @@ from openai import AsyncOpenAI, AsyncStream, OpenAI, OpenAIError, Stream from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding -from openai.types.chat import ChatCompletionChunk, ChatCompletionMessage +from openai.types.chat import ( + ChatCompletionChunk, + ChatCompletionCustomToolParam, + ChatCompletionFunctionToolParam, + ChatCompletionMessage, +) from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as DeltaChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta +from openai.types.chat.chat_completion_custom_tool_param import Custom from openai.types.create_embedding_response import Usage as EmbeddingTokenUsage +from openai.types.shared_params import FunctionDefinition SKIP_RESPONSES_TESTS = False @@ -107,6 +114,10 @@ async def __call__(self, *args, **kwargs): ) +@pytest.mark.skipif( + OPENAI_VERSION <= (1, 1, 0), + reason="OpenAI versions <=1.1.0 do not support the tools parameter.", +) @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_chat_completion_tool_definitions( @@ -151,12 +162,12 @@ def test_chat_completion_tool_definitions( {"role": "user", "content": "hello"}, ], tools=[ - { - "type": "function", - "function": { - "name": "name", - "description": "description", - "parameters": { + ChatCompletionFunctionToolParam( + type="function", + function=FunctionDefinition( + name="name", + description="description", + parameters={ "type": "object", "properties": { "city": {"type": "string"}, @@ -165,16 +176,16 @@ def test_chat_completion_tool_definitions( "required": ["city", "state"], "additionalProperties": False, }, - "strict": True, - }, - }, - { - "type": "custom", - "custom": { - "name": "name", - "description": "description", - }, - }, + strict=True, + ), + ), + ChatCompletionCustomToolParam( + type="custom", + custom=Custom( + name="name", + description="description", + ), + ), ], ) @@ -242,10 +253,8 @@ def test_chat_completion_tool_definitions( tx = events[0] assert tx["type"] == "transaction" - span = tx["spans"][0] - print(tx["spans"]) - assert json.loads(span["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + assert json.loads(tx["spans"][0]["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ { "type": "function", "name": "name", From 6d8546aaa32da6d2a4ad00fb3a0d24e34c1ccff5 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 11:53:46 +0200 Subject: [PATCH 05/12] clean up --- tests/integrations/openai/test_openai.py | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 2510ca970a..d3c88cc2b4 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -224,12 +224,12 @@ def test_chat_completion_tool_definitions( {"role": "user", "content": "hello"}, ], tools=[ - { - "type": "function", - "function": { - "name": "name", - "description": "description", - "parameters": { + ChatCompletionFunctionToolParam( + type="function", + function=FunctionDefinition( + name="name", + description="description", + parameters={ "type": "object", "properties": { "city": {"type": "string"}, @@ -238,16 +238,16 @@ def test_chat_completion_tool_definitions( "required": ["city", "state"], "additionalProperties": False, }, - "strict": True, - }, - }, - { - "type": "custom", - "custom": { - "name": "name", - "description": "description", - }, - }, + strict=True, + ), + ), + ChatCompletionCustomToolParam( + type="custom", + custom=Custom( + name="name", + description="description", + ), + ), ], ) From 54ca1d9e337782c637ac89b2f17a5358d76998d2 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 12:04:06 +0200 Subject: [PATCH 06/12] higher version bound --- tests/integrations/openai/test_openai.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index d3c88cc2b4..1a233986e3 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -30,6 +30,11 @@ from openai.types.create_embedding_response import Usage as EmbeddingTokenUsage from openai.types.shared_params import FunctionDefinition +try: + from openai.types.chat import ChatCompletionCustomToolParam +except ImportError: + pass + SKIP_RESPONSES_TESTS = False try: @@ -115,8 +120,8 @@ async def __call__(self, *args, **kwargs): @pytest.mark.skipif( - OPENAI_VERSION <= (1, 1, 0), - reason="OpenAI versions <=1.1.0 do not support the tools parameter.", + OPENAI_VERSION <= (2, 10, 0), + reason="ChatCompletionCustomToolParam is unavailable before.", ) @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) From f94e12e5a12ace4e587c5e7864a1cbdefaf0af58 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 13:15:27 +0200 Subject: [PATCH 07/12] fix import error --- tests/integrations/openai/test_openai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 1a233986e3..9dcadcfab0 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -19,7 +19,6 @@ from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding from openai.types.chat import ( ChatCompletionChunk, - ChatCompletionCustomToolParam, ChatCompletionFunctionToolParam, ChatCompletionMessage, ) From 8048f1dad3fb2dad19090d6f4bfe5bc59e58d55e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 13:18:35 +0200 Subject: [PATCH 08/12] fix import --- tests/integrations/openai/test_openai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 9dcadcfab0..aad6113d7e 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -19,7 +19,6 @@ from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding from openai.types.chat import ( ChatCompletionChunk, - ChatCompletionFunctionToolParam, ChatCompletionMessage, ) from openai.types.chat.chat_completion import Choice @@ -30,7 +29,10 @@ from openai.types.shared_params import FunctionDefinition try: - from openai.types.chat import ChatCompletionCustomToolParam + from openai.types.chat import ( + ChatCompletionCustomToolParam, + ChatCompletionFunctionToolParam, + ) except ImportError: pass From a9e263c2f142a3f0bec6f530be0869c5421c0673 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 13:21:55 +0200 Subject: [PATCH 09/12] more import problems --- tests/integrations/openai/test_openai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index aad6113d7e..2202d7ed42 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -24,15 +24,15 @@ from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as DeltaChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta -from openai.types.chat.chat_completion_custom_tool_param import Custom from openai.types.create_embedding_response import Usage as EmbeddingTokenUsage -from openai.types.shared_params import FunctionDefinition try: from openai.types.chat import ( ChatCompletionCustomToolParam, ChatCompletionFunctionToolParam, ) + from openai.types.chat.chat_completion_custom_tool_param import Custom + from openai.types.shared_params import FunctionDefinition except ImportError: pass From 2ceb5ef45af9ccaf52c8fdf13875db0cb18326f0 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 30 Jul 2026 13:23:10 +0200 Subject: [PATCH 10/12] restore import changes --- tests/integrations/openai/test_openai.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 2202d7ed42..53f993b5c0 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -17,10 +17,7 @@ from openai import AsyncOpenAI, AsyncStream, OpenAI, OpenAIError, Stream from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding -from openai.types.chat import ( - ChatCompletionChunk, - ChatCompletionMessage, -) +from openai.types.chat import ChatCompletionChunk, ChatCompletionMessage from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as DeltaChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta From 23696f302b299c257c471d2166e244f00aab8a0d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 31 Jul 2026 10:59:32 +0200 Subject: [PATCH 11/12] gate custom access --- sentry_sdk/ai/_openai_completions_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index a61a3e7814..213129b714 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -107,6 +107,10 @@ def _transform_tool_definitions( "type": "custom", } + if "custom" not in tool: + tool_definitions.append(tool_definition) + continue + if "name" in tool["custom"]: tool_definition["name"] = tool["custom"]["name"] From 3ff8a9f5293662929b3e3a86270c119d032f2562 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 31 Jul 2026 12:43:26 +0200 Subject: [PATCH 12/12] put types in quotes --- sentry_sdk/ai/_openai_completions_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 213129b714..3dc9290660 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -82,7 +82,7 @@ def _transform_tool_definitions( continue if tool["type"] == "function": - tool_definition: ToolDefinition = { + tool_definition: "ToolDefinition" = { "type": "function", }