diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index a43aad0bfc..f2a45b60ae 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -3,7 +3,7 @@ import time from collections.abc import Iterable from functools import wraps -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import sentry_sdk from sentry_sdk import consts @@ -48,6 +48,7 @@ from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, + has_data_collection_enabled, reraise, ) @@ -329,19 +330,11 @@ def _set_responses_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") - messages: "Optional[Union[str, ResponseInputParam]]" = kwargs.get("input") - set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data ) - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_responses(tools)), - ) + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") model = kwargs.get("model") if model is not None: @@ -376,40 +369,67 @@ def _set_responses_api_input_data( reasoning["effort"], ) - if not should_send_default_pii() or not integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") - return - - if ( - messages is None - and explicit_instructions is not None - and _is_given(explicit_instructions) - ): - set_on_span( - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - json.dumps( - [ - { - "type": "text", - "content": explicit_instructions, - } - ] - ), - ) + client_options = sentry_sdk.get_client().options + if has_data_collection_enabled(client_options): + if ( + integration.include_prompts + and client_options["data_collection"]["gen_ai"]["inputs"] + ): + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_responses(tools)), + ) + else: + # Pre-data collection this was always set, so this needs to be left here for now until + # we deprecate `send_default_pii`. Once we do, this 'else' branch should be removed, + # and the above branch placed below the "if not should_send_default_pii() or not integration.include_prompts" + # line below + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_responses(tools)), + ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + if has_data_collection_enabled(client_options): + # This takes precedence over the global data collection settings + if not integration.include_prompts: + return + if not client_options["data_collection"]["gen_ai"]["inputs"]: + return + elif not should_send_default_pii() or not integration.include_prompts: return + explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") + has_explicit_instructions = explicit_instructions is not None and _is_given( + explicit_instructions + ) + messages: "Optional[Union[str, ResponseInputParam]]" = kwargs.get("input") + instructions_text_parts: "list[TextPart]" = [] + if messages is None: - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + if has_explicit_instructions: + set_on_span( + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + json.dumps( + [ + { + "type": "text", + "content": explicit_instructions, + } + ] + ), + ) + # No messages to record (only instructions at most) return - instructions_text_parts: "list[TextPart]" = [] - if explicit_instructions is not None and _is_given(explicit_instructions): + if has_explicit_instructions: instructions_text_parts.append( { "type": "text", - "content": explicit_instructions, + "content": cast(str, explicit_instructions), } ) @@ -417,13 +437,13 @@ def _set_responses_api_input_data( # Deliberate use of function accepting completions API type because # of shared structure FOR THIS PURPOSE ONLY. instructions_text_parts += _transform_system_instructions(system_instructions) - if len(instructions_text_parts) > 0: set_on_span( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps(instructions_text_parts), ) + # Input was provided as a single string if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore client = sentry_sdk.get_client() @@ -437,10 +457,9 @@ def _set_responses_api_input_data( set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") return + # Input was provided as a list (potentially a multi-turn conversation) non_system_messages = [ message for message in messages if not _is_system_instruction_responses(message) ] @@ -458,8 +477,6 @@ def _set_responses_api_input_data( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") - def _set_completions_api_input_data( span: "Union[Span, StreamedSpan]", diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index b66d98ffc2..9ffa1b39cf 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -119,6 +119,24 @@ async def __call__(self, *args, **kwargs): ), ) +EXAMPLE_TOOLS = [ + { + "type": "function", + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + "required": ["location"], + }, + } +] + @pytest.mark.skipif( OPENAI_VERSION <= (2, 10, 0), @@ -4112,6 +4130,7 @@ def test_ai_client_span_responses_api_no_pii( temperature=0.7, top_p=0.9, reasoning={"effort": "high"}, + tools=EXAMPLE_TOOLS, ) spans = [item.payload for item in items] @@ -4694,6 +4713,231 @@ def test_ai_client_span_responses_api( assert spans[0]["data"][attr] == value +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,extra_kwargs,expected_present,expected_absent,include_prompts", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + True, + id="inputs-enabled-string-input", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + }, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + }, + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + True, + id="inputs-enabled-instructions-only", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + }, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + }, + {"type": "text", "content": "You are a helpful assistant."}, + ] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + }, + [SPANDATA.GEN_AI_TOOL_DEFINITIONS], + True, + id="inputs-enabled-list-input-with-system-message", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + {}, + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + True, + id="inputs-disabled", + ), + pytest.param( + {}, + { + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS], + True, + id="gen-ai-omitted-defaults-to-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + {}, + {}, + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + True, + id="inputs-enabled-no-input-provided", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + {}, + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + False, + id="include-prompts-disabled-overrides-inputs-enabled", + ), + ], +) +@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") +def test_responses_api_data_collection( + sentry_init, + capture_events, + capture_items, + data_collection, + extra_kwargs, + expected_present, + expected_absent, + include_prompts, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + _experiments={"data_collection": data_collection}, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + + client = OpenAI(api_key="z") + client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + + create_kwargs = { + "model": "gpt-4o", + "max_output_tokens": 100, + "temperature": 0.7, + "top_p": 0.9, + "reasoning": {"effort": "high"}, + } + create_kwargs.update(extra_kwargs) + + if span_streaming: + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="openai tx"): + client.responses.create(**create_kwargs) + + sentry_sdk.flush() + spans = [item.payload for item in items] + + assert len(spans) == 2 + span_data = spans[0]["attributes"] + elif stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + client.responses.create(**create_kwargs) + + spans = [item.payload for item in items] + + assert len(spans) == 1 + span_data = spans[0]["attributes"] + else: + events = capture_events() + + with start_transaction(name="openai tx"): + client.responses.create(**create_kwargs) + + (transaction,) = events + spans = transaction["spans"] + + assert len(spans) == 1 + assert spans[0]["op"] == "gen_ai.responses" + span_data = spans[0]["data"] + + # Non-input data is always collected, regardless of data collection config + assert span_data["gen_ai.operation.name"] == "responses" + assert span_data["gen_ai.request.model"] == "gpt-4o" + assert span_data["gen_ai.request.max_tokens"] == 100 + assert span_data["gen_ai.request.temperature"] == 0.7 + assert span_data["gen_ai.request.top_p"] == 0.9 + assert span_data["gen_ai.request.reasoning.level"] == "high" + assert span_data["gen_ai.system"] == "openai" + + for key, value in expected_present.items(): + assert span_data[key] == value + + for key in expected_absent: + assert key not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize(