From 782eb1aced6e6af86d9c386f387565f21ee6e28b Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:09:06 -0700 Subject: [PATCH] FIX Select response piece by data type in GPT-5 Responses tests PR #2283 made `OpenAIResponseTarget` sort reasoning pieces after the actionable response pieces, so for reasoning models the answer moved from `message_pieces[1]` to `message_pieces[0]`. The GPT-5 Responses integration tests and the paired Responses doc notebook still indexed position 1 and therefore validated the reasoning blob instead of the answer. Select the piece via `Message.get_piece_by_type(data_type="text")` so the assertions no longer depend on piece ordering, while still asserting that a reasoning piece is present. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: a008124c-220e-491c-9ea1-2144867ba53d --- doc/code/targets/2_openai_responses_target.ipynb | 5 ++++- doc/code/targets/2_openai_responses_target.py | 5 ++++- .../targets/test_openai_responses_gpt5.py | 16 +++++++++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/code/targets/2_openai_responses_target.ipynb b/doc/code/targets/2_openai_responses_target.ipynb index 561f2d0026..c4a1f7b2b6 100644 --- a/doc/code/targets/2_openai_responses_target.ipynb +++ b/doc/code/targets/2_openai_responses_target.ipynb @@ -395,7 +395,10 @@ "response = await target.send_prompt_async(message=message) # type: ignore\n", "\n", "# Validate and print the response\n", - "response_json = json.loads(response[0].message_pieces[1].converted_value)\n", + "# Reasoning models return an extra \"reasoning\" piece, so select the text piece explicitly\n", + "# instead of relying on the position of the pieces.\n", + "text_piece = response[0].get_piece_by_type(data_type=\"text\")\n", + "response_json = json.loads(text_piece.converted_value)\n", "print(json.dumps(response_json, indent=2))\n", "jsonschema.validate(instance=response_json, schema=person_schema)" ] diff --git a/doc/code/targets/2_openai_responses_target.py b/doc/code/targets/2_openai_responses_target.py index dd961257e4..25a8e3cbac 100644 --- a/doc/code/targets/2_openai_responses_target.py +++ b/doc/code/targets/2_openai_responses_target.py @@ -135,7 +135,10 @@ response = await target.send_prompt_async(message=message) # type: ignore # Validate and print the response -response_json = json.loads(response[0].message_pieces[1].converted_value) +# Reasoning models return an extra "reasoning" piece, so select the text piece explicitly +# instead of relying on the position of the pieces. +text_piece = response[0].get_piece_by_type(data_type="text") +response_json = json.loads(text_piece.converted_value) print(json.dumps(response_json, indent=2)) jsonschema.validate(instance=response_json, schema=person_schema) diff --git a/tests/integration/targets/test_openai_responses_gpt5.py b/tests/integration/targets/test_openai_responses_gpt5.py index eae9f2b697..784ae16158 100644 --- a/tests/integration/targets/test_openai_responses_gpt5.py +++ b/tests/integration/targets/test_openai_responses_gpt5.py @@ -60,10 +60,12 @@ async def test_openai_responses_gpt5(sqlite_instance, gpt5_args): assert result is not None assert len(result) == 1 assert len(result[0].message_pieces) == 2 - assert result[0].message_pieces[0].api_role == "assistant" - assert result[0].message_pieces[1].api_role == "assistant" + assert all(piece.api_role == "assistant" for piece in result[0].message_pieces) + assert result[0].get_piece_by_type(data_type="reasoning") is not None + text_piece = result[0].get_piece_by_type(data_type="text") + assert text_piece is not None # Hope that the model manages to give the correct answer somewhere (GPT-5 really should) - assert "Paris" in result[0].message_pieces[1].converted_value + assert "Paris" in text_piece.converted_value async def test_openai_responses_gpt5_json_schema(sqlite_instance, gpt5_args): @@ -110,7 +112,9 @@ async def test_openai_responses_gpt5_json_schema(sqlite_instance, gpt5_args): assert len(response) == 1 assert len(response[0].message_pieces) == 2 - response_piece = response[0].message_pieces[1] + assert response[0].get_piece_by_type(data_type="reasoning") is not None + response_piece = response[0].get_piece_by_type(data_type="text") + assert response_piece is not None assert response_piece.api_role == "assistant" response_json = json.loads(response_piece.converted_value) jsonschema.validate(instance=response_json, schema=cat_schema) @@ -144,7 +148,9 @@ async def test_openai_responses_gpt5_json_object(sqlite_instance, gpt5_args): assert len(response) == 1 assert len(response[0].message_pieces) == 2 - response_piece = response[0].message_pieces[1] + assert response[0].get_piece_by_type(data_type="reasoning") is not None + response_piece = response[0].get_piece_by_type(data_type="text") + assert response_piece is not None assert response_piece.api_role == "assistant" _ = json.loads(response_piece.converted_value) # Can't assert more, since the failure could be due to a bad generation by the model