From 938f4533ef18a91dc9e49890c598870bf1c0306c Mon Sep 17 00:00:00 2001 From: Joshua Wilshere Date: Fri, 18 Sep 2026 21:49:10 +0000 Subject: [PATCH 1/2] Fix group chat uploads for workspace owners, admins, and document managers. Include group user roles in the chat bootstrap payload so owners, admins, and document managers are correctly recognized as eligible group uploaders. Add regression coverage, documentation, and release notes --- application/single_app/config.py | 2 +- .../single_app/route_frontend_chats.py | 9 +++++++- .../fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md | 23 +++++++++++++++++++ docs/explanation/release_notes.md | 9 ++++++++ ...est_chat_upload_group_workspace_handoff.py | 7 +++--- 5 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 docs/explanation/fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md diff --git a/application/single_app/config.py b/application/single_app/config.py index 140c40b08..b56b998d7 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -98,7 +98,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.028" +VERSION = "0.261.029" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 97c872c3a..f81ef71c4 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -868,7 +868,14 @@ def chats(): user_groups_raw = [] try: user_groups_raw = get_user_groups(user_id) - user_groups_simple = [{'id': g['id'], 'name': g.get('name', 'Unnamed')} for g in user_groups_raw] + user_groups_simple = [ + { + 'id': group['id'], + 'name': group.get('name', 'Unnamed'), + 'userRole': get_user_role_in_group(group, user_id), + } + for group in user_groups_raw + ] except Exception as e: logger.warning(f"Failed to load user groups for chats page: {e}") diff --git a/docs/explanation/fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md b/docs/explanation/fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md new file mode 100644 index 000000000..3c23c99f8 --- /dev/null +++ b/docs/explanation/fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md @@ -0,0 +1,23 @@ +# Chat Group Upload Owner Role Fix + +## Issue + +Owners of group-scoped conversations were told that no group workspace was available for upload, even though owners are permitted to upload group documents. + +## Root Cause + +The chat page passed group IDs and names to the upload JavaScript but omitted each user's resolved group role. The client therefore treated owners as users without a group upload role before the upload request was sent. + +## Version + +Fixed/Implemented in version: **0.261.029** + +The application version was updated in `application/single_app/config.py`. + +## Technical Details + +The chat bootstrap payload now includes `userRole` for each group. This allows the existing `Owner`, `Admin`, and `DocumentManager` upload allowlist to operate correctly. Server-side group membership, status, and upload-role validation remain authoritative. + +## Validation + +The group upload handoff functional test includes a contract assertion that the resolved role is passed to the frontend. The full existing test currently has two unrelated stale assertions for older frontend/search contracts; the route syntax and version/role contract were validated separately. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 5516c8d98..0f92f7a11 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,15 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.029)** + +#### Bug Fixes + +* **Group Chat Uploads Now Recognize Workspace Owners** + * Fixed group-scoped chat uploads incorrectly reporting that no group workspace was available when the signed-in user was the group owner. + * Chat bootstrap data now includes each group's resolved user role, allowing owners, admins, and document managers to use the existing group upload permissions while preserving server-side authorization checks. + * (Ref: `route_frontend_chats.py`, `chat-input-actions.js`, [Chat Group Upload Owner Role Fix](fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md)) + ### **(v0.261.028)** Tracking: [#1489](https://github.com/microsoft/simplechat/issues/1489); implementation: [PR #1488](https://github.com/microsoft/simplechat/pull/1488). diff --git a/functional_tests/test_chat_upload_group_workspace_handoff.py b/functional_tests/test_chat_upload_group_workspace_handoff.py index 325cf57ab..0a7cd0d28 100644 --- a/functional_tests/test_chat_upload_group_workspace_handoff.py +++ b/functional_tests/test_chat_upload_group_workspace_handoff.py @@ -2,8 +2,8 @@ #!/usr/bin/env python3 """ Functional test for chat upload group workspace handoff. -Version: 0.241.176 -Implemented in: 0.241.176 +Version: 0.261.029 +Implemented in: 0.261.029 This test ensures group-scoped chat uploads are queued into group workspaces, respect group write roles, avoid accidental group document revisions through @@ -54,6 +54,7 @@ def test_upload_route_group_scope_contract(): route_frontend_chats = read_repo_file("application/single_app/route_frontend_chats.py") assert_contains(route_frontend_chats, "GROUP_CHAT_UPLOAD_ROLES = ('Owner', 'Admin', 'DocumentManager')", "group upload role allowlist") + assert_contains(route_frontend_chats, "'userRole': get_user_role_in_group(group, user_id)", "chat group role passed to frontend") assert_contains(route_frontend_chats, "def _resolve_group_workspace_upload_target", "group upload target resolver") assert_contains(route_frontend_chats, "check_group_status_allows_operation(group_doc, 'upload')", "group upload status validation") assert_contains(route_frontend_chats, "assert_group_role(user_id, normalized_selected_group_id, allowed_roles=GROUP_CHAT_UPLOAD_ROLES)", "server-side write role enforcement") @@ -106,7 +107,7 @@ def test_group_uploaded_documents_are_linked_to_chat_search_and_delete_contract( def test_version_contract(): """Validate the implementation version was bumped consistently.""" config = read_repo_file("application/single_app/config.py") - assert_contains(config, 'VERSION = "0.241.176"', "application version bump") + assert_contains(config, 'VERSION = "0.261.029"', "application version bump") def main(): From 899da951987693ab8bbc0c6583d8c53939ba724f Mon Sep 17 00:00:00 2001 From: Joshua Wilshere Date: Sat, 19 Sep 2026 01:12:18 +0000 Subject: [PATCH 2/2] fix(tabular): support ANSI CSVs and reroute repeated tool failures Add shared CSV decoding with UTF-8, UTF-8 BOM, Windows-1252, and Latin-1 fallbacks across tabular processing paths. Preserve the broader automatic tool-invocation budget while detecting repeated equivalent failures, emitting visible retry lifecycle updates, and instructing subsequent model attempts to use a different query path. --- application/single_app/config.py | 2 +- application/single_app/functions_content.py | 3 +- application/single_app/functions_documents.py | 7 +- .../single_app/functions_tabular_csv_query.py | 47 +++++++++-- application/single_app/route_backend_chats.py | 83 ++++++++++++++++++- .../single_app/route_backend_documents.py | 3 +- .../single_app/route_enhanced_citations.py | 3 +- .../tabular_processing_plugin.py | 3 +- ...TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md | 23 +++++ docs/explanation/release_notes.md | 10 +++ .../test_tabular_csv_ansi_encoding.py | 63 ++++++++++++++ 11 files changed, 233 insertions(+), 14 deletions(-) create mode 100644 docs/explanation/fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md create mode 100644 functional_tests/test_tabular_csv_ansi_encoding.py diff --git a/application/single_app/config.py b/application/single_app/config.py index b56b998d7..cc55f92ec 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -98,7 +98,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.029" +VERSION = "0.261.030" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index d236839a9..9f57cb90d 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -696,7 +696,8 @@ def extract_content_with_extraction_engine( def extract_table_file(file_path, file_ext): try: if file_ext == '.csv': - df = pandas.read_csv(file_path) + from functions_tabular_csv_query import read_tabular_csv + df = read_tabular_csv(file_path) elif file_ext in ['.xls', '.xlsx', '.xlsm']: df = pandas.read_excel(file_path) else: diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index c70adee4a..c3c49d0c8 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -61,6 +61,7 @@ get_model_endpoint_api_type, resolve_model_endpoint_request_model, ) +from functions_tabular_csv_query import read_tabular_csv from model_endpoint_clients import MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, infer_model_endpoint_protocol import azure.cognitiveservices.speech as speechsdk @@ -8418,7 +8419,7 @@ def _build_minimal_tabular_summary(temp_file_path, original_filename, file_ext): if file_ext == '.csv': column_summary = "Column discovery unavailable" try: - header_df = pandas.read_csv(temp_file_path, keep_default_na=False, dtype=str, nrows=0) + header_df = read_tabular_csv(temp_file_path, keep_default_na=False, dtype=str, nrows=0) compact_columns = _compact_tabular_columns(header_df.columns.tolist()) if compact_columns: column_summary = ", ".join(compact_columns) @@ -8465,7 +8466,7 @@ def _build_tabular_schema_summary(temp_file_path, original_filename, file_ext): plugin_note = "This file is available for detailed analysis via the Tabular Processing plugin." if file_ext == '.csv': - df_preview = pandas.read_csv( + df_preview = read_tabular_csv( temp_file_path, keep_default_na=False, dtype=str, @@ -8703,7 +8704,7 @@ def process_tabular(document_id, user_id, temp_file_path, original_filename, fil if total_chunks_saved == 0 and not enable_enhanced_citations: try: if file_ext == '.csv': - df = pandas.read_csv( + df = read_tabular_csv( temp_file_path, keep_default_na=False, dtype=str diff --git a/application/single_app/functions_tabular_csv_query.py b/application/single_app/functions_tabular_csv_query.py index e8b748687..cf7a68c0d 100644 --- a/application/single_app/functions_tabular_csv_query.py +++ b/application/single_app/functions_tabular_csv_query.py @@ -2,10 +2,50 @@ """Shared bounded CSV query evaluation for foreground tools and durable exports.""" import ast +import io +import os import pandas +TABULAR_CSV_ENCODINGS = ('utf-8-sig', 'utf-8', 'cp1252', 'latin-1') + + +def read_tabular_csv(source, **read_options): + """Read a CSV after resolving UTF-8 and common ANSI encodings.""" + if isinstance(source, (str, os.PathLike)): + with open(source, 'rb') as source_file: + source_bytes = source_file.read() + elif hasattr(source, 'read'): + current_position = source.tell() if hasattr(source, 'tell') else None + if hasattr(source, 'seek'): + source.seek(0) + source_value = source.read() + if current_position is not None and hasattr(source, 'seek'): + source.seek(current_position) + source_bytes = source_value.encode('utf-8') if isinstance(source_value, str) else source_value + else: + source_bytes = source + + if not isinstance(source_bytes, bytes): + raise TypeError('CSV source must be a path, byte stream, or bytes') + + for encoding in TABULAR_CSV_ENCODINGS: + try: + decoded_csv = source_bytes.decode(encoding) + return pandas.read_csv(io.StringIO(decoded_csv), **read_options) + except UnicodeDecodeError: + continue + + raise UnicodeDecodeError( + 'tabular_csv', + source_bytes, + 0, + len(source_bytes), + 'Unable to decode CSV using supported encodings', + ) + + TABULAR_ROW_LOCAL_QUERY_AST_NODES = ( ast.Expression, ast.BoolOp, @@ -240,8 +280,7 @@ def validate_tabular_csv_query_expression(query_expression): def detect_tabular_csv_numeric_columns(csv_stream, source_chunk_rows, tabular_plugin): """Find columns that pandas can convert to numeric across every bounded chunk.""" numeric_columns = None - csv_stream.seek(0) - for source_chunk in pandas.read_csv( + for source_chunk in read_tabular_csv( csv_stream, keep_default_na=False, dtype=str, @@ -255,7 +294,6 @@ def detect_tabular_csv_numeric_columns(csv_stream, source_chunk_rows, tabular_pl pandas.to_numeric(source_chunk[column_name]) except (TypeError, ValueError): numeric_columns.discard(column_name) - csv_stream.seek(0) return numeric_columns or set() @@ -279,7 +317,6 @@ def iter_tabular_csv_query_rows( ) parsed_return_columns = tabular_plugin._parse_optional_column_list_argument(return_columns) - csv_stream.seek(0) read_options = { 'keep_default_na': False, 'dtype': str, @@ -289,7 +326,7 @@ def iter_tabular_csv_query_rows( read_options['skiprows'] = lambda row_index: 0 < row_index <= start_source_row source_row_offset = start_source_row - for source_chunk in pandas.read_csv(csv_stream, **read_options): + for source_chunk in read_tabular_csv(csv_stream, **read_options): source_chunk = tabular_plugin._normalize_dataframe_columns(source_chunk) source_chunk.index = range(source_row_offset, source_row_offset + len(source_chunk)) source_row_offset += len(source_chunk) diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 3c017fcbf..caf8f11b6 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -9534,6 +9534,50 @@ def summarize_tabular_invocation_errors(invocations): return unique_errors +def build_tabular_invocation_failure_signature(invocation): + """Build a stable signature for detecting repeated equivalent tool failures.""" + error_message = get_tabular_invocation_error_message(invocation) + if not error_message: + return None + + parameters = getattr(invocation, 'parameters', {}) or {} + comparable_parameters = { + str(parameter_name): parameter_value + for parameter_name, parameter_value in parameters.items() + if parameter_name not in {'user_id', 'conversation_id'} + } + normalized_error = re.sub(r'\s+', ' ', str(error_message).strip()).casefold() + return ( + str(getattr(invocation, 'function_name', '') or '').strip(), + json.dumps(comparable_parameters, sort_keys=True, default=str), + normalized_error, + ) + + +def get_repeated_tabular_invocation_failures(invocations, minimum_repeats=2): + """Return repeated equivalent failures with their function and safe error text.""" + failures_by_signature = {} + for invocation in invocations or []: + signature = build_tabular_invocation_failure_signature(invocation) + if signature is None: + continue + failures_by_signature.setdefault(signature, []).append(invocation) + + repeated_failures = [] + for (function_name, _parameters, normalized_error), matching_invocations in failures_by_signature.items(): + if len(matching_invocations) < minimum_repeats: + continue + error_message = get_tabular_invocation_error_message(matching_invocations[0]) + repeated_failures.append({ + 'function_name': function_name, + 'count': len(matching_invocations), + 'error_message': error_message, + 'normalized_error': normalized_error, + }) + + return repeated_failures + + def summarize_tabular_discovery_invocations(invocations, max_sheet_names=6): """Return compact workbook-discovery summaries for retry prompts.""" discovery_summaries = [] @@ -12730,6 +12774,42 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, else: successful_schema_summary_invocations.append(invocation) + repeated_failures = get_repeated_tabular_invocation_failures( + failed_analytical_invocations + failed_schema_summary_invocations, + ) + repeated_failure_feedback_messages = [] + if repeated_failures: + repeated_failure = repeated_failures[0] + safe_repeated_failure_error = sanitize_plugin_invocation_value( + repeated_failure['error_message'] + ) + repeated_failure_feedback_messages.append( + f"The tool call {repeated_failure['function_name']} produced the same error repeatedly. Do not repeat that exact call. Change the filename, sheet, column, arguments, or use a different analytical function that addresses the user's question." + ) + log_event( + '[TABULAR_SK_ANALYSIS] Repeated equivalent tool failure detected; routing away from the failed call', + extra={ + 'function_name': repeated_failure['function_name'], + 'repeat_count': repeated_failure['count'], + 'error_message': repeated_failure['error_message'], + 'attempt_number': attempt_number, + }, + level=logging.ERROR, + ) + await emit_tabular_analysis_lifecycle_thought( + thought_callback, + f"Tabular tool {repeated_failure['function_name']} failed repeatedly", + detail=( + f"Failed {repeated_failure['count']} times with the same error: " + f"{safe_repeated_failure_error}" + ), + title='Tabular analysis needs a different query path', + state='running', + phase='retry', + attempt_number=attempt_number, + attempt_count=3, + ) + if synthesis_exception is not None: raw_tool_fallback = None if not schema_summary_mode: @@ -12785,6 +12865,7 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, if failed_schema_summary_invocations: previous_tool_error_messages = summarize_tabular_invocation_errors(failed_schema_summary_invocations) + previous_execution_gap_messages = repeated_failure_feedback_messages log_event( f"[TABULAR_SK_ANALYSIS] Attempt {attempt_number} used workbook schema tool(s) but all returned errors; retrying", extra={ @@ -12890,7 +12971,7 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, if failed_analytical_invocations: previous_tool_error_messages = summarize_tabular_invocation_errors(failed_analytical_invocations) - previous_execution_gap_messages = [] + previous_execution_gap_messages = repeated_failure_feedback_messages retry_sheet_overrides = get_tabular_retry_sheet_overrides(failed_analytical_invocations) for workbook_name, override_payload in retry_sheet_overrides.items(): blob_location = workbook_blob_locations.get(workbook_name) diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index 71d358417..d85c95588 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -444,7 +444,8 @@ def get_file_content(): file_ext = os.path.splitext(filename)[1].lower() if file_ext == '.csv': import pandas - df = pandas.read_csv(io.BytesIO(blob_data)) + from functions_tabular_csv_query import read_tabular_csv + df = read_tabular_csv(io.BytesIO(blob_data)) combined_content = df.to_csv(index=False) elif file_ext in ['.xlsx', '.xlsm']: import pandas diff --git a/application/single_app/route_enhanced_citations.py b/application/single_app/route_enhanced_citations.py index 081404c92..efa738a52 100644 --- a/application/single_app/route_enhanced_citations.py +++ b/application/single_app/route_enhanced_citations.py @@ -929,7 +929,8 @@ def get_enhanced_citation_tabular_preview(): selected_sheet = None sheet_names = [] if ext == 'csv': - df = pandas.read_csv(io.BytesIO(data), keep_default_na=False, dtype=str, nrows=nrows_limit) + from functions_tabular_csv_query import read_tabular_csv + df = read_tabular_csv(io.BytesIO(data), keep_default_na=False, dtype=str, nrows=nrows_limit) elif ext in ('xlsx', 'xlsm'): excel_file = pandas.ExcelFile(io.BytesIO(data), engine='openpyxl') sheet_names = list(excel_file.sheet_names) diff --git a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py index 7d93afa50..3ccf9a1ac 100644 --- a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py +++ b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py @@ -27,6 +27,7 @@ from functions_authentication import get_current_user_id from functions_tabular_csv_query import ( iter_tabular_csv_query_rows, + read_tabular_csv, validate_tabular_csv_query_expression, ) from functions_group import find_group_by_id, get_user_role_in_group @@ -3113,7 +3114,7 @@ def _read_tabular_blob_to_dataframe( name_lower = blob_name.lower() if name_lower.endswith('.csv'): - df = pandas.read_csv(io.BytesIO(data), keep_default_na=False, dtype=str) + df = read_tabular_csv(io.BytesIO(data), keep_default_na=False, dtype=str) elif name_lower.endswith('.xlsx') or name_lower.endswith('.xlsm'): df = pandas.read_excel( io.BytesIO(data), diff --git a/docs/explanation/fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md b/docs/explanation/fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md new file mode 100644 index 000000000..a07214e99 --- /dev/null +++ b/docs/explanation/fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md @@ -0,0 +1,23 @@ +# Tabular CSV ANSI Encoding and Retry Fix + +## Issue + +ANSI-encoded CSV files could upload successfully while producing unreadable metadata and failed `search_rows` calls. Repeated tool failures could also consume a long automatic-invocation sequence before the outer tabular retry logic completed, without clearly reporting the failure in the live analysis stream. + +## Root Cause + +CSV readers passed raw files directly to pandas, which defaults to UTF-8 and does not reliably decode Windows-1252 content. Tabular analysis had no targeted circuit breaker for equivalent failed calls, so the model could repeat the same invalid request while consuming its automatic-invocation budget. + +## Implemented in version: **0.261.030** + +## Technical Details + +- Added shared CSV decoding that prefers UTF-8, then UTF-8 with BOM, Windows-1252, and Latin-1. +- Applied the decoder to tabular metadata, indexing, citation previews, foreground queries, and durable row replay. +- Restored the broader 20-call automatic-invocation budget so complex analysis is not cut short by a universal cap. +- Detect repeated equivalent failures by function, arguments, and normalized error, then route the next model pass away from the failed call shape. +- Emit an explicit tabular retry lifecycle thought and server-side failure event when repeated failures are detected. + +## Validation + +The regression test in `functional_tests/test_tabular_csv_ansi_encoding.py` verifies Windows-1252 characters are preserved, the 20-call budget remains available for complex analysis, and repeated-failure routing is present. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 0f92f7a11..e4938075a 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,16 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.030)** + +#### Bug Fixes + +* **ANSI-Encoded CSV Files Are Now Read Correctly** + * CSV metadata extraction, indexing, citations, row searches, and durable tabular replay now support UTF-8, UTF-8 with BOM, Windows-1252, and Latin-1 files, preserving characters that previously made uploaded content appear unreadable. + * Tabular analysis retains the broader automatic-invocation budget for complex questions while detecting repeated equivalent failures and routing the next model pass to a different call shape instead of repeating the same error. + * Repeated tabular tool failures now emit an explicit retry lifecycle thought and server-side diagnostic event, making the failure visible while recovery continues. + * (Ref: `functions_tabular_csv_query.py`, `functions_documents.py`, `tabular_processing_plugin.py`, `route_backend_chats.py`, [Tabular CSV ANSI Encoding and Retry Fix](fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md)) + ### **(v0.261.029)** #### Bug Fixes diff --git a/functional_tests/test_tabular_csv_ansi_encoding.py b/functional_tests/test_tabular_csv_ansi_encoding.py new file mode 100644 index 000000000..86d1c153e --- /dev/null +++ b/functional_tests/test_tabular_csv_ansi_encoding.py @@ -0,0 +1,63 @@ +# test_tabular_csv_ansi_encoding.py +#!/usr/bin/env python3 +""" +Functional test for ANSI-encoded CSV tabular analysis. +Version: 0.261.030 +Implemented in: 0.261.030 + +This test ensures Windows-1252 CSV content is decoded correctly for metadata +and bounded tabular row queries, and that repeated tool failures are rerouted. +""" + +import io +import os +import sys + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +APPLICATION_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if APPLICATION_DIR not in sys.path: + sys.path.insert(0, APPLICATION_DIR) + +from functions_tabular_csv_query import read_tabular_csv +from test_support.versioning import assert_app_version_at_least + + +ROUTE_BACKEND_CHATS_FILE = os.path.join( + APPLICATION_DIR, + "route_backend_chats.py", +) + + +def test_windows1252_csv_content_is_read_without_replacement_characters(): + """Verify ANSI bytes survive CSV parsing as their intended characters.""" + csv_bytes = "Name,Comment\nAndre,Crème brûlée\n".encode("cp1252") + + dataframe = read_tabular_csv( + io.BytesIO(csv_bytes), + keep_default_na=False, + dtype=str, + ) + + assert dataframe.iloc[0]["Comment"] == "Crème brûlée" + assert "�" not in dataframe.iloc[0]["Comment"] + + +def test_tabular_analysis_routes_repeated_failures_without_cutting_analysis_depth(): + """Verify analysis retains depth while repeated calls receive routing feedback.""" + with open(ROUTE_BACKEND_CHATS_FILE, "r", encoding="utf-8") as file_handle: + route_source = file_handle.read() + + execution_settings_start = route_source.index("execution_settings = AzureChatPromptExecutionSettings(") + execution_settings_end = route_source.index("result = None", execution_settings_start) + tabular_analysis_source = route_source[execution_settings_start:execution_settings_end] + assert tabular_analysis_source.count("maximum_auto_invoke_attempts=20") == 2 + assert "get_repeated_tabular_invocation_failures" in route_source + assert "Do not repeat that exact call." in route_source + assert "Tabular analysis needs a different query path" in route_source + assert_app_version_at_least("0.261.030") + + +if __name__ == "__main__": + test_windows1252_csv_content_is_read_without_replacement_characters() + test_tabular_analysis_routes_repeated_failures_without_cutting_analysis_depth() + print("All ANSI CSV tabular regression tests passed")