diff --git a/application/single_app/config.py b/application/single_app/config.py index b56b998d7..cc55f92ec 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -98,7 +98,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.029" +VERSION = "0.261.030" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index d236839a9..9f57cb90d 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -696,7 +696,8 @@ def extract_content_with_extraction_engine( def extract_table_file(file_path, file_ext): try: if file_ext == '.csv': - df = pandas.read_csv(file_path) + from functions_tabular_csv_query import read_tabular_csv + df = read_tabular_csv(file_path) elif file_ext in ['.xls', '.xlsx', '.xlsm']: df = pandas.read_excel(file_path) else: diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index c70adee4a..c3c49d0c8 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -61,6 +61,7 @@ get_model_endpoint_api_type, resolve_model_endpoint_request_model, ) +from functions_tabular_csv_query import read_tabular_csv from model_endpoint_clients import MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, infer_model_endpoint_protocol import azure.cognitiveservices.speech as speechsdk @@ -8418,7 +8419,7 @@ def _build_minimal_tabular_summary(temp_file_path, original_filename, file_ext): if file_ext == '.csv': column_summary = "Column discovery unavailable" try: - header_df = pandas.read_csv(temp_file_path, keep_default_na=False, dtype=str, nrows=0) + header_df = read_tabular_csv(temp_file_path, keep_default_na=False, dtype=str, nrows=0) compact_columns = _compact_tabular_columns(header_df.columns.tolist()) if compact_columns: column_summary = ", ".join(compact_columns) @@ -8465,7 +8466,7 @@ def _build_tabular_schema_summary(temp_file_path, original_filename, file_ext): plugin_note = "This file is available for detailed analysis via the Tabular Processing plugin." if file_ext == '.csv': - df_preview = pandas.read_csv( + df_preview = read_tabular_csv( temp_file_path, keep_default_na=False, dtype=str, @@ -8703,7 +8704,7 @@ def process_tabular(document_id, user_id, temp_file_path, original_filename, fil if total_chunks_saved == 0 and not enable_enhanced_citations: try: if file_ext == '.csv': - df = pandas.read_csv( + df = read_tabular_csv( temp_file_path, keep_default_na=False, dtype=str diff --git a/application/single_app/functions_tabular_csv_query.py b/application/single_app/functions_tabular_csv_query.py index e8b748687..cf7a68c0d 100644 --- a/application/single_app/functions_tabular_csv_query.py +++ b/application/single_app/functions_tabular_csv_query.py @@ -2,10 +2,50 @@ """Shared bounded CSV query evaluation for foreground tools and durable exports.""" import ast +import io +import os import pandas +TABULAR_CSV_ENCODINGS = ('utf-8-sig', 'utf-8', 'cp1252', 'latin-1') + + +def read_tabular_csv(source, **read_options): + """Read a CSV after resolving UTF-8 and common ANSI encodings.""" + if isinstance(source, (str, os.PathLike)): + with open(source, 'rb') as source_file: + source_bytes = source_file.read() + elif hasattr(source, 'read'): + current_position = source.tell() if hasattr(source, 'tell') else None + if hasattr(source, 'seek'): + source.seek(0) + source_value = source.read() + if current_position is not None and hasattr(source, 'seek'): + source.seek(current_position) + source_bytes = source_value.encode('utf-8') if isinstance(source_value, str) else source_value + else: + source_bytes = source + + if not isinstance(source_bytes, bytes): + raise TypeError('CSV source must be a path, byte stream, or bytes') + + for encoding in TABULAR_CSV_ENCODINGS: + try: + decoded_csv = source_bytes.decode(encoding) + return pandas.read_csv(io.StringIO(decoded_csv), **read_options) + except UnicodeDecodeError: + continue + + raise UnicodeDecodeError( + 'tabular_csv', + source_bytes, + 0, + len(source_bytes), + 'Unable to decode CSV using supported encodings', + ) + + TABULAR_ROW_LOCAL_QUERY_AST_NODES = ( ast.Expression, ast.BoolOp, @@ -240,8 +280,7 @@ def validate_tabular_csv_query_expression(query_expression): def detect_tabular_csv_numeric_columns(csv_stream, source_chunk_rows, tabular_plugin): """Find columns that pandas can convert to numeric across every bounded chunk.""" numeric_columns = None - csv_stream.seek(0) - for source_chunk in pandas.read_csv( + for source_chunk in read_tabular_csv( csv_stream, keep_default_na=False, dtype=str, @@ -255,7 +294,6 @@ def detect_tabular_csv_numeric_columns(csv_stream, source_chunk_rows, tabular_pl pandas.to_numeric(source_chunk[column_name]) except (TypeError, ValueError): numeric_columns.discard(column_name) - csv_stream.seek(0) return numeric_columns or set() @@ -279,7 +317,6 @@ def iter_tabular_csv_query_rows( ) parsed_return_columns = tabular_plugin._parse_optional_column_list_argument(return_columns) - csv_stream.seek(0) read_options = { 'keep_default_na': False, 'dtype': str, @@ -289,7 +326,7 @@ def iter_tabular_csv_query_rows( read_options['skiprows'] = lambda row_index: 0 < row_index <= start_source_row source_row_offset = start_source_row - for source_chunk in pandas.read_csv(csv_stream, **read_options): + for source_chunk in read_tabular_csv(csv_stream, **read_options): source_chunk = tabular_plugin._normalize_dataframe_columns(source_chunk) source_chunk.index = range(source_row_offset, source_row_offset + len(source_chunk)) source_row_offset += len(source_chunk) diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 3c017fcbf..caf8f11b6 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -9534,6 +9534,50 @@ def summarize_tabular_invocation_errors(invocations): return unique_errors +def build_tabular_invocation_failure_signature(invocation): + """Build a stable signature for detecting repeated equivalent tool failures.""" + error_message = get_tabular_invocation_error_message(invocation) + if not error_message: + return None + + parameters = getattr(invocation, 'parameters', {}) or {} + comparable_parameters = { + str(parameter_name): parameter_value + for parameter_name, parameter_value in parameters.items() + if parameter_name not in {'user_id', 'conversation_id'} + } + normalized_error = re.sub(r'\s+', ' ', str(error_message).strip()).casefold() + return ( + str(getattr(invocation, 'function_name', '') or '').strip(), + json.dumps(comparable_parameters, sort_keys=True, default=str), + normalized_error, + ) + + +def get_repeated_tabular_invocation_failures(invocations, minimum_repeats=2): + """Return repeated equivalent failures with their function and safe error text.""" + failures_by_signature = {} + for invocation in invocations or []: + signature = build_tabular_invocation_failure_signature(invocation) + if signature is None: + continue + failures_by_signature.setdefault(signature, []).append(invocation) + + repeated_failures = [] + for (function_name, _parameters, normalized_error), matching_invocations in failures_by_signature.items(): + if len(matching_invocations) < minimum_repeats: + continue + error_message = get_tabular_invocation_error_message(matching_invocations[0]) + repeated_failures.append({ + 'function_name': function_name, + 'count': len(matching_invocations), + 'error_message': error_message, + 'normalized_error': normalized_error, + }) + + return repeated_failures + + def summarize_tabular_discovery_invocations(invocations, max_sheet_names=6): """Return compact workbook-discovery summaries for retry prompts.""" discovery_summaries = [] @@ -12730,6 +12774,42 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, else: successful_schema_summary_invocations.append(invocation) + repeated_failures = get_repeated_tabular_invocation_failures( + failed_analytical_invocations + failed_schema_summary_invocations, + ) + repeated_failure_feedback_messages = [] + if repeated_failures: + repeated_failure = repeated_failures[0] + safe_repeated_failure_error = sanitize_plugin_invocation_value( + repeated_failure['error_message'] + ) + repeated_failure_feedback_messages.append( + f"The tool call {repeated_failure['function_name']} produced the same error repeatedly. Do not repeat that exact call. Change the filename, sheet, column, arguments, or use a different analytical function that addresses the user's question." + ) + log_event( + '[TABULAR_SK_ANALYSIS] Repeated equivalent tool failure detected; routing away from the failed call', + extra={ + 'function_name': repeated_failure['function_name'], + 'repeat_count': repeated_failure['count'], + 'error_message': repeated_failure['error_message'], + 'attempt_number': attempt_number, + }, + level=logging.ERROR, + ) + await emit_tabular_analysis_lifecycle_thought( + thought_callback, + f"Tabular tool {repeated_failure['function_name']} failed repeatedly", + detail=( + f"Failed {repeated_failure['count']} times with the same error: " + f"{safe_repeated_failure_error}" + ), + title='Tabular analysis needs a different query path', + state='running', + phase='retry', + attempt_number=attempt_number, + attempt_count=3, + ) + if synthesis_exception is not None: raw_tool_fallback = None if not schema_summary_mode: @@ -12785,6 +12865,7 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, if failed_schema_summary_invocations: previous_tool_error_messages = summarize_tabular_invocation_errors(failed_schema_summary_invocations) + previous_execution_gap_messages = repeated_failure_feedback_messages log_event( f"[TABULAR_SK_ANALYSIS] Attempt {attempt_number} used workbook schema tool(s) but all returned errors; retrying", extra={ @@ -12890,7 +12971,7 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, if failed_analytical_invocations: previous_tool_error_messages = summarize_tabular_invocation_errors(failed_analytical_invocations) - previous_execution_gap_messages = [] + previous_execution_gap_messages = repeated_failure_feedback_messages retry_sheet_overrides = get_tabular_retry_sheet_overrides(failed_analytical_invocations) for workbook_name, override_payload in retry_sheet_overrides.items(): blob_location = workbook_blob_locations.get(workbook_name) diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index 71d358417..d85c95588 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -444,7 +444,8 @@ def get_file_content(): file_ext = os.path.splitext(filename)[1].lower() if file_ext == '.csv': import pandas - df = pandas.read_csv(io.BytesIO(blob_data)) + from functions_tabular_csv_query import read_tabular_csv + df = read_tabular_csv(io.BytesIO(blob_data)) combined_content = df.to_csv(index=False) elif file_ext in ['.xlsx', '.xlsm']: import pandas diff --git a/application/single_app/route_enhanced_citations.py b/application/single_app/route_enhanced_citations.py index 081404c92..efa738a52 100644 --- a/application/single_app/route_enhanced_citations.py +++ b/application/single_app/route_enhanced_citations.py @@ -929,7 +929,8 @@ def get_enhanced_citation_tabular_preview(): selected_sheet = None sheet_names = [] if ext == 'csv': - df = pandas.read_csv(io.BytesIO(data), keep_default_na=False, dtype=str, nrows=nrows_limit) + from functions_tabular_csv_query import read_tabular_csv + df = read_tabular_csv(io.BytesIO(data), keep_default_na=False, dtype=str, nrows=nrows_limit) elif ext in ('xlsx', 'xlsm'): excel_file = pandas.ExcelFile(io.BytesIO(data), engine='openpyxl') sheet_names = list(excel_file.sheet_names) diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 97c872c3a..f81ef71c4 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -868,7 +868,14 @@ def chats(): user_groups_raw = [] try: user_groups_raw = get_user_groups(user_id) - user_groups_simple = [{'id': g['id'], 'name': g.get('name', 'Unnamed')} for g in user_groups_raw] + user_groups_simple = [ + { + 'id': group['id'], + 'name': group.get('name', 'Unnamed'), + 'userRole': get_user_role_in_group(group, user_id), + } + for group in user_groups_raw + ] except Exception as e: logger.warning(f"Failed to load user groups for chats page: {e}") diff --git a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py index 7d93afa50..3ccf9a1ac 100644 --- a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py +++ b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py @@ -27,6 +27,7 @@ from functions_authentication import get_current_user_id from functions_tabular_csv_query import ( iter_tabular_csv_query_rows, + read_tabular_csv, validate_tabular_csv_query_expression, ) from functions_group import find_group_by_id, get_user_role_in_group @@ -3113,7 +3114,7 @@ def _read_tabular_blob_to_dataframe( name_lower = blob_name.lower() if name_lower.endswith('.csv'): - df = pandas.read_csv(io.BytesIO(data), keep_default_na=False, dtype=str) + df = read_tabular_csv(io.BytesIO(data), keep_default_na=False, dtype=str) elif name_lower.endswith('.xlsx') or name_lower.endswith('.xlsm'): df = pandas.read_excel( io.BytesIO(data), diff --git a/docs/explanation/fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md b/docs/explanation/fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md new file mode 100644 index 000000000..3c23c99f8 --- /dev/null +++ b/docs/explanation/fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md @@ -0,0 +1,23 @@ +# Chat Group Upload Owner Role Fix + +## Issue + +Owners of group-scoped conversations were told that no group workspace was available for upload, even though owners are permitted to upload group documents. + +## Root Cause + +The chat page passed group IDs and names to the upload JavaScript but omitted each user's resolved group role. The client therefore treated owners as users without a group upload role before the upload request was sent. + +## Version + +Fixed/Implemented in version: **0.261.029** + +The application version was updated in `application/single_app/config.py`. + +## Technical Details + +The chat bootstrap payload now includes `userRole` for each group. This allows the existing `Owner`, `Admin`, and `DocumentManager` upload allowlist to operate correctly. Server-side group membership, status, and upload-role validation remain authoritative. + +## Validation + +The group upload handoff functional test includes a contract assertion that the resolved role is passed to the frontend. The full existing test currently has two unrelated stale assertions for older frontend/search contracts; the route syntax and version/role contract were validated separately. diff --git a/docs/explanation/fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md b/docs/explanation/fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md new file mode 100644 index 000000000..a07214e99 --- /dev/null +++ b/docs/explanation/fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md @@ -0,0 +1,23 @@ +# Tabular CSV ANSI Encoding and Retry Fix + +## Issue + +ANSI-encoded CSV files could upload successfully while producing unreadable metadata and failed `search_rows` calls. Repeated tool failures could also consume a long automatic-invocation sequence before the outer tabular retry logic completed, without clearly reporting the failure in the live analysis stream. + +## Root Cause + +CSV readers passed raw files directly to pandas, which defaults to UTF-8 and does not reliably decode Windows-1252 content. Tabular analysis had no targeted circuit breaker for equivalent failed calls, so the model could repeat the same invalid request while consuming its automatic-invocation budget. + +## Implemented in version: **0.261.030** + +## Technical Details + +- Added shared CSV decoding that prefers UTF-8, then UTF-8 with BOM, Windows-1252, and Latin-1. +- Applied the decoder to tabular metadata, indexing, citation previews, foreground queries, and durable row replay. +- Restored the broader 20-call automatic-invocation budget so complex analysis is not cut short by a universal cap. +- Detect repeated equivalent failures by function, arguments, and normalized error, then route the next model pass away from the failed call shape. +- Emit an explicit tabular retry lifecycle thought and server-side failure event when repeated failures are detected. + +## Validation + +The regression test in `functional_tests/test_tabular_csv_ansi_encoding.py` verifies Windows-1252 characters are preserved, the 20-call budget remains available for complex analysis, and repeated-failure routing is present. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 5516c8d98..e4938075a 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,25 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.030)** + +#### Bug Fixes + +* **ANSI-Encoded CSV Files Are Now Read Correctly** + * CSV metadata extraction, indexing, citations, row searches, and durable tabular replay now support UTF-8, UTF-8 with BOM, Windows-1252, and Latin-1 files, preserving characters that previously made uploaded content appear unreadable. + * Tabular analysis retains the broader automatic-invocation budget for complex questions while detecting repeated equivalent failures and routing the next model pass to a different call shape instead of repeating the same error. + * Repeated tabular tool failures now emit an explicit retry lifecycle thought and server-side diagnostic event, making the failure visible while recovery continues. + * (Ref: `functions_tabular_csv_query.py`, `functions_documents.py`, `tabular_processing_plugin.py`, `route_backend_chats.py`, [Tabular CSV ANSI Encoding and Retry Fix](fixes/TABULAR_CSV_ANSI_ENCODING_AND_RETRY_FIX.md)) + +### **(v0.261.029)** + +#### Bug Fixes + +* **Group Chat Uploads Now Recognize Workspace Owners** + * Fixed group-scoped chat uploads incorrectly reporting that no group workspace was available when the signed-in user was the group owner. + * Chat bootstrap data now includes each group's resolved user role, allowing owners, admins, and document managers to use the existing group upload permissions while preserving server-side authorization checks. + * (Ref: `route_frontend_chats.py`, `chat-input-actions.js`, [Chat Group Upload Owner Role Fix](fixes/CHAT_GROUP_UPLOAD_OWNER_ROLE_FIX.md)) + ### **(v0.261.028)** Tracking: [#1489](https://github.com/microsoft/simplechat/issues/1489); implementation: [PR #1488](https://github.com/microsoft/simplechat/pull/1488). diff --git a/functional_tests/test_chat_upload_group_workspace_handoff.py b/functional_tests/test_chat_upload_group_workspace_handoff.py index 325cf57ab..0a7cd0d28 100644 --- a/functional_tests/test_chat_upload_group_workspace_handoff.py +++ b/functional_tests/test_chat_upload_group_workspace_handoff.py @@ -2,8 +2,8 @@ #!/usr/bin/env python3 """ Functional test for chat upload group workspace handoff. -Version: 0.241.176 -Implemented in: 0.241.176 +Version: 0.261.029 +Implemented in: 0.261.029 This test ensures group-scoped chat uploads are queued into group workspaces, respect group write roles, avoid accidental group document revisions through @@ -54,6 +54,7 @@ def test_upload_route_group_scope_contract(): route_frontend_chats = read_repo_file("application/single_app/route_frontend_chats.py") assert_contains(route_frontend_chats, "GROUP_CHAT_UPLOAD_ROLES = ('Owner', 'Admin', 'DocumentManager')", "group upload role allowlist") + assert_contains(route_frontend_chats, "'userRole': get_user_role_in_group(group, user_id)", "chat group role passed to frontend") assert_contains(route_frontend_chats, "def _resolve_group_workspace_upload_target", "group upload target resolver") assert_contains(route_frontend_chats, "check_group_status_allows_operation(group_doc, 'upload')", "group upload status validation") assert_contains(route_frontend_chats, "assert_group_role(user_id, normalized_selected_group_id, allowed_roles=GROUP_CHAT_UPLOAD_ROLES)", "server-side write role enforcement") @@ -106,7 +107,7 @@ def test_group_uploaded_documents_are_linked_to_chat_search_and_delete_contract( def test_version_contract(): """Validate the implementation version was bumped consistently.""" config = read_repo_file("application/single_app/config.py") - assert_contains(config, 'VERSION = "0.241.176"', "application version bump") + assert_contains(config, 'VERSION = "0.261.029"', "application version bump") def main(): diff --git a/functional_tests/test_tabular_csv_ansi_encoding.py b/functional_tests/test_tabular_csv_ansi_encoding.py new file mode 100644 index 000000000..86d1c153e --- /dev/null +++ b/functional_tests/test_tabular_csv_ansi_encoding.py @@ -0,0 +1,63 @@ +# test_tabular_csv_ansi_encoding.py +#!/usr/bin/env python3 +""" +Functional test for ANSI-encoded CSV tabular analysis. +Version: 0.261.030 +Implemented in: 0.261.030 + +This test ensures Windows-1252 CSV content is decoded correctly for metadata +and bounded tabular row queries, and that repeated tool failures are rerouted. +""" + +import io +import os +import sys + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +APPLICATION_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if APPLICATION_DIR not in sys.path: + sys.path.insert(0, APPLICATION_DIR) + +from functions_tabular_csv_query import read_tabular_csv +from test_support.versioning import assert_app_version_at_least + + +ROUTE_BACKEND_CHATS_FILE = os.path.join( + APPLICATION_DIR, + "route_backend_chats.py", +) + + +def test_windows1252_csv_content_is_read_without_replacement_characters(): + """Verify ANSI bytes survive CSV parsing as their intended characters.""" + csv_bytes = "Name,Comment\nAndre,Crème brûlée\n".encode("cp1252") + + dataframe = read_tabular_csv( + io.BytesIO(csv_bytes), + keep_default_na=False, + dtype=str, + ) + + assert dataframe.iloc[0]["Comment"] == "Crème brûlée" + assert "�" not in dataframe.iloc[0]["Comment"] + + +def test_tabular_analysis_routes_repeated_failures_without_cutting_analysis_depth(): + """Verify analysis retains depth while repeated calls receive routing feedback.""" + with open(ROUTE_BACKEND_CHATS_FILE, "r", encoding="utf-8") as file_handle: + route_source = file_handle.read() + + execution_settings_start = route_source.index("execution_settings = AzureChatPromptExecutionSettings(") + execution_settings_end = route_source.index("result = None", execution_settings_start) + tabular_analysis_source = route_source[execution_settings_start:execution_settings_end] + assert tabular_analysis_source.count("maximum_auto_invoke_attempts=20") == 2 + assert "get_repeated_tabular_invocation_failures" in route_source + assert "Do not repeat that exact call." in route_source + assert "Tabular analysis needs a different query path" in route_source + assert_app_version_at_least("0.261.030") + + +if __name__ == "__main__": + test_windows1252_csv_content_is_read_without_replacement_characters() + test_tabular_analysis_routes_repeated_failures_without_cutting_analysis_depth() + print("All ANSI CSV tabular regression tests passed")