From f2fded483e636abf3469bc67272e31591445abc7 Mon Sep 17 00:00:00 2001 From: Andrew Leverette Date: Mon, 10 Aug 2026 15:50:32 -0500 Subject: [PATCH] feat(googleSheets): auto-collapse pivot table outer row groups on export Collapse every pivot table's outer-most row-field group before exporting a Google Sheets template report as xlsx, so the rendered file shows only header/Total rows instead of every expanded detail row. - Read each pivot table's rendered cell text to discover current group values (freshly written pivots have no pre-existing ValueMetadata). - Merge Collapsed = true into the outer-most PivotGroup's ValueMetadata per discovered value, preserving entries for values not currently rendered, and write the whole PivotTable back via UpdateCells. - Batch collapse requests (chunks of 100) via batchUpdate. - Wire into GoogleSheetsTemplateReportControl as a best-effort step: a collapse failure is logged but doesn't block export. Mirrors the pivot-collapsing algorithm and Sheets API call shapes used in the node-utility project's googleSheetsClient. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../GoogleSheets/GoogleSheetsClient.cs | 302 ++++++++++++++++++ .../GoogleSheetsClientException.cs | 1 + .../GoogleSheetsTemplateReportControl.ascx.cs | 12 + 3 files changed, 315 insertions(+) diff --git a/Components/Services/GoogleSheets/GoogleSheetsClient.cs b/Components/Services/GoogleSheets/GoogleSheetsClient.cs index ac990e5..50171bc 100644 --- a/Components/Services/GoogleSheets/GoogleSheetsClient.cs +++ b/Components/Services/GoogleSheets/GoogleSheetsClient.cs @@ -3,6 +3,7 @@ using System.Configuration; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Google.Apis.Auth.OAuth2; using Google.Apis.Drive.v3; @@ -28,6 +29,21 @@ public class GoogleSheetsClient private static readonly TimeSpan CredentialLifetime = TimeSpan.FromHours(1); private static readonly string[] Scopes = { DriveService.Scope.Drive, SheetsService.Scope.Spreadsheets }; + /// Max columns to scan rightward from a pivot table's anchor cell when detecting its header width. + private const int PivotMaxColumnScan = 50; + + /// Max rows to scan downward from a pivot table's anchor cell when detecting its extent, bounding pathologically large sheets. + private const int PivotMaxRowScan = 50000; + + /// Consecutive fully-blank rows required before treating a pivot's rendered output as finished, so a single stray blank separator row doesn't end the scan early. + private const int PivotBlankRowConfirmation = 2; + + /// Max collapse requests per batchUpdate call, so workbooks with many pivot tables can't produce an oversized single payload. + private const int PivotBatchChunkSize = 100; + + /// Matches a pivot subtotal row's rendered label, e.g. "Driver: John Smith Total". + private static readonly Regex TotalRowPattern = new Regex(@"\bTotal$", RegexOptions.IgnoreCase); + private static readonly object CredentialLock = new object(); private static DriveService _cachedDriveService; private static SheetsService _cachedSheetsService; @@ -235,6 +251,292 @@ public byte[] ExportAsXlsx(string spreadsheetId) } } + /// + /// Collapse every pivot table's outer-most row-field group by writing + /// Collapsed = true into that group's ValueMetadata, one entry per unique + /// group value, so the pivot renders with only header/Total rows visible. + /// + /// This reads each pivot table's rendered cell text to discover the actual group values + /// currently present (since a freshly written pivot has no pre-existing + /// ValueMetadata), merges Collapsed = true into the outer-most + /// 's ValueMetadata for each discovered value (preserving + /// any existing entries for values not currently rendered), and writes the entire + /// PivotTable object back via UpdateCells (the Sheets API requires the + /// whole pivot table definition on write, not a partial patch). Spreadsheets with no + /// pivot tables, or pivot tables with no row fields, are a no-op. + /// + public void CollapsePivotTables(string spreadsheetId) + { + try + { + var getRequest = Sheets.Spreadsheets.Get(spreadsheetId); + getRequest.Fields = "sheets(properties.sheetId,properties.gridProperties.rowCount,data(startRow,startColumn,rowData(values(formattedValue,pivotTable))))"; + var spreadsheet = getRequest.Execute(); + + var collapseRequests = new List(); + + foreach (var sheet in spreadsheet.Sheets ?? new List()) + { + var sheetId = sheet.Properties != null ? sheet.Properties.SheetId : null; + if (sheetId == null) + { + continue; + } + + var maxRow = Math.Min( + sheet.Properties.GridProperties != null && sheet.Properties.GridProperties.RowCount.HasValue ? sheet.Properties.GridProperties.RowCount.Value : 0, + PivotMaxRowScan); + var dataChunks = sheet.Data ?? new List(); + var cellText = BuildCellTextLookup(dataChunks); + + foreach (var dataChunk in dataChunks) + { + var startRow = dataChunk.StartRow ?? 0; + var startColumn = dataChunk.StartColumn ?? 0; + var rowDataList = dataChunk.RowData ?? new List(); + + for (var rowIndex = 0; rowIndex < rowDataList.Count; rowIndex++) + { + var values = rowDataList[rowIndex].Values ?? new List(); + for (var colIndex = 0; colIndex < values.Count; colIndex++) + { + var pivotTable = values[colIndex].PivotTable; + var outerRowField = pivotTable != null && pivotTable.Rows != null && pivotTable.Rows.Count > 0 ? pivotTable.Rows[0] : null; + if (pivotTable == null || outerRowField == null) + { + continue; + } + + var anchorRow = startRow + rowIndex; + var anchorColumn = startColumn + colIndex; + + var width = DetectPivotWidth(cellText, anchorRow, anchorColumn, pivotTable); + var extentEnd = DetectPivotExtent(cellText, anchorRow, anchorColumn, width, maxRow); + + if (extentEnd <= anchorRow) + { + continue; + } + + // Only the outer-most row field needs to be collapsed: + // collapsing its groups collapses everything nested beneath. + var groupValues = DetectOuterGroupValues( + cellText, + anchorColumn, + anchorRow + 1, + extentEnd, + outerRowField.ShowTotals != false); + + if (groupValues.Count == 0) + { + continue; + } + + var updatedRows = new List(pivotTable.Rows); + updatedRows[0] = new PivotGroup + { + GroupRule = outerRowField.GroupRule, + Label = outerRowField.Label, + RepeatHeadings = outerRowField.RepeatHeadings, + ShowTotals = outerRowField.ShowTotals, + SortOrder = outerRowField.SortOrder, + SourceColumnOffset = outerRowField.SourceColumnOffset, + ValueBucket = outerRowField.ValueBucket, + ValueMetadata = MergeCollapsedValueMetadata(outerRowField.ValueMetadata, groupValues), + }; + + var updatedPivotTable = new PivotTable + { + Columns = pivotTable.Columns, + Criteria = pivotTable.Criteria, + Rows = updatedRows, + Source = pivotTable.Source, + ValueLayout = pivotTable.ValueLayout, + Values = pivotTable.Values, + }; + + collapseRequests.Add(new Request + { + UpdateCells = new UpdateCellsRequest + { + Start = new GridCoordinate { SheetId = sheetId, RowIndex = anchorRow, ColumnIndex = anchorColumn }, + Rows = new List + { + new RowData + { + Values = new List + { + new CellData { PivotTable = updatedPivotTable }, + }, + }, + }, + Fields = "pivotTable", + }, + }); + } + } + } + } + + if (collapseRequests.Count == 0) + { + return; + } + + for (var i = 0; i < collapseRequests.Count; i += PivotBatchChunkSize) + { + var chunk = collapseRequests.Skip(i).Take(PivotBatchChunkSize).ToList(); + var batchRequest = Sheets.Spreadsheets.BatchUpdate(new BatchUpdateSpreadsheetRequest { Requests = chunk }, spreadsheetId); + batchRequest.Execute(); + } + } + catch (Google.GoogleApiException ex) + { + throw GoogleSheetsClientException.FromGoogleApiException(GoogleSheetsErrorType.Collapse, string.Format("Error collapsing pivot tables in spreadsheet '{0}'.", spreadsheetId), ex); + } + } + + /// Build a fast (row, col) -> formattedValue lookup over one or more fetched GridData chunks. + private static Func BuildCellTextLookup(IList dataChunks) + { + var map = new Dictionary(); + + foreach (var chunk in dataChunks) + { + var startRow = chunk.StartRow ?? 0; + var startColumn = chunk.StartColumn ?? 0; + var rowDataList = chunk.RowData ?? new List(); + + for (var rowIndex = 0; rowIndex < rowDataList.Count; rowIndex++) + { + var values = rowDataList[rowIndex].Values ?? new List(); + for (var colIndex = 0; colIndex < values.Count; colIndex++) + { + if (!string.IsNullOrEmpty(values[colIndex].FormattedValue)) + { + map[string.Format("{0}:{1}", startRow + rowIndex, startColumn + colIndex)] = values[colIndex].FormattedValue; + } + } + } + } + + return (row, col) => + { + string text; + return map.TryGetValue(string.Format("{0}:{1}", row, col), out text) ? text : string.Empty; + }; + } + + /// Count contiguous non-empty header columns starting at the pivot table's anchor cell, to bound scans. + private static int DetectPivotWidth(Func cellText, int anchorRow, int anchorColumn, PivotTable pivotTable) + { + var structuralMinWidth = (pivotTable.Rows != null ? pivotTable.Rows.Count : 0) + (pivotTable.Values != null ? pivotTable.Values.Count : 0); + + var width = 0; + while (width < PivotMaxColumnScan && cellText(anchorRow, anchorColumn + width) != string.Empty) + { + width++; + } + + return Math.Max(width, Math.Max(structuralMinWidth, 1)); + } + + /// Finds the last row (inclusive) belonging to a rendered pivot table's output, scanning down from its anchor. + private static int DetectPivotExtent(Func cellText, int anchorRow, int anchorColumn, int width, int maxRow) + { + var extentEnd = anchorRow; + var consecutiveBlankRows = 0; + + for (var row = anchorRow + 1; row < maxRow; row++) + { + var rowHasContent = false; + for (var col = anchorColumn; col < anchorColumn + width; col++) + { + if (cellText(row, col) != string.Empty) + { + rowHasContent = true; + break; + } + } + + if (!rowHasContent) + { + consecutiveBlankRows++; + if (consecutiveBlankRows >= PivotBlankRowConfirmation) + { + break; + } + continue; + } + + consecutiveBlankRows = 0; + extentEnd = row; + } + + return extentEnd; + } + + /// + /// Scans the outer-most pivot row-field column and returns the distinct group values + /// rendered there. When is true (the field has + /// ShowTotals enabled), rows matching are treated as + /// generated subtotal rows and skipped; otherwise every non-blank value is kept, even + /// if it happens to end in "Total". + /// + private static List DetectOuterGroupValues(Func cellText, int column, int rangeStart, int rangeEnd, bool excludeTotals) + { + var values = new List(); + + for (var row = rangeStart; row <= rangeEnd; row++) + { + var text = cellText(row, column); + if (text == string.Empty || (excludeTotals && TotalRowPattern.IsMatch(text.Trim()))) + { + continue; + } + + values.Add(text); + } + + return values; + } + + /// + /// Merges Collapsed = true into a row field's existing ValueMetadata for + /// each newly-discovered group value, keyed by Value.StringValue. Existing + /// entries for values not currently rendered (e.g. filtered out) are preserved rather + /// than discarded. + /// + private static List MergeCollapsedValueMetadata(IList existing, List groupValues) + { + var merged = new Dictionary(); + var order = new List(); + + foreach (var entry in existing ?? new List()) + { + var key = entry.Value != null ? entry.Value.StringValue : null; + if (key != null) + { + if (!merged.ContainsKey(key)) + { + order.Add(key); + } + merged[key] = entry; + } + } + + foreach (var value in groupValues) + { + if (!merged.ContainsKey(value)) + { + order.Add(value); + } + merged[value] = new PivotGroupValueMetadata { Value = new ExtendedValue { StringValue = value }, Collapsed = true }; + } + + return order.Select(key => merged[key]).ToList(); + } + /// /// Lists every (non-trashed) folder anywhere within the given shared drive - not just /// the folders directly under the drive's root - sorted by name. Used to populate the diff --git a/Components/Services/GoogleSheets/GoogleSheetsClientException.cs b/Components/Services/GoogleSheets/GoogleSheetsClientException.cs index f1fc09d..981fe7c 100644 --- a/Components/Services/GoogleSheets/GoogleSheetsClientException.cs +++ b/Components/Services/GoogleSheets/GoogleSheetsClientException.cs @@ -11,6 +11,7 @@ public enum GoogleSheetsErrorType Export, Delete, FolderList, + Collapse, RateLimit, Unknown } diff --git a/Reports/GoogleSheets/Report/GoogleSheetsTemplate/GoogleSheetsTemplateReportControl.ascx.cs b/Reports/GoogleSheets/Report/GoogleSheetsTemplate/GoogleSheetsTemplateReportControl.ascx.cs index 21470ea..a656f79 100644 --- a/Reports/GoogleSheets/Report/GoogleSheetsTemplate/GoogleSheetsTemplateReportControl.ascx.cs +++ b/Reports/GoogleSheets/Report/GoogleSheetsTemplate/GoogleSheetsTemplateReportControl.ascx.cs @@ -109,6 +109,18 @@ private void RenderGoogleSheetsTemplate(DataTable dt) client.WriteData(spreadsheetId, string.Format("{0}!A1", dataSheetName), BuildValueRows(dt, includeHeader: true)); } + try + { + client.CollapsePivotTables(spreadsheetId); + } + catch (Exception ex) + { + // Best-effort: a failure here shouldn't block the report from being + // generated - it just means the pivot table(s) will be fully expanded + // instead of collapsed in this export. + DotNetNuke.Services.Exceptions.Exceptions.LogException(ex); + } + var xlsxBytes = client.ExportAsXlsx(spreadsheetId); var details = new ExportDetails();