From a5b253a4c7fe1c727abaf19b4d55ef271ed2a278 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Thu, 30 Jul 2026 23:44:08 -0400 Subject: [PATCH 1/6] Fix export widget in Python Signed-off-by: Andrew Stein --- .../templates/exported_widget.html.template | 12 +- .../tests/widget/test_widget_html_export.py | 113 ++++++++++++++++++ .../perspective/widget/__init__.py | 44 +++---- 3 files changed, 145 insertions(+), 24 deletions(-) create mode 100644 rust/perspective-python/perspective/tests/widget/test_widget_html_export.py diff --git a/rust/perspective-python/perspective/templates/exported_widget.html.template b/rust/perspective-python/perspective/templates/exported_widget.html.template index 080f46de31..219c4367b1 100644 --- a/rust/perspective-python/perspective/templates/exported_widget.html.template +++ b/rust/perspective-python/perspective/templates/exported_widget.html.template @@ -17,17 +17,21 @@ } import * as perspective from "$psp_cdn_perspective"; - const viewerId = $viewer_id; + const viewerId = "$viewer_id"; const currentScript = document.scripts[document.scripts.length - 1]; const envelope = document.getElementById(`perspective-envelope-$${viewerId}`); - const dataScript = envelope.querySelector('script[type="application/vnd.apache.arrow.file"]');; + const dataScript = envelope.querySelector('script[type="application/vnd.apache.arrow.file"]'); if (!dataScript) throw new Error('data script missing for viewer', viewerId); const data = base64ToBytes(dataScript.textContent); const viewerAttrs = $viewer_attrs; - // Create a new worker, then a new table promise on that worker. - const table = await perspective.worker().table(data.buffer); + // Create a new worker, then a new table on that worker. `worker()` + // sources the client wasm from the registered `` + // Custom Element, which registers asynchronously. + await customElements.whenDefined("perspective-viewer"); + const client = await perspective.worker(); + const table = await client.table(data.buffer); const viewer = envelope.querySelector('perspective-viewer'); viewer.load(table); viewer.restore(viewerAttrs); diff --git a/rust/perspective-python/perspective/tests/widget/test_widget_html_export.py b/rust/perspective-python/perspective/tests/widget/test_widget_html_export.py new file mode 100644 index 0000000000..2364cda67a --- /dev/null +++ b/rust/perspective-python/perspective/tests/widget/test_widget_html_export.py @@ -0,0 +1,113 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +# Regression tests for static HTML export, based on the repro in +# https://github.com/perspective-dev/perspective/issues/2846 + +import base64 +import json +import re + +import pandas as pd +import pytest + +from perspective.widget import PerspectiveWidget, __version__ + +pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning") + + +def make_widget(): + df = pd.DataFrame(dict(x=[1.0, 2.0, 3.0], y=[1.0, 3.0, 2.0])) + return PerspectiveWidget(df) + + +def export_html(widget): + bundle = widget._repr_mimebundle_() + data = bundle[0] if isinstance(bundle, tuple) else bundle + return data["text/html"] + + +class TestWidgetHtmlExport: + @pytest.fixture(autouse=True) + def _enable_export(self, monkeypatch): + monkeypatch.setenv("PSP_JUPYTER_HTML_EXPORT", "1") + + def test_disabled_without_env(self, monkeypatch): + monkeypatch.delenv("PSP_JUPYTER_HTML_EXPORT") + bundle = make_widget()._repr_mimebundle_() + data = bundle[0] if isinstance(bundle, tuple) else bundle + assert data is None or "text/html" not in data + + def test_bundle_preserves_anywidget_view(self): + bundle = make_widget()._repr_mimebundle_() + assert isinstance(bundle, tuple) + data, _metadata = bundle + assert "text/html" in data + assert "application/vnd.jupyter.widget-view+json" in data + + def test_viewer_id_is_quoted(self): + widget = make_widget() + html = export_html(widget) + assert f'const viewerId = "{widget.model_id}";' in html + assert f'id="perspective-envelope-{widget.model_id}"' in html + + def test_viewer_attrs_is_json(self): + widget = make_widget() + html = export_html(widget) + match = re.search(r"const viewerAttrs = (\{.*?\});", html, re.S) + assert match is not None + attrs = json.loads(match.group(1)) + assert attrs["columns"] == ["index", "x", "y"] + assert attrs == json.loads(json.dumps(widget.save())) + + def test_worker_is_awaited_before_table(self): + html = export_html(make_widget()) + assert 'await customElements.whenDefined("perspective-viewer");' in html + assert "const client = await perspective.worker();" in html + assert "const table = await client.table(data.buffer);" in html + assert "perspective.worker().table" not in html + + def test_cdn_urls_use_5x_package_names(self): + html = export_html(make_widget()) + urls = re.findall(r'(?:src|href)="([^"]+)"', html) + prefix = f"https://cdn.jsdelivr.net/npm/@perspective-dev" + assert urls == [ + f"{prefix}/client@{__version__}/dist/cdn/perspective.js", + f"{prefix}/viewer@{__version__}/dist/cdn/perspective-viewer.js", + f"{prefix}/viewer-datagrid@{__version__}/dist/cdn/perspective-viewer-datagrid.js", + f"{prefix}/viewer-charts@{__version__}/dist/cdn/perspective-viewer-charts.js", + f"{prefix}/viewer@{__version__}/dist/css/themes.css", + ] + + def test_arrow_payload_round_trips(self): + widget = make_widget() + html = export_html(widget) + match = re.search( + r'', + html, + re.S, + ) + assert match is not None + data = base64.b64decode("".join(match.group(1).split())) + table = widget.table.get_client().table(data) + try: + view = table.view() + try: + assert view.to_columns() == { + "index": [0, 1, 2], + "x": [1.0, 2.0, 3.0], + "y": [1.0, 3.0, 2.0], + } + finally: + view.delete() + finally: + table.delete() diff --git a/rust/perspective-python/perspective/widget/__init__.py b/rust/perspective-python/perspective/widget/__init__.py index e87dc1bef9..51159b4fd4 100644 --- a/rust/perspective-python/perspective/widget/__init__.py +++ b/rust/perspective-python/perspective/widget/__init__.py @@ -11,6 +11,7 @@ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ import base64 +import json import logging import os import pathlib @@ -309,30 +310,33 @@ def _repr_mimebundle_(self, **kwargs): with open(template_path, "r") as template_data: template = Template(template_data.read()) - def psp_cdn(module, path=None): - if path is None: - path = f"cdn/{module}.js" - + def psp_cdn(module, path): # perspective developer affordance: works with your local `pnpm run start blocks` # return f"http://localhost:8080/node_modules/@perspective-dev/{module}/dist/{path}" return f"https://cdn.jsdelivr.net/npm/@perspective-dev/{module}@{__version__}/dist/{path}" - return super()._repr_mimebundle_(**kwargs) | { - "text/html": template.substitute( - psp_cdn_perspective=psp_cdn("perspective"), - psp_cdn_perspective_viewer=psp_cdn("perspective-viewer"), - psp_cdn_perspective_viewer_datagrid=psp_cdn( - "perspective-viewer-datagrid" - ), - psp_cdn_perspective_viewer_charts=psp_cdn("perspective-viewer-charts"), - psp_cdn_perspective_viewer_themes=psp_cdn( - "perspective-viewer-themes", "css/themes.css" - ), - viewer_id=self.model_id, - viewer_attrs=viewer_attrs, - b64_data=b64_data.decode("utf-8"), - ) - } + html = template.substitute( + psp_cdn_perspective=psp_cdn("client", "cdn/perspective.js"), + psp_cdn_perspective_viewer=psp_cdn("viewer", "cdn/perspective-viewer.js"), + psp_cdn_perspective_viewer_datagrid=psp_cdn( + "viewer-datagrid", "cdn/perspective-viewer-datagrid.js" + ), + psp_cdn_perspective_viewer_charts=psp_cdn( + "viewer-charts", "cdn/perspective-viewer-charts.js" + ), + psp_cdn_perspective_viewer_themes=psp_cdn("viewer", "css/themes.css"), + viewer_id=self.model_id, + viewer_attrs=json.dumps(viewer_attrs), + b64_data=b64_data.decode("utf-8"), + ) + + # anywidget's `_repr_mimebundle_` returns `tuple[dict, dict] | None` + # (data, metadata) rather than the plain dict `ipywidgets` returns. + if isinstance(super_bundle, tuple): + data, metadata = super_bundle + return dict(data) | {"text/html": html}, metadata + + return (super_bundle or {}) | {"text/html": html} def _jupyter_html_export_enabled(): From 75d8fb96c6f1e7785426844c0f80092d13556d31 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 1 Aug 2026 12:44:25 -0400 Subject: [PATCH 2/6] Chart UI Fixes Signed-off-by: Andrew Stein Chart UI fixes Signed-off-by: Andrew Stein --- .../src/ts/charts/series/glyphs/draw-areas.ts | 13 +- .../src/ts/charts/series/glyphs/draw-lines.ts | 13 +- .../ts/charts/series/glyphs/draw-scatter.ts | 22 +- .../src/ts/charts/series/series-interact.ts | 215 ++++++++----- .../src/ts/charts/series/series-render.ts | 291 ++++++++++++------ .../src/ts/charts/series/series.ts | 113 +++++++ .../viewer-charts/src/ts/plugin/plugin.ts | 14 +- .../src/ts/transport/protocol.ts | 9 + .../src/ts/transport/renderer-transport.ts | 13 + .../src/ts/worker/renderer.worker.ts | 3 + .../viewer-charts/test/ts/alt-axis.spec.ts | 140 +++++++++ .../test/ts/glyph-z-order.spec.ts | 178 +++++++++++ .../test/ts/hover-restore.spec.ts | 168 ++++++++++ 13 files changed, 1004 insertions(+), 188 deletions(-) create mode 100644 packages/viewer-charts/test/ts/alt-axis.spec.ts create mode 100644 packages/viewer-charts/test/ts/glyph-z-order.spec.ts create mode 100644 packages/viewer-charts/test/ts/hover-restore.spec.ts diff --git a/packages/viewer-charts/src/ts/charts/series/glyphs/draw-areas.ts b/packages/viewer-charts/src/ts/charts/series/glyphs/draw-areas.ts index caf739c357..1708f7d94f 100644 --- a/packages/viewer-charts/src/ts/charts/series/glyphs/draw-areas.ts +++ b/packages/viewer-charts/src/ts/charts/series/glyphs/draw-areas.ts @@ -188,7 +188,9 @@ export class AreaGlyph { /** * Bind persistent strip buffers and dispatch one TRIANGLE_STRIP per * series-run. Skips hidden series. `splitFilter` (faceted frames) - * draws only the series whose `splitIdx` matches. + * draws only the series whose `splitIdx` matches; `aggRange` (mixed + * glyph-run frames) only those whose `aggIdx` lies in the inclusive + * run span. */ draw( chart: SeriesChart, @@ -198,6 +200,7 @@ export class AreaGlyph { projRight: Float32Array, opacity: number, splitFilter?: number, + aggRange?: { start: number; end: number }, ): void { const buf = this._buffers; const cache = this._program; @@ -221,6 +224,14 @@ export class AreaGlyph { continue; } + const aggIdx = chart._series[s.seriesId].aggIdx; + if ( + aggRange !== undefined && + (aggIdx < aggRange.start || aggIdx > aggRange.end) + ) { + continue; + } + gl.uniformMatrix4fv( cache.u_projection, false, diff --git a/packages/viewer-charts/src/ts/charts/series/glyphs/draw-lines.ts b/packages/viewer-charts/src/ts/charts/series/glyphs/draw-lines.ts index 3ed76b9fa5..8013a0e30a 100644 --- a/packages/viewer-charts/src/ts/charts/series/glyphs/draw-lines.ts +++ b/packages/viewer-charts/src/ts/charts/series/glyphs/draw-lines.ts @@ -272,7 +272,9 @@ export class LineGlyph { * per series. Skips hidden series via `_hiddenSeries`. Gap / * transparency rendering is governed by `u_interp_alpha`, set per * series. `splitFilter` (faceted frames) draws only the series - * whose `splitIdx` matches — one call per facet. + * whose `splitIdx` matches — one call per facet. `aggRange` (mixed + * glyph-run frames) draws only the series whose `aggIdx` lies in + * the inclusive run span. */ draw( chart: SeriesChart, @@ -281,6 +283,7 @@ export class LineGlyph { projLeft: Float32Array, projRight: Float32Array, splitFilter?: number, + aggRange?: { start: number; end: number }, ): void { const buf = this._buffers; const cache = this._program; @@ -319,6 +322,14 @@ export class LineGlyph { continue; } + const aggIdx = chart._series[s.seriesId].aggIdx; + if ( + aggRange !== undefined && + (aggIdx < aggRange.start || aggIdx > aggRange.end) + ) { + continue; + } + gl.uniformMatrix4fv( cache.u_projection, false, diff --git a/packages/viewer-charts/src/ts/charts/series/glyphs/draw-scatter.ts b/packages/viewer-charts/src/ts/charts/series/glyphs/draw-scatter.ts index a8a35e1ad6..9dbe732cf6 100644 --- a/packages/viewer-charts/src/ts/charts/series/glyphs/draw-scatter.ts +++ b/packages/viewer-charts/src/ts/charts/series/glyphs/draw-scatter.ts @@ -268,9 +268,9 @@ export class ScatterGlyph { /** * Bind the persistent left/right buffers and issue up to two draw * calls. No per-frame allocations or buffer uploads. `splitFilter` - * (faceted frames) instead draws each matching series' contiguous - * bucket sub-range — one `drawArrays(POINTS, first, count)` per - * series of the facet. + * (faceted frames) and/or `aggRange` (mixed glyph-run frames) + * instead draw each matching series' contiguous bucket sub-range — + * one `drawArrays(POINTS, first, count)` per matching series. */ draw( chart: SeriesChart, @@ -279,6 +279,7 @@ export class ScatterGlyph { projLeft: Float32Array, projRight: Float32Array, splitFilter?: number, + aggRange?: { start: number; end: number }, ): void { const buf = this._buffers; const cache = this._program; @@ -297,9 +298,20 @@ export class ScatterGlyph { chart._pluginConfig.point_size_px * dpr, ); - if (splitFilter !== undefined) { + if (splitFilter !== undefined || aggRange !== undefined) { for (const r of buf.seriesRanges) { - if (chart._series[r.seriesId].splitIdx !== splitFilter) { + if ( + splitFilter !== undefined && + chart._series[r.seriesId].splitIdx !== splitFilter + ) { + continue; + } + + const aggIdx = chart._series[r.seriesId].aggIdx; + if ( + aggRange !== undefined && + (aggIdx < aggRange.start || aggIdx > aggRange.end) + ) { continue; } diff --git a/packages/viewer-charts/src/ts/charts/series/series-interact.ts b/packages/viewer-charts/src/ts/charts/series/series-interact.ts index 9d56cc9386..4200c63d02 100644 --- a/packages/viewer-charts/src/ts/charts/series/series-interact.ts +++ b/packages/viewer-charts/src/ts/charts/series/series-interact.ts @@ -47,8 +47,9 @@ export function getHoveredBar(chart: SeriesChart): SeriesChartRecord | null { } /** - * Handle mouse-move across all glyph types. Tests (in reverse paint order - * so top glyphs win): scatter points → line points → bars → areas. + * Handle mouse-move across all glyph types. Tests glyph runs in reverse + * paint order so top glyphs win — paint order is the `columns` + * declaration order (see `_glyphRuns` / `drawGlyphRuns`). * Updates `_hoveredBarIdx` or `_hoveredSample` and re-renders on change. * * Faceted frames first resolve the cell under the cursor: that cell's @@ -153,96 +154,131 @@ export function handleBarHover( let nextBarIdx = -1; let nextSample: SeriesChartRecord | null = null; - // 1. Scatter (top). - nextSample = hitTestPoints( - chart, - "scatter", - dataX, - dataYLeft, - dataYRight, - pxPerDataX, - pxPerDataYLeft, - pxPerDataYRight, - splitFilter, - ); + // Test topmost-first: the REVERSE of the paint order, which is the + // `columns` declaration order (`_glyphRuns`, ascending `aggIdx` — + // see `drawGlyphRuns`). A single-run chart passes no `aggRange`, + // reducing to the legacy whole-type scan. + const runs = chart._glyphRuns; + const single = runs.length <= 1; + for (let r = runs.length - 1; r >= 0; r--) { + const run = runs[r]; + const aggRange = single + ? undefined + : { start: run.aggStart, end: run.aggEnd }; + switch (run.chartType) { + case "scatter": + case "line": + nextSample = hitTestPoints( + chart, + run.chartType, + dataX, + dataYLeft, + dataYRight, + pxPerDataX, + pxPerDataYLeft, + pxPerDataYRight, + splitFilter, + aggRange, + ); + break; + case "bar": + nextBarIdx = hitTestBars( + chart, + dataX, + dataYLeft, + dataYRight, + splitFilter, + aggRange, + ); + break; + case "area": { + const areaHit = hitTestAreas( + chart, + dataX, + dataYLeft, + dataYRight, + splitFilter, + aggRange, + ); + if (areaHit) { + if (areaHit.idx >= 0) { + nextBarIdx = areaHit.idx; + } else { + nextSample = areaHit.bar; + } + } - // 2. Line points (still above bars; treat as point hits). - if (!nextSample) { - nextSample = hitTestPoints( - chart, - "line", - dataX, - dataYLeft, - dataYRight, - pxPerDataX, - pxPerDataYLeft, - pxPerDataYRight, - splitFilter, - ); + break; + } + } + + if (nextBarIdx >= 0 || nextSample) { + break; + } } - // 3. Bars (rect intersect). - if (!nextSample) { - const bars = chart._bars; - const ct = bars.chartType; - const sid = bars.seriesId; - const xC = bars.xCenter; - const hw = bars.halfWidth; - const by0 = bars.y0; - const by1 = bars.y1; - const ax = bars.axis; - const hidden = chart._hiddenSeries; - const P = chart._splitPrefixes.length; - for (let i = 0; i < bars.count; i++) { - if (ct[i] !== BAR_TYPE_BAR) { - continue; - } + applyHover(chart, nextBarIdx, nextSample); +} - if (hidden.has(sid[i])) { - continue; - } +/** + * Rect-intersect hit-test over the bar-typed `_bars` records. Returns + * the record index, or `-1`. + */ +function hitTestBars( + chart: SeriesChart, + dataX: number, + dataYLeft: number, + dataYRight: number, + splitFilter?: number, + aggRange?: { start: number; end: number }, +): number { + const bars = chart._bars; + const ct = bars.chartType; + const sid = bars.seriesId; + const xC = bars.xCenter; + const hw = bars.halfWidth; + const by0 = bars.y0; + const by1 = bars.y1; + const ax = bars.axis; + const hidden = chart._hiddenSeries; + const P = Math.max(1, chart._splitPrefixes.length); + for (let i = 0; i < bars.count; i++) { + if (ct[i] !== BAR_TYPE_BAR) { + continue; + } - if (splitFilter !== undefined && sid[i] % P !== splitFilter) { - continue; - } + if (hidden.has(sid[i])) { + continue; + } - const xc = xC[i]; - const halfW = hw[i]; - if (dataX < xc - halfW || dataX > xc + halfW) { + if (splitFilter !== undefined && sid[i] % P !== splitFilter) { + continue; + } + + if (aggRange !== undefined) { + const aggIdx = Math.floor(sid[i] / P); + if (aggIdx < aggRange.start || aggIdx > aggRange.end) { continue; } + } - const dy = ax[i] === 0 ? dataYLeft : dataYRight; - const y0 = by0[i]; - const y1 = by1[i]; - const lo = y0 < y1 ? y0 : y1; - const hi = y0 < y1 ? y1 : y0; - if (dy >= lo && dy <= hi) { - nextBarIdx = i; - break; - } + const xc = xC[i]; + const halfW = hw[i]; + if (dataX < xc - halfW || dataX > xc + halfW) { + continue; } - } - // 4. Areas (strip hit — stacked records via `_bars`, unstacked via samples). - if (nextBarIdx < 0 && !nextSample) { - const areaHit = hitTestAreas( - chart, - dataX, - dataYLeft, - dataYRight, - splitFilter, - ); - if (areaHit) { - if (areaHit.idx >= 0) { - nextBarIdx = areaHit.idx; - } else { - nextSample = areaHit.bar; - } + const dy = ax[i] === 0 ? dataYLeft : dataYRight; + const y0 = by0[i]; + const y1 = by1[i]; + const lo = y0 < y1 ? y0 : y1; + const hi = y0 < y1 ? y1 : y0; + if (dy >= lo && dy <= hi) { + return i; } } - applyHover(chart, nextBarIdx, nextSample); + return -1; } function hitTestPoints( @@ -255,6 +291,7 @@ function hitTestPoints( pxPerDataYLeft: number, pxPerDataYRight: number, splitFilter?: number, + aggRange?: { start: number; end: number }, ): SeriesChartRecord | null { const N = chart._numCategories; const S = chart._series.length; @@ -283,6 +320,13 @@ function hitTestPoints( continue; } + if ( + aggRange !== undefined && + (s.aggIdx < aggRange.start || s.aggIdx > aggRange.end) + ) { + continue; + } + const dataY = s.axis === 1 ? dataYRight : dataYLeft; const pyPerData = s.axis === 1 ? pxPerDataYRight : pxPerDataYLeft; @@ -340,6 +384,7 @@ function hitTestAreas( dataYLeft: number, dataYRight: number, splitFilter?: number, + aggRange?: { start: number; end: number }, ): { idx: number; bar: SeriesChartRecord | null } | null { // Closest category to the mouse; an area covers every [cat - 0.5, cat + 0.5] // slot, so use `round(dataX)` as the candidate index. @@ -382,6 +427,13 @@ function hitTestAreas( continue; } + if (aggRange !== undefined) { + const aggIdx = Math.floor(sid[i] / Math.max(1, P)); + if (aggIdx < aggRange.start || aggIdx > aggRange.end) { + continue; + } + } + const dy = ax[i] === 0 ? dataYLeft : dataYRight; const y0 = by0[i]; const y1 = by1[i]; @@ -406,6 +458,13 @@ function hitTestAreas( continue; } + if ( + aggRange !== undefined && + (s.aggIdx < aggRange.start || s.aggIdx > aggRange.end) + ) { + continue; + } + const idx = cat * S + s.seriesId; if (!((valid[idx >> 3] >> (idx & 7)) & 1)) { continue; diff --git a/packages/viewer-charts/src/ts/charts/series/series-render.ts b/packages/viewer-charts/src/ts/charts/series/series-render.ts index 64028f78c4..731183ac7a 100644 --- a/packages/viewer-charts/src/ts/charts/series/series-render.ts +++ b/packages/viewer-charts/src/ts/charts/series/series-render.ts @@ -14,6 +14,7 @@ import type { Context2D } from "../canvas-types"; import type { WebGLContextManager } from "../../webgl/context-manager"; import { ensurePalette, + type GlyphRun, type SeriesChart, type SeriesAutoFitCache, } from "./series"; @@ -114,6 +115,8 @@ export function uploadBarInstances( let n = 0; chart._facetBarRanges = null; + chart._facetBarAggRanges = null; + chart._barAggRanges = null; if (total > 0) { const scratch = ensureBarInstanceScratch(total); if ( @@ -137,33 +140,60 @@ export function uploadBarInstances( const by1 = bars.y1; const ax = bars.axis; - // Facet mode: counting sort by splitIdx (`seriesId % P`). Pass - // 1 counts eligible instances per split; prefix sums become the - // per-split write cursors AND the published ranges. - let writeAt: ((seriesId: number) => number) | null = null; - if (faceted) { - const counts = new Array(P).fill(0); - for (let i = 0; i < total; i++) { - if (ct[i] !== BAR_TYPE_BAR || hidden.has(sid[i])) { - continue; - } - - counts[sid[i] % P]++; + // Counting sort so contiguous instance ranges exist for every + // draw grouping. Overlay: key = aggIdx, publishing + // `_barAggRanges` — a glyph run `[aggStart, aggEnd]` is one + // contiguous slice, and within-call overlap Z follows `columns` + // declaration order. Faceted: key = splitIdx-major then aggIdx, + // publishing both the per-split `_facetBarRanges` (facet + // dispatch) and the per-(split, agg) `_facetBarAggRanges` (run + // slices within a facet). Pass 1 counts eligible instances per + // bucket; prefix sums become the write cursors AND the ranges. + const M = Math.max(1, chart._aggregates.length); + const bucketOf = faceted + ? (seriesId: number) => + (seriesId % P) * M + Math.floor(seriesId / P) + : (seriesId: number) => + P > 0 ? Math.floor(seriesId / P) : seriesId; + const numBuckets = faceted ? P * M : M; + const counts = new Array(numBuckets).fill(0); + for (let i = 0; i < total; i++) { + if (ct[i] !== BAR_TYPE_BAR || hidden.has(sid[i])) { + continue; } - const ranges: { start: number; count: number }[] = []; - const cursors = new Int32Array(P); - let acc = 0; - for (let p = 0; p < P; p++) { - ranges.push({ start: acc, count: counts[p] }); - cursors[p] = acc; - acc += counts[p]; - } + counts[bucketOf(sid[i])]++; + } - chart._facetBarRanges = ranges; - writeAt = (seriesId: number) => cursors[seriesId % P]++; + const ranges: { start: number; count: number }[] = []; + const cursors = new Int32Array(numBuckets); + let acc = 0; + for (let b = 0; b < numBuckets; b++) { + ranges.push({ start: acc, count: counts[b] }); + cursors[b] = acc; + acc += counts[b]; } + if (faceted) { + chart._barAggRanges = null; + chart._facetBarAggRanges = Array.from({ length: P }, (_, p) => + ranges.slice(p * M, (p + 1) * M), + ); + chart._facetBarRanges = Array.from({ length: P }, (_, p) => { + const first = ranges[p * M]; + const last = ranges[(p + 1) * M - 1]; + return { + start: first.start, + count: last.start + last.count - first.start, + }; + }); + } else { + chart._barAggRanges = ranges; + chart._facetBarAggRanges = null; + } + + const writeAt = (seriesId: number) => cursors[bucketOf(seriesId)]++; + for (let i = 0; i < total; i++) { if (ct[i] !== BAR_TYPE_BAR) { continue; @@ -174,7 +204,7 @@ export function uploadBarInstances( continue; } - const w = writeAt ? writeAt(seriesId) : n; + const w = writeAt(seriesId); scratch.xCenters[w] = xC[i] - xOrigin; scratch.halfWidths[w] = hw[i]; scratch.y0s[w] = by0[i]; @@ -643,41 +673,18 @@ export function renderBarFrame( ); } + const hovered = chart._series.length > 1 ? getHoveredBar(chart) : null; renderInPlotFrame(gl, layout, glManager.dpr, () => { - // Paint order: areas behind bars (so bar borders stay crisp), - // bars above, lines above those, scatter points on top. X Bar - // only paints bars — the other glyphs bake in vertical geometry - // and aren't supported for horizontal orientation. - if (!horizontal) { - chart._glyphs.areas.draw( - chart, - gl, - glManager, - projLeft, - projRight, - theme.areaOpacity, - ); - } - - gl.useProgram(chart._program!); - const loc = chart._locations!; - gl.uniformMatrix4fv(loc.u_proj_left, false, projLeft); - gl.uniformMatrix4fv(loc.u_proj_right, false, projRight); - gl.uniform1f(loc.u_horizontal, horizontal ? 1.0 : 0.0); - const hovered = chart._series.length > 1 ? getHoveredBar(chart) : null; - gl.uniform1f(loc.u_hover_series, hovered ? hovered.seriesId : -1); - drawBars(chart, gl, glManager); - - if (!horizontal) { - chart._glyphs.lines.draw(chart, gl, glManager, projLeft, projRight); - chart._glyphs.scatter.draw( - chart, - gl, - glManager, - projLeft, - projRight, - ); - } + drawGlyphRuns( + chart, + gl, + glManager, + projLeft, + projRight, + theme.areaOpacity, + horizontal, + hovered ? hovered.seriesId : -1, + ); }); chart._lastXDomain = catDomain; @@ -694,6 +701,124 @@ export function renderBarFrame( chart._defer2D(() => renderBarChromeOverlay(chart)); } +/** + * Paint every glyph in `columns` declaration Z-order — one pass per + * {@link SeriesChart._glyphRuns} entry, ascending `aggIdx`, later + * columns on top; splits within an aggregate paint in `splitIdx` + * order. A homogeneous chart is a single run and takes exactly the + * legacy one-pass path (no per-series filtering, single instanced bar + * draw). X Bar (horizontal) paints bars only — the other glyphs bake + * vertical geometry. + * + * The caller wraps this in its plot clip (`renderInPlotFrame` / + * `withScissor`); `splitFilter` is the facet index in faceted frames. + */ +function drawGlyphRuns( + chart: SeriesChart, + gl: WebGL2RenderingContext | WebGLRenderingContext, + glManager: WebGLContextManager, + projLeft: Float32Array, + projRight: Float32Array, + areaOpacity: number, + horizontal: boolean, + hoveredSeriesId: number, + splitFilter?: number, +): void { + const runs = chart._glyphRuns; + const single = runs.length <= 1; + for (const run of runs) { + const aggRange = single + ? undefined + : { start: run.aggStart, end: run.aggEnd }; + switch (run.chartType) { + case "area": + if (!horizontal) { + chart._glyphs.areas.draw( + chart, + gl, + glManager, + projLeft, + projRight, + areaOpacity, + splitFilter, + aggRange, + ); + } + + break; + case "bar": { + gl.useProgram(chart._program!); + const loc = chart._locations!; + gl.uniformMatrix4fv(loc.u_proj_left, false, projLeft); + gl.uniformMatrix4fv(loc.u_proj_right, false, projRight); + gl.uniform1f(loc.u_horizontal, horizontal ? 1.0 : 0.0); + gl.uniform1f(loc.u_hover_series, hoveredSeriesId); + drawBars( + chart, + gl, + glManager, + barRunRange(chart, run, splitFilter), + ); + break; + } + + case "line": + if (!horizontal) { + chart._glyphs.lines.draw( + chart, + gl, + glManager, + projLeft, + projRight, + splitFilter, + aggRange, + ); + } + + break; + case "scatter": + if (!horizontal) { + chart._glyphs.scatter.draw( + chart, + gl, + glManager, + projLeft, + projRight, + splitFilter, + aggRange, + ); + } + + break; + } + } +} + +/** + * The contiguous uploaded-instance slice for a bar run — per-aggregate + * ranges are adjacent by construction (`uploadBarInstances` counting + * sort), so the run `[aggStart, aggEnd]` spans from the first range's + * start through the last range's end. Faceted frames slice within the + * facet's split-major block. + */ +function barRunRange( + chart: SeriesChart, + run: GlyphRun, + splitFilter?: number, +): { start: number; count: number } | undefined { + const table = + splitFilter !== undefined + ? chart._facetBarAggRanges?.[splitFilter] + : chart._barAggRanges; + if (!table) { + return splitFilter !== undefined ? { start: 0, count: 0 } : undefined; + } + + const first = table[run.aggStart]; + const last = table[run.aggEnd]; + return { start: first.start, count: last.start + last.count - first.start }; +} + /** * Domain window computed by `renderBarFrame`'s shared prologue (zoom * window + auto-fit + `include_zero`), handed to the faceted branch. @@ -873,55 +998,17 @@ function renderFacetedBarFrame( } withScissor(gl, layout, dpr, () => { - // Same paint order as the single-plot path: areas behind - // bars, lines above, scatter on top. X Bar draws bars only - // (the other glyphs bake vertical geometry). - if (!horizontal) { - chart._glyphs.areas.draw( - chart, - gl, - glManager, - projLeft, - projRight, - theme.areaOpacity, - p, - ); - } - - gl.useProgram(chart._program!); - const loc = chart._locations!; - gl.uniformMatrix4fv(loc.u_proj_left, false, projLeft); - gl.uniformMatrix4fv(loc.u_proj_right, false, projRight); - gl.uniform1f(loc.u_horizontal, horizontal ? 1.0 : 0.0); - gl.uniform1f( - loc.u_hover_series, - hovered && hovered.splitIdx === p ? hovered.seriesId : -1, - ); - drawBars( + drawGlyphRuns( chart, gl, glManager, - chart._facetBarRanges?.[p] ?? { start: 0, count: 0 }, + projLeft, + projRight, + theme.areaOpacity, + horizontal, + hovered && hovered.splitIdx === p ? hovered.seriesId : -1, + p, ); - - if (!horizontal) { - chart._glyphs.lines.draw( - chart, - gl, - glManager, - projLeft, - projRight, - p, - ); - chart._glyphs.scatter.draw( - chart, - gl, - glManager, - projLeft, - projRight, - p, - ); - } }); } diff --git a/packages/viewer-charts/src/ts/charts/series/series.ts b/packages/viewer-charts/src/ts/charts/series/series.ts index 4c497af133..1ddd2396bf 100644 --- a/packages/viewer-charts/src/ts/charts/series/series.ts +++ b/packages/viewer-charts/src/ts/charts/series/series.ts @@ -24,6 +24,7 @@ import { type SeriesChartRecord, type NumericCategoryDomain, type SeriesInfo, + type SeriesPipelineResult, type BarColumns, emptyBarColumns, } from "./series-build"; @@ -66,6 +67,16 @@ export interface SeriesAutoFitCache { hasRight: boolean; } +/** + * One paint pass: the contiguous span of aggregates (`columns` indices, + * inclusive) sharing a glyph type. See {@link SeriesChart._glyphRuns}. + */ +export interface GlyphRun { + chartType: SeriesInfo["chartType"]; + aggStart: number; + aggEnd: number; +} + export interface CachedLocations { u_proj_left: WebGLUniformLocation | null; u_proj_right: WebGLUniformLocation | null; @@ -153,6 +164,32 @@ export class SeriesChart extends CategoricalYChart { */ _facetBarRanges: { start: number; count: number }[] | null = null; + /** + * Paint order of the current build: consecutive aggregates sharing + * a glyph type, merged into runs, in `columns` declaration order + * (`aggIdx` ascending — later columns paint on top). The frame + * draws one pass per run; a homogeneous chart is a single run and + * degenerates to the legacy one-pass-per-type path. `aggEnd` is + * inclusive. + */ + _glyphRuns: GlyphRun[] = []; + + /** + * Per-aggregate contiguous instance ranges in the uploaded bar + * buffers, indexed by `aggIdx` — the overlay-mode counterpart of + * `_facetBarRanges`, populated by `uploadBarInstances`' aggregate + * counting sort. `null` until first upload. + */ + _barAggRanges: { start: number; count: number }[] | null = null; + + /** + * Faceted per-(split, aggregate) instance ranges, + * `[splitIdx][aggIdx]` — instances are emitted split-major then + * aggregate-ordered, so a facet's glyph-run slice is contiguous. + * `null` in overlay mode. + */ + _facetBarAggRanges: { start: number; count: number }[][] | null = null; + /** * Columnar bar/area record storage. Indexed by bar slot in * `[0, _bars.count)`. Replaces the legacy `SeriesChartRecord[]` to @@ -249,6 +286,20 @@ export class SeriesChart extends CategoricalYChart { _expandedLeftDomain: { min: number; max: number } | null = null; _expandedRightDomain: { min: number; max: number } | null = null; + /** + * The axis partition (per-aggregate axis side) the expand + * accumulators were accumulated under. An accumulator is only + * meaningful for the partition that produced it: when an aggregate + * moves sides (`columns_config.alt_axis` pin via a config-only + * `update()`, or a data-driven `auto_alt_y_axis` flip), the side it + * LEFT would otherwise retain its extent forever — the departed + * column's range never leaves the primary axis. A signature + * mismatch resets BOTH accumulators before the union. Keyed per + * AGGREGATE (not per series) so split-group growth on a streaming + * update never spuriously resets. + */ + _expandedAxisSig: string | null = null; + /** * Numeric category-axis state. Populated only when `group_by` has * exactly one level and that level is `date | datetime | integer | @@ -564,6 +615,13 @@ export class SeriesChart extends CategoricalYChart { // `"fit"` (or a fresh reset) leaves the result untouched and // clears the accumulators so the next toggle starts fresh. if (this._pluginConfig.domain_mode === "expand") { + const axisSig = expandAxisSignature(result); + if (this._expandedAxisSig !== axisSig) { + this._expandedLeftDomain = null; + this._expandedRightDomain = null; + this._expandedAxisSig = axisSig; + } + this._expandedLeftDomain = expandDomainInPlace( this._expandedLeftDomain, result.leftDomain, @@ -578,6 +636,7 @@ export class SeriesChart extends CategoricalYChart { } else { this._expandedLeftDomain = null; this._expandedRightDomain = null; + this._expandedAxisSig = null; } this._aggregates = result.aggregates; @@ -639,6 +698,25 @@ export class SeriesChart extends CategoricalYChart { this._primaryValueLabel = uniqueAggLabels(result.series, 0); this._altValueLabel = uniqueAggLabels(result.series, 1); + // Paint-order runs: every series of an aggregate shares one + // glyph type (`resolveChartType` is per-aggName), so sampling + // the aggregate's first series suffices. MUST be computed + // before `uploadBarInstances` below — the aggregate counting + // sort publishes ranges the run passes draw from. + this._glyphRuns.length = 0; + { + const P = Math.max(1, result.splitPrefixes.length); + for (let k = 0; k < result.aggregates.length; k++) { + const chartType = result.series[k * P].chartType; + const last = this._glyphRuns[this._glyphRuns.length - 1]; + if (last && last.chartType === chartType) { + last.aggEnd = k; + } else { + this._glyphRuns.push({ chartType, aggStart: k, aggEnd: k }); + } + } + } + // Pre-build the area-strip lookup index (seriesId * 1e9 + catIdx // → bar slot). Legacy code rebuilt this every frame inside // `drawAreas`. The index is derived purely from `_bars` and is @@ -657,6 +735,17 @@ export class SeriesChart extends CategoricalYChart { this._paletteCacheKey = null; this._catExtentsHidden = null; this._lastUploadedColors = null; + + // Hover records index into `_bars` / `_series` and MUST NOT + // survive a build that replaces them: bar-column capacity is + // reused across builds, so a stale `_hoveredBarIdx` reads a + // leftover `seriesId` past the new series set (and a retained + // `_hoveredSample` carries one verbatim), crashing the tooltip + // pass of the next present when the pointer rests over a glyph + // through a host `restore`. The next mousemove re-hit-tests + // against the new data. + this._hoveredBarIdx = -1; + this._hoveredSample = null; this._sampleValid = result.sampleValid; this._leftDomain = result.leftDomain; this._rightDomain = result.rightDomain; @@ -691,6 +780,7 @@ export class SeriesChart extends CategoricalYChart { override resetExpandedDomain(): void { this._expandedLeftDomain = null; this._expandedRightDomain = null; + this._expandedAxisSig = null; } protected destroyInternal(): void { @@ -709,6 +799,9 @@ export class SeriesChart extends CategoricalYChart { this._facetActive = false; this._facetGrid = null; this._facetBarRanges = null; + this._facetBarAggRanges = null; + this._barAggRanges = null; + this._glyphRuns.length = 0; this._bars = emptyBarColumns(); this._series = []; this._barSeries = []; @@ -724,6 +817,8 @@ export class SeriesChart extends CategoricalYChart { this._rowPaths = []; this._numCategories = 0; this._hiddenSeries.clear(); + this._hoveredBarIdx = -1; + this._hoveredSample = null; } } @@ -770,6 +865,24 @@ function uniqueAggLabels(series: SeriesInfo[], axis: 0 | 1): string { return ordered.join(", "); } +/** + * Per-aggregate axis-partition signature for the `domain_mode: + * "expand"` accumulators. Invariant: every series of an aggregate + * shares one axis side (both axis-override loops in the build pipeline + * assign by `aggIdx`), so sampling the aggregate's first series + * suffices. NUL-joined — aggregate names may contain any printable + * separator. + */ +function expandAxisSignature(result: SeriesPipelineResult): string { + const P = Math.max(1, result.splitPrefixes.length); + const parts: string[] = []; + for (let k = 0; k < result.aggregates.length; k++) { + parts.push(`${result.aggregates[k]}:${result.series[k * P].axis}`); + } + + return parts.join("\u0000"); +} + /** * Resolve the per-series palette and stamp it onto `_series[i].color`. * Cached on `_paletteCache` keyed by reference identity of the theme diff --git a/packages/viewer-charts/src/ts/plugin/plugin.ts b/packages/viewer-charts/src/ts/plugin/plugin.ts index a21c3b6e78..ad0049ddb9 100644 --- a/packages/viewer-charts/src/ts/plugin/plugin.ts +++ b/packages/viewer-charts/src/ts/plugin/plugin.ts @@ -171,6 +171,16 @@ export class HTMLPerspectiveViewerWebGLPluginElement */ private _pluginConfigStore: PluginConfig | null = null; + /** + * Per-column config (`alt_axis`, `chart_type`, formats, …), held on + * the element for the same reason as `_pluginConfigStore`: the host + * calls `restore()` BEFORE the first draw builds the renderer, so + * forwarding only to a live renderer silently drops the initial + * `columns_config` — `_buildRenderer` ships it in the `InitMsg` + * instead. + */ + private _columnsConfig: Record = {}; + private get _pluginConfig(): PluginConfig { if (!this._pluginConfigStore) { this._pluginConfigStore = this._effectiveDefaults(); @@ -396,6 +406,7 @@ export class HTMLPerspectiveViewerWebGLPluginElement zoom_mode: this._pluginConfig.facet_zoom_mode, }, pluginConfig: this._pluginConfig, + columnsConfig: this._columnsConfig, defaultChartType: this._chartType.default_chart_type, renderBlitMode: BLIT_MODE, }); @@ -693,8 +704,9 @@ export class HTMLPerspectiveViewerWebGLPluginElement ...config, }; + this._columnsConfig = columns_config ?? {}; this._renderer?.setPluginConfig(this._pluginConfig); - this._renderer?.setColumnsConfig(columns_config ?? {}); + this._renderer?.setColumnsConfig(this._columnsConfig); } delete() { diff --git a/packages/viewer-charts/src/ts/transport/protocol.ts b/packages/viewer-charts/src/ts/transport/protocol.ts index 21e4662aa1..ad38ff2711 100644 --- a/packages/viewer-charts/src/ts/transport/protocol.ts +++ b/packages/viewer-charts/src/ts/transport/protocol.ts @@ -162,6 +162,15 @@ export interface InitMsg { * a `setPluginConfig` control msg. */ pluginConfig: PluginConfig; + + /** + * Initial per-column config (`alt_axis`, `chart_type`, formats, …). + * Seeds the chart impl before the first `loadAndRender` — the host + * calls `plugin.restore` before the renderer exists, so without + * this the initial `columns_config` never reaches the worker. + * Later changes arrive as `setColumnsConfig` control msgs. + */ + columnsConfig?: Record; defaultChartType?: string; /** diff --git a/packages/viewer-charts/src/ts/transport/renderer-transport.ts b/packages/viewer-charts/src/ts/transport/renderer-transport.ts index 8552a9440e..0f2881ed6a 100644 --- a/packages/viewer-charts/src/ts/transport/renderer-transport.ts +++ b/packages/viewer-charts/src/ts/transport/renderer-transport.ts @@ -258,6 +258,7 @@ export class RendererTransport { chrome: HTMLCanvasElement; facetConfig: FacetConfig; pluginConfig: PluginConfig; + columnsConfig?: Record; defaultChartType?: string; renderBlitMode: "blit" | "direct"; }): Promise { @@ -339,6 +340,7 @@ export class RendererTransport { tableName: this._tableName, facetConfig: opts.facetConfig, pluginConfig: opts.pluginConfig, + columnsConfig: opts.columnsConfig, defaultChartType: opts.defaultChartType, themeVars, fontFaces, @@ -577,6 +579,17 @@ export class RendererTransport { return Promise.resolve(); } + const dpr = window.devicePixelRatio || 1; + const last = this._lastPostedSize; + if ( + last && + Math.abs(last.cssWidth - cssWidth) <= 0.5 && + Math.abs(last.cssHeight - cssHeight) <= 0.5 && + last.dpr === dpr + ) { + return Promise.resolve(); + } + this._holdPresent = true; return this._postResize(cssWidth, cssHeight).then( () => () => this._presentStaged(), diff --git a/packages/viewer-charts/src/ts/worker/renderer.worker.ts b/packages/viewer-charts/src/ts/worker/renderer.worker.ts index a95ca29cd1..dae225e4a9 100644 --- a/packages/viewer-charts/src/ts/worker/renderer.worker.ts +++ b/packages/viewer-charts/src/ts/worker/renderer.worker.ts @@ -204,6 +204,9 @@ export class WorkerRenderer { this.chartImpl.setFacetConfig?.(msg.facetConfig); this.chartImpl.setPluginConfig?.(msg.pluginConfig); + if (msg.columnsConfig) { + this.chartImpl.setColumnsConfig?.(msg.columnsConfig); + } if (this.chartImpl.setZoomController) { this.zoomController = new ZoomController(); diff --git a/packages/viewer-charts/test/ts/alt-axis.spec.ts b/packages/viewer-charts/test/ts/alt-axis.spec.ts new file mode 100644 index 0000000000..770dfc73d7 --- /dev/null +++ b/packages/viewer-charts/test/ts/alt-axis.spec.ts @@ -0,0 +1,140 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +/** + * `alt_axis` primary-domain regressions (.plan/ALT_AXIS_DOMAIN_PLAN.md). + * + * Two independent bugs share one symptom fixture — a tiny column + * (`Discount`, sums to single digits per Category) next to a huge one + * (`Sales`, sums to ~10K) with `Sales` pinned to the alt axis: + * + * A. `domain_mode` defaults to `"expand"`, and its accumulators were + * keyed by axis SIDE only. A `columns_config`-only change arrives + * as `plugin.update()` (no `resetExpandedDomain`), so after pinning + * `Sales` to alt the primary accumulator retained Sales's extent — + * the primary axis never excluded the departed column and the + * Discount bars rendered invisibly small. Fixed by the per-aggregate + * axis-partition signature on the accumulators. + * + * B. A one-shot `restore` (`columns` + `columns_config` together) + * dropped `columns_config` entirely: the host calls + * `plugin.restore` before the first draw builds the renderer, and + * the old forwarding was `this._renderer?.setColumnsConfig(...)` — + * a silent no-op pre-renderer. Fixed by storing `_columnsConfig` + * on the element and shipping it in the `InitMsg` handshake. + * + * Assertions are reference-normalized plot-pixel counts: the two-step + * `domain_mode: "fit"` flow renders this fixture CORRECTLY on both + * sides of the fixes (verified by screenshot 2026-08-01), so each test + * demands its variant paint approximately as many plot pixels as that + * reference. Both bugs cut the visible glyph area roughly in half + * (Discount bars collapse to invisibility), far outside the tolerance. + */ + +import type { Page } from "@playwright/test"; +import { expect, test } from "@perspective-dev/test"; +import { calibratePlotBaseline, gotoBasic, restoreChart } from "./helpers"; + +const ALT_ON_SALES = { Sales: { alt_axis: true } }; + +const BASE_CONFIG = { + plugin: "Y Bar", + columns: ["Discount", "Sales"], + group_by: ["Category"], +}; + +/** Settle a restore's async draw before sampling pixels. */ +const SETTLE_MS = 500; + +/** + * The known-good render of the fixture: draw WITHOUT alt, flip + * `alt_axis` via a second restore, under `domain_mode: "fit"` so no + * accumulator is involved. Returns its plot-pixel count. + */ +async function referencePixels(page: Page): Promise { + await gotoBasic(page); + await restoreChart(page, { + ...BASE_CONFIG, + plugin_config: { domain_mode: "fit" }, + } as never); + await page.waitForTimeout(SETTLE_MS); + await restoreChart(page, { columns_config: ALT_ON_SALES } as never); + await page.waitForTimeout(SETTLE_MS); + return await calibratePlotBaseline(page); +} + +function expectNearReference(actual: number, reference: number): void { + expect(actual).toBeGreaterThan(reference * 0.7); + expect(actual).toBeLessThan(reference * 1.3); +} + +test.describe("alt_axis primary-domain exclusion", () => { + test("one-shot restore honors columns_config alt_axis", async ({ + page, + }) => { + const reference = await referencePixels(page); + + // Fresh page: single restore carrying columns + columns_config + // together, exactly as a saved workspace loads. Pre-fix B the + // per-column config never reached the worker — no alt axis, + // Discount invisible under Sales's unioned domain. + await gotoBasic(page); + await restoreChart(page, { + ...BASE_CONFIG, + columns_config: ALT_ON_SALES, + } as never); + await page.waitForTimeout(SETTLE_MS); + + expectNearReference(await calibratePlotBaseline(page), reference); + }); + + test("alt_axis flip refits primary under default domain_mode", async ({ + page, + }) => { + const reference = await referencePixels(page); + + // Fresh page: same two-step flow as the reference but under the + // DEFAULT `domain_mode: "expand"`. Pre-fix A the primary + // accumulator retained Sales's pre-flip extent, so Discount + // stayed invisible even though the alt axis was correct. + await gotoBasic(page); + await restoreChart(page, BASE_CONFIG as never); + await page.waitForTimeout(SETTLE_MS); + await restoreChart(page, { columns_config: ALT_ON_SALES } as never); + await page.waitForTimeout(SETTLE_MS); + + expectNearReference(await calibratePlotBaseline(page), reference); + }); + + test("same-partition restore keeps the render stable", async ({ page }) => { + // Guard against an over-eager signature: re-sending an + // IDENTICAL columns_config must not perturb the render (the + // signature matches, the accumulators survive, domains are + // unchanged). + const reference = await referencePixels(page); + + await gotoBasic(page); + await restoreChart(page, BASE_CONFIG as never); + await page.waitForTimeout(SETTLE_MS); + await restoreChart(page, { columns_config: ALT_ON_SALES } as never); + await page.waitForTimeout(SETTLE_MS); + const before = await calibratePlotBaseline(page); + + await restoreChart(page, { columns_config: ALT_ON_SALES } as never); + await page.waitForTimeout(SETTLE_MS); + const after = await calibratePlotBaseline(page); + + expect(after).toBeGreaterThan(before * 0.95); + expect(after).toBeLessThan(before * 1.05); + expectNearReference(after, reference); + }); +}); diff --git a/packages/viewer-charts/test/ts/glyph-z-order.spec.ts b/packages/viewer-charts/test/ts/glyph-z-order.spec.ts new file mode 100644 index 0000000000..686f3ec09d --- /dev/null +++ b/packages/viewer-charts/test/ts/glyph-z-order.spec.ts @@ -0,0 +1,178 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +/** + * Mixed-glyph Z-order = `columns` declaration order + * (.plan/GLYPH_Z_ORDER_PLAN.md). Legacy behavior painted a FIXED type + * sequence (areas → bars → lines → scatter) regardless of declaration + * order; the fix paints glyph runs in ascending `aggIdx` — later + * columns on top. + * + * Fixture: two constant expression columns, `avg`-aggregated so every + * category lands exactly at the constant — a bar column at 100 and a + * line column at 50. With `include_zero` (Y Bar default) the bars span + * the full value range, so the horizontal line at 50 crosses EVERY bar + * body, and the gaps between bars show the bare line. + * + * Assertion (palette- and coordinate-free): on the visible GL canvas + * (glyph fragments only — gridlines/chrome are separate canvases, and + * glyph pixels are the only `alpha > 0` pixels), the row with the most + * OPAQUE pixels is the line's center row (the line spans the whole + * plot; bars alone cover only the band fraction). The dominant-color + * share of that row discriminates the stack: + * + * - line on top → the whole row is line-colored → share ≈ 1.0 + * - bars on top → the row alternates bar-color runs (inside bodies) + * with line-color runs (gaps) → share ≈ the band fraction, well + * under 0.85. + */ + +import type { Page } from "@playwright/test"; +import { expect, test } from "@perspective-dev/test"; +import { gotoBasic, restoreChart, waitOneFrame } from "./helpers"; + +const SETTLE_MS = 500; + +const FIXTURE = { + plugin: "Y Bar", + group_by: ["Category"], + expressions: { b100: "100", l50: "50" }, + aggregates: { b100: "avg", l50: "avg" }, + columns_config: { + b100: { chart_type: "bar" }, + l50: { chart_type: "line" }, + }, +}; + +/** + * Dominant-color share of the widest fully-opaque row of the visible + * `.webgl-canvas` (colors quantized >>3 per channel to absorb AA). + */ +async function lineRowModeFraction(page: Page): Promise { + return await page.evaluate(() => { + const visit = ( + root: Document | ShadowRoot, + ): HTMLCanvasElement | null => { + const direct = root.querySelector( + ".webgl-canvas", + ) as HTMLCanvasElement | null; + if (direct) { + return direct; + } + + for (const el of Array.from(root.querySelectorAll("*"))) { + const sr = (el as Element & { shadowRoot?: ShadowRoot }) + .shadowRoot; + if (sr) { + const found = visit(sr); + if (found) { + return found; + } + } + } + + return null; + }; + + const canvas = visit(document); + if (!canvas || canvas.width === 0 || canvas.height === 0) { + throw new Error("glyph-z-order: no .webgl-canvas found"); + } + + const sampler = document.createElement("canvas"); + sampler.width = canvas.width; + sampler.height = canvas.height; + const ctx = sampler.getContext("2d", { willReadFrequently: true })!; + ctx.drawImage(canvas, 0, 0); + const { data } = ctx.getImageData(0, 0, sampler.width, sampler.height); + + const W = sampler.width; + const H = sampler.height; + + // Row with the most opaque pixels = the line's center row: the + // line crosses the entire plot width while bars cover only the + // band fraction, and the line's AA edge rows aren't opaque. + let bestY = -1; + let bestCount = 0; + for (let y = 0; y < H; y++) { + let count = 0; + for (let x = 0; x < W; x++) { + if (data[(y * W + x) * 4 + 3] > 200) { + count++; + } + } + + if (count > bestCount) { + bestCount = count; + bestY = y; + } + } + + if (bestY < 0 || bestCount < W * 0.3) { + throw new Error( + `glyph-z-order: no line row found (best ${bestCount}/${W})`, + ); + } + + const histogram = new Map(); + for (let x = 0; x < W; x++) { + const i = (bestY * W + x) * 4; + if (data[i + 3] <= 200) { + continue; + } + + const key = + ((data[i] >> 3) << 10) | + ((data[i + 1] >> 3) << 5) | + (data[i + 2] >> 3); + histogram.set(key, (histogram.get(key) ?? 0) + 1); + } + + let mode = 0; + for (const count of histogram.values()) { + mode = Math.max(mode, count); + } + + return mode / bestCount; + }); +} + +async function renderAndMeasure( + page: Page, + columns: string[], +): Promise { + await gotoBasic(page); + await restoreChart(page, { ...FIXTURE, columns } as never); + await page.waitForTimeout(SETTLE_MS); + await waitOneFrame(page); + return await lineRowModeFraction(page); +} + +test.describe("Mixed-glyph Z-order follows columns order", () => { + test("bar declared after line occludes it", async ({ page }) => { + // `columns: [line, bar]` — the bar column is declared later, so + // bars must paint OVER the line inside their bodies, leaving + // the line visible only in the gaps. Pre-fix the line always + // painted on top (fixed type order) and the row is uniformly + // line-colored. + const share = await renderAndMeasure(page, ["l50", "b100"]); + expect(share).toBeLessThan(0.85); + }); + + test("line declared after bar stays on top", async ({ page }) => { + // `columns: [bar, line]` — declaration order agrees with the + // legacy fixed order; the line crosses every bar uncovered. + // Guards the run path against over-occluding. + const share = await renderAndMeasure(page, ["b100", "l50"]); + expect(share).toBeGreaterThan(0.9); + }); +}); diff --git a/packages/viewer-charts/test/ts/hover-restore.spec.ts b/packages/viewer-charts/test/ts/hover-restore.spec.ts new file mode 100644 index 0000000000..bbffe4180c --- /dev/null +++ b/packages/viewer-charts/test/ts/hover-restore.spec.ts @@ -0,0 +1,168 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +/** + * Stale-hover-across-restore regression. + * + * Hover state (`_hoveredBarIdx` / `_hoveredSample`) indexes into the + * chart's `_series` / `_bars` build products. A host `restore` that + * shrinks the series set rebuilds both, but the hover survives until + * the next mousemove — and bar-column capacity is reused across builds, + * so a stale bar index reads a leftover `seriesId` past the new series + * set. The first present after the rebuild then crashes in the canvas- + * tooltip pass (`buildBarTooltipLines` reads + * `chart._series[b.seriesId].aggName` → "Cannot read properties of + * undefined (reading 'aggName')", logged as "scheduler: present + * failed") whenever the pointer is resting over an upper-stack / + * non-first-series glyph through the restore. + * + * Repro shape: hover a glyph in a many-series config, then — without + * moving the mouse — restore a single-series config. The hovered + * position is data-dependent, so each test sweeps a coarse grid of + * plot positions and performs the shrink-restore at every one; any + * position that lands on a `seriesId >= 1` glyph reproduces pre-fix. + * + * Hard gates: the shrink `restore` must resolve (the scheduler rejects + * the draw's present waiters on failure) and the chart must still + * paint afterwards. The console/pageerror capture is best-effort + * added signal — worker console messages may or may not surface via + * `page.on("console")`. + */ + +import type { ConsoleMessage, Page } from "@playwright/test"; +import type { ViewerConfigUpdate } from "@perspective-dev/viewer"; +import { expect, test } from "@perspective-dev/test"; +import { + calibratePlotBaseline, + gotoBasic, + restoreChart, + waitOneFrame, +} from "./helpers"; + +/** Matches the scheduler's present-failure log and the crash itself. */ +const PRESENT_FAILED = /present failed|reading 'aggName'/i; + +/** + * Hover dispatch is RAF-throttled in the worker; 200ms covers the + * mousemove → hover-state hop reliably under swiftshader (same margin + * as `tooltip.spec.ts`). + */ +const HOVER_SETTLE_MS = 200; + +/** + * Coarse 3×3 grid over the 1280×720 viewport. Columns straddle the + * three `Category` band centers; rows cover upper / middle / lower + * glyph bodies so at least one position lands on a non-first-series + * glyph regardless of stack heights. + */ +const HOVER_XS = [320, 640, 960]; +const HOVER_YS = [216, 396, 576]; + +function collectPresentErrors(page: Page): string[] { + const hits: string[] = []; + page.on("console", (m: ConsoleMessage) => { + if (PRESENT_FAILED.test(m.text())) { + hits.push(m.text()); + } + }); + + page.on("pageerror", (e: Error) => { + if (PRESENT_FAILED.test(String(e))) { + hits.push(String(e)); + } + }); + + return hits; +} + +async function sweepShrinkRestore( + page: Page, + manySeries: ViewerConfigUpdate, + oneSeries: ViewerConfigUpdate, +): Promise { + const errors = collectPresentErrors(page); + for (const x of HOVER_XS) { + for (const y of HOVER_YS) { + await restoreChart(page, manySeries); + await waitOneFrame(page); + await page.mouse.move(x, y); + await page.waitForTimeout(HOVER_SETTLE_MS); + + // The shrink rebuilds `_series` / `_bars` while the pointer + // rests on the old glyph; pre-fix the scheduler rejects the + // present waiters and this `restore` throws. + await restoreChart(page, oneSeries); + await waitOneFrame(page); + } + } + + return errors; +} + +test.describe("Hover across shrinking restore", () => { + test.beforeEach(async ({ page }) => { + await gotoBasic(page); + }); + + test("stacked bar hover survives restore to fewer series", async ({ + page, + }) => { + test.setTimeout(120_000); + const errors = await sweepShrinkRestore( + page, + { + plugin: "Y Bar", + columns: ["Sales"], + group_by: ["Category"], + split_by: ["Region"], + }, + { + plugin: "Y Bar", + columns: ["Sales"], + group_by: ["Category"], + split_by: [], + }, + ); + + expect(errors).toEqual([]); + expect(await calibratePlotBaseline(page)).toBeGreaterThan(0); + }); + + test("line hover sample survives restore to fewer series", async ({ + page, + }) => { + test.setTimeout(120_000); + + // Lines hit-test within a point radius rather than a bar body, + // so this sweep is best-effort at reproducing pre-fix; dense + // `State` categories maximize the chance a grid position lands + // within radius of a `seriesId >= 1` vertex. + const errors = await sweepShrinkRestore( + page, + { + plugin: "Y Line", + columns: ["Sales", "Profit", "Quantity"], + group_by: ["State"], + split_by: [], + }, + { + plugin: "Y Line", + columns: ["Sales"], + group_by: ["State"], + split_by: [], + }, + ); + + expect(errors).toEqual([]); + expect(await calibratePlotBaseline(page)).toBeGreaterThan(0); + }); +}); From a78fa3bdd5a6c85d8f99292297cbf94843c42f5a Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 2 Aug 2026 18:11:06 -0400 Subject: [PATCH 3/6] Datagrid UI fixes Signed-off-by: Andrew Stein --- .../src/ts/custom_elements/toolbar.ts | 15 +- .../viewer-datagrid/src/ts/model/create.ts | 9 +- .../src/ts/style_handlers/body.ts | 8 +- .../ts/style_handlers/table_cell/numeric.ts | 58 ++--- .../test/js/column_style.spec.ts | 231 ++++++++++++++++++ .../test/js/row_tree_selection.spec.ts | 186 ++++++++++++++ 6 files changed, 458 insertions(+), 49 deletions(-) create mode 100644 packages/viewer-datagrid/test/js/row_tree_selection.spec.ts diff --git a/packages/viewer-datagrid/src/ts/custom_elements/toolbar.ts b/packages/viewer-datagrid/src/ts/custom_elements/toolbar.ts index 39d8edf1b1..fc70057274 100644 --- a/packages/viewer-datagrid/src/ts/custom_elements/toolbar.ts +++ b/packages/viewer-datagrid/src/ts/custom_elements/toolbar.ts @@ -12,7 +12,7 @@ import type { HTMLPerspectiveViewerElement } from "@perspective-dev/viewer"; import TOOLBAR_STYLE from "../../../dist/css/perspective-viewer-datagrid-toolbar.css"; -import { toggle_edit_mode, toggle_scroll_lock } from "../model/toolbar.js"; +import { toggle_edit_mode } from "../model/toolbar.js"; import type { DatagridPluginElement } from "../types.js"; const stylesheet = new CSSStyleSheet(); @@ -42,11 +42,6 @@ export class HTMLPerspectiveViewerDatagridToolbarElement extends HTMLElement { this.shadowRoot!.adoptedStyleSheets.push(stylesheet); this.shadowRoot!.innerHTML = `
- - - - - @@ -63,14 +58,6 @@ export class HTMLPerspectiveViewerDatagridToolbarElement extends HTMLElement { : viewer.getPlugin("Datagrid") ) as DatagridPluginElement; - plugin._scroll_lock = this.shadowRoot!.querySelector( - "#scroll_lock", - ) as HTMLElement; - - plugin._scroll_lock.addEventListener("click", () => - toggle_scroll_lock.call(plugin), - ); - plugin._edit_button = this.shadowRoot!.querySelector( "#edit_mode", ) as HTMLElement; diff --git a/packages/viewer-datagrid/src/ts/model/create.ts b/packages/viewer-datagrid/src/ts/model/create.ts index 75eef5d11d..e5fc2b3212 100644 --- a/packages/viewer-datagrid/src/ts/model/create.ts +++ b/packages/viewer-datagrid/src/ts/model/create.ts @@ -225,11 +225,14 @@ export async function createModel( const _column_types: ColumnType[] = []; let _edit_mode: EditMode = this._edit_mode || "READ_ONLY"; - if ( + if (_edit_mode === "SELECT_ROW_TREE" && config.group_by.length === 0) { + _edit_mode = "READ_ONLY"; + this._edit_mode = _edit_mode; + } else if ( _edit_mode === "SELECT_ROW_TREE" && - (config.group_by.length === 0 || config.group_rollup_mode === "flat") + config.group_rollup_mode === "flat" ) { - _edit_mode = "READ_ONLY"; + _edit_mode = "SELECT_ROW"; this._edit_mode = _edit_mode; } diff --git a/packages/viewer-datagrid/src/ts/style_handlers/body.ts b/packages/viewer-datagrid/src/ts/style_handlers/body.ts index 7b370724da..cf37eef17c 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/body.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/body.ts @@ -143,10 +143,16 @@ export function applyBodyCellStyles( const isSub = id.length !== selectedId.length && key_match; if (isHeader) { + // A row-header `` is "inert" when its level lies + // within the row's path depth — those cells are the + // merged/rowspan'd group headers whose metadata row is + // merely the first row of their span. Compare indices, + // not values: a falsy group key (0, "", false, null) + // is still a real path segment. if ( metadata.type === "row_header" && metadata.row_header_x !== undefined && - !!id[metadata.row_header_x] + metadata.row_header_x < id.length ) { td.classList.toggle("psp-select-region", false); } else { diff --git a/packages/viewer-datagrid/src/ts/style_handlers/table_cell/numeric.ts b/packages/viewer-datagrid/src/ts/style_handlers/table_cell/numeric.ts index f61c6b0b14..f8824c3cdc 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/table_cell/numeric.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/table_cell/numeric.ts @@ -121,37 +121,33 @@ export function cell_style_numeric( } } - const fg_tuple: ColorRecord = (() => { - if (plugin?.pos_fg_color !== undefined) { - return is_positive - ? plugin.pos_fg_color - : is_negative - ? plugin.neg_fg_color! - : [ - "", - model._plugin_background[0], - model._plugin_background[1], - model._plugin_background[2], - "", - "", - "", - ]; - } else { - return is_positive - ? model._pos_fg_color - : is_negative - ? model._neg_fg_color - : [ - "", - model._plugin_background[0], - model._plugin_background[1], - model._plugin_background[2], - "", - "", - "", - ]; - } - })(); + let pos_fg_color: ColorRecord; + if (plugin?.pos_fg_color !== undefined) { + pos_fg_color = plugin.pos_fg_color; + } else { + pos_fg_color = model._pos_fg_color; + } + + let neg_fg_color: ColorRecord; + if (plugin?.neg_fg_color !== undefined) { + neg_fg_color = plugin.neg_fg_color; + } else { + neg_fg_color = model._neg_fg_color; + } + + const fg_tuple: ColorRecord = is_positive + ? pos_fg_color + : is_negative + ? neg_fg_color + : [ + "", + model._plugin_background[0], + model._plugin_background[1], + model._plugin_background[2], + "", + "", + "", + ]; const [hex, , , , gradhex] = fg_tuple; diff --git a/packages/viewer-datagrid/test/js/column_style.spec.ts b/packages/viewer-datagrid/test/js/column_style.spec.ts index 505cd76ca4..f2848dad79 100644 --- a/packages/viewer-datagrid/test/js/column_style.spec.ts +++ b/packages/viewer-datagrid/test/js/column_style.spec.ts @@ -680,6 +680,237 @@ test.describe("Column Style Tests", () => { }); }); + // Regression: the sidebar's ColorRange control emits a sparse config — + // only the side(s) that differ from the theme default are written. A + // one-sided fg pair (e.g. `pos_fg_color` alone) used to crash + // `cell_style_numeric` on negative cells ("d is not iterable"), and the + // mirror case (`neg_fg_color` alone) was silently ignored. Each side + // must fall back to the theme color independently, like the bg pair. + test("pos_fg_color alone renders negatives with theme fallback (no crash)", async ({ + page, + }) => { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); + + const { cells, rejections } = await page.evaluate(async () => { + const rejections: string[] = []; + window.addEventListener("unhandledrejection", (e) => { + rejections.push(String(e.reason)); + }); + + const viewer = document.querySelector("perspective-viewer")!; + await viewer.restore({ + plugin: "Datagrid", + columns: ["Profit"], + sort: [["Profit", "asc"]], + columns_config: { + Profit: { + number_fg_mode: "color", + pos_fg_color: "#00ff00", + }, + }, + }); + + await viewer.flush(); + await new Promise((x) => setTimeout(x, 100)); + const tds = Array.from( + ( + viewer.querySelector("perspective-viewer-datagrid") as any + ).shadowRoot.querySelectorAll("regular-table tbody td"), + ); + + return { + rejections, + cells: tds.map((td: any) => ({ + text: td.textContent.trim(), + color: td.style.color, + })), + }; + }); + + expect(rejections).toEqual([]); + const negatives = cells.filter((c) => c.text.startsWith("-")); + expect(negatives.length).toBeGreaterThan(0); + for (const cell of negatives) { + expect(cell.color).not.toEqual(""); + expect(cell.color).not.toEqual("rgb(0, 255, 0)"); + } + }); + + test("neg_fg_color alone renders negatives with the custom color", async ({ + page, + }) => { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); + + const errors: string[] = []; + page.on("pageerror", (e) => errors.push(e.message)); + + const cells = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")!; + await viewer.restore({ + plugin: "Datagrid", + columns: ["Profit"], + sort: [["Profit", "asc"]], + columns_config: { + Profit: { + number_fg_mode: "color", + neg_fg_color: "#ff0000", + }, + }, + }); + + await viewer.flush(); + const tds = Array.from( + ( + viewer.querySelector("perspective-viewer-datagrid") as any + ).shadowRoot.querySelectorAll("regular-table tbody td"), + ); + + return tds.map((td: any) => ({ + text: td.textContent.trim(), + color: td.style.color, + })); + }); + + expect(errors).toEqual([]); + const negatives = cells.filter((c) => c.text.startsWith("-")); + expect(negatives.length).toBeGreaterThan(0); + for (const cell of negatives) { + expect(cell.color).toEqual("rgb(255, 0, 0)"); + } + + const positives = cells.filter( + (c) => !c.text.startsWith("-") && c.text !== "", + ); + for (const cell of positives) { + expect(cell.color).not.toEqual("rgb(255, 0, 0)"); + } + }); + + test("color input node is replaced on column switch and stale events write nothing", async ({ + page, + }) => { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); + + await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")!; + await viewer.restore({ + plugin: "Datagrid", + group_by: ["State"], + settings: true, + }); + await viewer.flush(); + }); + + const open_editor = async (nth: number) => { + const target = await page.evaluate(async (nth) => { + const viewer = document.querySelector("perspective-viewer")!; + const datagrid = viewer.querySelector( + "perspective-viewer-datagrid", + ) as any; + const rt = datagrid.shadowRoot.querySelector("regular-table"); + const ths = Array.from( + rt.querySelectorAll( + "#psp-column-edit-buttons th.psp-menu-enabled", + ), + ) as any[]; + const th = ths[nth]; + const meta = rt.getMeta(th); + const rect = th.getBoundingClientRect(); + return { + column: meta?.column_header?.[0], + x: Math.floor(rect.left + rect.width / 2), + y: Math.floor(rect.top + rect.height / 2), + }; + }, nth); + + await page.mouse.click(target.x, target.y); + await page + .locator( + "perspective-viewer #column_settings_sidebar #style-tab input[type=color]", + ) + .first() + .waitFor(); + return target.column; + }; + + const col_a = await open_editor(2); + await page.evaluate(() => { + const viewer = document.querySelector("perspective-viewer")!; + const root = (viewer.shadowRoot ?? viewer) as any; + (window as any).__held_input__ = root.querySelector( + "#column_settings_sidebar #style-tab input[type=color]", + ); + }); + + const col_b = await open_editor(4); + expect(col_b).not.toEqual(col_a); + + // The old column's input node must have been REPLACED, which + // dismisses any open native color chooser bound to it. + const held = await page.evaluate(() => { + const input = (window as any).__held_input__; + const viewer = document.querySelector("perspective-viewer")!; + const root = (viewer.shadowRoot ?? viewer) as any; + const current = root.querySelector( + "#column_settings_sidebar #style-tab input[type=color]", + ); + const same_node = input === current; + input.value = "#aa00aa"; + input.dispatchEvent(new Event("input", { bubbles: true })); + return { same_node, connected: input.isConnected }; + }); + + expect(held.same_node).toEqual(false); + expect(held.connected).toEqual(false); + + // The stale event must not have written ANY column's config. + await new Promise((x) => setTimeout(x, 300)); + const saved = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")!; + await viewer.flush(); + return (await viewer.save()).columns_config ?? {}; + }); + + expect(JSON.stringify(saved)).not.toContain("#aa00aa"); + + // Sanity: the CURRENT column's input still works and writes the + // currently-open column. + await page + .locator( + "perspective-viewer #column_settings_sidebar #style-tab input[type=color]", + ) + .first() + .evaluate((el: any) => { + el.value = "#00ff00"; + el.dispatchEvent(new Event("input", { bubbles: true })); + }); + + await new Promise((x) => setTimeout(x, 300)); + const saved2 = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")!; + await viewer.flush(); + return (await viewer.save()).columns_config ?? {}; + }); + + expect(saved2[col_b]?.pos_fg_color).toEqual("#00ff00"); + expect(saved2[col_a]).toBeUndefined(); + }); + test("repeated restore with plugin_config + columns_config is stable", async ({ page, }) => { diff --git a/packages/viewer-datagrid/test/js/row_tree_selection.spec.ts b/packages/viewer-datagrid/test/js/row_tree_selection.spec.ts new file mode 100644 index 0000000000..8a3019779a --- /dev/null +++ b/packages/viewer-datagrid/test/js/row_tree_selection.spec.ts @@ -0,0 +1,186 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import type { Page } from "@playwright/test"; + +interface CellState { + tag: string; + rowspan: number; + text: string; + sel: boolean; + sub: boolean; +} + +async function await_ready(page: Page): Promise { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); +} + +async function click_row(page: Page, row_index: number): Promise { + const { x, y } = await page.evaluate(async (row_index) => { + const viewer = document.querySelector("perspective-viewer")!; + const datagrid = viewer.querySelector( + "perspective-viewer-datagrid", + ) as any; + const rows = datagrid.shadowRoot.querySelectorAll( + "regular-table tbody tr", + ); + + const td = rows[row_index].querySelector("td"); + const rect = td.getBoundingClientRect(); + return { + x: Math.floor(rect.left + rect.width / 2), + y: Math.floor(rect.top + rect.height / 2), + }; + }, row_index); + + await page.mouse.click(x, y); + await page.evaluate(async () => { + await document.querySelector("perspective-viewer")!.flush(); + }); +} + +async function wait_for_selection_class( + page: Page, + class_name: string, +): Promise { + await page.waitForFunction((class_name) => { + const datagrid = document + .querySelector("perspective-viewer") + ?.querySelector("perspective-viewer-datagrid") as any; + return !!datagrid?.shadowRoot?.querySelector( + `regular-table tbody .${class_name}`, + ); + }, class_name); +} + +async function read_rows(page: Page, limit: number): Promise { + return await page.evaluate(async (limit) => { + const viewer = document.querySelector("perspective-viewer")!; + const datagrid = viewer.querySelector( + "perspective-viewer-datagrid", + ) as any; + const rows = Array.from( + datagrid.shadowRoot.querySelectorAll("regular-table tbody tr"), + ).slice(0, limit); + + return rows.map((tr: any) => + Array.from(tr.children).map((cell: any) => ({ + tag: cell.tagName, + rowspan: cell.rowSpan, + text: cell.textContent.trim(), + sel: cell.classList.contains("psp-select-region"), + sub: cell.classList.contains("psp-select-region-inactive"), + })), + ); + }, limit); +} + +test.describe("SELECT_ROW_TREE selection styling", () => { + // Regression: the row-header "inert th" guard compared the path + // _value_ at the th's level truthily, so a falsy group key (the + // `Discount` value 0) routed the merged rowspan'd group header into + // the styling branch. Selecting the first row of its span (the first + // row in the sub group) lit the whole inert header in the selection + // color. + test("falsy group key does not style the inert rowspan th", async ({ + page, + }) => { + await await_ready(page); + await page.evaluate(async () => { + await document.querySelector("perspective-viewer")!.restore({ + plugin: "Datagrid", + group_by: ["Category", "Discount"], + columns: ["Sales", "Profit"], + plugin_config: { edit_mode: "SELECT_ROW_TREE" }, + }); + }); + + await click_row(page, 2); + await wait_for_selection_class(page, "psp-select-region"); + const rows = await read_rows(page, 4); + + const [inert_th, content_th, ...tds] = rows[2]; + expect(inert_th.tag).toEqual("TH"); + expect(inert_th.rowspan).toBeGreaterThan(1); + expect(inert_th.text).toEqual(""); + expect(inert_th.sel).toEqual(false); + expect(inert_th.sub).toEqual(false); + + expect(content_th.sel).toEqual(true); + for (const td of tds) { + expect(td.sel).toEqual(true); + } + + for (const cell of rows[3]) { + expect(cell.sel).toEqual(false); + expect(cell.sub).toEqual(false); + } + }); + + test("group selection styles descendants with the secondary class", async ({ + page, + }) => { + await await_ready(page); + await page.evaluate(async () => { + await document.querySelector("perspective-viewer")!.restore({ + plugin: "Datagrid", + group_by: ["Category", "Sub-Category"], + columns: ["Sales", "Profit"], + plugin_config: { edit_mode: "SELECT_ROW_TREE" }, + }); + }); + + await click_row(page, 1); + await wait_for_selection_class(page, "psp-select-region-inactive"); + const rows = await read_rows(page, 6); + const furniture = rows[1].find((c) => c.text === "Furniture")!; + expect(furniture.sel).toEqual(true); + for (const row of rows.slice(2, 6)) { + const content = row.filter((c) => c.text !== ""); + expect(content.length).toBeGreaterThan(0); + for (const cell of content) { + expect(cell.sub).toEqual(true); + expect(cell.sel).toEqual(false); + } + } + }); + + test("flat rollup mode downgrades SELECT_ROW_TREE to SELECT_ROW", async ({ + page, + }) => { + await await_ready(page); + const edit_mode = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")!; + await viewer.restore({ + plugin: "Datagrid", + group_by: ["Category", "Sub-Category"], + group_rollup_mode: "flat", + columns: ["Sales", "Profit"], + plugin_config: { edit_mode: "SELECT_ROW_TREE" }, + }); + + await viewer.flush(); + const datagrid = viewer.querySelector( + "perspective-viewer-datagrid", + ) as any; + return datagrid.model._edit_mode; + }); + + expect(edit_mode).toEqual("SELECT_ROW"); + }); +}); From a9e60c99e843ce28bed2fff1ad89ad97ecff09f9 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 1 Aug 2026 00:01:28 -0400 Subject: [PATCH 4/6] Window functions Signed-off-by: Andrew Stein --- rust/metadata/main.rs | 2 + rust/perspective-client/perspective.proto | 44 + rust/perspective-client/src/rust/client.rs | 26 + .../perspective-client/src/rust/config/mod.rs | 2 + .../src/rust/config/view_config.rs | 39 + .../src/rust/config/windows.rs | 389 ++++ .../src/rust/virtual_server/data.rs | 19 +- .../src/rust/virtual_server/features.rs | 39 +- .../rust/virtual_server/generic_sql_model.rs | 2 +- .../generic_sql_model/table_make_view.rs | 202 ++- .../virtual_server/generic_sql_model/tests.rs | 226 ++- .../src/rust/virtual_server/server.rs | 18 +- rust/perspective-js/src/rust/lib.rs | 6 + .../perspective-js/src/rust/virtual_server.rs | 100 +- rust/perspective-js/src/ts/virtual_server.ts | 72 +- .../src/ts/virtual_servers/clickhouse.ts | 46 +- .../src/ts/virtual_servers/duckdb.ts | 46 +- .../test/js/duckdb/group_by.spec.js | 24 + .../test/js/duckdb/windows.spec.js | 202 +++ rust/perspective-js/test/js/windows.spec.js | 1000 ++++++++++ .../perspective/virtual_servers/clickhouse.py | 35 + .../perspective/virtual_servers/duckdb.py | 32 + .../src/server/virtual_server_sync.rs | 2 +- .../cpp/perspective/CMakeLists.txt | 1 + .../cpp/perspective/src/cpp/config.cpp | 19 +- .../src/cpp/context_grouped_pkey.cpp | 33 +- .../cpp/perspective/src/cpp/context_one.cpp | 33 +- .../cpp/perspective/src/cpp/context_two.cpp | 33 +- .../cpp/perspective/src/cpp/context_unit.cpp | 12 +- .../cpp/perspective/src/cpp/context_zero.cpp | 73 +- .../perspective/src/cpp/expression_tables.cpp | 11 +- .../cpp/perspective/src/cpp/gnode.cpp | 214 ++- .../cpp/perspective/src/cpp/server.cpp | 300 ++- .../cpp/perspective/src/cpp/view_config.cpp | 17 +- .../cpp/perspective/src/cpp/window.cpp | 1603 +++++++++++++++++ .../src/include/perspective/config.h | 13 +- .../perspective/context_common_decls.h | 9 + .../perspective/context_grouped_pkey.h | 1 + .../src/include/perspective/context_one.h | 1 + .../src/include/perspective/context_two.h | 1 + .../src/include/perspective/context_zero.h | 1 + .../include/perspective/expression_tables.h | 4 +- .../src/include/perspective/gnode.h | 29 +- .../src/include/perspective/view_config.h | 7 +- .../src/include/perspective/window.h | 299 +++ .../rust/session/replace_expression_update.rs | 1 + tools/bench/basic_suite.mjs | 1 + tools/bench/cross_platform_suite.mjs | 85 + tools/bench/package.json | 1 + tools/bench/puppeteer_suite.mjs | 1 + tools/bench/python_suite.mjs | 1 + tools/bench/windows_suite.mjs | 116 ++ 52 files changed, 5374 insertions(+), 119 deletions(-) create mode 100644 rust/perspective-client/src/rust/config/windows.rs create mode 100644 rust/perspective-js/test/js/duckdb/windows.spec.js create mode 100644 rust/perspective-js/test/js/windows.spec.js create mode 100644 rust/perspective-server/cpp/perspective/src/cpp/window.cpp create mode 100644 rust/perspective-server/cpp/perspective/src/include/perspective/window.h create mode 100644 tools/bench/windows_suite.mjs diff --git a/rust/metadata/main.rs b/rust/metadata/main.rs index 2210ec7b42..b3145d96f3 100644 --- a/rust/metadata/main.rs +++ b/rust/metadata/main.rs @@ -31,6 +31,7 @@ use std::fmt::Write; use std::fs; use perspective_client::config::*; +use perspective_client::virtual_server::Features; use perspective_client::{ ColumnWindow, DeleteOptions, JoinOptions, OnUpdateData, OnUpdateOptions, SystemInfo, TableInitOptions, UpdateOptions, ViewWindow, @@ -81,6 +82,7 @@ pub fn generate_type_bindings_js() -> Result<(), Box> { ColumnType::export_all_to(&path)?; ColumnWindow::export_all_to(&path)?; DeleteOptions::export_all_to(&path)?; + Features::export_all_to(&path)?; JoinOptions::export_all_to(&path)?; OnUpdateData::export_all_to(&path)?; OnUpdateOptions::export_all_to(&path)?; diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index 515bc090d1..35a1e36ec4 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -229,6 +229,13 @@ message GetFeaturesResp { map filter_ops = 6; map aggregates = 7; repeated GroupRollupMode group_rollup_mode = 8; + map window_aggregates = 9; + + bool unordered = 10; + + message WindowAggregateOptions { + repeated WindowAggregate options = 1; + } message ColumnTypeOptions { repeated string options = 1; @@ -539,6 +546,42 @@ message ServerSystemInfoResp { } +enum WindowAggregate { + WINDOW_AGGREGATE_SUM = 0; + WINDOW_AGGREGATE_AVG = 1; + WINDOW_AGGREGATE_COUNT = 2; + WINDOW_AGGREGATE_MIN = 3; + WINDOW_AGGREGATE_MAX = 4; + WINDOW_AGGREGATE_STDDEV = 5; + WINDOW_AGGREGATE_VAR = 6; + WINDOW_AGGREGATE_FIRST = 7; + WINDOW_AGGREGATE_LAST = 8; + WINDOW_AGGREGATE_LAG = 9; + WINDOW_AGGREGATE_LEAD = 10; + WINDOW_AGGREGATE_DIFF = 11; + WINDOW_AGGREGATE_RATE = 12; + WINDOW_AGGREGATE_EMA = 13; +} + +message WindowSpec { + string source = 2; + WindowAggregate op = 3; + repeated string partition_by = 4; + Order order_by = 5; + oneof frame { + uint32 rows = 6; + double range = 7; + google.protobuf.NullValue cumulative = 8; + } + optional uint32 offset = 9; + optional double alpha = 10; + + message Order { + string column = 1; + bool desc = 2; + } +} + message ViewConfig { repeated string group_by = 1; repeated string split_by = 2; @@ -550,6 +593,7 @@ message ViewConfig { FilterReducer filter_op = 8; optional uint32 group_by_depth = 9; optional GroupRollupMode group_rollup_mode = 10; + map windows = 11; message AggList { repeated string aggregations = 1; diff --git a/rust/perspective-client/src/rust/client.rs b/rust/perspective-client/src/rust/client.rs index b100160268..8f07eb4f2a 100644 --- a/rust/perspective-client/src/rust/client.rs +++ b/rust/perspective-client/src/rust/client.rs @@ -121,6 +121,32 @@ impl GetFeaturesResp { .first() .map(|x| x.as_str()) } + + /// The window aggregates this server supports for a `col_type` SOURCE + /// column, in the server's declared (menu) order. + pub fn get_window_aggregates( + &self, + col_type: ColumnType, + ) -> Vec { + self.window_aggregates + .get(&(col_type as u32)) + .map(|x| { + x.options + .iter() + .filter_map(|x| crate::proto::WindowAggregate::try_from(*x).ok()) + .map(|x| x.into()) + .collect() + }) + .unwrap_or_default() + } + + /// Whether this server supports window columns at all - the + /// `window_aggregates` declaration is the single source of truth. + pub fn has_window_aggregates(&self) -> bool { + self.window_aggregates + .values() + .any(|x| !x.options.is_empty()) + } } type BoxFn = Box O + Send + Sync + 'static>; diff --git a/rust/perspective-client/src/rust/config/mod.rs b/rust/perspective-client/src/rust/config/mod.rs index 6a84aadfa8..9ab65f1ebf 100644 --- a/rust/perspective-client/src/rust/config/mod.rs +++ b/rust/perspective-client/src/rust/config/mod.rs @@ -26,6 +26,7 @@ mod filters; mod plugin; mod sort; mod view_config; +pub mod windows; pub use aggregates::*; pub use expressions::*; @@ -33,5 +34,6 @@ pub use filters::*; pub use plugin::*; pub use sort::*; pub use view_config::*; +pub use windows::*; pub use crate::proto::{ColumnType, SortOp}; diff --git a/rust/perspective-client/src/rust/config/view_config.rs b/rust/perspective-client/src/rust/config/view_config.rs index 42f5cff2eb..da430fc5ab 100644 --- a/rust/perspective-client/src/rust/config/view_config.rs +++ b/rust/perspective-client/src/rust/config/view_config.rs @@ -20,6 +20,7 @@ use super::aggregates::*; use super::expressions::*; use super::filters::*; use super::sort::*; +use super::windows::*; use crate::proto; use crate::proto::columns_update; @@ -92,6 +93,10 @@ pub struct ViewConfig { #[serde(default)] pub expressions: Expressions, + #[serde(default)] + #[serde(skip_serializing_if = "is_default_value")] + pub windows: Windows, + #[serde(default)] pub columns: Vec>, @@ -187,6 +192,16 @@ pub struct ViewConfigUpdate { #[ts(optional)] pub expressions: Option, + /// The `windows` property declares ordered, partitioned rolling + /// computations (moving aggregates, cumulative sums) as _new_ columns + /// keyed by output alias (`{"name": {...spec}}`, symmetric with + /// `expressions`), analogous to SQL window functions. See + /// [`crate::config::WindowSpec`]. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + #[ts(optional)] + pub windows: Option, + /// Aggregates perform a calculation over an entire column, and are /// displayed when one or more [Group By](#group-by) are applied to the /// `View`. Aggregates can be specified by the user, or Perspective will @@ -248,6 +263,13 @@ impl From for proto::ViewConfig { .map(|x| x.into()) .collect(), expressions: value.expressions.unwrap_or_default().0, + windows: value + .windows + .unwrap_or_default() + .0 + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), aggregates: value .aggregates .unwrap_or_default() @@ -290,6 +312,7 @@ impl From for ViewConfigUpdate { filter_op: Some(value.filter_op), sort: Some(value.sort), expressions: Some(value.expressions), + windows: Some(value.windows), aggregates: Some(value.aggregates), group_by_depth: value.group_by_depth, group_rollup_mode: Some(value.group_rollup_mode), @@ -316,6 +339,13 @@ impl From for ViewConfig { .into(), sort: value.sort.into_iter().map(|x| x.into()).collect(), expressions: Expressions(value.expressions), + windows: Windows( + value + .windows + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + ), aggregates: value .aggregates .into_iter() @@ -342,6 +372,7 @@ impl From for ViewConfig { filter_op: value.filter_op.unwrap_or_default(), sort: value.sort.unwrap_or_default(), expressions: value.expressions.unwrap_or_default(), + windows: value.windows.unwrap_or_default(), aggregates: value.aggregates.unwrap_or_default(), group_by_depth: value.group_by_depth, group_rollup_mode: value.group_rollup_mode.unwrap_or_default(), @@ -368,6 +399,13 @@ impl From for ViewConfigUpdate { ), sort: Some(value.sort.into_iter().map(|x| x.into()).collect()), expressions: Some(Expressions(value.expressions)), + windows: Some(Windows( + value + .windows + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + )), aggregates: Some( value .aggregates @@ -434,6 +472,7 @@ impl ViewConfig { changed = Self::_apply(&mut self.sort, update.sort) || changed; changed = Self::_apply(&mut self.aggregates, update.aggregates) || changed; changed = Self::_apply(&mut self.expressions, update.expressions) || changed; + changed = Self::_apply(&mut self.windows, update.windows) || changed; changed = Self::_apply(&mut self.group_rollup_mode, update.group_rollup_mode) || changed; if self.group_rollup_mode == GroupRollupMode::Total && !self.group_by.is_empty() { tracing::info!("`total` incompatible with `group_by`"); diff --git a/rust/perspective-client/src/rust/config/windows.rs b/rust/perspective-client/src/rust/config/windows.rs new file mode 100644 index 0000000000..4f30b932ea --- /dev/null +++ b/rust/perspective-client/src/rust/config/windows.rs @@ -0,0 +1,389 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +//! Window columns are ordered, partitioned rolling computations over the rows +//! of a [`crate::Table`], declared per-[`crate::View`] like expression +//! columns - the analogue of SQL window functions. Each [`WindowSpec`] +//! produces a new output column, named by its key in the `windows` map +//! (`{"name": {...spec}}`, symmetric with `expressions`), which may be +//! used anywhere a `Table` column can: `columns`, `filter`, `sort`, +//! `group_by`, etc. Window columns update incrementally as the `Table` +//! updates, including rows _outside_ an update batch whose window frames +//! were affected by it. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +use crate::proto; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, TS)] +#[serde(rename_all = "snake_case")] +pub enum WindowAggregate { + Sum, + Avg, + Count, + Min, + Max, + Stddev, + Var, + First, + Last, + Lag, + Lead, + Diff, + Rate, + Ema, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFrame { + Rows(u32), + Range(f64), + Cumulative, +} + +/// A window's order direction. A dedicated two-variant enum rather than +/// [`crate::config::SortDir`] - the column-sort extras (`col asc`, `abs`, +/// `none`) are meaningless inside a window frame and unrepresentable here. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize, TS)] +#[serde(rename_all = "snake_case")] +pub enum WindowSortDir { + #[default] + Asc, + Desc, +} + +impl std::fmt::Display for WindowSortDir { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Asc => "asc", + Self::Desc => "desc", + }) + } +} + +/// The `Table` column which orders each partition, with its direction - +/// serialized as a two-element array (`["ts", "desc"]`), symmetric with the +/// `ViewConfig`'s `sort` field. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +pub struct WindowSort(pub String, pub WindowSortDir); + +/// The window columns of a `ViewConfig`, keyed by output column alias - +/// symmetric with `expressions` (`{"name": {...spec}}`). An alias must not +/// collide with a `Table` column, expression alias, or another window's +/// key. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)] +pub struct Windows(#[ts(as = "HashMap")] pub HashMap); + +impl std::ops::Deref for Windows { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for Windows { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// A declaration of a single window column, named by its key in +/// [`Windows`]. See the [module docs](self) for the computation model. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(try_from = "RawWindowSpec", into = "RawWindowSpec")] +pub struct WindowSpec { + pub column: String, + pub aggregate: WindowAggregate, + pub partition_by: Vec, + pub order_by: Option, + pub frame: Option, + pub offset: Option, + pub alpha: Option, +} + +/// The serialized form of [`WindowSpec`]. +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[serde(deny_unknown_fields)] +#[ts(rename = "WindowSpec")] +struct RawWindowSpec { + /// The input column - either a real `Table` column or an expression + /// alias from the same `ViewConfig`. + column: String, + + aggregate: WindowAggregate, + + /// Columns whose distinct value tuples partition the rows; empty + /// partitions the whole `Table` as one group. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + partition_by: Vec, + + /// The `Table` column which orders each partition, and the direction. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + order_by: Option, + + /// A frame of the `rows` preceding each row, plus the row itself. + /// Mutually exclusive with `range` and `cumulative`. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + rows: Option, + + /// A frame of the rows whose `order_by` value lies within `range` of + /// each row's, requiring a numeric or temporal `order_by`. Mutually + /// exclusive with `rows` and `cumulative`. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + range: Option, + + /// `true` frames all rows from the partition start through each row. + /// Default can be omitted. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + cumulative: Option, + + /// Row offset for `lag`/`lead` (default 1). + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + offset: Option, + + /// Smoothing factor in `(0, 1]` for `ema`. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + alpha: Option, +} + +impl From for RawWindowSpec { + fn from(value: WindowSpec) -> Self { + let (rows, range, cumulative) = match value.frame { + Some(WindowFrame::Rows(n)) => (Some(n), None, None), + Some(WindowFrame::Range(x)) => (None, Some(x), None), + Some(WindowFrame::Cumulative) => (None, None, Some(true)), + None => (None, None, None), + }; + + RawWindowSpec { + column: value.column, + aggregate: value.aggregate, + partition_by: value.partition_by, + order_by: value.order_by, + rows, + range, + cumulative, + offset: value.offset, + alpha: value.alpha, + } + } +} + +impl TryFrom for WindowSpec { + type Error = String; + + fn try_from(value: RawWindowSpec) -> Result { + let frame = match (value.rows, value.range, value.cumulative) { + (None, None, None) => None, + (Some(n), None, None) => Some(WindowFrame::Rows(n)), + (None, Some(x), None) => Some(WindowFrame::Range(x)), + (None, None, Some(true)) => Some(WindowFrame::Cumulative), + (None, None, Some(false)) => { + return Err("`cumulative` must be `true` when present".to_string()); + }, + _ => { + return Err("`rows`, `range` and `cumulative` are mutually exclusive".to_string()); + }, + }; + + Ok(WindowSpec { + column: value.column, + aggregate: value.aggregate, + partition_by: value.partition_by, + order_by: value.order_by, + frame, + offset: value.offset, + alpha: value.alpha, + }) + } +} + +impl From for proto::WindowAggregate { + fn from(value: WindowAggregate) -> Self { + match value { + WindowAggregate::Sum => Self::Sum, + WindowAggregate::Avg => Self::Avg, + WindowAggregate::Count => Self::Count, + WindowAggregate::Min => Self::Min, + WindowAggregate::Max => Self::Max, + WindowAggregate::Stddev => Self::Stddev, + WindowAggregate::Var => Self::Var, + WindowAggregate::First => Self::First, + WindowAggregate::Last => Self::Last, + WindowAggregate::Lag => Self::Lag, + WindowAggregate::Lead => Self::Lead, + WindowAggregate::Diff => Self::Diff, + WindowAggregate::Rate => Self::Rate, + WindowAggregate::Ema => Self::Ema, + } + } +} + +impl From for WindowAggregate { + fn from(value: proto::WindowAggregate) -> Self { + match value { + proto::WindowAggregate::Sum => Self::Sum, + proto::WindowAggregate::Avg => Self::Avg, + proto::WindowAggregate::Count => Self::Count, + proto::WindowAggregate::Min => Self::Min, + proto::WindowAggregate::Max => Self::Max, + proto::WindowAggregate::Stddev => Self::Stddev, + proto::WindowAggregate::Var => Self::Var, + proto::WindowAggregate::First => Self::First, + proto::WindowAggregate::Last => Self::Last, + proto::WindowAggregate::Lag => Self::Lag, + proto::WindowAggregate::Lead => Self::Lead, + proto::WindowAggregate::Diff => Self::Diff, + proto::WindowAggregate::Rate => Self::Rate, + proto::WindowAggregate::Ema => Self::Ema, + } + } +} + +impl From for proto::window_spec::Frame { + fn from(value: WindowFrame) -> Self { + match value { + WindowFrame::Rows(n) => Self::Rows(n), + WindowFrame::Range(x) => Self::Range(x), + WindowFrame::Cumulative => Self::Cumulative(0), + } + } +} + +impl From for WindowFrame { + fn from(value: proto::window_spec::Frame) -> Self { + match value { + proto::window_spec::Frame::Rows(n) => Self::Rows(n), + proto::window_spec::Frame::Range(x) => Self::Range(x), + proto::window_spec::Frame::Cumulative(_) => Self::Cumulative, + } + } +} + +impl From for proto::window_spec::Order { + fn from(value: WindowSort) -> Self { + proto::window_spec::Order { + column: value.0, + desc: value.1 == WindowSortDir::Desc, + } + } +} + +impl From for WindowSort { + fn from(value: proto::window_spec::Order) -> Self { + WindowSort( + value.column, + if value.desc { + WindowSortDir::Desc + } else { + WindowSortDir::Asc + }, + ) + } +} + +impl From for proto::WindowSpec { + fn from(value: WindowSpec) -> Self { + proto::WindowSpec { + source: value.column, + op: proto::WindowAggregate::from(value.aggregate) as i32, + partition_by: value.partition_by, + order_by: value.order_by.map(|x| x.into()), + frame: value.frame.map(|x| x.into()), + offset: value.offset, + alpha: value.alpha, + } + } +} + +impl From for WindowSpec { + fn from(value: proto::WindowSpec) -> Self { + WindowSpec { + column: value.source, + aggregate: proto::WindowAggregate::try_from(value.op) + .unwrap_or(proto::WindowAggregate::Sum) + .into(), + partition_by: value.partition_by, + order_by: value.order_by.map(WindowSort::from), + frame: value.frame.map(|x| x.into()), + offset: value.offset, + alpha: value.alpha, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec(frame: Option) -> WindowSpec { + WindowSpec { + column: "price".to_string(), + aggregate: WindowAggregate::Sum, + partition_by: vec![], + order_by: None, + frame, + offset: None, + alpha: None, + } + } + + #[test] + fn test_frame_roundtrips_flattened() { + for (frame, json) in [ + (None, r#"{"column":"price","aggregate":"sum"}"#), + ( + Some(WindowFrame::Rows(19)), + r#"{"column":"price","aggregate":"sum","rows":19}"#, + ), + ( + Some(WindowFrame::Range(5000.0)), + r#"{"column":"price","aggregate":"sum","range":5000.0}"#, + ), + ( + Some(WindowFrame::Cumulative), + r#"{"column":"price","aggregate":"sum","cumulative":true}"#, + ), + ] { + assert_eq!(serde_json::to_string(&spec(frame)).unwrap(), json); + assert_eq!( + serde_json::from_str::(json).unwrap(), + spec(frame) + ); + } + } + + #[test] + fn test_frame_rejects_invalid_combinations() { + for json in [ + r#"{"column":"price","aggregate":"sum","rows":19,"range":1.0}"#, + r#"{"column":"price","aggregate":"sum","rows":19,"cumulative":true}"#, + r#"{"column":"price","aggregate":"sum","cumulative":false}"#, + r#"{"column":"price","aggregate":"sum","frame":"cumulative"}"#, + r#"{"column":"price","aggregate":"sum","rowz":19}"#, + ] { + assert!(serde_json::from_str::(json).is_err(), "{json}"); + } + } +} diff --git a/rust/perspective-client/src/rust/virtual_server/data.rs b/rust/perspective-client/src/rust/virtual_server/data.rs index 459cb4ae0a..47b5b3924c 100644 --- a/rust/perspective-client/src/rust/virtual_server/data.rs +++ b/rust/perspective-client/src/rust/virtual_server/data.rs @@ -894,18 +894,29 @@ impl VirtualDataSlice { /// `__ROW_PATH_N__` columns (`PerLevel`, currently unused — reserved /// for the future deprecation of `__ROW_PATH__`). See /// [`RowPathStyle`] for context. + /// + /// `id` emits an `__ID__` column of per-row identities, matching the + /// native engine's `to_columns(id = true)` shape for grouped views + /// (each row's identity is its `__ROW_PATH__` prefix). Ungrouped + /// views have no `row_path` and emit no `__ID__` — consumers fall + /// back to positional identity, as before. pub fn render_to_columns_json( &mut self, style: RowPathStyle, + id: bool, ) -> Result> { let batch = self.freeze().clone(); let schema = batch.schema(); let mut map = serde_json::Map::new(); - if style == RowPathStyle::Sidecar - && let Some(ref rp) = self.row_path - { - map.insert("__ROW_PATH__".to_string(), serde_json::to_value(rp)?); + if let Some(ref rp) = self.row_path { + if style == RowPathStyle::Sidecar { + map.insert("__ROW_PATH__".to_string(), serde_json::to_value(rp)?); + } + + if id { + map.insert("__ID__".to_string(), serde_json::to_value(rp)?); + } } for (col_idx, field) in schema.fields().iter().enumerate() { diff --git a/rust/perspective-client/src/rust/virtual_server/features.rs b/rust/perspective-client/src/rust/virtual_server/features.rs index 03399dd556..c361fc276c 100644 --- a/rust/perspective-client/src/rust/virtual_server/features.rs +++ b/rust/perspective-client/src/rust/virtual_server/features.rs @@ -14,6 +14,7 @@ use std::borrow::Cow; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; +use ts_rs::TS; use crate::config::GroupRollupMode; use crate::proto::get_features_resp::{AggregateArgs, AggregateOptions, ColumnTypeOptions}; @@ -24,46 +25,64 @@ use crate::proto::{ColumnType, GetFeaturesResp}; /// This struct is returned by /// [`VirtualServerHandler::get_features`](super::VirtualServerHandler::get_features) /// to inform clients about which operations are available. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)] pub struct Features<'a> { /// Whether group-by aggregation is supported. #[serde(default)] + #[ts(optional, as = "Option<_>")] pub group_by: bool, /// Which `group_by_rollup_mode` options are supported #[serde(default)] + #[ts(optional, as = "Option<_>")] pub group_rollup_mode: Vec, /// Whether split-by (pivot) operations are supported. #[serde(default)] + #[ts(optional, as = "Option<_>")] pub split_by: bool, /// Available filter operators per column type. #[serde(default)] + #[ts(optional, as = "Option<_>")] pub filter_ops: IndexMap>>, /// Available aggregate functions per column type. #[serde(default)] + #[ts(optional, as = "Option<_>")] pub aggregates: IndexMap>>, /// Whether sorting is supported. #[serde(default)] + #[ts(optional, as = "Option<_>")] pub sort: bool, /// Whether computed expressions are supported. #[serde(default)] + #[ts(optional, as = "Option<_>")] pub expressions: bool, + /// Available window aggregates. + #[serde(default)] + #[ts(optional, as = "Option<_>")] + pub window_aggregates: IndexMap>, + /// Whether update callbacks are supported. #[serde(default)] + #[ts(optional, as = "Option<_>")] pub on_update: bool, + + /// The data store has no reliable natural row order + #[serde(default)] + #[ts(optional, as = "Option<_>")] + pub unordered: bool, } /// Specification for an aggregate function. /// /// Aggregates can either take no additional arguments ([`AggSpec::Single`]) /// or require column type arguments ([`AggSpec::Multiple`]). -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] #[serde(untagged)] pub enum AggSpec<'a> { /// An aggregate function with no additional arguments. @@ -85,6 +104,22 @@ impl<'a> From> for GetFeaturesResp { expressions: value.expressions, on_update: value.on_update, sort: value.sort, + unordered: value.unordered, + window_aggregates: value + .window_aggregates + .iter() + .map(|(ty, aggs)| { + ( + *ty as u32, + crate::proto::get_features_resp::WindowAggregateOptions { + options: aggs + .iter() + .map(|x| crate::proto::WindowAggregate::from(*x) as i32) + .collect(), + }, + ) + }) + .collect(), aggregates: value .aggregates .iter() diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs index d0f90f95f5..b2c0019ab7 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs @@ -229,7 +229,7 @@ impl GenericSQLVirtualServerModel { view_id: &str, config: &ViewConfig, ) -> GenericSQLResult { - let ctx = ViewQueryContext::new(self, table_id, config); + let ctx = ViewQueryContext::new(self, table_id, config)?; let query = ctx.build_query(); let template = self.0.create_entity.as_deref().unwrap_or("TABLE"); Ok(format!("CREATE {} {} AS ({})", template, view_id, query)) diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs index 1b0e24af88..a04ddbd644 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs @@ -10,7 +10,11 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -use crate::config::{Aggregate, GroupRollupMode, Sort, SortDir, ViewConfig}; +use super::GenericSQLError; +use crate::config::{ + Aggregate, GroupRollupMode, Sort, SortDir, ViewConfig, WindowAggregate, WindowFrame, + WindowSortDir, WindowSpec, +}; fn aggregate_to_string(agg: &Aggregate) -> String { match agg { @@ -54,6 +58,149 @@ enum QueryOrientation { TotalPivoted, } +fn window_over_clause(w: &WindowSpec, frame: Option<&str>) -> String { + let mut parts: Vec = Vec::new(); + if !w.partition_by.is_empty() { + parts.push(format!( + "PARTITION BY {}", + w.partition_by + .iter() + .map(|c| format!("\"{}\"", quote_ident(c))) + .collect::>() + .join(", ") + )); + } + + // NULLS FIRST matches the engine's invalid-keys-sort-first ordering (in + // both directions). The engine additionally breaks order-key ties by + // primary key; SQL row order within ties is dialect-defined, so tied + // ROWS frames may differ. An OMITTED `order_by` takes the model's + // natural row order - `rowid`, the same identity unsorted view results + // are already ordered by. + match &w.order_by { + Some(order_by) => parts.push(format!( + "ORDER BY \"{}\" {} NULLS FIRST", + quote_ident(&order_by.0), + match order_by.1 { + WindowSortDir::Asc => "ASC", + WindowSortDir::Desc => "DESC", + } + )), + None => parts.push("ORDER BY rowid ASC".to_string()), + } + if let Some(f) = frame { + parts.push(f.to_string()); + } + + parts.join(" ") +} + +fn window_frame_sql(frame: Option<&WindowFrame>) -> String { + match frame { + Some(WindowFrame::Rows(n)) => { + format!("ROWS BETWEEN {} PRECEDING AND CURRENT ROW", n) + }, + Some(WindowFrame::Range(x)) => { + format!("RANGE BETWEEN {} PRECEDING AND CURRENT ROW", x) + }, + Some(WindowFrame::Cumulative) | None => { + "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string() + }, + } +} + +/// One `WindowSpec` as a SQL window-function expression - the 1:1 `OVER` +/// mapping that keeps hot-tier and virtual-server semantics interchangeable +/// (WINDOW_FUNCTIONS_PLAN Phase 5). `resolve` inlines expression-alias +/// sources. `ema` is recursive and has no SQL window equivalent. +fn window_sql(w: &WindowSpec, resolve: &dyn Fn(&str) -> String) -> Result { + // `range` frame interval arithmetic is defined on the order key's + // units - the natural (`rowid`) fallback is meaningless for it, so an + // explicit `order_by` is required (mirrors the engine's validation). + if w.order_by.is_none() && matches!(w.frame, Some(WindowFrame::Range(_))) { + return Err(GenericSQLError::UnsupportedOperation( + "window `range` frames require an explicit `order_by`".to_string(), + )); + } + + let src = resolve(&w.column); + let agg_fn = match w.aggregate { + WindowAggregate::Sum => Some("SUM"), + WindowAggregate::Avg => Some("AVG"), + WindowAggregate::Count => Some("COUNT"), + WindowAggregate::Min => Some("MIN"), + WindowAggregate::Max => Some("MAX"), + WindowAggregate::Stddev => Some("STDDEV_SAMP"), + WindowAggregate::Var => Some("VAR_SAMP"), + _ => None, + }; + + if let Some(agg_fn) = agg_fn { + let frame = window_frame_sql(w.frame.as_ref()); + return Ok(format!( + "{}({}) OVER ({})", + agg_fn, + src, + window_over_clause(w, Some(&frame)) + )); + } + + match w.aggregate { + WindowAggregate::Lag | WindowAggregate::Lead => Ok(format!( + "{}({}, {}) OVER ({})", + if w.aggregate == WindowAggregate::Lag { + "LAG" + } else { + "LEAD" + }, + src, + w.offset.unwrap_or(1), + window_over_clause(w, None) + )), + WindowAggregate::Diff => Ok(format!( + "({} - LAG({}, {}) OVER ({}))", + src, + src, + w.offset.unwrap_or(1), + window_over_clause(w, None) + )), + WindowAggregate::Rate => { + let Some(order_by) = &w.order_by else { + return Err(GenericSQLError::UnsupportedOperation( + "window `rate` requires an explicit `order_by`".to_string(), + )); + }; + + // The engine's validation matrix rejects frameless `rate` + // (an omitted frame means cumulative only for aggregating + // ops) - mirror it rather than diverge. + if !matches!(w.frame, Some(WindowFrame::Range(_))) { + return Err(GenericSQLError::UnsupportedOperation( + "window `rate` requires a `range` frame".to_string(), + )); + } + + let frame = window_frame_sql(w.frame.as_ref()); + let over = window_over_clause(w, Some(&frame)); + let okey = format!("\"{}\"", quote_ident(&order_by.0)); + Ok(format!( + "(({} - FIRST_VALUE({}) OVER ({})) / NULLIF(CAST({} AS DOUBLE) - \ + CAST(FIRST_VALUE({}) OVER ({}) AS DOUBLE), 0))", + src, src, over, okey, okey, over + )) + }, + WindowAggregate::Ema => Err(GenericSQLError::UnsupportedOperation( + "`ema` windows cannot be translated to a SQL window function (recursive); compute it \ + in the Perspective engine instead" + .to_string(), + )), + _ => Err(GenericSQLError::UnsupportedOperation(format!( + "window op {:?} is not supported by the SQL translation", + w.aggregate + ))), + } +} + fn quote_ident(name: &str) -> String { name.replace('"', "\"\"") } @@ -68,7 +215,7 @@ fn quote_literal(value: &str) -> String { /// needed to emit the correct `SELECT`, `GROUP BY`, `PIVOT`, `ORDER BY`, and /// `WINDOW` clauses for every combination of `group_by` / `split_by`. pub(crate) struct ViewQueryContext<'a> { - table: &'a str, + from_expr: String, config: &'a ViewConfig, group_col_names: Vec, grouping_fn: &'a str, @@ -83,7 +230,7 @@ impl<'a> ViewQueryContext<'a> { model: &'a super::GenericSQLVirtualServerModel, table: &'a str, config: &'a ViewConfig, - ) -> Self { + ) -> Result { let expressions = &config.expressions.0; let col_name_resolve = |col: &str| -> String { expressions @@ -92,6 +239,33 @@ impl<'a> ViewQueryContext<'a> { .unwrap_or_else(|| format!("\"{}\"", col)) }; + // Window columns materialize in a wrapping sub-select, so every + // downstream clause (filter, group_by, aggregate, sort) sees them as + // plain columns in all four query orientations - mirroring the + // engine, where windows compute over raw rows before pivoting. + let from_expr = if config.windows.is_empty() { + table.to_string() + } else { + // Sorted by alias so the generated SQL is deterministic (the + // `windows` map itself is unordered). + let mut windows = config.windows.iter().collect::>(); + windows.sort_by_key(|(name, _)| name.as_str()); + let mut selects = Vec::with_capacity(windows.len()); + for (name, w) in windows { + selects.push(format!( + "{} AS \"{}\"", + window_sql(w, &col_name_resolve)?, + quote_ident(name) + )); + } + + format!( + "(SELECT *, {} FROM {}) AS __PSP_WINDOW_SRC__", + selects.join(", "), + table + ) + }; + let grouping_fn = model.0.grouping_fn.as_deref().unwrap_or("GROUPING_ID"); let column_separator = model.0.column_separator.as_deref().unwrap_or("|"); let group_col_names: Vec = config @@ -104,14 +278,14 @@ impl<'a> ViewQueryContext<'a> { .map(|i| format!("__ROW_PATH_{}__", i)) .collect(); - Self { - table, + Ok(Self { + from_expr, config, group_col_names, grouping_fn, column_separator, row_path_aliases, - } + }) } /// Builds the inner `SELECT` query (without the outer `CREATE TABLE` @@ -124,7 +298,7 @@ impl<'a> ViewQueryContext<'a> { let mut query = match self.query_orientation() { QueryOrientation::Flat => { let select = self.select_clauses().join(", "); - format!("SELECT {} FROM {}{}", select, self.table, where_sql) + format!("SELECT {} FROM {}{}", select, self.from_expr, where_sql) }, QueryOrientation::Grouped => { let mut clauses = self.select_clauses(); @@ -133,7 +307,7 @@ impl<'a> ViewQueryContext<'a> { format!( "SELECT {} FROM {}{} GROUP BY {}", clauses.join(", "), - self.table, + self.from_expr, where_sql, self.group_col_names.join(", ") ) @@ -142,7 +316,7 @@ impl<'a> ViewQueryContext<'a> { format!( "SELECT {} FROM {}{} GROUP BY ROLLUP({})", clauses.join(", "), - self.table, + self.from_expr, where_sql, self.group_col_names.join(", ") ) @@ -159,7 +333,7 @@ impl<'a> ViewQueryContext<'a> { let src = format!( "SELECT {} FROM {}{}", src_clauses.join(", "), - self.table, + self.from_expr, where_sql ); @@ -214,7 +388,7 @@ impl<'a> ViewQueryContext<'a> { format!( "SELECT {} FROM {}{} GROUP BY {}, {}", inner_clauses.join(", "), - self.table, + self.from_expr, where_sql, groups_joined, split_cols_joined, @@ -223,7 +397,7 @@ impl<'a> ViewQueryContext<'a> { format!( "SELECT {} FROM {}{} GROUP BY ROLLUP({}), {}", inner_clauses.join(", "), - self.table, + self.from_expr, where_sql, groups_joined, split_cols_joined, @@ -254,7 +428,7 @@ impl<'a> ViewQueryContext<'a> { }, QueryOrientation::Total => { let select = self.select_clauses().join(", "); - format!("SELECT {} FROM {}{}", select, self.table, where_sql) + format!("SELECT {} FROM {}{}", select, self.from_expr, where_sql) }, QueryOrientation::TotalPivoted => { let mut src_clauses: Vec = self @@ -269,7 +443,7 @@ impl<'a> ViewQueryContext<'a> { let src = format!( "SELECT {} FROM {}{}", src_clauses.join(", "), - self.table, + self.from_expr, where_sql ); diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs index 99a1049922..c78a097669 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs @@ -13,7 +13,10 @@ use std::collections::HashMap; use super::*; -use crate::config::{Aggregate, GroupRollupMode}; +use crate::config::{ + Aggregate, GroupRollupMode, WindowAggregate, WindowFrame, WindowSort, WindowSortDir, + WindowSpec, Windows, +}; #[test] fn test_get_hosted_tables() { @@ -860,3 +863,224 @@ fn test_table_make_view_total_with_split_by() { sql ); } + +fn window_spec( + name: &str, + op: WindowAggregate, + frame: Option, +) -> (String, WindowSpec) { + (name.to_string(), WindowSpec { + column: "price".to_string(), + aggregate: op, + partition_by: vec!["sym".to_string()], + order_by: Some(WindowSort("t".to_string(), WindowSortDir::Asc)), + frame, + offset: None, + alpha: None, + }) +} + +#[test] +fn test_table_make_view_window_natural_order() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("t".to_string()), Some("cumsum".to_string())]; + let (name, mut spec) = window_spec( + "cumsum", + WindowAggregate::Sum, + Some(WindowFrame::Cumulative), + ); + spec.order_by = None; + config.windows = Windows(HashMap::from([(name, spec)])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + // An omitted `order_by` takes the model's natural order - the same + // `rowid` identity unsorted view results are ordered by. + assert!( + sql.contains("PARTITION BY \"sym\" ORDER BY rowid ASC"), + "natural order in OVER clause: {}", + sql + ); +} + +#[test] +fn test_table_make_view_window_range_requires_order_by() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("rs".to_string())]; + let (name, mut spec) = window_spec("rs", WindowAggregate::Sum, Some(WindowFrame::Range(10.0))); + spec.order_by = None; + config.windows = Windows(HashMap::from([(name, spec)])); + let result = builder.table_make_view("source_table", "dest_view", &config); + assert!(matches!( + result, + Err(GenericSQLError::UnsupportedOperation(_)) + )); +} + +#[test] +fn test_table_make_view_window_order_desc() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("t".to_string()), Some("cumsum".to_string())]; + let (name, mut spec) = window_spec( + "cumsum", + WindowAggregate::Sum, + Some(WindowFrame::Cumulative), + ); + spec.order_by.as_mut().unwrap().1 = WindowSortDir::Desc; + config.windows = Windows(HashMap::from([(name, spec)])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + assert!( + sql.contains("ORDER BY \"t\" DESC NULLS FIRST"), + "desc order in OVER clause: {}", + sql + ); +} + +#[test] +fn test_table_make_view_window_cumulative_sum() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("t".to_string()), Some("cumsum".to_string())]; + config.windows = Windows(HashMap::from([window_spec( + "cumsum", + WindowAggregate::Sum, + Some(WindowFrame::Cumulative), + )])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + assert!(sql.contains( + "SUM(\"price\") OVER (PARTITION BY \"sym\" ORDER BY \"t\" ASC NULLS FIRST ROWS BETWEEN \ + UNBOUNDED PRECEDING AND CURRENT ROW) AS \"cumsum\"" + )); + assert!(sql.contains("FROM (SELECT *,")); + assert!(sql.contains("FROM source_table) AS __PSP_WINDOW_SRC__")); +} + +#[test] +fn test_table_make_view_window_rows_and_range_frames() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("sma".to_string()), Some("rsum".to_string())]; + config.windows = Windows(HashMap::from([ + window_spec("sma", WindowAggregate::Avg, Some(WindowFrame::Rows(20))), + window_spec( + "rsum", + WindowAggregate::Sum, + Some(WindowFrame::Range(100.0)), + ), + ])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + assert!(sql.contains("AVG(\"price\") OVER")); + assert!(sql.contains("ROWS BETWEEN 20 PRECEDING AND CURRENT ROW")); + assert!(sql.contains("RANGE BETWEEN 100 PRECEDING AND CURRENT ROW")); +} + +#[test] +fn test_table_make_view_window_lag_diff() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("lg".to_string()), Some("df".to_string())]; + let (lag_name, mut lag) = window_spec("lg", WindowAggregate::Lag, None); + lag.offset = Some(2); + config.windows = Windows(HashMap::from([ + (lag_name, lag), + window_spec("df", WindowAggregate::Diff, None), + ])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + assert!(sql.contains( + "LAG(\"price\", 2) OVER (PARTITION BY \"sym\" ORDER BY \"t\" ASC NULLS FIRST) AS \"lg\"" + )); + assert!(sql.contains("(\"price\" - LAG(\"price\", 1) OVER")); +} + +#[test] +fn test_table_make_view_window_rate() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("rt".to_string())]; + config.windows = Windows(HashMap::from([window_spec( + "rt", + WindowAggregate::Rate, + Some(WindowFrame::Range(10.0)), + )])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + assert!(sql.contains("FIRST_VALUE(\"price\") OVER")); + assert!(sql.contains("NULLIF(CAST(\"t\" AS DOUBLE)")); + assert!(sql.contains("RANGE BETWEEN 10 PRECEDING AND CURRENT ROW")); +} + +#[test] +fn test_table_make_view_window_over_expression_source() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("w".to_string())]; + config.expressions = crate::config::Expressions(HashMap::from([( + "double_price".to_string(), + "\"price\" * 2".to_string(), + )])); + let (w_name, mut w) = window_spec("w", WindowAggregate::Sum, Some(WindowFrame::Cumulative)); + w.column = "double_price".to_string(); + config.windows = Windows(HashMap::from([(w_name, w)])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + assert!(sql.contains("SUM(\"price\" * 2) OVER")); +} + +#[test] +fn test_table_make_view_window_group_by_over_window_column() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("cumsum".to_string())]; + config.group_by = vec!["sym".to_string()]; + config.aggregates = HashMap::from([( + "cumsum".to_string(), + Aggregate::SingleAggregate("max".to_string()), + )]); + config.windows = Windows(HashMap::from([window_spec( + "cumsum", + WindowAggregate::Sum, + Some(WindowFrame::Cumulative), + )])); + let sql = builder + .table_make_view("source_table", "dest_view", &config) + .unwrap(); + + assert!(sql.contains("GROUP BY")); + assert!(sql.contains("__PSP_WINDOW_SRC__")); + assert!(sql.contains("max(\"cumsum\")")); +} + +#[test] +fn test_table_make_view_window_ema_unsupported() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("e".to_string())]; + let (w_name, mut w) = window_spec("e", WindowAggregate::Ema, None); + w.alpha = Some(0.5); + config.windows = Windows(HashMap::from([(w_name, w)])); + let result = builder.table_make_view("source_table", "dest_view", &config); + assert!(matches!( + result, + Err(GenericSQLError::UnsupportedOperation(_)) + )); +} diff --git a/rust/perspective-client/src/rust/virtual_server/server.rs b/rust/perspective-client/src/rust/virtual_server/server.rs index a593faaaa6..166b8b7322 100644 --- a/rust/perspective-client/src/rust/virtual_server/server.rs +++ b/rust/perspective-client/src/rust/virtual_server/server.rs @@ -178,6 +178,19 @@ impl VirtualServer { .insert(req.view_id.clone(), msg.entity_id.clone()); let mut config: ViewConfigUpdate = req.config.clone().unwrap_or_default().into(); + + // An UNORDERED store has no natural row order to fall back + // on, so every window must carry an explicit `order_by`. + if let Some(windows) = &config.windows + && windows.values().any(|w| w.order_by.is_none()) + && self.handler.get_features().await?.unordered + { + return Err(VirtualServerError::Other( + "This data store is unordered - windows require an explicit `order_by`" + .to_string(), + )); + } + let bytes = respond!(msg, TableMakeViewResp { view_id: self .handler @@ -394,7 +407,10 @@ impl VirtualServer { .await?; let json_string = cols - .render_to_columns_json(RowPathStyle::Sidecar) + .render_to_columns_json( + RowPathStyle::Sidecar, + view_to_columns_string_req.id.unwrap_or_default(), + ) .map_err(|e| VirtualServerError::Other(e.to_string()))?; respond!(msg, ViewToColumnsStringResp { json_string }) diff --git a/rust/perspective-js/src/rust/lib.rs b/rust/perspective-js/src/rust/lib.rs index 7099c35403..a7aa0df477 100644 --- a/rust/perspective-js/src/rust/lib.rs +++ b/rust/perspective-js/src/rust/lib.rs @@ -64,6 +64,9 @@ export type * from "../../src/ts/ts-rs/ViewConfig.d.ts"; export type * from "../../src/ts/ts-rs/JoinOptions.ts"; export type * from "../../src/ts/ts-rs/JoinType.ts"; export type * from "../../src/ts/ts-rs/TypedArrayWindow.ts"; +export type * from "../../src/ts/ts-rs/Features.ts"; +export type * from "../../src/ts/ts-rs/AggSpec.ts"; +export type * from "../../src/ts/ts-rs/WindowAggregate.ts"; import type {ColumnWindow} from "../../src/ts/ts-rs/ColumnWindow.d.ts"; import type {ColumnType} from "../../src/ts/ts-rs/ColumnType.d.ts"; @@ -78,6 +81,9 @@ import type {OnUpdateOptions} from "../../src/ts/ts-rs/OnUpdateOptions.d.ts"; import type {UpdateOptions} from "../../src/ts/ts-rs/UpdateOptions.d.ts"; import type {DeleteOptions} from "../../src/ts/ts-rs/DeleteOptions.d.ts"; import type {SystemInfo} from "../../src/ts/ts-rs/SystemInfo.d.ts"; +import type {ViewConfig} from "../../src/ts/ts-rs/ViewConfig.d.ts"; +import type {Scalar} from "../../src/ts/ts-rs/Scalar.d.ts"; +import type {Features} from "../../src/ts/ts-rs/Features.ts"; "#; #[cfg(feature = "export-init")] diff --git a/rust/perspective-js/src/rust/virtual_server.rs b/rust/perspective-js/src/rust/virtual_server.rs index d3f6a74324..32c7524e4a 100644 --- a/rust/perspective-js/src/rust/virtual_server.rs +++ b/rust/perspective-js/src/rust/virtual_server.rs @@ -74,6 +74,102 @@ fn jsvalue_to_scalar(val: &JsValue) -> perspective_client::config::Scalar { } } +// This interface is the TypeScript contract for [`JsServerHandler`] below. +// There is no codegen tying the two together - every method dispatched via +// `Reflect::get` in this file MUST be declared here, with the exact argument +// and return types the `Reflect` call sites accept. Keep them in sync when +// editing either. +#[wasm_bindgen(typescript_custom_section)] +const TS_VIRTUAL_SERVER_HANDLER: &'static str = r#" +/** + * A table hosted by a `VirtualServerHandler`, as returned by + * `getHostedTables()`. A plain `string` is shorthand for `{ name }`. + */ +export interface VirtualHostedTable { + name: string; + index?: string; + limit?: number; +} + +/** + * Handler interface that you implement to provide custom data sources. + * + * All methods will be called by the `VirtualServer` when handling protocol + * messages from Perspective clients. Methods can return values directly or + * return Promises for asynchronous operations (e.g., database queries). + * Optional methods fall back to defaults documented per-method. + */ +export interface VirtualServerHandler { + getHostedTables(): + | (string | VirtualHostedTable)[] + | Promise<(string | VirtualHostedTable)[]>; + tableSchema( + tableId: string, + ): Record | Promise>; + tableSize(tableId: string): number | Promise; + tableMakeView( + tableId: string, + viewId: string, + config: ViewConfigUpdate, + ): void | Promise; + viewDelete(viewId: string): void | Promise; + viewGetData( + viewId: string, + config: ViewConfig, + schema: Record, + viewport: ViewWindow, + dataSlice: VirtualDataSlice, + ): void | Promise; + + /** Defaults to `tableSchema(viewId)`. */ + viewSchema?( + viewId: string, + config: ViewConfig, + ): Record | Promise>; + + /** Defaults to `tableSize(viewId)`. */ + viewSize?(viewId: string): number | Promise; + + /** Defaults to the length of `tableSchema(tableId)`. */ + tableColumnsSize?(tableId: string): number | Promise; + + /** Defaults to the length of `viewSchema(viewId, config)`. */ + viewColumnSize?( + viewId: string, + config: ViewConfig, + ): number | Promise; + + /** Required when `getFeatures()` reports `expressions: true`. */ + tableValidateExpression?( + tableId: string, + expression: string, + ): ColumnType | Promise; + + viewGetMinMax?( + viewId: string, + columnName: string, + config: ViewConfig, + ): { min: Scalar; max: Scalar } | Promise<{ min: Scalar; max: Scalar }>; + + /** Defaults to no optional features. */ + getFeatures?(): Features | Promise; + + /** Defaults to port `0`. */ + tableMakePort?(): number | Promise; + + makeTable?( + tableId: string, + data: string | Uint8Array, + ): void | Promise; +} +"#; + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(typescript_type = "VirtualServerHandler")] + pub type JsVirtualServerHandler; +} + pub struct JsServerHandler(Object); impl JsServerHandler { @@ -785,9 +881,9 @@ pub struct VirtualServer(Rc Result { + pub fn new(handler: JsVirtualServerHandler) -> Result { Ok(VirtualServer(Rc::new(UnsafeCell::new( - virtual_server::VirtualServer::new(JsServerHandler(handler)), + virtual_server::VirtualServer::new(JsServerHandler(handler.unchecked_into())), )))) } diff --git a/rust/perspective-js/src/ts/virtual_server.ts b/rust/perspective-js/src/ts/virtual_server.ts index 0b91659061..11cd29c424 100644 --- a/rust/perspective-js/src/ts/virtual_server.ts +++ b/rust/perspective-js/src/ts/virtual_server.ts @@ -10,10 +10,6 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -import { ColumnType } from "./ts-rs/ColumnType.ts"; -import { ViewConfig } from "./ts-rs/ViewConfig.ts"; -import { ViewWindow } from "./ts-rs/ViewWindow.ts"; - import type * as perspective from "../../dist/wasm/perspective-js.js"; /** @@ -26,63 +22,29 @@ import type * as perspective from "../../dist/wasm/perspective-js.js"; * - Implementing custom aggregation or transformation logic * - Creating data adapters without copying data into Perspective tables * + * The `VirtualServerHandler` interface and `Features` struct are declared + * in Rust alongside the bridge that invokes them (`perspective-js`'s + * `virtual_server.rs` and `perspective-client`'s `features.rs` + * respectively) and re-exported here. `ServerFeatures` is a legacy alias + * for `Features`. + * * @module virtual_server */ -export interface ServerFeatures { - expressions?: boolean; -} - -/** - * Handler interface that you implement to provide custom data sources. - * - * All methods will be called by the VirtualServer when handling protocol - * messages from Perspective clients. Methods can return values directly or - * return Promises for asynchronous operations (e.g., database queries). - */ -export interface VirtualServerHandler { - getHostedTables(): string[] | Promise; - tableSchema( - tableId: string, - ): Record | Promise>; - tableSize(tableId: string): number | Promise; - tableMakeView( - tableId: string, - viewId: string, - config: ViewConfig, - ): void | Promise; - viewDelete(viewId: string): void | Promise; - viewGetData( - viewId: string, - config: ViewConfig, - schema: Record, - viewport: ViewWindow, - dataSlice: perspective.VirtualDataSlice, - ): void | Promise; - viewSchema?( - viewId: string, - config?: ViewConfig, - ): Record | Promise>; - viewSize?(viewId: string): number | Promise; - tableValidateExpression?( - tableId: string, - expression: string, - ): ColumnType | Promise; - viewGetMinMax?( - viewId: string, - columnName: string, - config: ViewConfig, - ): { min: any; max: any } | Promise<{ min: any; max: any }>; - getFeatures?(): ServerFeatures | Promise; - makeTable?( - tableId: string, - data: string | Uint8Array, - ): void | Promise; -} +// `Features` must re-export from the wasm `.d.ts` (NOT "./ts-rs/Features.ts") +// - `perspective.browser.ts` star-exports both this module and the wasm +// `.d.ts`, and star exports of the same name only merge when they resolve to +// the same declaration. +export type { + VirtualServerHandler, + VirtualHostedTable, + Features, + Features as ServerFeatures, +} from "../../dist/wasm/perspective-js.js"; export function createMessageHandler( mod: typeof perspective, - handler: VirtualServerHandler, + handler: perspective.VirtualServerHandler, ) { let virtualServer: perspective.VirtualServer; async function postMessage(port: MessagePort, msg: MessageEvent) { diff --git a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts index 1dd15a2526..eb1da682be 100644 --- a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts +++ b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts @@ -24,7 +24,9 @@ import type * as perspective from "@perspective-dev/client"; import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnType.d.ts"; import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; +import type { ViewConfigUpdate } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfigUpdate.d.ts"; import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; +import type { WindowAggregate } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggregate.d.ts"; import type * as clickhouse from "@clickhouse/client-web"; const NUMBER_AGGS = [ @@ -63,6 +65,30 @@ const STRING_AGGS = [ "string_agg", ]; +// The window aggregates the SQL translation supports, per source +// column type (`ema` is recursive - no SQL window equivalent). +const WINDOW_AGGREGATES: WindowAggregate[] = [ + "sum", + "avg", + "count", + "min", + "max", + "stddev", + "var", + "lag", + "lead", + "diff", + "rate", +]; + +const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [ + "count", + "min", + "max", + "lag", + "lead", +]; + const FILTER_OPS = [ "==", "!=", @@ -218,12 +244,24 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler { }); } - getFeatures() { + getFeatures(): perspective.Features { return { group_by: true, split_by: false, sort: true, expressions: true, + // ClickHouse has no stable `rowid`, so natural-order windows + // are unsupported. + unordered: true, + window_aggregates: { + // `ema` is recursive and has no SQL window translation. + integer: WINDOW_AGGREGATES, + float: WINDOW_AGGREGATES, + string: WINDOW_AGGREGATES_ANY, + date: WINDOW_AGGREGATES_ANY, + datetime: WINDOW_AGGREGATES_ANY, + boolean: WINDOW_AGGREGATES_ANY, + }, group_rollup_mode: ["rollup", "flat", "total"], filter_ops: { integer: FILTER_OPS, @@ -284,7 +322,11 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler { return Number(results[0]["COUNT()"]); } - async tableMakeView(tableId: string, viewId: string, config: ViewConfig) { + async tableMakeView( + tableId: string, + viewId: string, + config: ViewConfigUpdate, + ) { const query = this.sqlBuilder.tableMakeView(tableId, viewId, config); await runQuery(this.db, query, { execute: true }); } diff --git a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts index a8a9599983..cabc35b543 100644 --- a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts +++ b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts @@ -24,7 +24,10 @@ import type * as perspective from "@perspective-dev/client"; import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnType.d.ts"; import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; +import type { ViewConfigUpdate } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfigUpdate.d.ts"; import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; +import type { WindowAggregate } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggregate.d.ts"; +import type { Scalar } from "@perspective-dev/client/dist/esm/ts-rs/Scalar.d.ts"; import type * as duckdb from "@duckdb/duckdb-wasm"; const NUMBER_AGGS = [ @@ -63,6 +66,30 @@ const STRING_AGGS = [ "string_agg", ]; +// The window aggregates the SQL translation supports, per source +// column type (`ema` is recursive - no SQL window equivalent). +const WINDOW_AGGREGATES: WindowAggregate[] = [ + "sum", + "avg", + "count", + "min", + "max", + "stddev", + "var", + "lag", + "lead", + "diff", + "rate", +]; + +const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [ + "count", + "min", + "max", + "lag", + "lead", +]; + const FILTER_OPS = [ "==", "!=", @@ -189,12 +216,21 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { }); } - getFeatures() { + getFeatures(): perspective.Features { return { group_by: true, split_by: true, sort: true, expressions: true, + window_aggregates: { + // `ema` is recursive and has no SQL window translation. + integer: WINDOW_AGGREGATES, + float: WINDOW_AGGREGATES, + string: WINDOW_AGGREGATES_ANY, + date: WINDOW_AGGREGATES_ANY, + datetime: WINDOW_AGGREGATES_ANY, + boolean: WINDOW_AGGREGATES_ANY, + }, group_rollup_mode: ["rollup", "flat", "total"], filter_ops: { integer: FILTER_OPS, @@ -256,7 +292,11 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { return Number(results[0].toJSON()["count_star()"]); } - async tableMakeView(tableId: string, viewId: string, config: ViewConfig) { + async tableMakeView( + tableId: string, + viewId: string, + config: ViewConfigUpdate, + ) { const query = this.sqlBuilder.tableMakeView(tableId, viewId, config); await runQuery(this.db, query); } @@ -288,7 +328,7 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { let [min, max] = Object.values(row); if (typeof min === "bigint") min = Number(min); if (typeof max === "bigint") max = Number(max); - return { min: min ?? null, max: max ?? null }; + return { min: (min ?? null) as Scalar, max: (max ?? null) as Scalar }; } async viewGetData( diff --git a/rust/perspective-js/test/js/duckdb/group_by.spec.js b/rust/perspective-js/test/js/duckdb/group_by.spec.js index 1e0d09ef6e..0e4fe4b6a4 100644 --- a/rust/perspective-js/test/js/duckdb/group_by.spec.js +++ b/rust/perspective-js/test/js/duckdb/group_by.spec.js @@ -174,6 +174,30 @@ describeDuckDB("group_by", (getClient) => { await view.delete(); }); + // `__ID__` drives identity-based consumers (e.g. the Datagrid's + // `SELECT_ROW_TREE` mode, which prefix-matches row identities to + // style descendants). For grouped views each row's identity is its + // `__ROW_PATH__` prefix, matching the native engine's + // `to_columns(id = true)` shape. + test("to_columns with id emits __ID__ matching __ROW_PATH__", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Region", "Category"], + aggregates: { Sales: "sum" }, + }); + + const cols = await view.to_columns({ id: true }); + expect(cols.__ID__).toEqual(cols.__ROW_PATH__); + expect(cols.__ID__[0]).toEqual([]); + expect(cols.__ID__[1]).toEqual(["Central"]); + expect(cols.__ID__[2]).toEqual(["Central", "Furniture"]); + + const cols2 = await view.to_columns(); + expect(cols2.__ID__).toBeUndefined(); + await view.delete(); + }); + test("group_by with max aggregate", async function () { const table = await getClient().open_table("memory.superstore"); const view = await table.view({ diff --git a/rust/perspective-js/test/js/duckdb/windows.spec.js b/rust/perspective-js/test/js/duckdb/windows.spec.js new file mode 100644 index 0000000000..cf5173494c --- /dev/null +++ b/rust/perspective-js/test/js/duckdb/windows.spec.js @@ -0,0 +1,202 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import { describeDuckDB } from "./setup.js"; + +// Fetch the raw columns once and brute-force the expected window outputs in +// JS - "Row ID" is unique, so the ordering is tie-free and deterministic on +// both sides. +async function raw_rows(client) { + const table = await client.open_table("memory.superstore"); + const view = await table.view({ + columns: ["Row ID", "Region", "Sales"], + sort: [["Row ID", "asc"]], + }); + const cols = await view.to_columns(); + await view.delete(); + const rows = cols["Row ID"].map((id, i) => ({ + id, + region: cols["Region"][i], + sales: cols["Sales"][i], + })); + return rows; +} + +describeDuckDB("windows", (getClient) => { + test("cumulative sum partitioned by Region", async function () { + const rows = await raw_rows(getClient()); + const running = {}; + const expected = rows.map((r) => { + running[r.region] = (running[r.region] ?? 0) + r.sales; + return running[r.region]; + }); + + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["Row ID", "cumsum"], + sort: [["Row ID", "asc"]], + windows: { + cumsum: { + column: "Sales", + aggregate: "sum", + order_by: ["Row ID", "asc"], + partition_by: ["Region"], + cumulative: true, + }, + }, + }); + + const result = await view.to_columns(); + expect(result["cumsum"].length).toBe(expected.length); + for (let i = 0; i < expected.length; i++) { + expect(result["cumsum"][i]).toBeCloseTo(expected[i], 6); + } + await view.delete(); + }); + + test("lag copies the previous partition row exactly", async function () { + const rows = await raw_rows(getClient()); + const prev = {}; + const expected = rows.map((r) => { + const out = prev[r.region] ?? null; + prev[r.region] = r.sales; + return out; + }); + + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["Row ID", "lg"], + sort: [["Row ID", "asc"]], + windows: { + lg: { + column: "Sales", + aggregate: "lag", + order_by: ["Row ID", "asc"], + partition_by: ["Region"], + }, + }, + }); + + const result = await view.to_columns(); + expect(result["lg"]).toEqual(expected); + await view.delete(); + }); + + test("descending cumulative sum matches the reversed running sum", async function () { + const rows = await raw_rows(getClient()); + const running = {}; + const expected = rows.map(() => 0); + for (let i = rows.length - 1; i >= 0; i--) { + const r = rows[i]; + running[r.region] = (running[r.region] ?? 0) + r.sales; + expected[i] = running[r.region]; + } + + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["Row ID", "cumsum"], + sort: [["Row ID", "asc"]], + windows: { + cumsum: { + column: "Sales", + aggregate: "sum", + order_by: ["Row ID", "desc"], + partition_by: ["Region"], + cumulative: true, + }, + }, + }); + + const result = await view.to_columns(); + expect(result["cumsum"].length).toBe(expected.length); + for (let i = 0; i < expected.length; i++) { + expect(result["cumsum"][i]).toBeCloseTo(expected[i], 6); + } + await view.delete(); + }); + + test("omitted order_by uses natural (rowid) order", async function () { + // Natural order for the SQL model is `rowid` - the same identity + // an unsorted view's results are already ordered by, so the + // expected values are the running sums over an UNSORTED fetch. + const table = await getClient().open_table("memory.superstore"); + const raw_view = await table.view({ + columns: ["Row ID", "Region", "Sales"], + }); + const raw = await raw_view.to_columns(); + await raw_view.delete(); + + const running = {}; + const expected_by_id = {}; + for (let i = 0; i < raw["Row ID"].length; i++) { + const region = raw["Region"][i]; + running[region] = (running[region] ?? 0) + raw["Sales"][i]; + expected_by_id[raw["Row ID"][i]] = running[region]; + } + + const view = await table.view({ + columns: ["Row ID", "cumsum"], + sort: [["Row ID", "asc"]], + windows: { + cumsum: { + column: "Sales", + aggregate: "sum", + partition_by: ["Region"], + cumulative: true, + }, + }, + }); + + const result = await view.to_columns(); + for (let i = 0; i < result["Row ID"].length; i++) { + expect(result["cumsum"][i]).toBeCloseTo( + expected_by_id[result["Row ID"][i]], + 6, + ); + } + await view.delete(); + }); + + test("window column as group_by aggregate input", async function () { + const rows = await raw_rows(getClient()); + const running = {}; + for (const r of rows) { + running[r.region] = (running[r.region] ?? 0) + r.sales; + } + + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["cumsum"], + group_by: ["Region"], + aggregates: { cumsum: "max" }, + windows: { + cumsum: { + column: "Sales", + aggregate: "sum", + order_by: ["Row ID", "asc"], + partition_by: ["Region"], + cumulative: true, + }, + }, + }); + + const result = await view.to_json(); + for (const row of result) { + const path = row["__ROW_PATH__"]; + if (path.length === 1) { + expect(row["cumsum"]).toBeCloseTo(running[path[0]], 6); + } + } + await view.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/windows.spec.js b/rust/perspective-js/test/js/windows.spec.js new file mode 100644 index 0000000000..2b3852b139 --- /dev/null +++ b/rust/perspective-js/test/js/windows.spec.js @@ -0,0 +1,1000 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "./perspective_client"; + +// Rows arrive deliberately out of `t` order to prove windows order by +// `order_by`, not insertion order. +const data = [ + { t: 3, sym: "a", price: 30 }, + { t: 1, sym: "a", price: 10 }, + { t: 2, sym: "b", price: 100 }, + { t: 4, sym: "b", price: 200 }, + { t: 2, sym: "a", price: 20 }, +]; + +test.describe("Window columns", function () { + test("cumulative sum over one partition", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "price", "cumsum"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "asc"], + partition_by: ["sym"], + cumulative: true, + }, + }, + }); + + const result = await view.to_columns(); + expect(result["cumsum"]).toEqual([10, 30, 60]); + await view.delete(); + await table.delete(); + }); + + test("omitted frame defaults to cumulative for aggregating ops", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "price", "cumsum"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "asc"], + partition_by: ["sym"], + }, + }, + }); + + const result = await view.to_columns(); + expect(result["cumsum"]).toEqual([10, 30, 60]); + await view.delete(); + await table.delete(); + }); + + test("moving sum and avg with rows frame, partitioned", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["sym", "t", "sma"], + sort: [ + ["sym", "asc"], + ["t", "asc"], + ], + windows: { + sma: { + column: "price", + aggregate: "avg", + order_by: ["t", "asc"], + partition_by: ["sym"], + rows: 1, + }, + }, + }); + + const result = await view.to_columns(); + // frame = 1 preceding + current: per-partition trailing pairs + expect(result["sma"]).toEqual([10, 15, 25, 100, 150]); + await view.delete(); + await table.delete(); + }); + + test("streaming append extends cumulative sum", async function () { + const table = await perspective.table( + { t: "integer", sym: "string", price: "float" }, + { index: "t" }, + ); + await table.update([ + { t: 1, sym: "a", price: 1 }, + { t: 2, sym: "a", price: 2 }, + ]); + const view = await table.view({ + columns: ["t", "cumsum"], + sort: [["t", "asc"]], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "asc"], + cumulative: true, + }, + }, + }); + + expect((await view.to_columns())["cumsum"]).toEqual([1, 3]); + + await table.update([{ t: 3, sym: "a", price: 3 }]); + expect((await view.to_columns())["cumsum"]).toEqual([1, 3, 6]); + + // A mid-history edit re-bases every later cumulative value. + await table.update([{ t: 1, sym: "a", price: 10 }]); + expect((await view.to_columns())["cumsum"]).toEqual([10, 12, 15]); + + await view.delete(); + await table.delete(); + }); + + test("window column works as group_by aggregate input", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + group_by: ["sym"], + columns: ["last_cumsum"], + aggregates: { last_cumsum: "max" }, + windows: { + last_cumsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "asc"], + partition_by: ["sym"], + cumulative: true, + }, + }, + }); + + const result = await view.to_columns(); + // [TOTAL, a, b]: partition cumsums end at 60 (a) and 300 (b) + expect(result["last_cumsum"]).toEqual([300, 60, 300]); + await view.delete(); + await table.delete(); + }); + + test("min/max rows frame and count cumulative", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "mn", "mx", "n"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + windows: { + mn: { + column: "price", + aggregate: "min", + order_by: ["t", "asc"], + partition_by: ["sym"], + rows: 1, + }, + mx: { + column: "price", + aggregate: "max", + order_by: ["t", "asc"], + partition_by: ["sym"], + rows: 1, + }, + n: { + column: "price", + aggregate: "count", + order_by: ["t", "asc"], + partition_by: ["sym"], + cumulative: true, + }, + }, + }); + + const result = await view.to_columns(); + expect(result["mn"]).toEqual([10, 10, 20]); + expect(result["mx"]).toEqual([10, 20, 30]); + expect(result["n"]).toEqual([1, 2, 3]); + await view.delete(); + await table.delete(); + }); + + test("row delta includes out-of-batch rows whose window outputs changed", async function () { + const table = await perspective.table( + { t: "integer", price: "float" }, + { index: "t" }, + ); + await table.update([ + { t: 1, price: 10 }, + { t: 2, price: 2 }, + { t: 3, price: 3 }, + ]); + const view = await table.view({ + columns: ["t", "price", "cumsum"], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "asc"], + cumulative: true, + }, + }, + }); + // Settle the registration notify before subscribing. + await view.to_columns(); + + const delta = new Promise((resolve) => { + view.on_update((updated) => resolve(updated.delta), { + mode: "row", + }); + }); + await table.update([{ t: 1, price: 100 }]); + + const delta_table = await perspective.table(await delta); + const delta_view = await delta_table.view({ sort: [["t", "asc"]] }); + const result = await delta_view.to_columns(); + // The batch touched only t=1, but every later cumulative value + // changed - the widening pass must surface t=2 and t=3. + expect(result["t"]).toEqual([1, 2, 3]); + expect(result["cumsum"]).toEqual([100, 102, 105]); + await delta_view.delete(); + await delta_table.delete(); + await view.delete(); + await table.delete(); + }); + + test("row delta suppresses widened rows whose outputs did not change", async function () { + const table = await perspective.table( + { t: "integer", price: "float" }, + { index: "t" }, + ); + await table.update([ + { t: 1, price: 5 }, + { t: 2, price: 10 }, + { t: 3, price: 1 }, + ]); + const view = await table.view({ + columns: ["t", "price", "mn"], + windows: { + mn: { + column: "price", + aggregate: "min", + order_by: ["t", "asc"], + rows: 1, + }, + }, + }); + await view.to_columns(); + + const delta = new Promise((resolve) => { + view.on_update((updated) => resolve(updated.delta), { + mode: "row", + }); + }); + // 10 -> 7 leaves both trailing-pair minimums unchanged + // (min(5,7) == 5, min(7,1) == 1), so the widened t=3 row must be + // suppressed by the pipeline's prev/current diff. + await table.update([{ t: 2, price: 7 }]); + + const delta_table = await perspective.table(await delta); + const delta_view = await delta_table.view({ sort: [["t", "asc"]] }); + const result = await delta_view.to_columns(); + expect(result["t"]).toEqual([2]); + expect(result["mn"]).toEqual([5]); + await delta_view.delete(); + await delta_table.delete(); + await view.delete(); + await table.delete(); + }); + + test("random ops match a fresh-view oracle", async function () { + // mulberry32: seeded so failures reproduce + let seed = 0x9e3779b9; + const rand = () => { + seed |= 0; + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + + const config = { + columns: [ + "id", + "sym", + "t", + "cumsum", + "sma", + "lg", + "ld", + "df", + "rsum", + "rt", + "ema1", + "sd", + ], + sort: [["id", "asc"]], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "asc"], + partition_by: ["sym"], + cumulative: true, + }, + sma: { + column: "price", + aggregate: "avg", + order_by: ["t", "asc"], + partition_by: ["sym"], + rows: 2, + }, + lg: { + column: "price", + aggregate: "lag", + order_by: ["t", "asc"], + partition_by: ["sym"], + }, + ld: { + column: "price", + aggregate: "lead", + order_by: ["t", "asc"], + partition_by: ["sym"], + }, + df: { + column: "price", + aggregate: "diff", + order_by: ["t", "asc"], + partition_by: ["sym"], + }, + rsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "asc"], + partition_by: ["sym"], + range: 3, + }, + rt: { + column: "price", + aggregate: "rate", + order_by: ["t", "asc"], + partition_by: ["sym"], + range: 5, + }, + ema1: { + column: "price", + aggregate: "ema", + order_by: ["t", "asc"], + partition_by: ["sym"], + alpha: 0.5, + }, + sd: { + column: "price", + aggregate: "stddev", + order_by: ["t", "asc"], + partition_by: ["sym"], + cumulative: true, + }, + }, + }; + const schema = { + id: "integer", + sym: "string", + t: "integer", + price: "float", + }; + + const table = await perspective.table(schema, { index: "id" }); + const view = await table.view(config); + const rows = new Map(); + let next_id = 0; + + for (let op_idx = 0; op_idx < 40; op_idx++) { + const roll = rand(); + if (roll < 0.4 || rows.size === 0) { + const row = { + id: next_id++, + sym: rand() < 0.5 ? "a" : "b", + t: Math.floor(rand() * 20), + price: Math.floor(rand() * 100), + }; + rows.set(row.id, row); + await table.update([row]); + } else { + const ids = [...rows.keys()]; + const id = ids[Math.floor(rand() * ids.length)]; + if (roll < 0.55) { + await table.remove([id]); + rows.delete(id); + } else { + // mutate price, order key, or partition - the last two + // exercise relocation and partition migration + const row = { ...rows.get(id) }; + const which = rand(); + if (which < 0.4) { + row.price = Math.floor(rand() * 100); + } else if (which < 0.7) { + row.t = Math.floor(rand() * 20); + } else { + row.sym = row.sym === "a" ? "b" : "a"; + } + rows.set(id, row); + await table.update([row]); + } + } + + const incremental = await view.to_columns(); + const oracle_table = await perspective.table(schema, { + index: "id", + }); + if (rows.size > 0) { + await oracle_table.update([...rows.values()]); + } + const oracle_view = await oracle_table.view(config); + const expected = await oracle_view.to_columns(); + await oracle_view.delete(); + await oracle_table.delete(); + + expect({ op: op_idx, cols: incremental }).toEqual({ + op: op_idx, + cols: expected, + }); + } + + await view.delete(); + await table.delete(); + }); + + test("lag, lead and diff", async function () { + const table = await perspective.table({ t: "integer", p: "float" }); + await table.update([ + { t: 3, p: 40 }, + { t: 1, p: 10 }, + { t: 4, p: 80 }, + { t: 2, p: 20 }, + ]); + const view = await table.view({ + columns: ["t", "lg", "ld", "df"], + sort: [["t", "asc"]], + windows: { + lg: { column: "p", aggregate: "lag", order_by: ["t", "asc"] }, + ld: { column: "p", aggregate: "lead", order_by: ["t", "asc"] }, + df: { column: "p", aggregate: "diff", order_by: ["t", "asc"] }, + }, + }); + + const result = await view.to_columns(); + expect(result["lg"]).toEqual([null, 10, 20, 40]); + expect(result["ld"]).toEqual([20, 40, 80, null]); + expect(result["df"]).toEqual([null, 10, 20, 40]); + await view.delete(); + await table.delete(); + }); + + test("ema", async function () { + const table = await perspective.table({ t: "integer", p: "float" }); + await table.update([ + { t: 1, p: 10 }, + { t: 2, p: 20 }, + { t: 3, p: 40 }, + ]); + const view = await table.view({ + columns: ["t", "ema"], + sort: [["t", "asc"]], + windows: { + ema: { + column: "p", + aggregate: "ema", + order_by: ["t", "asc"], + alpha: 0.5, + }, + }, + }); + + const result = await view.to_columns(); + expect(result["ema"]).toEqual([10, 15, 27.5]); + await view.delete(); + await table.delete(); + }); + + test("range frame sum over sparse keys", async function () { + const table = await perspective.table({ t: "integer", p: "float" }); + await table.update([ + { t: 5, p: 4 }, + { t: 1, p: 1 }, + { t: 6, p: 8 }, + { t: 2, p: 2 }, + ]); + const view = await table.view({ + columns: ["t", "rsum"], + sort: [["t", "asc"]], + windows: { + rsum: { + column: "p", + aggregate: "sum", + order_by: ["t", "asc"], + range: 1, + }, + }, + }); + + const result = await view.to_columns(); + // frames by key interval [t - 1, t]: {1}, {1,2}, {4}, {4,8} + expect(result["rsum"]).toEqual([1, 3, 4, 12]); + await view.delete(); + await table.delete(); + }); + + test("rate over a range frame", async function () { + const table = await perspective.table({ t: "integer", p: "float" }); + await table.update([ + { t: 0, p: 0 }, + { t: 10, p: 5 }, + { t: 20, p: 20 }, + ]); + const view = await table.view({ + columns: ["t", "rate"], + sort: [["t", "asc"]], + windows: { + rate: { + column: "p", + aggregate: "rate", + order_by: ["t", "asc"], + range: 10, + }, + }, + }); + + const result = await view.to_columns(); + // (Δvalue / Δkey) over each 10-unit trailing window; the first row + // has no span. + expect(result["rate"]).toEqual([null, 0.5, 1.5]); + await view.delete(); + await table.delete(); + }); + + test("rolling sample stddev", async function () { + const table = await perspective.table({ t: "integer", p: "float" }); + await table.update([ + { t: 1, p: 10 }, + { t: 2, p: 20 }, + { t: 3, p: 40 }, + ]); + const view = await table.view({ + columns: ["t", "sd"], + sort: [["t", "asc"]], + windows: { + sd: { + column: "p", + aggregate: "stddev", + order_by: ["t", "asc"], + rows: 2, + }, + }, + }); + + const result = await view.to_columns(); + expect(result["sd"][0]).toBeNull(); + expect(result["sd"][1]).toBeCloseTo(Math.sqrt(50), 10); + expect(result["sd"][2]).toBeCloseTo(Math.sqrt(700 / 3), 10); + await view.delete(); + await table.delete(); + }); + + test("streaming mid-edit re-bases range and ema windows", async function () { + const table = await perspective.table( + { t: "integer", p: "float" }, + { index: "t" }, + ); + await table.update([ + { t: 1, p: 10 }, + { t: 2, p: 20 }, + { t: 3, p: 40 }, + ]); + const view = await table.view({ + columns: ["t", "rsum", "ema"], + sort: [["t", "asc"]], + windows: { + rsum: { + column: "p", + aggregate: "sum", + order_by: ["t", "asc"], + range: 1, + }, + ema: { + column: "p", + aggregate: "ema", + order_by: ["t", "asc"], + alpha: 0.5, + }, + }, + }); + + expect((await view.to_columns())["rsum"]).toEqual([10, 30, 60]); + expect((await view.to_columns())["ema"]).toEqual([10, 15, 27.5]); + + // Mid-history edit: every downstream range frame containing t=2 and + // the whole ema suffix re-base. + await table.update([{ t: 2, p: 100 }]); + expect((await view.to_columns())["rsum"]).toEqual([10, 110, 140]); + expect((await view.to_columns())["ema"]).toEqual([10, 55, 47.5]); + + await view.delete(); + await table.delete(); + }); + + test("sliding frames match brute force over a long series", async function () { + // The C++ sliding-window pass (Phase 4) is exercised by both the + // view and the property-test oracle, so pin it against an + // independent JS brute-force instead. Integer prices keep sliding + // add/subtract exact. + let seed = 0xc0ffee; + const rand = () => { + seed |= 0; + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + + const N = 200; + const FRAME = 7; + const RANGE = 5; + const rows = []; + for (let i = 0; i < N; i++) { + rows.push({ + t: i * 2 + Math.floor(rand() * 2), + p: Math.floor(rand() * 1000), + }); + } + + const table = await perspective.table( + { t: "integer", p: "float" }, + { index: "t" }, + ); + await table.update(rows); + const view = await table.view({ + columns: ["t", "s", "mn", "mx", "n", "sd"], + sort: [["t", "asc"]], + windows: { + s: { + column: "p", + aggregate: "sum", + order_by: ["t", "asc"], + rows: FRAME, + }, + mn: { + column: "p", + aggregate: "min", + order_by: ["t", "asc"], + rows: FRAME, + }, + mx: { + column: "p", + aggregate: "max", + order_by: ["t", "asc"], + rows: FRAME, + }, + n: { + column: "p", + aggregate: "count", + order_by: ["t", "asc"], + range: RANGE, + }, + sd: { + column: "p", + aggregate: "stddev", + order_by: ["t", "asc"], + rows: FRAME, + }, + }, + }); + + const brute = (sorted) => { + const out = { s: [], mn: [], mx: [], n: [], sd: [] }; + for (let i = 0; i < sorted.length; i++) { + const rows_frame = sorted.slice(Math.max(0, i - FRAME), i + 1); + const vals = rows_frame.map((r) => r.p); + out.s.push(vals.reduce((a, b) => a + b, 0)); + out.mn.push(Math.min(...vals)); + out.mx.push(Math.max(...vals)); + const range_vals = sorted + .filter( + (r) => r.t >= sorted[i].t - RANGE && r.t <= sorted[i].t, + ) + .map((r) => r.p); + out.n.push(range_vals.length); + if (vals.length < 2) { + out.sd.push(null); + } else { + const mean = vals.reduce((a, b) => a + b, 0) / vals.length; + out.sd.push( + Math.sqrt( + vals.reduce((a, b) => a + (b - mean) ** 2, 0) / + (vals.length - 1), + ), + ); + } + } + return out; + }; + + const check = async () => { + const sorted = [...rows].sort((a, b) => a.t - b.t); + const expected = brute(sorted); + const result = await view.to_columns(); + expect(result["s"]).toEqual(expected.s); + expect(result["mn"]).toEqual(expected.mn); + expect(result["mx"]).toEqual(expected.mx); + expect(result["n"]).toEqual(expected.n); + for (let i = 0; i < N; i++) { + if (expected.sd[i] === null) { + expect(result["sd"][i]).toBeNull(); + } else { + expect(result["sd"][i]).toBeCloseTo(expected.sd[i], 8); + } + } + }; + + await check(); + + // Mid-history edits drive the incremental sliding path over dirty + // ranges (the initial load computed via the full-rebuild path). + for (let e = 0; e < 5; e++) { + const victim = rows[Math.floor(rand() * rows.length)]; + victim.p = Math.floor(rand() * 1000); + await table.update([{ t: victim.t, p: victim.p }]); + await check(); + } + + await view.delete(); + await table.delete(); + }); + + test("window over an expression source", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "cumsum2"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + expressions: { double_price: `"price" * 2` }, + windows: { + cumsum2: { + column: "double_price", + aggregate: "sum", + order_by: ["t", "asc"], + partition_by: ["sym"], + cumulative: true, + }, + }, + }); + + const result = await view.to_columns(); + expect(result["cumsum2"]).toEqual([20, 60, 120]); + await view.delete(); + await table.delete(); + }); + + test("omitted order_by uses natural (insertion) order on an unindexed table", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "cumsum"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + partition_by: ["sym"], + cumulative: true, + }, + }, + }); + + // An unindexed table's natural (pkey) order is INSERTION order: + // partition "a" arrived as (t=3, 30), (t=1, 10), (t=2, 20), so the + // running sums are 30, 40, 60 - read back in ascending t order: + // t=1 -> 40, t=2 -> 60, t=3 -> 30. + const result = await view.to_columns(); + expect(result["cumsum"]).toEqual([40, 60, 30]); + await view.delete(); + await table.delete(); + }); + + test("omitted order_by uses natural (index) order on an indexed table", async function () { + const table = await perspective.table( + { t: "integer", sym: "string", price: "float" }, + { index: "t" }, + ); + + // Inserted out of t order - an indexed table's natural (pkey) + // order is the INDEX column's order, not arrival order. + await table.update([ + { t: 3, sym: "a", price: 3 }, + { t: 1, sym: "a", price: 1 }, + { t: 2, sym: "a", price: 2 }, + ]); + const view = await table.view({ + columns: ["t", "cumsum"], + sort: [["t", "asc"]], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + cumulative: true, + }, + }, + }); + + expect((await view.to_columns())["cumsum"]).toEqual([1, 3, 6]); + + // Incremental appends maintain natural order. + await table.update([{ t: 4, sym: "a", price: 4 }]); + expect((await view.to_columns())["cumsum"]).toEqual([1, 3, 6, 10]); + + await view.delete(); + await table.delete(); + }); + + test("descending cumulative sum accumulates in reverse order", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "cumsum_desc"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + windows: { + cumsum_desc: { + column: "price", + aggregate: "sum", + order_by: ["t", "desc"], + partition_by: ["sym"], + cumulative: true, + }, + }, + }); + + // Partition "a" in desc t order is t=3,2,1 - the running sum read + // back in asc t order is the asc column reversed over the same + // rows: [60, 50, 30]. + const result = await view.to_columns(); + expect(result["cumsum_desc"]).toEqual([60, 50, 30]); + await view.delete(); + await table.delete(); + }); + + test("descending lag reads the next-larger order key's value", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "prev"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + windows: { + prev: { + column: "price", + aggregate: "lag", + order_by: ["t", "desc"], + partition_by: ["sym"], + }, + }, + }); + + // Partition "a" in desc order is t=3,2,1; lag(1) reads the previous + // row IN THAT ORDER, i.e. the next-larger t: t=3 has none, t=2 sees + // 30, t=1 sees 20. (The partition matters: windows compute over + // TABLE rows, so without it the view's `sym` filter would not scope + // the lag and sym "b" rows would interleave.) + const result = await view.to_columns(); + expect(result["prev"]).toEqual([20, 30, null]); + await view.delete(); + await table.delete(); + }); + + test("descending range frame spans the preceding interval in sort order", async function () { + const table = await perspective.table({ + t: "integer", + sym: "string", + price: "float", + }); + await table.update(data); + const view = await table.view({ + columns: ["t", "rsum"], + sort: [["t", "asc"]], + filter: [["sym", "==", "a"]], + windows: { + rsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "desc"], + partition_by: ["sym"], + range: 1, + }, + }, + }); + + // Desc order t=3,2,1 with range 1: each frame is keys in + // [t, t + 1] - t=3 -> {3}, t=2 -> {3,2}, t=1 -> {2,1}. + const result = await view.to_columns(); + expect(result["rsum"]).toEqual([30, 50, 30]); + await view.delete(); + await table.delete(); + }); + + test("streaming append maintains a descending cumulative sum", async function () { + const table = await perspective.table( + { t: "integer", sym: "string", price: "float" }, + { index: "t" }, + ); + await table.update([ + { t: 1, sym: "a", price: 1 }, + { t: 2, sym: "a", price: 2 }, + ]); + const view = await table.view({ + columns: ["t", "cumsum"], + sort: [["t", "asc"]], + windows: { + cumsum: { + column: "price", + aggregate: "sum", + order_by: ["t", "desc"], + cumulative: true, + }, + }, + }); + + // Desc accumulation read back in asc order: t=1 sums {2,1}, t=2 + // sums {2}. + expect((await view.to_columns())["cumsum"]).toEqual([3, 2]); + + // Incremental insert exercises sorted-index insertion under the + // DESC comparator - a mismatch with the sort comparator corrupts + // positions silently. + await table.update([{ t: 3, sym: "a", price: 3 }]); + expect((await view.to_columns())["cumsum"]).toEqual([6, 5, 3]); + + // Mid-history edit re-bases the desc-suffix (asc-prefix) rows. + await table.update([{ t: 3, sym: "a", price: 30 }]); + expect((await view.to_columns())["cumsum"]).toEqual([33, 32, 30]); + + await view.delete(); + await table.delete(); + }); +}); diff --git a/rust/perspective-python/perspective/virtual_servers/clickhouse.py b/rust/perspective-python/perspective/virtual_servers/clickhouse.py index 67d00a1897..0265abc2a3 100644 --- a/rust/perspective-python/perspective/virtual_servers/clickhouse.py +++ b/rust/perspective-python/perspective/virtual_servers/clickhouse.py @@ -55,6 +55,30 @@ "string_agg", ] +# The window aggregates the SQL translation supports, per source column +# type (`ema` is recursive - no SQL window equivalent). +WINDOW_AGGREGATES = [ + "sum", + "avg", + "count", + "min", + "max", + "stddev", + "var", + "lag", + "lead", + "diff", + "rate", +] + +WINDOW_AGGREGATES_ANY = [ + "count", + "min", + "max", + "lag", + "lead", +] + FILTER_OPS = [ "==", "!=", @@ -119,6 +143,17 @@ def get_features(self): "date": STRING_AGGS, "datetime": STRING_AGGS, }, + # ClickHouse has no stable `rowid`, so natural-order windows are + # unsupported. + "unordered": True, + "window_aggregates": { + "integer": WINDOW_AGGREGATES, + "float": WINDOW_AGGREGATES, + "string": WINDOW_AGGREGATES_ANY, + "boolean": WINDOW_AGGREGATES_ANY, + "date": WINDOW_AGGREGATES_ANY, + "datetime": WINDOW_AGGREGATES_ANY, + }, } def get_hosted_tables(self): diff --git a/rust/perspective-python/perspective/virtual_servers/duckdb.py b/rust/perspective-python/perspective/virtual_servers/duckdb.py index a2e4f792ba..83a1469ee4 100644 --- a/rust/perspective-python/perspective/virtual_servers/duckdb.py +++ b/rust/perspective-python/perspective/virtual_servers/duckdb.py @@ -68,6 +68,30 @@ "string_agg", ] +# The window aggregates the SQL translation supports, per source column +# type (`ema` is recursive - no SQL window equivalent). +WINDOW_AGGREGATES = [ + "sum", + "avg", + "count", + "min", + "max", + "stddev", + "var", + "lag", + "lead", + "diff", + "rate", +] + +WINDOW_AGGREGATES_ANY = [ + "count", + "min", + "max", + "lag", + "lead", +] + FILTER_OPS = [ "==", "!=", @@ -130,6 +154,14 @@ def get_features(self): "date": STRING_AGGS, "datetime": STRING_AGGS, }, + "window_aggregates": { + "integer": WINDOW_AGGREGATES, + "float": WINDOW_AGGREGATES, + "string": WINDOW_AGGREGATES_ANY, + "boolean": WINDOW_AGGREGATES_ANY, + "date": WINDOW_AGGREGATES_ANY, + "datetime": WINDOW_AGGREGATES_ANY, + }, } def get_hosted_tables(self): diff --git a/rust/perspective-python/src/server/virtual_server_sync.rs b/rust/perspective-python/src/server/virtual_server_sync.rs index 710f4dc20f..abb9b8e170 100644 --- a/rust/perspective-python/src/server/virtual_server_sync.rs +++ b/rust/perspective-python/src/server/virtual_server_sync.rs @@ -416,7 +416,7 @@ impl PyVirtualDataSlice { self.0 .lock() .unwrap() - .render_to_columns_json(RowPathStyle::Sidecar) + .render_to_columns_json(RowPathStyle::Sidecar, false) .map_err(|e| PyValueError::new_err(e.to_string())) } diff --git a/rust/perspective-server/cpp/perspective/CMakeLists.txt b/rust/perspective-server/cpp/perspective/CMakeLists.txt index 53f25aaad7..67a35a1000 100644 --- a/rust/perspective-server/cpp/perspective/CMakeLists.txt +++ b/rust/perspective-server/cpp/perspective/CMakeLists.txt @@ -461,6 +461,7 @@ set(SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/compat_impl_wasm.cpp ${PSP_CPP_SRC}/src/cpp/compat_impl_win.cpp ${PSP_CPP_SRC}/src/cpp/computed_expression.cpp + ${PSP_CPP_SRC}/src/cpp/window.cpp ${PSP_CPP_SRC}/src/cpp/computed_function.cpp ${PSP_CPP_SRC}/src/cpp/config.cpp ${PSP_CPP_SRC}/src/cpp/context_base.cpp diff --git a/rust/perspective-server/cpp/perspective/src/cpp/config.cpp b/rust/perspective-server/cpp/perspective/src/cpp/config.cpp index 53c5636e07..b8292a5dd5 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/config.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/config.cpp @@ -24,18 +24,20 @@ t_config::t_config( const std::vector& detail_columns, const std::vector& fterms, t_filter_op combiner, - const std::vector>& expressions + const std::vector>& expressions, + const std::vector& windows ) : m_detail_columns(detail_columns), m_fterms(fterms), m_expressions(expressions), + m_windows(windows), m_combiner(combiner), m_fmode(FMODE_SIMPLE_CLAUSES) { setup(m_detail_columns); m_is_trivial_config = m_row_pivots.empty() && m_col_pivots.empty() && m_sortby.empty() && m_sortspecs.empty() && m_col_sortspecs.empty() && m_detail_columns.empty() && m_fterms.empty() - && m_expressions.empty(); + && m_expressions.empty() && m_windows.empty(); } // t_ctx1 @@ -44,11 +46,13 @@ t_config::t_config( const std::vector& aggregates, const std::vector& fterms, t_filter_op combiner, - const std::vector>& expressions + const std::vector>& expressions, + const std::vector& windows ) : m_aggregates(aggregates), m_fterms(fterms), m_expressions(expressions), + m_windows(windows), m_combiner(combiner), m_is_trivial_config(false), m_totals(TOTALS_BEFORE), @@ -70,11 +74,13 @@ t_config::t_config( const std::vector& fterms, t_filter_op combiner, const std::vector>& expressions, - bool column_only + bool column_only, + const std::vector& windows ) : m_aggregates(aggregates), m_fterms(fterms), m_expressions(expressions), + m_windows(windows), m_combiner(combiner), m_column_only(column_only), m_is_trivial_config(false), @@ -425,6 +431,11 @@ t_config::get_expressions() const { return m_expressions; } +const std::vector& +t_config::get_windows() const { + return m_windows; +} + t_filter_op t_config::get_combiner() const { return m_combiner; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp index 737df3ef64..78e6d4a802 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp @@ -55,8 +55,10 @@ t_ctx_grouped_pkey::init() { // and do not affect other contexts when they are calculated. const auto& expressions = m_config.get_expressions(); m_expression_tables = std::make_shared( - expressions, m_config.get_backing_store() + expressions, m_config.get_backing_store(), m_config.get_windows() ); + m_window_engine = + std::make_shared(m_config.get_windows()); m_init = true; } @@ -748,6 +750,10 @@ t_ctx_grouped_pkey::compute_expressions( regex_mapping ); } + + // Windows read expression-alias sources from the master expression + // table, so they must compute after the expression loop. + m_window_engine->compute_master(master, pkey_map, master_expression_table); } void @@ -825,10 +831,35 @@ t_ctx_grouped_pkey::compute_expressions( ); } + // Windows must compute after the expression loop (expression-alias + // sources) and before `calculate_transitions` (which diffs the window + // columns of `m_prev`/`m_current` like any other column). + m_window_engine->compute_update( + master, + pkey_map, + m_expression_tables->m_master, + m_expression_tables->m_flattened, + m_expression_tables->m_prev, + m_expression_tables->m_current, + m_expression_tables->m_delta, + flattened, + existed + ); + // Calculate the transitions now that the intermediate tables are computed m_expression_tables->calculate_transitions(existed); } +std::shared_ptr +t_ctx_grouped_pkey::get_window_engine() const { + return m_window_engine; +} + +bool +t_ctx_grouped_pkey::has_derived_columns() const { + return m_expression_tables->m_master->get_schema().size() > 0; +} + t_uindex t_ctx_grouped_pkey::num_expressions() const { const auto& expressions = m_config.get_expressions(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp index 909f503045..55b61b1cc8 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp @@ -49,8 +49,10 @@ t_ctx1::init() { // and do not affect other contexts when they are calculated. const auto& expressions = m_config.get_expressions(); m_expression_tables = std::make_shared( - expressions, m_config.get_backing_store() + expressions, m_config.get_backing_store(), m_config.get_windows() ); + m_window_engine = + std::make_shared(m_config.get_windows()); m_init = true; } @@ -712,6 +714,10 @@ t_ctx1::compute_expressions( regex_mapping ); } + + // Windows read expression-alias sources from the master expression + // table, so they must compute after the expression loop. + m_window_engine->compute_master(master, pkey_map, master_expression_table); } void @@ -789,6 +795,21 @@ t_ctx1::compute_expressions( ); } + // Windows must compute after the expression loop (expression-alias + // sources) and before `calculate_transitions` (which diffs the window + // columns of `m_prev`/`m_current` like any other column). + m_window_engine->compute_update( + master, + pkey_map, + m_expression_tables->m_master, + m_expression_tables->m_flattened, + m_expression_tables->m_prev, + m_expression_tables->m_current, + m_expression_tables->m_delta, + flattened, + existed + ); + // Calculate the transitions now that the intermediate tables are computed m_expression_tables->calculate_transitions(existed); } @@ -799,6 +820,16 @@ t_ctx1::is_expression_column(const std::string& colname) const { return schema.has_column(colname); } +std::shared_ptr +t_ctx1::get_window_engine() const { + return m_window_engine; +} + +bool +t_ctx1::has_derived_columns() const { + return m_expression_tables->m_master->get_schema().size() > 0; +} + t_uindex t_ctx1::num_expressions() const { const auto& expressions = m_config.get_expressions(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp index 15f0589c8d..bd9f0229c5 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp @@ -101,12 +101,24 @@ t_ctx2::init() { // and do not affect other contexts when they are calculated. const auto& expressions = m_config.get_expressions(); m_expression_tables = std::make_shared( - expressions, m_config.get_backing_store() + expressions, m_config.get_backing_store(), m_config.get_windows() ); + m_window_engine = + std::make_shared(m_config.get_windows()); m_init = true; } +std::shared_ptr +t_ctx2::get_window_engine() const { + return m_window_engine; +} + +bool +t_ctx2::has_derived_columns() const { + return m_expression_tables->m_master->get_schema().size() > 0; +} + t_uindex t_ctx2::num_expressions() const { const auto& expressions = m_config.get_expressions(); @@ -1317,6 +1329,10 @@ t_ctx2::compute_expressions( regex_mapping ); } + + // Windows read expression-alias sources from the master expression + // table, so they must compute after the expression loop. + m_window_engine->compute_master(master, pkey_map, master_expression_table); } void @@ -1394,6 +1410,21 @@ t_ctx2::compute_expressions( ); } + // Windows must compute after the expression loop (expression-alias + // sources) and before `calculate_transitions` (which diffs the window + // columns of `m_prev`/`m_current` like any other column). + m_window_engine->compute_update( + master, + pkey_map, + m_expression_tables->m_master, + m_expression_tables->m_flattened, + m_expression_tables->m_prev, + m_expression_tables->m_current, + m_expression_tables->m_delta, + flattened, + existed + ); + // Calculate the transitions now that the intermediate tables are computed m_expression_tables->calculate_transitions(existed); } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp index e7918eab5d..7bf263552a 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp @@ -77,6 +77,11 @@ t_ctxunit::notify( const t_column* pkey_col = pkey_sptr.get(); const t_column* op_col = op_sptr.get(); + // A unit context has no derived columns, so rows appended by the window + // widening pass (for OTHER contexts on this gnode) are pure no-ops here + // and must not enter the row delta. + const t_column* widened_col = existed._get_const_column("psp_widened"); + bool delete_encountered = false; for (t_uindex idx = 0; idx < nrecs; ++idx) { @@ -97,8 +102,11 @@ t_ctxunit::notify( } break; } - // add the pkey for row delta - add_delta_pkey(pkey); + // add the pkey for row delta (value read: `get_nth` never returns + // nullptr in bounds; the column is written for every row) + if (!*(widened_col->get_nth(idx))) { + add_delta_pkey(pkey); + } } m_has_delta = !m_delta_pkeys.empty() || delete_encountered; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp index 2dde9d2f09..8e06c2829c 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp @@ -41,8 +41,10 @@ t_ctx0::init() { // and do not affect other contexts when they are calculated. const auto& expressions = m_config.get_expressions(); m_expression_tables = std::make_shared( - expressions, m_config.get_backing_store() + expressions, m_config.get_backing_store(), m_config.get_windows() ); + m_window_engine = + std::make_shared(m_config.get_windows()); m_init = true; } @@ -102,6 +104,38 @@ t_ctx0::notify( const t_column* pkey_col = flattened._get_const_column("psp_pkey"); const t_column* op_col = flattened._get_const_column("psp_op"); const t_column* existed_col = existed._get_const_column("psp_existed"); + const t_column* widened_col = existed._get_const_column("psp_widened"); + + // Widened rows (appended by the gnode window pass, outside the update + // batch) enter the row delta only when one of this context's derived + // columns actually transitioned; batch rows keep their unconditional + // row-delta semantics. + const t_schema& derived_schema = + m_expression_tables->m_transitions->get_schema(); + std::vector derived_trans_cols; + derived_trans_cols.reserve(derived_schema.size()); + for (const auto& cname : derived_schema.m_columns) { + derived_trans_cols.push_back(transitions._get_const_column(cname)); + } + + auto derived_changed = [&derived_trans_cols](t_uindex idx) { + for (const t_column* tcol : derived_trans_cols) { + auto tr = static_cast( + *(tcol->get_nth(idx)) + ); + switch (tr) { + case VALUE_TRANSITION_NVEQ_FT: + case VALUE_TRANSITION_NEQ_FT: + case VALUE_TRANSITION_NEQ_TT: + case VALUE_TRANSITION_NEQ_TF: + case VALUE_TRANSITION_NEQ_TDT: + return true; + default: + break; + } + } + return false; + }; bool delete_encountered = false; bool has_filters = m_config.has_filters(); @@ -172,7 +206,13 @@ t_ctx0::notify( } break; } - add_delta_pkey(pkey); + // Value read, not pointer: `get_nth` never returns nullptr in + // bounds, and `_process_mask_existed_rows` writes this column for + // every batch row so the value is deterministic. + bool widened = *(widened_col->get_nth(idx)); + if (!widened || derived_changed(idx)) { + add_delta_pkey(pkey); + } } m_has_delta = @@ -731,6 +771,10 @@ t_ctx0::compute_expressions( regex_mapping ); } + + // Windows read expression-alias sources from the master expression + // table, so they must compute after the expression loop. + m_window_engine->compute_master(master, pkey_map, master_expression_table); } // TODO rewrite const& @@ -809,6 +853,21 @@ t_ctx0::compute_expressions( ); } + // Windows must compute after the expression loop (expression-alias + // sources) and before `calculate_transitions` (which diffs the window + // columns of `m_prev`/`m_current` like any other column). + m_window_engine->compute_update( + master, + pkey_map, + m_expression_tables->m_master, + m_expression_tables->m_flattened, + m_expression_tables->m_prev, + m_expression_tables->m_current, + m_expression_tables->m_delta, + flattened, + existed + ); + // Calculate the transitions now that the intermediate tables are computed m_expression_tables->calculate_transitions(existed); } @@ -819,6 +878,16 @@ t_ctx0::is_expression_column(const std::string& colname) const { return schema.has_column(colname); } +std::shared_ptr +t_ctx0::get_window_engine() const { + return m_window_engine; +} + +bool +t_ctx0::has_derived_columns() const { + return m_expression_tables->m_master->get_schema().size() > 0; +} + t_uindex t_ctx0::num_expressions() const { const auto& expressions = m_config.get_expressions(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp b/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp index 988337a36e..5343d4dbf9 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp @@ -19,7 +19,8 @@ namespace perspective { t_expression_tables::t_expression_tables( const std::vector>& expressions, - t_backing_store backing_store + t_backing_store backing_store, + const std::vector& windows ) { t_schema schema; t_schema transitions_schema; @@ -30,12 +31,18 @@ t_expression_tables::t_expression_tables( transitions_schema.add_column(alias, DTYPE_UINT8); } + for (const auto& window : windows) { + schema.add_column(window.m_name, window.m_dtype); + transitions_schema.add_column(window.m_name, DTYPE_UINT8); + } + // Only the persistent `m_master` table honors on-disk backing; the // transitional tables are per-update scratch (cleared every update, sized // to the update batch) so disk-backing them is pure I/O churn with no // memory-relief benefit, and they stay in memory. std::string master_dirname; - if (backing_store == BACKING_STORE_DISK && !expressions.empty()) { + if (backing_store == BACKING_STORE_DISK + && (!expressions.empty() || !windows.empty())) { master_dirname = create_backing_store_dir("perspective_expr_"); } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp b/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp index 7245ffb796..3be3eaff85 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -77,9 +78,14 @@ t_gnode::t_gnode( } t_schema trans_schema(m_output_schema.columns(), trans_types); + // `psp_widened` marks rows appended by the window widening pass + // (`_process_windows`) - synthesized "unchanged" rows outside the update + // batch. Contexts exclude them from row deltas unless one of their + // derived (expression/window) columns actually transitioned. Batch rows + // never write this column, so its slot validity IS the mark. t_schema existed_schema( - std::vector{"psp_existed"}, - std::vector{DTYPE_BOOL} + std::vector{"psp_existed", "psp_widened"}, + std::vector{DTYPE_BOOL, DTYPE_BOOL} ); m_transitional_schemas = std::vector{ @@ -240,6 +246,15 @@ t_gnode::_process_mask_existed_rows(t_process_state& process_state) { t_column* existed_column = process_state.m_existed_data_table->_get_column("psp_existed"); + // `psp_widened` must be written for EVERY batch row: transitional-table + // buffers are reused across updates and `t_lstore::clear` does not zero + // memory on WASM, so an unwritten slot could hold a stale mark from a + // prior update's widening pass. `get_nth` never returns nullptr in + // bounds - readers must test the VALUE, and the value must be + // deterministic. + t_column* widened_column = + process_state.m_existed_data_table->_get_column("psp_widened"); + for (t_uindex idx = 0; idx < flattened_num_rows; ++idx) { t_tscalar pkey = pkey_col->get_scalar(idx); std::uint8_t op_ = process_state.m_op_base[idx]; @@ -260,12 +275,14 @@ t_gnode::_process_mask_existed_rows(t_process_state& process_state) { row_pre_existed && !process_state.m_prev_pkey_eq_vec[idx]; mask.set(idx, true); existed_column->set_nth(added_count, row_pre_existed); + widened_column->set_nth(added_count, false); ++added_count; } break; case OP_DELETE: { if (row_pre_existed) { mask.set(idx, true); existed_column->set_nth(added_count, row_pre_existed); + widened_column->set_nth(added_count, false); ++added_count; } else { mask.set(idx, false); @@ -599,6 +616,24 @@ t_gnode::_process_table(t_uindex port_id) { } #endif + // Window widening (WINDOW_FUNCTIONS_PLAN §2.3): must see the batch + // before contexts compute, and must run after the master update so new + // row locations are readable. `row_lookup` is aligned with the UNMASKED + // flattened - re-align it when the mask dropped rows, since the engine + // indexes it by masked row. + if (flattened_masked.get() == _process_state.m_flattened_data_table.get()) { + _process_windows(flattened_masked, row_lookup); + } else { + std::vector masked_lookup; + masked_lookup.reserve(flattened_masked->size()); + for (t_uindex idx = 0; idx < flattened_num_rows; ++idx) { + if (existed_mask.get(idx)) { + masked_lookup.push_back(row_lookup[idx]); + } + } + _process_windows(flattened_masked, masked_lookup); + } + m_oports[PSP_PORT_FLATTENED]->set_table(flattened_masked); _compute_expressions(get_table_sptr(), flattened_masked); @@ -1245,6 +1280,181 @@ t_gnode::_register_context( } } +void +t_gnode::_process_windows( + const std::shared_ptr& flattened, + const std::vector& lookup +) { + PSP_TRACE_SENTINEL(); + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + + std::shared_ptr master = get_table_sptr(); + const t_gstate::t_mapping& pkey_map = m_gstate->get_pkey_map(); + + std::vector extra; + tsl::hopscotch_set seen; + for (auto& kv : m_contexts) { + const t_ctx_handle& ctxh = kv.second; + std::shared_ptr engine; + switch (ctxh.get_type()) { + case TWO_SIDED_CONTEXT: { + engine = ctxh.get()->get_window_engine(); + } break; + case ONE_SIDED_CONTEXT: { + engine = ctxh.get()->get_window_engine(); + } break; + case ZERO_SIDED_CONTEXT: { + engine = ctxh.get()->get_window_engine(); + } break; + case GROUPED_PKEY_CONTEXT: { + engine = ctxh.get()->get_window_engine(); + } break; + default: + break; + } + + if (!engine || !engine->enabled()) { + continue; + } + + std::vector invalidated = + engine->collect_invalidations(*flattened, lookup, master, pkey_map); + for (const t_tscalar& pkey : invalidated) { + if (seen.insert(pkey).second) { + extra.push_back(pkey); + } + } + } + + if (extra.empty()) { + return; + } + + t_uindex n_old = flattened->size(); + t_uindex n_new = n_old + extra.size(); + + std::shared_ptr delta = m_oports[PSP_PORT_DELTA]->get_table(); + std::shared_ptr prev = m_oports[PSP_PORT_PREV]->get_table(); + std::shared_ptr current = + m_oports[PSP_PORT_CURRENT]->get_table(); + std::shared_ptr transitions = + m_oports[PSP_PORT_TRANSITIONS]->get_table(); + std::shared_ptr existed = + m_oports[PSP_PORT_EXISTED]->get_table(); + + flattened->extend(n_new); + delta->extend(n_new); + prev->extend(n_new); + current->extend(n_new); + transitions->extend(n_new); + existed->extend(n_new); + + const t_schema& master_schema = master->get_schema(); + + enum class t_wcol_kind : std::uint8_t { PKEY, OP, MASTER, CLEAR }; + struct t_wcol { + t_column* m_col; + t_wcol_kind m_kind; + const t_column* m_master; + }; + + auto plan_table = [&](const std::shared_ptr& table) { + std::vector plan; + const auto& columns = table->get_schema().m_columns; + plan.reserve(columns.size()); + for (const auto& cname : columns) { + t_column* col = table->get_column(cname).get(); + if (cname == "psp_pkey") { + plan.push_back({col, t_wcol_kind::PKEY, nullptr}); + } else if (cname == "psp_op") { + plan.push_back({col, t_wcol_kind::OP, nullptr}); + } else if (master_schema.has_column(cname)) { + plan.push_back( + {col, + t_wcol_kind::MASTER, + master->get_const_column(cname).get()} + ); + } else { + plan.push_back({col, t_wcol_kind::CLEAR, nullptr}); + } + } + return plan; + }; + + // flattened/prev/current get the current master values (an unchanged + // row: prev == current for every real column, so `_process_column`-style + // diffing yields no spurious real-column deltas downstream). + std::vector> copy_plans; + copy_plans.push_back(plan_table(flattened)); + copy_plans.push_back(plan_table(prev)); + copy_plans.push_back(plan_table(current)); + std::vector delta_plan = plan_table(delta); + std::vector transitions_plan = plan_table(transitions); + t_column* existed_col = existed->get_column("psp_existed").get(); + t_column* widened_col = existed->get_column("psp_widened").get(); + + for (std::size_t i = 0; i < extra.size(); ++i) { + const t_tscalar& pkey = extra[i]; + t_uindex row = n_old + i; + auto it = pkey_map.find(pkey); + if (it == pkey_map.end()) { + continue; + } + + t_uindex mridx = it->second; + for (const auto& plan : copy_plans) { + for (const auto& wcol : plan) { + switch (wcol.m_kind) { + case t_wcol_kind::PKEY: + wcol.m_col->set_scalar(row, pkey); + break; + case t_wcol_kind::OP: + wcol.m_col->set_nth(row, OP_INSERT); + break; + case t_wcol_kind::MASTER: + if (wcol.m_master->is_valid(mridx)) { + wcol.m_col->set_scalar( + row, wcol.m_master->get_scalar(mridx) + ); + } else { + wcol.m_col->clear(row); + } + break; + case t_wcol_kind::CLEAR: + wcol.m_col->clear(row); + break; + } + } + } + + for (const auto& wcol : delta_plan) { + switch (wcol.m_kind) { + case t_wcol_kind::PKEY: + wcol.m_col->set_scalar(row, pkey); + break; + case t_wcol_kind::OP: + wcol.m_col->set_nth(row, OP_INSERT); + break; + default: + wcol.m_col->clear(row); + break; + } + } + + for (const auto& wcol : transitions_plan) { + std::uint8_t code = VALUE_TRANSITION_EQ_FF; + if (wcol.m_kind == t_wcol_kind::MASTER + && wcol.m_master->is_valid(mridx)) { + code = VALUE_TRANSITION_EQ_TT; + } + wcol.m_col->set_nth(row, code); + } + + existed_col->set_nth(row, true); + widened_col->set_nth(row, true); + } +} + void t_gnode::_unregister_context(const std::string& name) { PSP_TRACE_SENTINEL(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index 7b477f8e9e..ecec9a9705 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -107,7 +107,9 @@ make_context( auto sortspec = view_config->get_sortspec(); auto expressions = view_config->get_used_expressions(); - auto cfg = t_config(columns, fterm, filter_op, expressions); + auto cfg = t_config( + columns, fterm, filter_op, expressions, view_config->get_windows() + ); cfg.set_backing_store(table->get_backing_store()); auto ctx0 = std::make_shared(*schema, cfg); ctx0->init(); @@ -140,7 +142,14 @@ make_context( auto row_pivot_depth = view_config->get_row_pivot_depth(); auto expressions = view_config->get_used_expressions(); - auto cfg = t_config(row_pivots, aggspecs, fterm, filter_op, expressions); + auto cfg = t_config( + row_pivots, + aggspecs, + fterm, + filter_op, + expressions, + view_config->get_windows() + ); cfg.set_backing_store(table->get_backing_store()); auto ctx1 = std::make_shared(*schema, cfg); @@ -199,7 +208,8 @@ make_context( fterm, filter_op, expressions, - column_only + column_only, + view_config->get_windows() ); cfg.set_backing_store(table->get_backing_store()); auto ctx2 = std::make_shared(*schema, cfg); @@ -1410,6 +1420,36 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { features->set_sort(true); features->set_on_update(true); features->set_expressions(true); + + proto::GetFeaturesResp_WindowAggregateOptions numeric_aggs; + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_SUM); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_AVG); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_COUNT); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_MIN); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_MAX); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_STDDEV); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_VAR); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_LAG); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_LEAD); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_DIFF); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_RATE); + numeric_aggs.add_options(proto::WINDOW_AGGREGATE_EMA); + + proto::GetFeaturesResp_WindowAggregateOptions any_aggs; + any_aggs.add_options(proto::WINDOW_AGGREGATE_COUNT); + any_aggs.add_options(proto::WINDOW_AGGREGATE_MIN); + any_aggs.add_options(proto::WINDOW_AGGREGATE_MAX); + any_aggs.add_options(proto::WINDOW_AGGREGATE_LAG); + any_aggs.add_options(proto::WINDOW_AGGREGATE_LEAD); + + auto& window_aggs = *features->mutable_window_aggregates(); + window_aggs[proto::ColumnType::INTEGER] = numeric_aggs; + window_aggs[proto::ColumnType::FLOAT] = std::move(numeric_aggs); + window_aggs[proto::ColumnType::STRING] = any_aggs; + window_aggs[proto::ColumnType::DATE] = any_aggs; + window_aggs[proto::ColumnType::DATETIME] = any_aggs; + window_aggs[proto::ColumnType::BOOLEAN] = std::move(any_aggs); + features->add_group_rollup_mode(proto::GroupRollupMode::ROLLUP); features->add_group_rollup_mode(proto::GroupRollupMode::FLAT); features->add_group_rollup_mode(proto::GroupRollupMode::TOTAL); @@ -2148,6 +2188,252 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { )); } + std::vector windows; + windows.reserve(cfg.windows_size()); + const t_schema& table_schema = table->get_schema(); + + // `windows` is a proto map keyed by output alias; iterate in + // sorted-name order so output column registration (and thus + // any error precedence) is deterministic. + std::vector window_names; + window_names.reserve(cfg.windows_size()); + for (const auto& it : cfg.windows()) { + window_names.push_back(it.first); + } + std::sort(window_names.begin(), window_names.end()); + for (const auto& name : window_names) { + const auto& w = cfg.windows().at(name); + if (name.empty()) { + PSP_COMPLAIN_AND_ABORT("Window `name` must not be empty"); + } + + if (schema->has_column(name)) { + PSP_COMPLAIN_AND_ABORT( + "Window `name` collides with an existing column: " + + name + ); + } + + if (!schema->has_column(w.source())) { + PSP_COMPLAIN_AND_ABORT( + "Window `source` column not found: " + w.source() + ); + } + + // `order_by`/`partition_by` must be real `Table` columns - + // the window engine reads them from the gnode master table, + // where expression aliases do not exist. An OMITTED + // `order_by` takes natural (primary key) order. + if (w.has_order_by() + && !table_schema.has_column(w.order_by().column())) { + PSP_COMPLAIN_AND_ABORT( + "Window `order_by` must be a `Table` column: " + + w.order_by().column() + ); + } + + for (const auto& p : w.partition_by()) { + if (!table_schema.has_column(p)) { + PSP_COMPLAIN_AND_ABORT( + "Window `partition_by` must be a `Table` column: " + + p + ); + } + } + + t_window_op op = t_window_op::WINDOW_OP_SUM; + switch (w.op()) { + case proto::WINDOW_AGGREGATE_SUM: + op = t_window_op::WINDOW_OP_SUM; + break; + case proto::WINDOW_AGGREGATE_AVG: + op = t_window_op::WINDOW_OP_AVG; + break; + case proto::WINDOW_AGGREGATE_COUNT: + op = t_window_op::WINDOW_OP_COUNT; + break; + case proto::WINDOW_AGGREGATE_MIN: + op = t_window_op::WINDOW_OP_MIN; + break; + case proto::WINDOW_AGGREGATE_MAX: + op = t_window_op::WINDOW_OP_MAX; + break; + case proto::WINDOW_AGGREGATE_STDDEV: + op = t_window_op::WINDOW_OP_STDDEV; + break; + case proto::WINDOW_AGGREGATE_VAR: + op = t_window_op::WINDOW_OP_VAR; + break; + case proto::WINDOW_AGGREGATE_LAG: + op = t_window_op::WINDOW_OP_LAG; + break; + case proto::WINDOW_AGGREGATE_LEAD: + op = t_window_op::WINDOW_OP_LEAD; + break; + case proto::WINDOW_AGGREGATE_DIFF: + op = t_window_op::WINDOW_OP_DIFF; + break; + case proto::WINDOW_AGGREGATE_RATE: + op = t_window_op::WINDOW_OP_RATE; + break; + case proto::WINDOW_AGGREGATE_EMA: + op = t_window_op::WINDOW_OP_EMA; + break; + default: + PSP_COMPLAIN_AND_ABORT( + "Window `op` not implemented in this build" + ); + } + + // An OMITTED frame means cumulative for aggregating + // ops - the initializer below IS that default. + t_window_frame_type frame_type = + t_window_frame_type::WINDOW_FRAME_CUMULATIVE; + t_uindex frame_rows = 0; + double frame_range = 0; + bool has_frame = true; + switch (w.frame_case()) { + case proto::WindowSpec::kRows: + frame_type = t_window_frame_type::WINDOW_FRAME_ROWS; + frame_rows = w.rows(); + break; + case proto::WindowSpec::kCumulative: + frame_type = + t_window_frame_type::WINDOW_FRAME_CUMULATIVE; + break; + case proto::WindowSpec::kRange: { + frame_type = t_window_frame_type::WINDOW_FRAME_RANGE; + frame_range = w.range(); + if (!(frame_range > 0)) { + PSP_COMPLAIN_AND_ABORT( + "Window `range` must be a positive interval" + ); + } + + // Interval arithmetic is defined on the order + // column's raw units (ms for datetime, days for + // date) - the natural-order fallback has no units, + // so `range` requires an explicit `order_by`. + if (!w.has_order_by()) { + PSP_COMPLAIN_AND_ABORT( + "Window `range` frames require an explicit " + "`order_by`" + ); + } + + t_dtype order_dtype = + table_schema.get_dtype(w.order_by().column()); + switch (order_dtype) { + case DTYPE_INT8: + case DTYPE_INT16: + case DTYPE_INT32: + case DTYPE_INT64: + case DTYPE_UINT8: + case DTYPE_UINT16: + case DTYPE_UINT32: + case DTYPE_UINT64: + case DTYPE_FLOAT32: + case DTYPE_FLOAT64: + case DTYPE_TIME: + case DTYPE_DATE: + break; + default: + PSP_COMPLAIN_AND_ABORT( + "Window `range` frames require a numeric " + "or temporal `order_by` column: " + + w.order_by().column() + ); + } + } break; + default: + has_frame = false; + break; + } + + switch (op) { + case t_window_op::WINDOW_OP_LAG: + case t_window_op::WINDOW_OP_LEAD: + case t_window_op::WINDOW_OP_DIFF: + if (has_frame) { + PSP_COMPLAIN_AND_ABORT( + "Window `frame` is not applicable to " + "`lag`/`lead`/`diff` (use `offset`)" + ); + } + break; + case t_window_op::WINDOW_OP_RATE: + if (frame_type + != t_window_frame_type::WINDOW_FRAME_RANGE + || !has_frame) { + PSP_COMPLAIN_AND_ABORT( + "Window `rate` requires a `range` frame" + ); + } + break; + case t_window_op::WINDOW_OP_EMA: + if (has_frame + && frame_type + != t_window_frame_type:: + WINDOW_FRAME_CUMULATIVE) { + PSP_COMPLAIN_AND_ABORT( + "Window `ema` is cumulative; it does not " + "accept a `rows` or `range` frame" + ); + } + + if (!w.has_alpha() || !(w.alpha() > 0) + || w.alpha() > 1) { + PSP_COMPLAIN_AND_ABORT( + "Window `ema` requires `alpha` in (0, 1]" + ); + } + break; + default: + break; + } + + if (!t_window_engine::is_implemented(op, frame_type)) { + PSP_COMPLAIN_AND_ABORT( + "Window op/frame combination not implemented" + ); + } + + t_dtype source_dtype = schema->get_dtype(w.source()); + t_dtype dtype = + t_window_engine::resolve_dtype(op, source_dtype); + if (dtype == DTYPE_NONE) { + PSP_COMPLAIN_AND_ABORT( + "Window op requires a numeric `source` column: " + + w.source() + ); + } + + t_window_spec spec; + spec.m_name = name; + spec.m_source = w.source(); + + // Empty `m_order_by` = natural (primary key) order; the + // engine's comparator degenerates to its pkey tiebreak when + // every order key is absent. + spec.m_order_by = + w.has_order_by() ? w.order_by().column() : ""; + spec.m_order_desc = + w.has_order_by() && w.order_by().desc(); + spec.m_partition_by = { + w.partition_by().begin(), w.partition_by().end() + }; + spec.m_op = op; + spec.m_frame_type = frame_type; + spec.m_frame_rows = frame_rows; + spec.m_frame_range = frame_range; + spec.m_offset = w.has_offset() ? w.offset() : 1; + spec.m_alpha = w.has_alpha() ? w.alpha() : 0; + spec.m_dtype = dtype; + + schema->add_column(spec.m_name, dtype); + windows.push_back(std::move(spec)); + } + t_vocab vocab; vocab.init(false); std::vector< @@ -2253,6 +2539,9 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { for (const auto& f : expressions) { columns.push_back(f->get_expression_alias()); } + for (const auto& w : windows) { + columns.push_back(w.m_name); + } } LOG_DEBUG( @@ -2301,7 +2590,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { filter_op, column_only, leaves_only, - total_only + total_only, + windows ); config->init(schema); @@ -2326,7 +2616,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { bool is_unit_context = table->get_index().empty() && sides == 0 && row_pivots.empty() && column_pivots.empty() && aggregates.empty() && columns.empty() && sort_str.empty() - && cfg.expressions().empty(); + && cfg.expressions().empty() && cfg.windows().empty(); std::shared_ptr erased_view; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp b/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp index f1b2bd5711..69297bece3 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp @@ -30,7 +30,8 @@ t_view_config::t_view_config( std::string filter_op, bool column_only, bool leaves_only, - bool total_only + bool total_only, + const std::vector& windows ) : m_init(false), m_vocab(std::move(vocab)), @@ -41,6 +42,7 @@ t_view_config::t_view_config( m_filter(filter), m_sort(sort), m_expressions(expressions), + m_windows(windows), m_row_pivot_depth(-1), m_column_pivot_depth(-1), m_filter_op(std::move(filter_op)), @@ -145,6 +147,13 @@ t_view_config::get_used_expressions() { used_cols.insert(i[0]); } + // A window's source expression must survive pruning even when the + // expression itself is not selected - the window reads it from the + // expression master table. + for (const auto& window : m_windows) { + used_cols.insert(window.m_source); + } + std::copy( m_column_pivots.begin(), m_column_pivots.end(), @@ -256,6 +265,12 @@ t_view_config::get_expressions() const { return m_expressions; } +const std::vector& +t_view_config::get_windows() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_windows; +} + t_filter_op t_view_config::get_filter_op() const { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/window.cpp b/rust/perspective-server/cpp/perspective/src/cpp/window.cpp new file mode 100644 index 0000000000..e3775dd514 --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/cpp/window.cpp @@ -0,0 +1,1603 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#include +#include +#include +#include +#include +#include +#include + +namespace perspective { + +namespace { + +// Strict-weak scalar order: invalid (null) sorts first, then by value. +bool +scalar_less(const t_tscalar& a, const t_tscalar& b) { + bool av = a.is_valid(); + bool bv = b.is_valid(); + if (av != bv) { + return !av; + } + + return av && a < b; +} + +// (order, pkey) pair order shared by sort, insert and search - the ONE +// comparator that defines a spec's sorted direction (a second definition +// anywhere would let insert and sort disagree and silently corrupt +// positions). Rows with invalid order keys sort first in EITHER direction; +// valid keys compare ascending or descending per `desc`; ties broken by +// pkey (always ascending) so replays are deterministic. +bool +key_less( + const t_tscalar& a_order, + const t_tscalar& a_pkey, + const t_tscalar& b_order, + const t_tscalar& b_pkey, + bool desc +) { + const t_tscalar& first = desc ? b_order : a_order; + const t_tscalar& second = desc ? a_order : b_order; + bool av = a_order.is_valid(); + bool bv = b_order.is_valid(); + if (av != bv) { + return !av; + } + + if (av) { + if (first < second) { + return true; + } + + if (second < first) { + return false; + } + } + + return a_pkey < b_pkey; +} + +template +std::size_t +lower_bound_row( + const ROWS_T& rows, + const t_tscalar& order, + const t_tscalar& pkey, + bool desc +) { + std::size_t lo = 0; + std::size_t hi = rows.size(); + while (lo < hi) { + std::size_t mid = lo + (hi - lo) / 2; + if (key_less(rows[mid].m_order, rows[mid].m_pkey, order, pkey, desc)) { + lo = mid + 1; + } else { + hi = mid; + } + } + + return lo; +} + +// First position whose order key is valid. Invalid keys sort first, so +// validity is monotone over a sorted partition. +template +std::size_t +partition_first_valid(const ROWS_T& rows) { + std::size_t lo = 0; + std::size_t hi = rows.size(); + while (lo < hi) { + std::size_t mid = lo + (hi - lo) / 2; + if (!rows[mid].m_order.is_valid()) { + lo = mid + 1; + } else { + hi = mid; + } + } + + return lo; +} + +// Range-frame interval arithmetic. Keys compared as doubles (`Range` +// frames require a numeric or temporal order key, so intervals are defined +// on the column's raw units); keys are sorted in the spec's direction, so +// "before" any boundary is a positional prefix in either direction. + +// First position in [first_valid, size) at or inside a boundary key: +// ascending the first key >= `boundary`, descending the first key <= +// `boundary`. A row's Range frame starts here with `boundary = key -+ +// range` (its preceding interval in sort order); the invalidation reach of +// a touched key starts here with `boundary = key`. +template +std::size_t +frame_lower_bound( + const ROWS_T& rows, double boundary, std::size_t first_valid, bool desc +) { + std::size_t lo = first_valid; + std::size_t hi = rows.size(); + while (lo < hi) { + std::size_t mid = lo + (hi - lo) / 2; + double v = rows[mid].m_order.to_double(); + if (desc ? v > boundary : v < boundary) { + lo = mid + 1; + } else { + hi = mid; + } + } + + return lo; +} + +// First position in [first_valid, size) PAST the invalidation reach of a +// touched key `k` - the rows whose Range frames can contain `k`: keys in +// `[k, k + range]` ascending, `[k - range, k]` descending. +template +std::size_t +reach_upper_bound( + const ROWS_T& rows, + double k, + double range, + std::size_t first_valid, + bool desc +) { + double limit = desc ? k - range : k + range; + std::size_t lo = first_valid; + std::size_t hi = rows.size(); + while (lo < hi) { + std::size_t mid = lo + (hi - lo) / 2; + double v = rows[mid].m_order.to_double(); + if (desc ? v >= limit : v <= limit) { + lo = mid + 1; + } else { + hi = mid; + } + } + + return lo; +} + +void +write_float(t_column& out, t_uindex ridx, double value) { + t_tscalar s; + s.clear(); + s.set(value); + out.set_scalar(ridx, s); +} + +void +write_int(t_column& out, t_uindex ridx, std::int64_t value) { + t_tscalar s; + s.clear(); + s.set(value); + out.set_scalar(ridx, s); +} + +// Aggregate the frame positions [lo, hi] of one ordered partition and write +// the result for the row at position `hi`. Direct recompute per output row - +// no accumulator state, no float drift (WINDOW_FUNCTIONS_PLAN §2.4). +// Aggregating ops only; positional/recursive ops dispatch in +// `recompute_range`. +template +void +apply_frame( + t_window_op op, + const ROWS_T& rows, + std::size_t lo, + std::size_t hi, + const t_column& src, + t_column& out +) { + t_uindex out_ridx = rows[hi].m_ridx; + switch (op) { + case t_window_op::WINDOW_OP_SUM: + case t_window_op::WINDOW_OP_AVG: { + double sum = 0; + double c = 0; + std::int64_t n = 0; + for (std::size_t pos = lo; pos <= hi; ++pos) { + t_uindex ridx = rows[pos].m_ridx; + if (!src.is_valid(ridx)) { + continue; + } + + double x = src.get_scalar(ridx).to_double(); + double y = x - c; + double t = sum + y; + c = (t - sum) - y; + sum = t; + ++n; + } + + if (n == 0) { + out.clear(out_ridx); + } else if (op == t_window_op::WINDOW_OP_SUM) { + write_float(out, out_ridx, sum); + } else { + write_float(out, out_ridx, sum / static_cast(n)); + } + } break; + case t_window_op::WINDOW_OP_COUNT: { + std::int64_t n = 0; + for (std::size_t pos = lo; pos <= hi; ++pos) { + if (src.is_valid(rows[pos].m_ridx)) { + ++n; + } + } + + write_int(out, out_ridx, n); + } break; + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: { + bool has = false; + t_tscalar best; + best.clear(); + for (std::size_t pos = lo; pos <= hi; ++pos) { + t_uindex ridx = rows[pos].m_ridx; + if (!src.is_valid(ridx)) { + continue; + } + + t_tscalar v = src.get_scalar(ridx); + if (!has) { + best = v; + has = true; + } else if (op == t_window_op::WINDOW_OP_MIN ? v < best : best < v) { + best = v; + } + } + + if (has) { + out.set_scalar(out_ridx, best); + } else { + out.clear(out_ridx); + } + } break; + case t_window_op::WINDOW_OP_STDDEV: + case t_window_op::WINDOW_OP_VAR: { + // Welford: numerically stable over small frames. + double mean = 0; + double m2 = 0; + std::int64_t n = 0; + for (std::size_t pos = lo; pos <= hi; ++pos) { + t_uindex ridx = rows[pos].m_ridx; + if (!src.is_valid(ridx)) { + continue; + } + + double x = src.get_scalar(ridx).to_double(); + ++n; + double d = x - mean; + mean += d / static_cast(n); + m2 += d * (x - mean); + } + + if (n < 2) { + out.clear(out_ridx); + } else { + double var = m2 / static_cast(n - 1); + write_float( + out, + out_ridx, + op == t_window_op::WINDOW_OP_VAR ? var : std::sqrt(var) + ); + } + } break; + default: + PSP_COMPLAIN_AND_ABORT( + "[t_window_engine] window op not implemented" + ); + } +} + +double +sample_var(double sum, double sumsq, std::int64_t n) { + double nn = static_cast(n); + double var = (sumsq - (sum * sum) / nn) / (nn - 1); + return var < 0 ? 0 : var; +} + +// Sliding float accumulators re-prime from a direct frame scan this often, +// bounding subtract-induced drift; the min/max deque is exact and never +// refreshes. Integer sources never drift (exact adds/subtracts), which is +// what keeps the JS property test's incremental-vs-oracle comparison +// bit-identical. +constexpr std::size_t WINDOW_REFRESH_INTERVAL = 4096; + +// One in-pass sliding aggregate over an ordered partition walk. State lives +// only for the duration of a single `recompute_range` call - nothing slides +// across updates, so a bug here cannot corrupt persistent index state. +template +struct t_sliding { + t_window_op m_op; + const ROWS_T& m_rows; + const t_column& m_src; + double m_sum = 0; + double m_c = 0; + double m_sumsq = 0; + double m_csq = 0; + std::int64_t m_n = 0; + std::deque m_minmax; + std::size_t m_since_refresh = 0; + + t_sliding(t_window_op op, const ROWS_T& rows, const t_column& src) : + m_op(op), + m_rows(rows), + m_src(src) {} + + bool + is_float_acc() const { + switch (m_op) { + case t_window_op::WINDOW_OP_SUM: + case t_window_op::WINDOW_OP_AVG: + case t_window_op::WINDOW_OP_STDDEV: + case t_window_op::WINDOW_OP_VAR: + return true; + default: + return false; + } + } + + static void + kadd(double& sum, double& c, double x) { + double y = x - c; + double t = sum + y; + c = (t - sum) - y; + sum = t; + } + + void + enter(std::size_t pos) { + t_uindex ridx = m_rows[pos].m_ridx; + if (!m_src.is_valid(ridx)) { + return; + } + + switch (m_op) { + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: { + t_tscalar x = m_src.get_scalar(ridx); + while (!m_minmax.empty()) { + t_tscalar back = + m_src.get_scalar(m_rows[m_minmax.back()].m_ridx); + bool dominated = m_op == t_window_op::WINDOW_OP_MIN + ? !(back < x) + : !(x < back); + if (!dominated) { + break; + } + m_minmax.pop_back(); + } + m_minmax.push_back(pos); + } break; + case t_window_op::WINDOW_OP_COUNT: + ++m_n; + break; + default: { + double x = m_src.get_scalar(ridx).to_double(); + kadd(m_sum, m_c, x); + kadd(m_sumsq, m_csq, x * x); + ++m_n; + } break; + } + } + + // Remove the row at `pos` (a position strictly below the new frame + // start). The min/max deque instead drops stale positions lazily in + // `settle`. + void + evict(std::size_t pos) { + t_uindex ridx = m_rows[pos].m_ridx; + if (!m_src.is_valid(ridx)) { + return; + } + + switch (m_op) { + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: + break; + case t_window_op::WINDOW_OP_COUNT: + --m_n; + break; + default: { + double x = m_src.get_scalar(ridx).to_double(); + kadd(m_sum, m_c, -x); + kadd(m_sumsq, m_csq, -(x * x)); + --m_n; + } break; + } + } + + void + settle(std::size_t frame_start) { + while (!m_minmax.empty() && m_minmax.front() < frame_start) { + m_minmax.pop_front(); + } + } + + // Direct rescan of the current frame - drift refresh for the float + // accumulators. + void + reprime(std::size_t frame_start, std::size_t pos) { + m_sum = 0; + m_c = 0; + m_sumsq = 0; + m_csq = 0; + m_n = 0; + for (std::size_t p = frame_start; p <= pos; ++p) { + t_uindex ridx = m_rows[p].m_ridx; + if (!m_src.is_valid(ridx)) { + continue; + } + + double x = m_src.get_scalar(ridx).to_double(); + kadd(m_sum, m_c, x); + kadd(m_sumsq, m_csq, x * x); + ++m_n; + } + m_since_refresh = 0; + } + + void + write(std::size_t pos, t_column& out) { + t_uindex out_ridx = m_rows[pos].m_ridx; + switch (m_op) { + case t_window_op::WINDOW_OP_SUM: + if (m_n == 0) { + out.clear(out_ridx); + } else { + write_float(out, out_ridx, m_sum); + } + break; + case t_window_op::WINDOW_OP_AVG: + if (m_n == 0) { + out.clear(out_ridx); + } else { + write_float( + out, out_ridx, m_sum / static_cast(m_n) + ); + } + break; + case t_window_op::WINDOW_OP_COUNT: + write_int(out, out_ridx, m_n); + break; + case t_window_op::WINDOW_OP_STDDEV: + case t_window_op::WINDOW_OP_VAR: + if (m_n < 2) { + out.clear(out_ridx); + } else { + double var = sample_var(m_sum, m_sumsq, m_n); + write_float( + out, + out_ridx, + m_op == t_window_op::WINDOW_OP_VAR ? var + : std::sqrt(var) + ); + } + break; + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: + if (m_minmax.empty()) { + out.clear(out_ridx); + } else { + out.set_scalar( + out_ridx, + m_src.get_scalar(m_rows[m_minmax.front()].m_ridx) + ); + } + break; + default: + PSP_COMPLAIN_AND_ABORT( + "[t_window_engine] window op not implemented" + ); + } + } +}; + +} // anonymous namespace + +bool +t_window_engine::t_scalar_vec_cmp::operator()( + const std::vector& a, const std::vector& b +) const { + return std::lexicographical_compare( + a.begin(), a.end(), b.begin(), b.end(), scalar_less + ); +} + +t_window_engine::t_window_engine(std::vector specs) : + m_collected(false) { + m_states.reserve(specs.size()); + for (auto& spec : specs) { + t_spec_state state; + state.m_needs_prefix = + class_of(spec) == t_window_class::WINDOW_CLASS_CUMULATIVE + && (spec.m_op == t_window_op::WINDOW_OP_SUM + || spec.m_op == t_window_op::WINDOW_OP_AVG + || spec.m_op == t_window_op::WINDOW_OP_COUNT + || spec.m_op == t_window_op::WINDOW_OP_STDDEV + || spec.m_op == t_window_op::WINDOW_OP_VAR); + state.m_spec = std::move(spec); + m_states.push_back(std::move(state)); + } +} + +bool +t_window_engine::enabled() const { + return !m_states.empty(); +} + +t_dtype +t_window_engine::resolve_dtype(t_window_op op, t_dtype source_dtype) { + bool numeric = false; + switch (source_dtype) { + case DTYPE_INT8: + case DTYPE_INT16: + case DTYPE_INT32: + case DTYPE_INT64: + case DTYPE_UINT8: + case DTYPE_UINT16: + case DTYPE_UINT32: + case DTYPE_UINT64: + case DTYPE_FLOAT32: + case DTYPE_FLOAT64: + numeric = true; + break; + default: + break; + } + + switch (op) { + case t_window_op::WINDOW_OP_SUM: + case t_window_op::WINDOW_OP_AVG: + case t_window_op::WINDOW_OP_STDDEV: + case t_window_op::WINDOW_OP_VAR: + case t_window_op::WINDOW_OP_RATE: + case t_window_op::WINDOW_OP_EMA: + case t_window_op::WINDOW_OP_DIFF: + return numeric ? DTYPE_FLOAT64 : DTYPE_NONE; + case t_window_op::WINDOW_OP_COUNT: + return DTYPE_INT64; + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: + case t_window_op::WINDOW_OP_FIRST: + case t_window_op::WINDOW_OP_LAST: + case t_window_op::WINDOW_OP_LAG: + case t_window_op::WINDOW_OP_LEAD: + return source_dtype; + default: + return DTYPE_NONE; + } +} + +bool +t_window_engine::is_implemented(t_window_op op, t_window_frame_type frame) { + switch (op) { + case t_window_op::WINDOW_OP_SUM: + case t_window_op::WINDOW_OP_AVG: + case t_window_op::WINDOW_OP_COUNT: + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: + case t_window_op::WINDOW_OP_STDDEV: + case t_window_op::WINDOW_OP_VAR: + // Aggregating ops accept every frame type. + return true; + case t_window_op::WINDOW_OP_LAG: + case t_window_op::WINDOW_OP_LEAD: + case t_window_op::WINDOW_OP_DIFF: + case t_window_op::WINDOW_OP_EMA: + // Frame-independent; frame legality is enforced at `View` + // construction. + return true; + case t_window_op::WINDOW_OP_RATE: + return frame == t_window_frame_type::WINDOW_FRAME_RANGE; + default: + // FIRST/LAST remain unimplemented. + return false; + } +} + +t_window_class +t_window_engine::class_of(const t_window_spec& spec) { + switch (spec.m_op) { + case t_window_op::WINDOW_OP_EMA: + return t_window_class::WINDOW_CLASS_CUMULATIVE; + case t_window_op::WINDOW_OP_LAG: + case t_window_op::WINDOW_OP_DIFF: + return t_window_class::WINDOW_CLASS_LAG_LIKE; + case t_window_op::WINDOW_OP_LEAD: + return t_window_class::WINDOW_CLASS_LEAD_LIKE; + case t_window_op::WINDOW_OP_RATE: + return t_window_class::WINDOW_CLASS_AGG_RANGE; + default: + break; + } + + switch (spec.m_frame_type) { + case t_window_frame_type::WINDOW_FRAME_ROWS: + return t_window_class::WINDOW_CLASS_AGG_ROWS; + case t_window_frame_type::WINDOW_FRAME_RANGE: + return t_window_class::WINDOW_CLASS_AGG_RANGE; + default: + return t_window_class::WINDOW_CLASS_CUMULATIVE; + } +} + +void +t_window_engine::reset_state() { + for (auto& state : m_states) { + state.m_partitions.clear(); + state.m_locations.clear(); + state.m_edits.clear(); + state.m_dead_ridxs.clear(); + state.m_dirty.clear(); + } + + m_collected = false; +} + +const t_column* +t_window_engine::source_column( + const t_spec_state& state, + const std::shared_ptr& master, + const std::shared_ptr& dst_master +) const { + const t_data_table* src_table = + master->get_schema().has_column(state.m_spec.m_source) + ? master.get() + : dst_master.get(); + return src_table->get_const_column(state.m_spec.m_source).get(); +} + +// Recompute one dirty range of one partition. +// +// - `AGG_ROWS`/`AGG_RANGE`: direct per-output-row frame recompute (a +// `Range` frame's start is found by key search; rows with invalid order +// keys frame together in the invalid prefix). +// - `CUMULATIVE`: one seeded pass to the partition end. sum/avg/count/ +// stddev/var seed from the prefix accumulators; min/max and ema seed from +// the last clean OUTPUT row (the output IS the running state). +// - `LAG_LIKE`/`LEAD_LIKE`: direct positional reads. +void +t_window_engine::recompute_range( + t_spec_state& state, + t_window_partition& partition, + std::size_t lo, + std::size_t hi, + const t_column& src, + t_column& out +) const { + auto& rows = partition.m_rows; + const t_window_spec& spec = state.m_spec; + switch (class_of(spec)) { + case t_window_class::WINDOW_CLASS_AGG_ROWS: { + std::size_t n = spec.m_frame_rows; + t_sliding> sliding( + spec.m_op, rows, src + ); + std::size_t frame_start = lo > n ? lo - n : 0; + for (std::size_t p = frame_start; p < lo; ++p) { + sliding.enter(p); + } + + for (std::size_t pos = lo; pos < hi; ++pos) { + std::size_t fs = pos > n ? pos - n : 0; + for (; frame_start < fs; ++frame_start) { + sliding.evict(frame_start); + } + + sliding.enter(pos); + sliding.settle(fs); + if (sliding.is_float_acc() + && ++sliding.m_since_refresh >= WINDOW_REFRESH_INTERVAL) { + sliding.reprime(fs, pos); + } + + sliding.write(pos, out); + } + } break; + case t_window_class::WINDOW_CLASS_AGG_RANGE: { + std::size_t first_valid = partition_first_valid(rows); + + // Invalid order keys frame together at partition start - a + // grow-only frame, computed directly (rare). + for (std::size_t pos = lo; pos < std::min(hi, first_valid); + ++pos) { + if (spec.m_op == t_window_op::WINDOW_OP_RATE) { + out.clear(rows[pos].m_ridx); + } else { + apply_frame(spec.m_op, rows, 0, pos, src, out); + } + } + + std::size_t start = std::max(lo, first_valid); + if (start >= hi) { + break; + } + + const bool desc = spec.m_order_desc; + if (spec.m_op == t_window_op::WINDOW_OP_RATE) { + for (std::size_t pos = start; pos < hi; ++pos) { + t_uindex out_ridx = rows[pos].m_ridx; + double key = rows[pos].m_order.to_double(); + std::size_t frame_lo = frame_lower_bound( + rows, + desc ? key + spec.m_frame_range + : key - spec.m_frame_range, + first_valid, + desc + ); + t_uindex lo_ridx = rows[frame_lo].m_ridx; + if (frame_lo >= pos || !src.is_valid(out_ridx) + || !src.is_valid(lo_ridx)) { + out.clear(out_ridx); + continue; + } + + // Δv/Δk is the same slope whichever end of the frame is + // "first" - for desc both deltas negate. + double dk = key - rows[frame_lo].m_order.to_double(); + if (dk == 0) { + out.clear(out_ridx); + continue; + } + + double dv = src.get_scalar(out_ridx).to_double() + - src.get_scalar(lo_ridx).to_double(); + write_float(out, out_ridx, dv / dk); + } + break; + } + + // Two-pointer sliding pass: keys are sorted (in the spec's + // direction), so each frame start is monotone non-decreasing + // in `pos`. The frame boundary is `key -+ range` - the + // preceding interval in sort order. + t_sliding> sliding( + spec.m_op, rows, src + ); + std::size_t frame_start = frame_lower_bound( + rows, + desc ? rows[start].m_order.to_double() + spec.m_frame_range + : rows[start].m_order.to_double() - spec.m_frame_range, + first_valid, + desc + ); + for (std::size_t p = frame_start; p < start; ++p) { + sliding.enter(p); + } + + for (std::size_t pos = start; pos < hi; ++pos) { + double key = rows[pos].m_order.to_double(); + double boundary = + desc ? key + spec.m_frame_range + : key - spec.m_frame_range; + for (; frame_start < pos + && (desc + ? rows[frame_start].m_order.to_double() + > boundary + : rows[frame_start].m_order.to_double() + < boundary); + ++frame_start) { + sliding.evict(frame_start); + } + + sliding.enter(pos); + sliding.settle(frame_start); + if (sliding.is_float_acc() + && ++sliding.m_since_refresh >= WINDOW_REFRESH_INTERVAL) { + sliding.reprime(frame_start, pos); + } + + sliding.write(pos, out); + } + } break; + case t_window_class::WINDOW_CLASS_LAG_LIKE: { + std::size_t k = spec.m_offset; + for (std::size_t pos = lo; pos < hi; ++pos) { + t_uindex out_ridx = rows[pos].m_ridx; + if (pos < k) { + out.clear(out_ridx); + continue; + } + + t_uindex src_ridx = rows[pos - k].m_ridx; + if (spec.m_op == t_window_op::WINDOW_OP_LAG) { + if (src.is_valid(src_ridx)) { + out.set_scalar(out_ridx, src.get_scalar(src_ridx)); + } else { + out.clear(out_ridx); + } + } else { + // DIFF + if (src.is_valid(src_ridx) && src.is_valid(out_ridx)) { + write_float( + out, + out_ridx, + src.get_scalar(out_ridx).to_double() + - src.get_scalar(src_ridx).to_double() + ); + } else { + out.clear(out_ridx); + } + } + } + } break; + case t_window_class::WINDOW_CLASS_LEAD_LIKE: { + std::size_t k = spec.m_offset; + for (std::size_t pos = lo; pos < hi; ++pos) { + t_uindex out_ridx = rows[pos].m_ridx; + if (pos + k >= rows.size()) { + out.clear(out_ridx); + continue; + } + + t_uindex src_ridx = rows[pos + k].m_ridx; + if (src.is_valid(src_ridx)) { + out.set_scalar(out_ridx, src.get_scalar(src_ridx)); + } else { + out.clear(out_ridx); + } + } + } break; + case t_window_class::WINDOW_CLASS_CUMULATIVE: { + if (state.m_needs_prefix) { + partition.m_prefix_sum.resize(rows.size()); + partition.m_prefix_sumsq.resize(rows.size()); + partition.m_prefix_count.resize(rows.size()); + } + + double sum = 0; + double c = 0; + double sumsq = 0; + double csq = 0; + std::int64_t n = 0; + bool has = false; + t_tscalar best; + best.clear(); + double ema = 0; + + if (lo > 0) { + if (state.m_needs_prefix) { + sum = partition.m_prefix_sum[lo - 1]; + sumsq = partition.m_prefix_sumsq[lo - 1]; + n = partition.m_prefix_count[lo - 1]; + } else { + // min/max/ema: the output of the last clean row IS the + // running state. + t_uindex prev_ridx = rows[lo - 1].m_ridx; + if (out.is_valid(prev_ridx)) { + best = out.get_scalar(prev_ridx); + ema = best.to_double(); + has = true; + } + } + } + + for (std::size_t pos = lo; pos < rows.size(); ++pos) { + t_uindex ridx = rows[pos].m_ridx; + if (src.is_valid(ridx)) { + t_tscalar v = src.get_scalar(ridx); + switch (spec.m_op) { + case t_window_op::WINDOW_OP_SUM: + case t_window_op::WINDOW_OP_AVG: + case t_window_op::WINDOW_OP_COUNT: + case t_window_op::WINDOW_OP_STDDEV: + case t_window_op::WINDOW_OP_VAR: { + double x = v.to_double(); + double y = x - c; + double t = sum + y; + c = (t - sum) - y; + sum = t; + double ysq = x * x - csq; + double tsq = sumsq + ysq; + csq = (tsq - sumsq) - ysq; + sumsq = tsq; + ++n; + } break; + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: { + if (!has) { + best = v; + has = true; + } else if ( + spec.m_op == t_window_op::WINDOW_OP_MIN + ? v < best + : best < v + ) { + best = v; + } + } break; + case t_window_op::WINDOW_OP_EMA: { + double x = v.to_double(); + ema = has + ? spec.m_alpha * x + (1 - spec.m_alpha) * ema + : x; + has = true; + } break; + default: + PSP_COMPLAIN_AND_ABORT( + "[t_window_engine] window op not implemented" + ); + } + } + + if (state.m_needs_prefix) { + partition.m_prefix_sum[pos] = sum; + partition.m_prefix_sumsq[pos] = sumsq; + partition.m_prefix_count[pos] = n; + } + + switch (spec.m_op) { + case t_window_op::WINDOW_OP_SUM: + if (n == 0) { + out.clear(ridx); + } else { + write_float(out, ridx, sum); + } + break; + case t_window_op::WINDOW_OP_AVG: + if (n == 0) { + out.clear(ridx); + } else { + write_float( + out, ridx, sum / static_cast(n) + ); + } + break; + case t_window_op::WINDOW_OP_COUNT: + write_int(out, ridx, n); + break; + case t_window_op::WINDOW_OP_STDDEV: + case t_window_op::WINDOW_OP_VAR: + if (n < 2) { + out.clear(ridx); + } else { + double var = sample_var(sum, sumsq, n); + write_float( + out, + ridx, + spec.m_op == t_window_op::WINDOW_OP_VAR + ? var + : std::sqrt(var) + ); + } + break; + case t_window_op::WINDOW_OP_MIN: + case t_window_op::WINDOW_OP_MAX: + if (has) { + out.set_scalar(ridx, best); + } else { + out.clear(ridx); + } + break; + case t_window_op::WINDOW_OP_EMA: + if (has) { + write_float(out, ridx, ema); + } else { + out.clear(ridx); + } + break; + default: + break; + } + } + } break; + } +} + +void +t_window_engine::compute_master( + const std::shared_ptr& master, + const t_gstate::t_mapping& pkey_map, + const std::shared_ptr& dst_master +) { + if (m_states.empty()) { + return; + } + + reset_state(); + + t_uindex nrows = master->size(); + + for (auto& state : m_states) { + const auto& spec = state.m_spec; + auto out = dst_master->add_column_sptr(spec.m_name, spec.m_dtype, true); + out->reserve(nrows); + + // Dead (freed) master slots are not present in `pkey_map`; + // pre-clearing every row leaves them invalid. + for (t_uindex ridx = 0; ridx < nrows; ++ridx) { + out->clear(ridx); + } + + const t_column* src = source_column(state, master, dst_master); + + // Empty `m_order_by` = NATURAL order: every order key is left + // invalid, so `key_less` degenerates to its (always-ascending) + // pkey tiebreak - index-column order for indexed tables, insertion + // order otherwise. + const t_column* ord = spec.m_order_by.empty() + ? nullptr + : master->get_const_column(spec.m_order_by).get(); + + std::vector> parts; + parts.reserve(spec.m_partition_by.size()); + for (const auto& colname : spec.m_partition_by) { + parts.push_back(master->get_const_column(colname)); + } + + std::vector key(parts.size()); + for (const auto& kv : pkey_map) { + t_uindex ridx = kv.second; + if (ridx >= nrows) { + continue; + } + + for (std::size_t i = 0; i < parts.size(); ++i) { + key[i] = parts[i]->get_scalar(ridx); + } + + t_tscalar order; + order.clear(); + if (ord != nullptr) { + order = ord->get_scalar(ridx); + } + auto part_it = state.m_partitions.try_emplace(key).first; + part_it->second.m_rows.push_back({ridx, order, kv.first}); + state.m_locations[kv.first] = {&part_it->first, order, ridx}; + } + + for (auto& partition : state.m_partitions) { + auto& rows = partition.second.m_rows; + const bool desc = spec.m_order_desc; + std::sort( + rows.begin(), + rows.end(), + [desc](const t_window_row& a, const t_window_row& b) { + return key_less( + a.m_order, a.m_pkey, b.m_order, b.m_pkey, desc + ); + } + ); + + recompute_range( + state, partition.second, 0, rows.size(), *src, *out + ); + } + } +} + +std::vector +t_window_engine::collect_invalidations( + const t_data_table& flattened, + const std::vector& lookup, + const std::shared_ptr& master, + const t_gstate::t_mapping& pkey_map +) { + std::vector invalidated; + if (m_states.empty()) { + return invalidated; + } + + for (auto& state : m_states) { + state.m_edits.clear(); + state.m_dead_ridxs.clear(); + state.m_dirty.clear(); + } + + m_collected = true; + + t_uindex num_rows = flattened.size(); + const auto pkey_col = flattened.get_const_column("psp_pkey"); + const auto op_col = flattened.get_const_column("psp_op"); + + tsl::hopscotch_set batch_pkeys; + batch_pkeys.reserve(num_rows); + + struct t_spec_cols { + const t_column* m_ord; + std::vector> m_parts; + + // Partition-key scratch, reused across every batch row. + std::vector m_key_buf; + }; + std::vector spec_cols; + spec_cols.reserve(m_states.size()); + for (const auto& state : m_states) { + t_spec_cols cols; + + // Empty `m_order_by` = natural order (see `compute_master`). + cols.m_ord = state.m_spec.m_order_by.empty() + ? nullptr + : master->get_const_column(state.m_spec.m_order_by).get(); + for (const auto& colname : state.m_spec.m_partition_by) { + cols.m_parts.push_back(master->get_const_column(colname)); + } + cols.m_key_buf.resize(cols.m_parts.size()); + spec_cols.push_back(std::move(cols)); + } + + // Pass 1: scan the batch, RECORDING each row's index mutation in its + // partition's `t_partition_edits` (no index vectors are touched here) - + // locations update immediately since they are keyed by pkey, not + // position. + for (t_uindex idx = 0; idx < num_rows; ++idx) { + t_tscalar pkey = pkey_col->get_scalar(idx); + const auto* op_ptr = op_col->get_nth(idx); + t_op op = static_cast(*op_ptr); + batch_pkeys.insert(pkey); + + for (std::size_t sidx = 0; sidx < m_states.size(); ++sidx) { + auto& state = m_states[sidx]; + + auto loc_it = state.m_locations.find(pkey); + if (loc_it != state.m_locations.end()) { + const t_window_location& loc = loc_it->second; + t_partition_edits& edits = state.m_edits[*loc.m_part_key]; + edits.m_removals.emplace_back(loc.m_order, pkey); + edits.m_touched.emplace_back(loc.m_order, pkey); + state.m_locations.erase(loc_it); + } + + if (op != OP_DELETE) { + auto pkey_it = pkey_map.find(pkey); + if (pkey_it == pkey_map.end()) { + continue; + } + + t_uindex ridx = pkey_it->second; + auto& cols = spec_cols[sidx]; + for (std::size_t i = 0; i < cols.m_parts.size(); ++i) { + cols.m_key_buf[i] = cols.m_parts[i]->get_scalar(ridx); + } + + t_tscalar order; + order.clear(); + if (cols.m_ord != nullptr) { + order = cols.m_ord->get_scalar(ridx); + } + + // `try_emplace` guarantees the partition node (and thus the + // stable key the location points at) exists even before + // pass 2 merges the row in. + auto part_it = + state.m_partitions.try_emplace(cols.m_key_buf).first; + t_partition_edits& edits = state.m_edits[cols.m_key_buf]; + edits.m_additions.push_back({ridx, order, pkey}); + edits.m_touched.emplace_back(order, pkey); + state.m_locations[pkey] = {&part_it->first, order, ridx}; + } else if (idx < lookup.size() && lookup[idx].m_exists) { + state.m_dead_ridxs.push_back(lookup[idx].m_idx); + } + } + } + + // Pass 2: apply each partition's buffered edits in ONE compaction + + // sorted-merge pass - O(n + k log k) per partition, where the removed + // per-row `erase`/`insert` was O(k * n). The prefix accumulators only + // track SIZE here: positions at or past the first edit are + // garbage-by-contract until `recompute_range` rewrites them, and its + // dirty range always starts at or before the first edited position; + // positions below it are untouched by the resize. + for (auto& state : m_states) { + const bool desc = state.m_spec.m_order_desc; + for (auto& kv : state.m_edits) { + auto part_it = state.m_partitions.find(kv.first); + if (part_it == state.m_partitions.end()) { + continue; + } + + t_window_partition& partition = part_it->second; + auto& rows = partition.m_rows; + t_partition_edits& edits = kv.second; + + // Removal positions, resolved against the PRE-edit index. + std::vector rpos; + rpos.reserve(edits.m_removals.size()); + for (const auto& rem : edits.m_removals) { + std::size_t pos = + lower_bound_row(rows, rem.first, rem.second, desc); + if (pos < rows.size() && rows[pos].m_pkey == rem.second) { + rpos.push_back(pos); + } + } + + std::sort(rpos.begin(), rpos.end()); + if (!rpos.empty()) { + std::size_t w = rpos[0]; + std::size_t next = 0; + for (std::size_t r = rpos[0]; r < rows.size(); ++r) { + if (next < rpos.size() && rpos[next] == r) { + ++next; + continue; + } + + rows[w++] = rows[r]; + } + + rows.resize(w); + } + + if (!edits.m_additions.empty()) { + auto& adds = edits.m_additions; + std::sort( + adds.begin(), + adds.end(), + [desc](const t_window_row& a, const t_window_row& b) { + return key_less( + a.m_order, a.m_pkey, b.m_order, b.m_pkey, desc + ); + } + ); + + // In-place backward merge; (order, pkey) keys are unique + // (a re-added pkey was removed above), so ties cannot + // occur. + std::size_t old_n = rows.size(); + rows.resize(old_n + adds.size()); + std::size_t i = old_n; + std::size_t j = adds.size(); + std::size_t w = rows.size(); + while (j > 0) { + if (i > 0 + && key_less( + adds[j - 1].m_order, + adds[j - 1].m_pkey, + rows[i - 1].m_order, + rows[i - 1].m_pkey, + desc + )) { + rows[--w] = rows[--i]; + } else { + rows[--w] = adds[--j]; + } + } + } + + if (state.m_needs_prefix) { + partition.m_prefix_sum.resize(rows.size()); + partition.m_prefix_sumsq.resize(rows.size()); + partition.m_prefix_count.resize(rows.size()); + } + } + } + + // Resolve touched keys to positions on the FINAL (post-maintenance) + // partition contents, extend to the spec class's invalidation reach, + // merge, and emit the pkeys outside the batch. Over-approximation is + // safe: the pipeline's prev/current diff suppresses rows whose outputs + // did not change. + tsl::hopscotch_set seen; + for (auto& state : m_states) { + const t_window_spec& spec = state.m_spec; + t_window_class klass = class_of(spec); + + for (const auto& touched : state.m_edits) { + auto part_it = state.m_partitions.find(touched.first); + if (part_it == state.m_partitions.end() + || part_it->second.m_rows.empty()) { + if (part_it != state.m_partitions.end()) { + state.m_partitions.erase(part_it); + } + continue; + } + + const auto& rows = part_it->second.m_rows; + std::size_t first_valid = partition_first_valid(rows); + std::vector> ranges; + std::size_t min_pos = rows.size(); + + for (const auto& key : touched.second.m_touched) { + std::size_t pos = lower_bound_row( + rows, key.first, key.second, spec.m_order_desc + ); + switch (klass) { + case t_window_class::WINDOW_CLASS_CUMULATIVE: + min_pos = std::min(min_pos, pos); + break; + case t_window_class::WINDOW_CLASS_AGG_ROWS: + if (pos < rows.size()) { + ranges.emplace_back( + pos, + std::min( + rows.size(), pos + spec.m_frame_rows + 1 + ) + ); + } + break; + case t_window_class::WINDOW_CLASS_AGG_RANGE: { + if (!key.first.is_valid()) { + // Rows with invalid order keys frame together + // in the invalid prefix. + if (pos < first_valid) { + ranges.emplace_back(pos, first_valid); + } + break; + } + + double k = key.first.to_double(); + std::size_t lo = frame_lower_bound( + rows, k, first_valid, spec.m_order_desc + ); + std::size_t hi = reach_upper_bound( + rows, + k, + spec.m_frame_range, + first_valid, + spec.m_order_desc + ); + if (lo < hi) { + ranges.emplace_back(lo, hi); + } + } break; + case t_window_class::WINDOW_CLASS_LAG_LIKE: + if (pos < rows.size()) { + ranges.emplace_back( + pos, + std::min( + rows.size(), pos + spec.m_offset + 1 + ) + ); + } + break; + case t_window_class::WINDOW_CLASS_LEAD_LIKE: { + std::size_t lo = + pos > spec.m_offset ? pos - spec.m_offset : 0; + std::size_t hi = std::min(rows.size(), pos + 1); + if (lo < hi) { + ranges.emplace_back(lo, hi); + } + } break; + } + } + + if (klass == t_window_class::WINDOW_CLASS_CUMULATIVE) { + if (min_pos < rows.size()) { + ranges.emplace_back(min_pos, rows.size()); + } + } else { + std::sort(ranges.begin(), ranges.end()); + std::vector> merged; + for (const auto& r : ranges) { + if (!merged.empty() && r.first <= merged.back().second) { + merged.back().second = + std::max(merged.back().second, r.second); + } else { + merged.push_back(r); + } + } + ranges = std::move(merged); + } + + for (const auto& r : ranges) { + state.m_dirty.push_back({touched.first, r.first, r.second}); + for (std::size_t pos = r.first; pos < r.second; ++pos) { + const t_tscalar& pk = rows[pos].m_pkey; + if (batch_pkeys.find(pk) == batch_pkeys.end() + && seen.insert(pk).second) { + invalidated.push_back(pk); + } + } + } + } + } + + return invalidated; +} + +void +t_window_engine::compute_update( + const std::shared_ptr& master, + const t_gstate::t_mapping& pkey_map, + const std::shared_ptr& expr_master, + const std::shared_ptr& expr_flattened, + const std::shared_ptr& expr_prev, + const std::shared_ptr& expr_current, + const std::shared_ptr& expr_delta, + const std::shared_ptr& flattened, + const std::shared_ptr& existed +) { + if (m_states.empty()) { + return; + } + + t_uindex num_rows = flattened->size(); + const auto pkey_col = flattened->get_const_column("psp_pkey"); + const auto existed_col = existed->get_const_column("psp_existed"); + + // Previous window values must be captured from `expr_master` BEFORE the + // recompute below overwrites it - it still holds the pre-update + // outputs. Rows-outer so the existed check and pkey hash lookup happen + // ONCE per row, shared by every spec. + struct t_prev_cols { + t_column* m_prev; + const t_column* m_old; + }; + std::vector prev_cols; + prev_cols.reserve(m_states.size()); + for (const auto& state : m_states) { + const auto& spec = state.m_spec; + auto prev_col = + expr_prev->add_column_sptr(spec.m_name, spec.m_dtype, true); + prev_col->reserve(num_rows); + auto old_col = + expr_master->add_column_sptr(spec.m_name, spec.m_dtype, true); + prev_cols.push_back({prev_col.get(), old_col.get()}); + } + + for (t_uindex ridx = 0; ridx < num_rows; ++ridx) { + // Value read: `get_nth` never returns nullptr in bounds. + bool row_existed = *(existed_col->get_nth(ridx)); + t_uindex mridx = 0; + bool found = false; + if (row_existed) { + t_tscalar pkey = pkey_col->get_scalar(ridx); + auto it = pkey_map.find(pkey); + if (it != pkey_map.end() && it->second < expr_master->size()) { + mridx = it->second; + found = true; + } + } + + for (const auto& cols : prev_cols) { + if (found && cols.m_old->is_valid(mridx)) { + cols.m_prev->set_scalar(ridx, cols.m_old->get_scalar(mridx)); + } else { + cols.m_prev->clear(ridx); + } + } + } + + if (!m_collected) { + // No widening pass preceded this update (e.g. a code path that does + // not run the gnode `_process_table` hook) - fall back to the full + // rebuild, which is always correct. + compute_master(master, pkey_map, expr_master); + } else { + for (auto& state : m_states) { + const auto& spec = state.m_spec; + auto out = + expr_master->add_column_sptr(spec.m_name, spec.m_dtype, true); + out->reserve(expr_master->size()); + + for (t_uindex dead : state.m_dead_ridxs) { + if (dead < expr_master->size()) { + out->clear(dead); + } + } + + const t_column* src = source_column(state, master, expr_master); + for (const auto& dirty : state.m_dirty) { + auto part_it = state.m_partitions.find(dirty.m_part_key); + if (part_it == state.m_partitions.end()) { + continue; + } + + recompute_range( + state, + part_it->second, + dirty.m_lo, + std::min(dirty.m_hi, part_it->second.m_rows.size()), + *src, + *out + ); + } + + state.m_edits.clear(); + state.m_dead_ridxs.clear(); + state.m_dirty.clear(); + } + + m_collected = false; + } + + fill_transitional( + pkey_map, + expr_master, + expr_flattened, + expr_prev, + expr_current, + expr_delta, + flattened + ); +} + +void +t_window_engine::fill_transitional( + const t_gstate::t_mapping& pkey_map, + const std::shared_ptr& expr_master, + const std::shared_ptr& expr_flattened, + const std::shared_ptr& expr_prev, + const std::shared_ptr& expr_current, + const std::shared_ptr& expr_delta, + const std::shared_ptr& flattened +) const { + t_uindex num_rows = flattened->size(); + const auto pkey_col = flattened->get_const_column("psp_pkey"); + + // Rows-outer so the pkey hash lookup happens ONCE per row, shared by + // every spec. + struct t_fill_cols { + t_dtype m_dtype; + const t_column* m_new; + const t_column* m_prev; + t_column* m_flattened; + t_column* m_current; + t_column* m_delta; + }; + std::vector fill_cols; + fill_cols.reserve(m_states.size()); + for (const auto& state : m_states) { + const auto& spec = state.m_spec; + auto new_col = expr_master->get_column(spec.m_name); + auto prev_col = expr_prev->get_column(spec.m_name); + auto flattened_col = + expr_flattened->add_column_sptr(spec.m_name, spec.m_dtype, true); + auto current_col = + expr_current->add_column_sptr(spec.m_name, spec.m_dtype, true); + auto delta_col = + expr_delta->add_column_sptr(spec.m_name, spec.m_dtype, true); + flattened_col->reserve(num_rows); + current_col->reserve(num_rows); + delta_col->reserve(num_rows); + fill_cols.push_back( + {spec.m_dtype, + new_col.get(), + prev_col.get(), + flattened_col.get(), + current_col.get(), + delta_col.get()} + ); + } + + for (t_uindex ridx = 0; ridx < num_rows; ++ridx) { + t_tscalar pkey = pkey_col->get_scalar(ridx); + auto it = pkey_map.find(pkey); + bool found = it != pkey_map.end() && it->second < expr_master->size(); + for (const auto& cols : fill_cols) { + if (!found || !cols.m_new->is_valid(it->second)) { + cols.m_flattened->clear(ridx); + cols.m_current->clear(ridx); + cols.m_delta->clear(ridx); + continue; + } + + t_tscalar value = cols.m_new->get_scalar(it->second); + cols.m_flattened->set_scalar(ridx, value); + cols.m_current->set_scalar(ridx, value); + + bool prev_valid = cols.m_prev->is_valid(ridx); + if (cols.m_dtype == DTYPE_FLOAT64 && prev_valid) { + write_float( + *cols.m_delta, + ridx, + value.to_double() + - cols.m_prev->get_scalar(ridx).to_double() + ); + } else if (cols.m_dtype == DTYPE_INT64 && prev_valid) { + write_int( + *cols.m_delta, + ridx, + value.get() + - cols.m_prev->get_scalar(ridx).get() + ); + } else { + cols.m_delta->set_scalar(ridx, value); + } + } + } +} + +} // end namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/config.h b/rust/perspective-server/cpp/perspective/src/include/perspective/config.h index 9ec4f6dbfa..04fca7ec01 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/config.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/config.h @@ -20,6 +20,7 @@ #include #include #include +#include namespace perspective { @@ -49,7 +50,8 @@ class PERSPECTIVE_EXPORT t_config { const std::vector& detail_columns, const std::vector& fterms, t_filter_op combiner, - const std::vector>& expressions + const std::vector>& expressions, + const std::vector& windows = {} ); /** @@ -66,7 +68,8 @@ class PERSPECTIVE_EXPORT t_config { const std::vector& aggregates, const std::vector& fterms, t_filter_op combiner, - const std::vector>& expressions + const std::vector>& expressions, + const std::vector& windows = {} ); /** @@ -89,7 +92,8 @@ class PERSPECTIVE_EXPORT t_config { const std::vector& fterms, t_filter_op combiner, const std::vector>& expressions, - bool column_only + bool column_only, + const std::vector& windows = {} ); // An empty config, used for the unit context. @@ -186,6 +190,8 @@ class PERSPECTIVE_EXPORT t_config { std::vector> get_expressions() const; + const std::vector& get_windows() const; + t_totals get_totals() const; t_filter_op get_combiner() const; @@ -232,6 +238,7 @@ class PERSPECTIVE_EXPORT t_config { std::vector m_col_sortspecs; std::vector m_fterms; std::vector> m_expressions; + std::vector m_windows; t_filter_op m_combiner; bool m_column_only; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h index f566ff5298..8cbb63667d 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h @@ -101,6 +101,15 @@ bool is_expression_column(const std::string& colname) const; t_uindex num_expressions() const; +/** + * @brief Whether this context's expression tables contribute any columns + * (expressions or windows) - i.e. whether the gnode must join them into the + * tables it notifies this context with. + */ +bool has_derived_columns() const; + +std::shared_ptr get_window_engine() const; + std::shared_ptr get_expression_tables() const; // Given shared pointers to data tables from the gnode, use them to diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_grouped_pkey.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_grouped_pkey.h index a3fa08d943..b98e23053e 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_grouped_pkey.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_grouped_pkey.h @@ -76,6 +76,7 @@ class PERSPECTIVE_EXPORT t_ctx_grouped_pkey t_depth m_depth; bool m_depth_set; std::shared_ptr m_expression_tables; + std::shared_ptr m_window_engine; }; typedef std::shared_ptr t_ctx_grouped_pkey_sptr; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_one.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_one.h index 6adf0807ca..e068ad14fc 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_one.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_one.h @@ -60,6 +60,7 @@ class PERSPECTIVE_EXPORT t_ctx1 : public t_ctxbase { std::shared_ptr m_tree; std::vector m_sortby; std::shared_ptr m_expression_tables; + std::shared_ptr m_window_engine; t_depth m_depth; bool m_depth_set; bool m_leaves_only = false; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_two.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_two.h index b93d3089ff..0e42e8f296 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_two.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_two.h @@ -99,6 +99,7 @@ class PERSPECTIVE_EXPORT t_ctx2 : public t_ctxbase { t_depth m_column_depth; bool m_column_depth_set; std::shared_ptr m_expression_tables; + std::shared_ptr m_window_engine; bool m_leaves_only = false; bool m_total_only = false; }; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h index 768cfd9036..0fd65d992f 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h @@ -106,6 +106,7 @@ class PERSPECTIVE_EXPORT t_ctx0 : public t_ctxbase { std::shared_ptr m_deltas; tsl::hopscotch_set m_delta_pkeys; std::shared_ptr m_expression_tables; + std::shared_ptr m_window_engine; t_symtable m_symtable; bool m_has_delta; }; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h b/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h index 0ff88af888..532ea88349 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h @@ -16,6 +16,7 @@ #include #include #include +#include namespace perspective { @@ -31,7 +32,8 @@ struct t_expression_tables { t_expression_tables( const std::vector>& expressions, - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + const std::vector& windows = {} ); /** diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h b/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h index 772b58ed2b..6fc933ee9f 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h @@ -379,6 +379,21 @@ class PERSPECTIVE_EXPORT t_gnode { const std::shared_ptr& flattened ); + /** + * @brief The window widening pass (WINDOW_FUNCTIONS_PLAN §2.3): apply + * the update batch to every registered context's window indexes, then + * append a synthesized "unchanged" row to `flattened` and the + * transitional port tables for each row OUTSIDE the batch whose window + * outputs may change. The ordinary pipeline then reports those rows' + * window deltas, and its per-row prev/current diffing suppresses the + * over-approximation. Must run after `m_gstate` is updated and before + * `_compute_expressions`. + */ + void _process_windows( + const std::shared_ptr& flattened, + const std::vector& lookup + ); + private: /** * @brief Process the input data table by flattening it, calculating @@ -460,10 +475,10 @@ t_gnode::notify_context( ctx->step_begin(); - if (ctx->num_expressions() > 0) { + if (ctx->has_derived_columns()) { // Join expression tables on the context with gnode tables and pass - // those into the context so there is no distinction between expression - // and real columns for the context. + // those into the context so there is no distinction between + // expression/window and real columns for the context. std::shared_ptr ctx_expression_tables = ctx->get_expression_tables(); @@ -535,10 +550,10 @@ t_gnode::update_context_from_state( // to update its registered contexts with the new data. `is_registration` // is `false` here — a subscriber may have attached between context // creation and the first update, and expects to see all rows as deltas. - if (ctx->num_expressions() > 0) { - // If the context has expression columns, it has already been computed - // in `process_table` and we can join the "real" and expression columns - // together and pass it to the context. + if (ctx->has_derived_columns()) { + // If the context has expression or window columns, they have already + // been computed in `process_table` and we can join the "real" and + // derived columns together and pass it to the context. std::shared_ptr ctx_expression_tables = ctx->get_expression_tables(); std::shared_ptr joined_flattened = diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h b/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h index 5a334b5def..17d894b76e 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -59,7 +60,8 @@ class PERSPECTIVE_EXPORT t_view_config { std::string filter_op, bool column_only, bool leaves_only = false, - bool total_only = false + bool total_only = false, + const std::vector& windows = {} ); /** @@ -117,6 +119,8 @@ class PERSPECTIVE_EXPORT t_view_config { std::vector> get_expressions() const; + const std::vector& get_windows() const; + t_filter_op get_filter_op() const; bool is_column_only() const; @@ -198,6 +202,7 @@ class PERSPECTIVE_EXPORT t_view_config { m_filter; std::vector> m_sort; std::vector> m_expressions; + std::vector m_windows; /** * @brief The ordered list of aggregate columns: diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/window.h b/rust/perspective-server/cpp/perspective/src/include/perspective/window.h new file mode 100644 index 0000000000..f4d4c7b8e9 --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/window.h @@ -0,0 +1,299 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace perspective { + +enum class t_window_op : std::uint8_t { + WINDOW_OP_SUM, + WINDOW_OP_AVG, + WINDOW_OP_COUNT, + WINDOW_OP_MIN, + WINDOW_OP_MAX, + WINDOW_OP_STDDEV, + WINDOW_OP_VAR, + WINDOW_OP_FIRST, + WINDOW_OP_LAST, + WINDOW_OP_LAG, + WINDOW_OP_LEAD, + WINDOW_OP_DIFF, + WINDOW_OP_RATE, + WINDOW_OP_EMA +}; + +enum class t_window_frame_type : std::uint8_t { + WINDOW_FRAME_ROWS, + WINDOW_FRAME_RANGE, + WINDOW_FRAME_CUMULATIVE +}; + +/** + * @brief A single window column declaration: `m_op` applied over an ordered, + * partitioned frame of rows, producing output column `m_name` in a context's + * expression tables. Validated and dtype-resolved at `View` construction + * (`server.cpp`), inert afterwards. + */ +struct PERSPECTIVE_EXPORT t_window_spec { + std::string m_name; + std::string m_source; + std::string m_order_by; + + // The sorted direction IS the window's row order: cumulative frames + // accumulate in it, lag/lead offsets and Range intervals are relative + // to it. Invalid order keys sort first in either direction; pkey + // tiebreaks stay ascending. + bool m_order_desc; + + std::vector m_partition_by; + t_window_op m_op; + t_window_frame_type m_frame_type; + t_uindex m_frame_rows; + double m_frame_range; + t_uindex m_offset; + double m_alpha; + t_dtype m_dtype; +}; + +/** + * @brief The computation shape of a spec, which keys both the recompute + * dispatch and the invalidation reach of an update: + * + * - `AGG_ROWS`/`AGG_RANGE`: an aggregating op over a positional/key-interval + * trailing frame; a touched row invalidates the rows whose frames can + * contain it (`n` successors / the `[key, key + range]` key interval). + * - `CUMULATIVE`: running aggregates and `ema` (recursive, so + * cumulative-class regardless of declared frame); a touched row + * invalidates the partition suffix. + * - `LAG_LIKE` (`lag`, `diff`): reads `offset` rows back, so a touched row + * invalidates `offset` successors. `LEAD_LIKE` reads forward and + * invalidates `offset` PREDECESSORS - the one backward reach. + */ +enum class t_window_class : std::uint8_t { + WINDOW_CLASS_AGG_ROWS, + WINDOW_CLASS_AGG_RANGE, + WINDOW_CLASS_CUMULATIVE, + WINDOW_CLASS_LAG_LIKE, + WINDOW_CLASS_LEAD_LIKE +}; + +/** + * @brief Per-context window computation engine. + * + * Owns, per spec, a sorted (order key, pkey) index of each partition plus + * prefix accumulators for cumulative frames, maintained incrementally from + * each update batch. An update is processed in two steps, both inside one + * gnode `_process_table` pass: + * + * 1. `collect_invalidations` (called by the gnode widening pass, before + * contexts compute): maintains the indexes from the batch, records the + * dirty recompute ranges, and returns the primary keys OUTSIDE the batch + * whose window outputs may change. The gnode appends those rows to the + * flattened/transitional tables so the ordinary pipeline reports their + * deltas; over-approximation is suppressed by the pipeline's own + * prev/current diffing. + * 2. `compute_update` (called by the context's `compute_expressions`): + * recomputes only the dirty ranges into the expression master table, then + * fills the transitional tables for the (widened) batch rows by pkey. + * + * `compute_master` performs a full state rebuild + full recompute; it is the + * registration path, the fallback when no `collect_invalidations` preceded + * `compute_update`, and the permanent correctness oracle. + */ +class PERSPECTIVE_EXPORT t_window_engine { +public: + explicit t_window_engine(std::vector specs); + + /** + * @brief The output dtype for `op` applied to a `source_dtype` column, + * or `DTYPE_NONE` if the combination is invalid. + */ + static t_dtype resolve_dtype(t_window_op op, t_dtype source_dtype); + + /** + * @brief Whether `op`/`frame` is implemented by this engine. + */ + static bool is_implemented(t_window_op op, t_window_frame_type frame); + + /** + * @brief The computation shape of a validated spec. + */ + static t_window_class class_of(const t_window_spec& spec); + + bool enabled() const; + + /** + * @brief Rebuild all index state and recompute every window column over + * the live rows of `master`, writing output columns into `dst_master` + * (which also supplies expression-alias `m_source` columns, so + * expressions must be computed into it first). + */ + void compute_master( + const std::shared_ptr& master, + const t_gstate::t_mapping& pkey_map, + const std::shared_ptr& dst_master + ); + + /** + * @brief Incremental step 1: apply the update batch to the indexes and + * return the invalidated pkeys outside the batch. `flattened` is the + * masked update batch (including `psp_op` deletes); `lookup` holds the + * PRE-update row lookups aligned with `flattened`'s rows (dead output + * slots of deleted rows are unknowable after the master update); + * `master`/`pkey_map` are POST-update. + */ + std::vector collect_invalidations( + const t_data_table& flattened, + const std::vector& lookup, + const std::shared_ptr& master, + const t_gstate::t_mapping& pkey_map + ); + + /** + * @brief Incremental step 2: capture per-row previous window values for + * the (widened) update batch into `expr_prev`, recompute the dirty + * ranges recorded by `collect_invalidations` into `expr_master`, then + * fill `expr_flattened`/`expr_current`/`expr_delta` rows by primary key. + * + * Falls back to `compute_master` when no collection preceded this call. + * Must run after the context's expression loop and before + * `t_expression_tables::calculate_transitions`. + */ + void compute_update( + const std::shared_ptr& master, + const t_gstate::t_mapping& pkey_map, + const std::shared_ptr& expr_master, + const std::shared_ptr& expr_flattened, + const std::shared_ptr& expr_prev, + const std::shared_ptr& expr_current, + const std::shared_ptr& expr_delta, + const std::shared_ptr& flattened, + const std::shared_ptr& existed + ); + +private: + struct t_window_row { + t_uindex m_ridx; + t_tscalar m_order; + t_tscalar m_pkey; + }; + + struct t_scalar_vec_cmp { + bool operator()( + const std::vector& a, const std::vector& b + ) const; + }; + + struct t_window_partition { + std::vector m_rows; + + // Prefix accumulators for cumulative sum/avg/count/stddev/var + // frames, aligned with `m_rows`; slots past a dirty range's start + // are garbage until that range is recomputed. + std::vector m_prefix_sum; + std::vector m_prefix_sumsq; + std::vector m_prefix_count; + }; + + struct t_window_location { + // Points at the owning partition's `m_partitions` map node KEY - + // stable for the node's lifetime (`std::map` nodes never move). A + // row's location is erased before its partition node can become + // empty (and be erased), so a live location never dangles. Storing + // the pointer instead of a key copy removes a per-row heap + // allocation per spec. + const std::vector* m_part_key; + t_tscalar m_order; + t_uindex m_ridx; + }; + + struct t_dirty_range { + std::vector m_part_key; + std::size_t m_lo; + std::size_t m_hi; // exclusive + }; + + // One partition's buffered index mutations for the current update - + // `collect_invalidations`' batch scan RECORDS removals/additions here + // and applies each partition's set in a single compaction + sorted + // merge pass, bounding maintenance at O(n + k log k) where per-row + // `erase`/`insert` was O(k * n). + struct t_partition_edits { + // (order key, pkey) of every row touched in this partition - both + // the old and new positions of moved rows - resolved to + // invalidation reaches against the POST-edit index. + std::vector> m_touched; + + // (order key, pkey) of rows to remove from the index. + std::vector> m_removals; + + // Rows to insert into the index. + std::vector m_additions; + }; + + struct t_spec_state { + t_window_spec m_spec; + bool m_needs_prefix; + std::map, t_window_partition, t_scalar_vec_cmp> + m_partitions; + tsl::hopscotch_map m_locations; + + // Per-update scratch, valid between `collect_invalidations` and + // `compute_update`. Keyed by partition (an ordered map so widened + // row emission order is deterministic across runs). + std::map, t_partition_edits, t_scalar_vec_cmp> + m_edits; + std::vector m_dead_ridxs; + std::vector m_dirty; + }; + + void reset_state(); + + const t_column* source_column( + const t_spec_state& state, + const std::shared_ptr& master, + const std::shared_ptr& dst_master + ) const; + + void recompute_range( + t_spec_state& state, + t_window_partition& partition, + std::size_t lo, + std::size_t hi, + const t_column& src, + t_column& out + ) const; + + void fill_transitional( + const t_gstate::t_mapping& pkey_map, + const std::shared_ptr& expr_master, + const std::shared_ptr& expr_flattened, + const std::shared_ptr& expr_prev, + const std::shared_ptr& expr_current, + const std::shared_ptr& expr_delta, + const std::shared_ptr& flattened + ) const; + + std::vector m_states; + bool m_collected; +}; + +} // end namespace perspective diff --git a/rust/perspective-viewer/src/rust/session/replace_expression_update.rs b/rust/perspective-viewer/src/rust/session/replace_expression_update.rs index ac1a4087ea..adc2839147 100644 --- a/rust/perspective-viewer/src/rust/session/replace_expression_update.rs +++ b/rust/perspective-viewer/src/rust/session/replace_expression_update.rs @@ -126,6 +126,7 @@ pub impl ViewConfig { filter_op: None, group_by_depth: None, group_rollup_mode: None, + windows: None, } } } diff --git a/tools/bench/basic_suite.mjs b/tools/bench/basic_suite.mjs index 75a95132c1..b8ed912695 100644 --- a/tools/bench/basic_suite.mjs +++ b/tools/bench/basic_suite.mjs @@ -71,5 +71,6 @@ perspective_bench.suite( await all_benchmarks.view_suite(client, metadata); await all_benchmarks.to_data_suite(client, metadata); await all_benchmarks.join_suite(client, metadata); + await all_benchmarks.window_suite(client, metadata); }, ); diff --git a/tools/bench/cross_platform_suite.mjs b/tools/bench/cross_platform_suite.mjs index 87cc5341b2..3625b1944a 100644 --- a/tools/bench/cross_platform_suite.mjs +++ b/tools/bench/cross_platform_suite.mjs @@ -60,6 +60,91 @@ export async function join_suite(perspective, metadata) { } } +export async function window_suite(perspective, metadata) { + if (!check_version_gte(metadata.version, "5.0.0")) { + return; + } + + async function before_all() { + const table = await perspective.table(new_superstore_table(metadata)); + const view = await table.view(); + const arrow = await view.to_arrow(); + await view.delete(); + await table.delete(); + return { arrow }; + } + + const WINDOWS = { + cumsum: { + column: "Sales", + aggregate: "sum", + order_by: ["Row ID", "asc"], + partition_by: ["Region"], + cumulative: true, + }, + sma20: { + column: "Sales", + aggregate: "avg", + order_by: ["Row ID", "asc"], + partition_by: ["Region"], + rows: 20, + }, + }; + + await benchmark({ + name: `.view({windows})`, + before_all, + metadata, + async before({ arrow }) { + return await perspective.table(arrow.slice()); + }, + async after(_, table, view) { + await view.delete(); + await table.delete(); + }, + async test(_, table) { + const view = await table.view({ + columns: ["Row ID", "cumsum", "sma20"], + windows: WINDOWS, + }); + + // Materialize so the master compute is actually included. + await view.to_columns(); + return view; + }, + }); + + await benchmark({ + name: `table.update(arrow) with windows`, + before_all, + metadata, + async before({ arrow }) { + // Indexed by the unique "Row ID", so re-updating with the same + // arrow touches EVERY row - the worst case for the incremental + // window index maintenance path. + const table = await perspective.table(arrow.slice(), { + index: "Row ID", + }); + + const view = await table.view({ + columns: ["Row ID", "cumsum", "sma20"], + windows: WINDOWS, + }); + + await view.to_columns(); + return { table, view }; + }, + async after(_, { table, view }) { + await view.delete(); + await table.delete(); + }, + async test({ arrow }, { table, view }) { + await table.update(arrow.slice()); + await view.to_columns(); + }, + }); +} + export async function to_data_suite(perspective, metadata) { async function before_all() { const table = await perspective.table(new_superstore_table(metadata)); diff --git a/tools/bench/package.json b/tools/bench/package.json index d8afd6f9a0..1b85988e53 100644 --- a/tools/bench/package.json +++ b/tools/bench/package.json @@ -15,6 +15,7 @@ }, "scripts": { "bench_js": "node --experimental-wasm-memory64 basic_suite.mjs", + "bench_windows": "node --experimental-wasm-memory64 windows_suite.mjs", "bench_python": "node --experimental-wasm-memory64 python_suite.mjs", "bench_charts": "node --experimental-wasm-memory64 charts_suite.mjs" }, diff --git a/tools/bench/puppeteer_suite.mjs b/tools/bench/puppeteer_suite.mjs index ede70f4ac4..ab9e29e2ad 100644 --- a/tools/bench/puppeteer_suite.mjs +++ b/tools/bench/puppeteer_suite.mjs @@ -81,5 +81,6 @@ perspective_bench.suite( await test_suite("view_suite"); await test_suite("to_data_suite"); await test_suite("join_suite"); + await test_suite("window_suite"); }, ); diff --git a/tools/bench/python_suite.mjs b/tools/bench/python_suite.mjs index 0ccd79c6d0..e04117f1f4 100644 --- a/tools/bench/python_suite.mjs +++ b/tools/bench/python_suite.mjs @@ -73,6 +73,7 @@ perspective_bench.suite( await all_benchmarks.view_suite(client, metadata); await all_benchmarks.to_data_suite(client, metadata); await all_benchmarks.join_suite(client, metadata); + await all_benchmarks.window_suite(client, metadata); }, python.start, python.stop, diff --git a/tools/bench/windows_suite.mjs b/tools/bench/windows_suite.mjs new file mode 100644 index 0000000000..615f2b5c6f --- /dev/null +++ b/tools/bench/windows_suite.mjs @@ -0,0 +1,116 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// Window-function streaming benchmark (WINDOW_FUNCTIONS_PLAN Phase 4): +// sustained tail-append and mid-edit throughput with k active window +// columns vs. a windowless baseline view on the same stream. Run manually: +// +// pnpm run --filter @perspective-dev/bench bench_windows + +import perspective from "@perspective-dev/client"; + +const SCHEMA = { + id: "integer", + sym: "string", + t: "integer", + price: "float", +}; + +const SYMS = ["a", "b", "c", "d", "e", "f", "g", "h"]; +const BATCH = 1_000; +const BATCHES = 200; +const EDIT_BATCHES = 50; + +function window_specs(k) { + const all = [ + ["w_cumsum", { aggregate: "sum", cumulative: true }], + ["w_sma", { aggregate: "avg", rows: 20 }], + ["w_ema", { aggregate: "ema", alpha: 0.1 }], + ["w_rsum", { aggregate: "sum", range: 100 }], + ["w_min", { aggregate: "min", rows: 50 }], + ["w_std", { aggregate: "stddev", rows: 20 }], + ]; + + return Object.fromEntries( + all.slice(0, k).map(([name, w]) => [ + name, + { + column: "price", + order_by: ["t", "asc"], + partition_by: ["sym"], + ...w, + }, + ]) + ); +} + +function batch(start, mutate) { + const rows = new Array(BATCH); + for (let i = 0; i < BATCH; i++) { + const id = mutate ? Math.floor(Math.random() * start) : start + i; + rows[i] = { + id, + sym: SYMS[id % SYMS.length], + t: mutate ? Math.floor(Math.random() * start) : start + i, + price: Math.random() * 100, + }; + } + + return rows; +} + +async function scenario(k) { + const table = await perspective.table(SCHEMA, { index: "id" }); + const windows = window_specs(k); + const window_names = Object.keys(windows); + const view = await table.view( + window_names.length > 0 + ? { columns: ["id", ...window_names], windows } + : {} + ); + + // tail appends + let t0 = performance.now(); + for (let b = 0; b < BATCHES; b++) { + await table.update(batch(b * BATCH, false)); + } + await view.num_rows(); + const append_ms = performance.now() - t0; + + // random mid-edits over the accumulated history + const size = BATCHES * BATCH; + t0 = performance.now(); + for (let b = 0; b < EDIT_BATCHES; b++) { + await table.update(batch(size, true)); + } + await view.num_rows(); + const edit_ms = performance.now() - t0; + + await view.delete(); + await table.delete(); + return { append_ms, edit_ms }; +} + +const results = []; +for (const k of [0, 1, 3, 6]) { + const { append_ms, edit_ms } = await scenario(k); + results.push({ + windows: k, + "append rows/s": Math.round((BATCHES * BATCH * 1000) / append_ms), + "append ms": Math.round(append_ms), + "edit rows/s": Math.round((EDIT_BATCHES * BATCH * 1000) / edit_ms), + "edit ms": Math.round(edit_ms), + }); + console.log(`k=${k} done`); +} + +console.table(results); From 76c285851870f70f39ee22339c348b1916a0c391 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 2 Aug 2026 18:10:37 -0400 Subject: [PATCH 5/6] Window functions UI Signed-off-by: Andrew Stein --- .../src/css/column-selector.css | 1 + .../src/css/column-settings-panel.css | 69 +- .../src/css/column-style.css | 7 +- .../src/css/config-selector.css | 23 +- .../src/css/containers/context-menu.css | 18 +- .../src/css/containers/tabs.css | 4 +- rust/perspective-viewer/src/css/viewer.css | 8 + .../src/rust/components/column_selector.rs | 23 +- .../column_selector/active_column.rs | 48 +- .../column_selector_column_row.rs | 73 ++ .../column_selector/config_selector.rs | 8 +- .../column_selector/expr_edit_button.rs | 8 +- .../column_selector/inactive_column.rs | 9 +- .../components/column_settings_sidebar.rs | 179 ++- .../column_settings_sidebar/style_tab.rs | 3 +- .../column_settings_sidebar/window_tab.rs | 34 + .../components/containers/dragdrop_list.rs | 76 +- .../src/rust/components/containers/sidebar.rs | 121 +- .../rust/components/containers/split_panel.rs | 6 + .../src/rust/components/editable_header.rs | 2 + .../src/rust/components/main_panel.rs | 5 + .../src/rust/components/mod.rs | 1 + .../src/rust/components/panel_menu.rs | 7 + .../src/rust/components/panel_tab.rs | 5 + .../src/rust/components/portal.rs | 7 + .../src/rust/components/viewer.rs | 2 + .../src/rust/components/viewer/msg.rs | 2 + .../src/rust/components/viewer/render.rs | 12 + .../src/rust/components/viewer/settings.rs | 34 +- .../src/rust/components/window_editor.rs | 1013 +++++++++++++++++ rust/perspective-viewer/src/rust/lib.rs | 5 +- .../src/rust/presentation/column_locator.rs | 17 +- .../src/rust/queries/columns_iter_set.rs | 8 +- rust/perspective-viewer/src/rust/renderer.rs | 35 +- .../src/rust/renderer/dispatch.rs | 143 ++- rust/perspective-viewer/src/rust/session.rs | 12 +- .../src/rust/session/drag_drop_update.rs | 18 + .../src/rust/session/metadata.rs | 75 +- .../src/rust/tasks/edit_window.rs | 155 +++ rust/perspective-viewer/src/rust/tasks/mod.rs | 2 + .../src/rust/utils/browser/dragdrop.rs | 12 + .../src/rust/utils/browser/tests/debounce.rs | 49 + .../src/rust/utils/debounce.rs | 103 +- rust/perspective-viewer/src/svg/pin-icon.svg | 5 + rust/perspective-viewer/src/themes/icons.css | 1 + rust/perspective-viewer/src/themes/intl.css | 11 +- .../perspective-viewer/src/themes/intl/de.css | 7 + .../perspective-viewer/src/themes/intl/es.css | 7 + .../perspective-viewer/src/themes/intl/fr.css | 7 + .../perspective-viewer/src/themes/intl/ja.css | 7 + .../perspective-viewer/src/themes/intl/pt.css | 7 + .../perspective-viewer/src/themes/intl/zh.css | 7 + .../multi_panel/context_menu_picker.spec.ts | 230 ++++ .../test/js/windows.spec.ts | 989 ++++++++++++++++ 54 files changed, 3526 insertions(+), 194 deletions(-) create mode 100644 rust/perspective-viewer/src/rust/components/column_selector/column_selector_column_row.rs create mode 100644 rust/perspective-viewer/src/rust/components/column_settings_sidebar/window_tab.rs create mode 100644 rust/perspective-viewer/src/rust/components/window_editor.rs create mode 100644 rust/perspective-viewer/src/rust/tasks/edit_window.rs create mode 100644 rust/perspective-viewer/src/svg/pin-icon.svg create mode 100644 rust/perspective-viewer/test/js/multi_panel/context_menu_picker.spec.ts create mode 100644 rust/perspective-viewer/test/js/windows.spec.ts diff --git a/rust/perspective-viewer/src/css/column-selector.css b/rust/perspective-viewer/src/css/column-selector.css index 5c8e5ba4ab..54fbf2f156 100644 --- a/rust/perspective-viewer/src/css/column-selector.css +++ b/rust/perspective-viewer/src/css/column-selector.css @@ -185,6 +185,7 @@ display: flex; flex-direction: row-reverse; align-items: center; + flex: 1 1 auto; /* Expression column toolbar buttons */ span.expression-edit-button, diff --git a/rust/perspective-viewer/src/css/column-settings-panel.css b/rust/perspective-viewer/src/css/column-settings-panel.css index 182c024353..753f88eadd 100644 --- a/rust/perspective-viewer/src/css/column-settings-panel.css +++ b/rust/perspective-viewer/src/css/column-settings-panel.css @@ -41,6 +41,7 @@ .item_title { flex: 0 0 auto; font-size: var(--label--font-size, 0.75em); + margin-top: 9px; } input { @@ -70,7 +71,7 @@ .sidebar_header_contents { display: flex; - margin: 8px; + margin: 7px 8px; align-items: center; border-radius: 3px; outline-width: 1px; @@ -117,6 +118,18 @@ content: var(--psp-label--color--content, "Color"); } + label#window-frame-label:before { + content: var(--psp-label--window-frame--content, "Frame"); + } + + label#window-offset-label:before { + content: var(--psp-label--window-offset--content, "Offset"); + } + + label#window-alpha-label:before { + content: var(--psp-label--window-alpha--content, "Alpha"); + } + label#format-label:before { content: var(--psp-label--format--content, "Format"); } @@ -296,6 +309,58 @@ } div.tab-title#Attributes:before { - content: var(--psp-label--attributes-tab--content, "Attributes"); + content: var(--psp-label--attributes-tab--content, "Expression"); + } + + div.tab-title#Window:before { + content: var(--psp-label--window-tab--content, "Window"); + } +} + +#window-tab { + display: flex; + flex-direction: column; + flex: 1; +} + +#window-editor-container .window-editor-error { + color: var(--error--color, #ff5942); + font-size: 11px; +} + +:host { + #column_settings_sidebar { + .sidebar_header { + padding-right: 24px; + } + } + + #window-order-by .column-selector-column-border { + padding-right: 8px; + } + + .sidebar_pin_button { + position: absolute; + right: 6px; + width: 14px; + height: 14px; + z-index: 8; + cursor: pointer; + background-color: var(--psp--color); + opacity: 0.5; + background-repeat: no-repeat; + -webkit-mask-size: cover; + mask-size: cover; + -webkit-mask-image: var(--psp-icon--pin--mask-image); + mask-image: var(--psp-icon--pin--mask-image); + transform: rotate(45deg); + &:hover { + opacity: 1; + } + + &.is-pinned { + transform: none; + opacity: 1; + } } } diff --git a/rust/perspective-viewer/src/css/column-style.css b/rust/perspective-viewer/src/css/column-style.css index 49ca734d88..0ae53ee328 100644 --- a/rust/perspective-viewer/src/css/column-style.css +++ b/rust/perspective-viewer/src/css/column-style.css @@ -13,7 +13,8 @@ :host { #column-style-container, - #plugin-config-container { + #plugin-config-container, + #window-editor-container { outline: none; user-select: none; @@ -81,8 +82,8 @@ label { display: block; font-size: var(--label--font-size, 0.75em); - margin: 4px 0 2px 0; width: 100%; + margin: 4px 0 2px 0; } span.reset-default-style-disabled { @@ -160,7 +161,7 @@ .column-style-label { display: flex; - padding: 4px 0px; + padding: 0px; } .indent { diff --git a/rust/perspective-viewer/src/css/config-selector.css b/rust/perspective-viewer/src/css/config-selector.css index 3bf34221f6..65a210a943 100644 --- a/rust/perspective-viewer/src/css/config-selector.css +++ b/rust/perspective-viewer/src/css/config-selector.css @@ -60,7 +60,8 @@ } } - #top_panel { + #top_panel, + #window-editor-slots { display: flex; flex-direction: column; justify-content: stretch; @@ -176,6 +177,7 @@ flex: 1 1 auto; align-self: stretch; position: relative; + min-height: 24px; &:before { content: " "; position: absolute; @@ -215,6 +217,21 @@ content: var(--psp-label--filter--content, "Where"); } + #window-source label.pivot-selector-label:before { + content: var(--psp-label--window-source--content, "Column"); + } + + #window-order-by label.pivot-selector-label:before { + content: var(--psp-label--window-order-by--content, "Order By"); + } + + #window-partition-by label.pivot-selector-label:before { + content: var( + --psp-label--window-partition-by--content, + "Partition By" + ); + } + .rrow { display: flex; min-height: 24px; @@ -341,6 +358,10 @@ } } + #window-editor-slots .pivot-column { + min-height: 24px; + } + .sort-icon { display: inline-flex; margin-left: auto; diff --git a/rust/perspective-viewer/src/css/containers/context-menu.css b/rust/perspective-viewer/src/css/containers/context-menu.css index 3f5feef281..6041032783 100644 --- a/rust/perspective-viewer/src/css/containers/context-menu.css +++ b/rust/perspective-viewer/src/css/containers/context-menu.css @@ -38,12 +38,13 @@ cursor: pointer; white-space: nowrap; + /* The dropdown-menu surface's inverted hover, from the same theme keys + * (`--psp--color`/`--psp--background-color` land on the host via the + * modal selector groups) — the Copy/Export pickers this menu spawns use + * exactly this idiom. */ &:hover { - background: var( - --psp-context-menu--hover--background, - var(--psp-active--background, rgba(0, 0, 0, 0.08)) - ); - color: var(--psp-context-menu--hover--color, inherit); + background-color: var(--psp--color); + color: var(--psp--background-color); } &.disabled { @@ -51,7 +52,7 @@ cursor: default; &:hover { - background: transparent; + background-color: transparent; color: inherit; } } @@ -86,6 +87,11 @@ max-height: 50vh; overflow-y: auto; background-color: var(--psp--background-color, white); + + /* The flyout is a CHILD of its (necessarily hovered, so inverted) parent + * item — un-hovered rows must not inherit the parent's inverted + * foreground. */ + color: var(--psp--color); border: 1px solid var(--psp-inactive--color); box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15); } diff --git a/rust/perspective-viewer/src/css/containers/tabs.css b/rust/perspective-viewer/src/css/containers/tabs.css index fa45c15a06..cb94629413 100644 --- a/rust/perspective-viewer/src/css/containers/tabs.css +++ b/rust/perspective-viewer/src/css/containers/tabs.css @@ -28,11 +28,11 @@ display: flex; flex-direction: column; .tab-section:last-child { - padding: 12px 0px 12px 8px; + padding: 13px 0px 12px 8px; } .tab-section { - padding: 12px 0px 0px 8px; + padding: 6px 0 0 8px; flex: 0 0 auto; overflow: hidden; } diff --git a/rust/perspective-viewer/src/css/viewer.css b/rust/perspective-viewer/src/css/viewer.css index f6e35aa297..d85df1e37e 100644 --- a/rust/perspective-viewer/src/css/viewer.css +++ b/rust/perspective-viewer/src/css/viewer.css @@ -303,6 +303,14 @@ } } + #modal_panel.pinned { + position: static; + flex: 0 0 auto; + width: auto; + pointer-events: all; + overflow: visible; + } + #modal_panel > .split-panel-divider { border-left: 1px solid var(--psp-inactive--color, #6e6e6e); margin-right: -5px; diff --git a/rust/perspective-viewer/src/rust/components/column_selector.rs b/rust/perspective-viewer/src/rust/components/column_selector.rs index f15b7c5199..7d21dce0d3 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector.rs @@ -13,6 +13,7 @@ mod active_column; mod add_expression_button; mod aggregate_selector; +mod column_selector_column_row; mod config_selector; mod empty_column; mod expr_edit_button; @@ -25,6 +26,7 @@ mod sort_column; use std::iter::*; use std::rc::Rc; +pub use column_selector_column_row::*; pub use empty_column::*; pub use invalid_column::*; use perspective_client::config::ViewConfig; @@ -367,7 +369,9 @@ impl Component for ColumnSelector { let column_dropdown = self.column_dropdown.clone(); let is_editing = matches!( &ctx.props().selected_column, - Some(ColumnLocator::Table(x)) | Some(ColumnLocator::Expression(x)) + Some(ColumnLocator::Table(x)) + | Some(ColumnLocator::Expression(x)) + | Some(ColumnLocator::Window(x)) if x == &key ); // Compute metadata-derived props here so that changes to @@ -392,10 +396,15 @@ impl Component for ColumnSelector { .map(|n| metadata.is_column_expression(n)) .unwrap_or(false); + let is_window = name + .get_name() + .map(|n| metadata.is_column_window(n)) + .unwrap_or(false); + let can_render_styles = name.get_name().is_some() && renderer.can_render_column_styles(); - let show_edit_btn = is_expression || can_render_styles; + let show_edit_btn = is_expression || is_window || can_render_styles; let on_open_expr_panel = &ctx.props().on_open_expr_panel; html_nested! { @@ -405,6 +414,7 @@ impl Component for ColumnSelector { {is_aggregated} {is_editing} {is_expression} + {is_window} {show_edit_btn} {col_type} view_config={config.clone()} @@ -425,12 +435,18 @@ impl Component for ColumnSelector { let mut inactive_children: Vec<_> = columns_iter .expression() + .chain(columns_iter.window()) .chain(columns_iter.inactive()) .enumerate() .map(|(idx, vc)| { let selected_column = ctx.props().selected_column.as_ref(); - let is_editing = matches!(selected_column, Some(ColumnLocator::Expression(x)) if x.as_str() == vc.name); + let is_editing = matches!( + selected_column, + Some(ColumnLocator::Expression(x)) | Some(ColumnLocator::Window(x)) + if x.as_str() == vc.name + ); let is_expression = metadata.is_column_expression(vc.name); + let is_window = metadata.is_column_window(vc.name); html_nested! { -
-
- - - if ctx.props().is_aggregated { + - } - - { name.clone() } - - if !ctx.props().is_aggregated { - - } + })} + trailing={html! { -
-
+ }} + />
} }, diff --git a/rust/perspective-viewer/src/rust/components/column_selector/column_selector_column_row.rs b/rust/perspective-viewer/src/rust/components/column_selector/column_selector_column_row.rs new file mode 100644 index 0000000000..152b7a2b60 --- /dev/null +++ b/rust/perspective-viewer/src/rust/components/column_selector/column_selector_column_row.rs @@ -0,0 +1,73 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use perspective_client::config::ColumnType; +use web_sys::DragEvent; +use yew::prelude::*; + +use crate::components::type_icon::TypeIcon; + +/// The draggable row of a `column-selector-column` +#[derive(Clone, PartialEq, Properties)] +pub struct ColumnSelectorColumnRowProps { + pub name: String, + + #[prop_or_default] + pub col_type: Option, + + #[prop_or_default] + pub aggregate: Option, + + /// Trailing affordance (e.g. the edit button in the active list). + #[prop_or_default] + pub trailing: Html, + + #[prop_or_default] + pub wrapper_class: Classes, + + #[prop_or_default] + pub wrapper_ref: NodeRef, + + #[prop_or_default] + pub ondragstart: Option>, + + #[prop_or_default] + pub ondragend: Option>, +} + +#[function_component] +pub fn ColumnSelectorColumnRow(p: &ColumnSelectorColumnRowProps) -> Html { + let mut classes = classes!["column-selector-draggable"]; + if p.aggregate.is_some() { + classes.push("show-aggregate"); + } + + classes.extend(p.wrapper_class.clone()); + html! { +
+
+ + + if let Some(aggregate) = &p.aggregate { { aggregate.clone() } } + { p.name.clone() } + if p.aggregate.is_none() { } + { p.trailing.clone() } +
+
+ } +} diff --git a/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs b/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs index 866db8ac4d..13e91dfbd4 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs @@ -235,7 +235,7 @@ impl Component for ConfigSelector { }, ConfigSelectorMsg::Close(..) => false, ConfigSelectorMsg::Drop(column, action, effect, index) - if action != DragTarget::Active => + if action != DragTarget::Active && !action.is_staged() => { let col_type = ctx .props() @@ -369,6 +369,12 @@ impl Component for ConfigSelector { ctx.props().onselect.emit(()); false }, + ConfigSelectorMsg::New( + DragTarget::WindowSource + | DragTarget::WindowOrderBy + | DragTarget::WindowPartitionBy, + _, + ) => false, ConfigSelectorMsg::New(DragTarget::Filter, InPlaceColumn::Column(column)) => { let mut view_config = (*ctx.props().view_config).clone(); let op = ctx.props().default_op(column.as_str()).unwrap_or_default(); diff --git a/rust/perspective-viewer/src/rust/components/column_selector/expr_edit_button.rs b/rust/perspective-viewer/src/rust/components/column_selector/expr_edit_button.rs index 394dbb6113..0fe2f53490 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector/expr_edit_button.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector/expr_edit_button.rs @@ -22,6 +22,10 @@ pub struct ExprEditButtonProps { /// Is this an expression column? pub is_expression: bool, + /// Is this a window column? + #[prop_or_default] + pub is_window: bool, + /// Fires when the config/expresison button is clicked. pub on_open_expr_panel: Callback, @@ -38,7 +42,9 @@ pub struct ExprEditButtonProps { #[function_component] pub fn ExprEditButton(p: &ExprEditButtonProps) -> Html { let onmousedown = yew::use_callback(p.clone(), |_, p| { - let name = if p.is_expression { + let name = if p.is_window { + ColumnLocator::Window(p.name.clone()) + } else if p.is_expression { ColumnLocator::Expression(p.name.clone()) } else { ColumnLocator::Table(p.name.clone()) diff --git a/rust/perspective-viewer/src/rust/components/column_selector/inactive_column.rs b/rust/perspective-viewer/src/rust/components/column_selector/inactive_column.rs index 3a9e784631..643098ad30 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector/inactive_column.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector/inactive_column.rs @@ -43,6 +43,10 @@ pub struct InactiveColumnProps { #[prop_or_default] pub is_expression: bool, + /// Whether this column is a window column. + #[prop_or_default] + pub is_window: bool, + /// Session metadata snapshot — threaded from `SessionProps`. pub metadata: SessionMetadataRc, @@ -72,6 +76,7 @@ impl PartialEq for InactiveColumnProps { && self.name == rhs.name && self.is_editing == rhs.is_editing && self.is_expression == rhs.is_expression + && self.is_window == rhs.is_window && self.metadata == rhs.metadata && self.view_config == rhs.view_config } @@ -148,6 +153,7 @@ impl Component for InactiveColumn { .callback(|event: MouseEvent| MouseEnter(event.which() == 0)); let is_expression = ctx.props().is_expression; + let is_window = ctx.props().is_window; let mut is_active_class = ctx.props().renderer.metadata().select_mode.css(); is_active_class.push("shift-alt-icon"); @@ -179,7 +185,8 @@ impl Component for InactiveColumn { name={ctx.props().name.clone()} on_open_expr_panel={&ctx.props().on_open_expr_panel} {is_expression} - is_disabled={!is_expression} + {is_window} + is_disabled={!(is_expression || is_window)} is_editing={ctx.props().is_editing} /> diff --git a/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs b/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs index dad3e3d2c5..3f46f29969 100644 --- a/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs +++ b/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs @@ -13,12 +13,13 @@ mod attributes_tab; mod save_settings; pub(crate) mod style_tab; +mod window_tab; use std::rc::Rc; use derivative::Derivative; use itertools::Itertools; -use perspective_client::config::{ColumnType, Expression, ViewConfig}; +use perspective_client::config::{ColumnType, Expression, ViewConfig, WindowSpec}; use perspective_client::utils::PerspectiveResultExt; use yew::{Callback, Component, Html, Properties, html, props}; @@ -27,15 +28,19 @@ use self::style_tab::StyleTabProps; use crate::components::column_settings_sidebar::attributes_tab::AttributesTab; use crate::components::column_settings_sidebar::save_settings::SaveSettingsProps; use crate::components::column_settings_sidebar::style_tab::StyleTab; +use crate::components::column_settings_sidebar::window_tab::{WindowTab, WindowTabProps}; use crate::components::containers::sidebar::Sidebar; use crate::components::containers::tab_list::TabList; use crate::components::editable_header::EditableHeaderProps; use crate::components::expression_editor::ExpressionEditorProps; use crate::components::type_icon::TypeIconType; +use crate::components::window_editor::WindowEditorProps; use crate::presentation::{ColumnLocator, ColumnSettingsTab, Presentation}; use crate::renderer::Renderer; use crate::session::{Session, SessionMetadataRc}; -use crate::tasks::{delete_expr, save_expr, update_expr}; +use crate::tasks::{ + delete_expr, delete_window, save_expr, save_window, update_expr, update_window, +}; use crate::utils::PtrEqRc; #[derive(Clone, Derivative, Properties)] @@ -47,9 +52,22 @@ pub struct ColumnSettingsPanelProps { pub width_override: Option, pub on_select_tab: Callback, - /// Active plugin name threaded as a value prop so that plugin changes - /// trigger re-initialization via `changed()` rather than a PubSub - /// `render_limits_changed` subscription. + /// Shared trap-door width across the drawer's Style/Attributes/Window + /// tabs. + #[prop_or_default] + pub auto_width: f64, + + #[prop_or_default] + pub on_auto_width: Callback, + + /// Whether the drawer is pinned into the layout. + #[prop_or_default] + pub is_pinned: bool, + + #[prop_or_default] + pub on_toggle_pin: Callback<()>, + + /// Active plugin name. pub plugin_name: Option, /// Session metadata snapshot — threaded from `SessionProps`. @@ -75,8 +93,10 @@ pub struct ColumnSettingsPanelProps { pub session: Session, } -impl PartialEq for ColumnSettingsPanelProps { - fn eq(&self, other: &Self) -> bool { +impl ColumnSettingsPanelProps { + /// Everything EXCEPT the trap-door `auto_width`: the props whose change + /// invalidates the drafts that `initialize` rebuilds. + fn identity_eq(&self, other: &Self) -> bool { self.selected_column == other.selected_column && self.selected_tab == other.selected_tab && self.plugin_name == other.plugin_name @@ -87,12 +107,21 @@ impl PartialEq for ColumnSettingsPanelProps { } } +impl PartialEq for ColumnSettingsPanelProps { + fn eq(&self, other: &Self) -> bool { + self.identity_eq(other) + && self.auto_width == other.auto_width + && self.is_pinned == other.is_pinned + } +} + #[derive(Debug)] pub enum ColumnSettingsPanelMsg { SetExprValue(Rc), SetExprValid(bool), SetHeaderValue(Option), SetHeaderValid(bool), + SetWindowValue(Option), SetSelectedTab((usize, ColumnSettingsTab)), OnSaveAttributes(()), OnResetAttributes(()), @@ -117,6 +146,8 @@ pub struct ColumnSettingsPanel { reset_enabled: bool, save_count: u8, save_enabled: bool, + initial_window_value: Option, + window_value: Option, tabs: Vec, } @@ -138,6 +169,8 @@ impl Component for ColumnSettingsPanel { reset_count: 0, column_name: "".to_owned(), maybe_ty: None, + initial_window_value: None, + window_value: None, tabs: vec![], on_input: Callback::default(), on_save: Callback::default(), @@ -149,12 +182,15 @@ impl Component for ColumnSettingsPanel { } fn changed(&mut self, ctx: &yew::prelude::Context, old_props: &Self::Properties) -> bool { - if ctx.props() != old_props { + // Only reached when props are unequal. Re-`initialize` (which wipes + // in-progress expression/window drafts) only on IDENTITY changes - + // a trap-door `auto_width` change re-renders the `Sidebar` sizer + // alone. + if !ctx.props().identity_eq(old_props) { self.initialize(ctx); - true - } else { - false } + + true } fn update(&mut self, ctx: &yew::prelude::Context, msg: Self::Message) -> bool { @@ -187,6 +223,15 @@ impl Component for ColumnSettingsPanel { self.save_enabled_effect(); true }, + ColumnSettingsPanelMsg::SetWindowValue(val) => { + if self.window_value != val { + self.window_value = val; + self.reset_enabled = true; + true + } else { + false + } + }, ColumnSettingsPanelMsg::SetSelectedTab((_, val)) => { let rerender = ctx.props().selected_tab != Some(val); ctx.props().on_select_tab.emit(val); @@ -195,19 +240,58 @@ impl Component for ColumnSettingsPanel { ColumnSettingsPanelMsg::OnResetAttributes(()) => { self.header_value.clone_from(&self.initial_header_value); self.expr_value.clone_from(&self.initial_expr_value); + self.window_value.clone_from(&self.initial_window_value); self.save_enabled = false; self.reset_enabled = false; self.reset_count += 1; true }, ColumnSettingsPanelMsg::OnSaveAttributes(()) => { + if matches!(ctx.props().selected_tab, Some(ColumnSettingsTab::Window)) { + if let Some(spec) = self.window_value.clone() { + let name = self + .header_value + .clone() + .unwrap_or_else(|| self.column_name.clone()); + match &ctx.props().selected_column { + ColumnLocator::Window(old_name) => update_window( + &ctx.props().session, + &ctx.props().renderer, + &ctx.props().presentation, + old_name.clone(), + name, + spec.clone(), + ), + _ => { + if let Err(err) = save_window( + &ctx.props().session, + &ctx.props().renderer, + &ctx.props().presentation, + name, + spec.clone(), + ) { + tracing::warn!("{}", err); + } + }, + } + + self.initial_window_value = Some(spec); + self.initial_header_value.clone_from(&self.header_value); + self.save_enabled = false; + self.reset_enabled = false; + self.save_count += 1; + } + + return true; + } + let new_expr = Expression::new( self.header_value.clone().map(|s| s.into()), (*(self.expr_value)).clone().into(), ); match &ctx.props().selected_column { - ColumnLocator::Table(_) => { + ColumnLocator::Table(_) | ColumnLocator::Window(_) => { tracing::error!("Tried to save non-expression column!") }, ColumnLocator::Expression(name) => update_expr( @@ -244,6 +328,13 @@ impl Component for ColumnSettingsPanel { &self.column_name, ) .unwrap_or_log(); + } else if ctx.props().selected_column.is_saved_window() { + delete_window( + &ctx.props().session, + &ctx.props().renderer, + &self.column_name, + ) + .unwrap_or_log(); } ctx.props().on_close.emit(()); @@ -253,15 +344,24 @@ impl Component for ColumnSettingsPanel { } fn view(&self, ctx: &yew::prelude::Context) -> Html { + let is_window_tab = matches!(ctx.props().selected_tab, Some(ColumnSettingsTab::Window)); + + let header_placeholder = if is_window_tab { + Rc::new(self.column_name.clone()) + } else { + self.expr_value.clone() + }; + let header_props = props!(EditableHeaderProps { initial_value: self.initial_header_value.clone(), - placeholder: self.expr_value.clone(), + placeholder: header_placeholder, reset_count: self.reset_count, - editable: ctx.props().selected_column.is_expr() + editable: (ctx.props().selected_column.is_expr() && matches!( ctx.props().selected_tab, Some(ColumnSettingsTab::Attributes) - ), + )) + || (ctx.props().selected_column.is_window_editable() && is_window_tab), update_on_input: true, icon_type: self .maybe_ty @@ -324,7 +424,26 @@ impl Component for ColumnSettingsPanel { let attrs_tab = AttributesTabProps { expr_editor, - save_section, + save_section: save_section.clone(), + }; + + let window_changed = self.window_value != self.initial_window_value + || self.header_value != self.initial_header_value; + let window_tab = WindowTabProps { + editor: WindowEditorProps { + metadata: ctx.props().metadata.clone(), + initial: self.initial_window_value.clone(), + on_change: ctx.link().callback(ColumnSettingsPanelMsg::SetWindowValue), + reset_count: self.reset_count, + presentation: ctx.props().presentation.clone(), + session: ctx.props().session.clone(), + selected_theme: ctx.props().selected_theme.clone(), + }, + save_section: SaveSettingsProps { + save_enabled: self.window_value.is_some() && window_changed && self.header_valid, + show_danger_zone: ctx.props().selected_column.is_saved_window(), + ..save_section + }, }; let style_tab = StyleTabProps { @@ -342,6 +461,7 @@ impl Component for ColumnSettingsPanel { let tab_children = self.tabs.iter().map(|tab| match tab { ColumnSettingsTab::Attributes => html! { }, + ColumnSettingsTab::Window => html! { }, ColumnSettingsTab::Style => html! { }, }); @@ -358,6 +478,10 @@ impl Component for ColumnSettingsPanel { on_close={ctx.props().on_close.clone()} id_prefix="column_settings" width_override={ctx.props().width_override} + auto_width={ctx.props().auto_width} + on_auto_width={ctx.props().on_auto_width.clone()} + is_pinned={ctx.props().is_pinned} + on_toggle_pin={Some(ctx.props().on_toggle_pin.clone())} selected_tab={selected_tab_idx} {header_props} > @@ -403,6 +527,16 @@ impl ColumnSettingsPanel { .metadata .locator_view_type(&ctx.props().selected_column); + // Specs are unnamed (the `windows` map key names them, and the + // editable header owns naming in the UI), so drafts and saved specs + // compare directly. + let initial_window_value = ctx + .props() + .selected_column + .name() + .and_then(|name| ctx.props().view_config.windows.get(name)) + .cloned(); + let tabs = { let mut tabs = vec![]; let is_new_expr = ctx.props().selected_column.is_new_expr(); @@ -424,6 +558,17 @@ impl ColumnSettingsPanel { tabs.push(ColumnSettingsTab::Attributes); } + let supports_windows = ctx + .props() + .metadata + .get_features() + .map(|x| x.has_window_aggregates()) + .unwrap_or_default(); + + if ctx.props().selected_column.is_window_editable() && supports_windows { + tabs.push(ColumnSettingsTab::Window); + } + tabs }; @@ -440,6 +585,8 @@ impl ColumnSettingsPanel { header_value: initial_header_value.clone(), initial_header_value, maybe_ty, + window_value: initial_window_value.clone(), + initial_window_value, tabs, header_valid: true, on_input, diff --git a/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs b/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs index cbef66587e..f3f0b4d577 100644 --- a/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs +++ b/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs @@ -326,7 +326,8 @@ pub fn StyleTab(props: &StyleTabProps) -> Html { }, }; - Some(html! {
{ component }
}) + let key = format!("{}::{}", props.column_name, keys.join("+")); + Some(html! {
{ component }
}) }) .collect_vec() }) diff --git a/rust/perspective-viewer/src/rust/components/column_settings_sidebar/window_tab.rs b/rust/perspective-viewer/src/rust/components/column_settings_sidebar/window_tab.rs new file mode 100644 index 0000000000..0e09569827 --- /dev/null +++ b/rust/perspective-viewer/src/rust/components/column_settings_sidebar/window_tab.rs @@ -0,0 +1,34 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use yew::prelude::*; + +use super::save_settings::{SaveSettings, SaveSettingsProps}; +use crate::components::window_editor::{WindowEditor, WindowEditorProps}; + +#[derive(PartialEq, Properties, Clone)] +pub struct WindowTabProps { + pub editor: WindowEditorProps, + pub save_section: SaveSettingsProps, +} + +#[function_component] +pub fn WindowTab(p: &WindowTabProps) -> Html { + html! { +
+
+ +
+
+
+ } +} diff --git a/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs b/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs index f0a7ba9c9c..84295a6919 100644 --- a/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs +++ b/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs @@ -52,6 +52,21 @@ where #[prop_or_default] pub allow_duplicates: bool, + + /// Single-slot mode: the list holds at most one item, a dragover + /// preview REPLACES the current item rather than inserting beside it, + /// and the trailing `EmptyColumn` autocomplete renders only while the + /// slot is empty. + #[prop_or_default] + pub single_slot: bool, + + /// The in-flight drag is INVALID for this list (a parent-defined rule, + /// e.g. the window editor's Table-columns-only slots): the dragover + /// preview is suppressed and the invalid-X overlay renders instead, + /// like a duplicate drag over `group_by`/`split_by`. The parent's drop + /// handler is still responsible for ignoring the drop itself. + #[prop_or_default] + pub is_invalid: bool, } impl PartialEq for DragDropListProps @@ -66,6 +81,8 @@ where && self.allow_duplicates == other.allow_duplicates && self.is_dragover == other.is_dragover && self.disabled == other.disabled + && self.single_slot == other.single_slot + && self.is_invalid == other.is_invalid } } @@ -202,7 +219,51 @@ where let invalid_drag: bool; let mut valid_duplicate_drag = false; - let columns_html = { + let columns_html = if ctx.props().single_slot { + invalid_drag = ctx.props().is_invalid && ctx.props().is_dragover.is_some(); + + // Dragging the slot's own pill over its own slot is a no-op + // move - keep showing the pill instead of the drop preview + // (mirrors the multi-column branch's `is_self_move` handling). + let is_self_move = ctx + .props() + .presentation + .get_drag_target() + .map(|x| V::is_self_move(x)) + .unwrap_or_default(); + + let close = ctx.props().parent.callback(|_| V::close(0)); + let dragenter = ctx.props().parent.callback({ + let container_noderef = container_noderef.clone(); + move |event: DragEvent| { + event.stop_propagation(); + event.prevent_default(); + if event.related_target().is_none() + && let Some(elem) = container_noderef.cast::() + { + let _ = elem.dataset().set("safaridragleave", "true"); + } + V::dragenter(0) + } + }); + + if ctx.props().is_dragover.is_some() && !is_self_move && !invalid_drag { + html! { +
+
+
+ } + } else if let Some(column) = ctx.props().children.iter().next() { + html! { +
+ { Html::from(column) } + +
+ } + } else { + html! {} + } + } else { let mut columns = ctx .props() .children @@ -211,7 +272,10 @@ where .enumerate() .collect::>(); - invalid_drag = if let Some((x, column)) = &ctx.props().is_dragover { + invalid_drag = if ctx.props().is_invalid && ctx.props().is_dragover.is_some() { + // Parent-defined invalidity: no preview, X overlay only. + true + } else if let Some((x, column)) = &ctx.props().is_dragover { let index = *x; let is_append = index == columns.len(); let is_self_move = ctx @@ -314,6 +378,12 @@ where .collect::() }; + let show_empty = if ctx.props().single_slot { + ctx.props().children.is_empty() && ctx.props().is_dragover.is_none() + } else { + ctx.props().is_dragover.is_none() | (!invalid_drag && valid_duplicate_drag) + }; + let column_dropdown = ctx.props().column_dropdown.clone(); let exclude = ctx.props().exclude.clone(); let on_select = ctx.props().parent.callback(V::create); @@ -345,7 +415,7 @@ where onmousedown={ctx.props().parent.callback(move |_| V::close(0))} />
- } else if ctx.props().is_dragover.is_none() | (!invalid_drag && valid_duplicate_drag) { + } else if show_empty { } else if invalid_drag { diff --git a/rust/perspective-viewer/src/rust/components/containers/sidebar.rs b/rust/perspective-viewer/src/rust/components/containers/sidebar.rs index 7d534ff215..bc17aeb40f 100644 --- a/rust/perspective-viewer/src/rust/components/containers/sidebar.rs +++ b/rust/perspective-viewer/src/rust/components/containers/sidebar.rs @@ -11,14 +11,17 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ use perspective_client::clone; -use web_sys::Element; +use wasm_bindgen::JsCast; +use wasm_bindgen::prelude::Closure; +use web_sys::HtmlElement; use yew::{ - Callback, Children, Html, Properties, function_component, html, use_effect_with, use_node_ref, - use_state_eq, + Callback, Children, Html, Properties, function_component, html, use_effect_with, use_mut_ref, + use_node_ref, }; use crate::components::containers::sidebar_close_button::SidebarCloseButton; use crate::components::editable_header::{EditableHeader, EditableHeaderProps}; +use crate::js::{ResizeObserver, ResizeObserverEntry}; #[derive(PartialEq, Clone, Properties)] pub struct SidebarProps { @@ -31,6 +34,28 @@ pub struct SidebarProps { pub width_override: Option, pub selected_tab: Option, pub header_props: EditableHeaderProps, + + /// Trap-door width shared across this sidebar's tabs: the lifted + /// running max of the widths this component reports through + /// `on_auto_width`. Held by the parent (ultimately + /// `PerspectiveViewer`'s geometry state, like the settings panel's + /// Query/Plugin/Debug trap-door) so it survives tab switches AND + /// sidebar re-mounts, and clears on divider reset. + #[prop_or_default] + pub auto_width: f64, + + /// Fires with the sidebar's rendered width after each render; the + /// owner keeps the running max threaded back as `auto_width`. + #[prop_or_default] + pub on_auto_width: Callback, + + /// Pinned state for the header's pin toggle; the button renders only + /// when `on_toggle_pin` is provided. + #[prop_or_default] + pub is_pinned: bool, + + #[prop_or_default] + pub on_toggle_pin: Option>, } /// Sidebars are designed to live in a @@ -39,33 +64,85 @@ pub struct SidebarProps { pub fn Sidebar(p: &SidebarProps) -> Html { let id = &p.id_prefix; let noderef = use_node_ref(); - let auto_width = use_state_eq(|| 0f64); - - // this gets the last calculated width and ensures that - // the auto-width element is at least that big. - // this ensures the panel grows but does not shrink. - use_effect_with(p.clone(), { - clone!(noderef, auto_width); - move |p| { - if p.width_override.is_none() { - let updated_width = noderef - .cast::() - .map(|el| el.get_bounding_client_rect().width()) - .unwrap_or_default(); - let new_auto_width = (*auto_width).max(updated_width); - auto_width.set(new_auto_width); - } else { - auto_width.set(0f64); + + // The trap-door reports the sidebar's rendered width to its owner via + // a `ResizeObserver` on the sidebar element, NOT a render effect: + // width changes are driven by DOM mutations anywhere in the tab + // subtree (e.g. the window editor staging a long column name into a + // slot), which need not re-render this component at all. The observer + // sees every one. `contentRect` (not the border box) is load-bearing: + // the sizer below is a content child, so ratcheting the border box + // would feed any sidebar padding back into unbounded growth. A manual + // divider drag (`width_override`) disables the ratchet until the + // divider resets. + let live_props = use_mut_ref(|| (Callback::::default(), None::)); + *live_props.borrow_mut() = (p.on_auto_width.clone(), p.width_override); + use_effect_with((), { + clone!(noderef, live_props); + move |_| { + let closure = + Closure::::new(move |entries: js_sys::Array| { + let (on_auto_width, width_override) = live_props.borrow().clone(); + if width_override.is_none() { + for entry in entries.iter() { + let entry: ResizeObserverEntry = entry.unchecked_into(); + on_auto_width.emit(entry.content_rect().width()); + } + } + }); + + let observer = ResizeObserver::new(closure.as_ref().unchecked_ref()); + let elem = noderef.cast::(); + if let Some(elem) = &elem { + observer.observe(elem); + } + + move || { + if let Some(elem) = &elem { + observer.unobserve(elem); + } + + drop(closure); } } }); - let width_style = format!("min-width: 200px; width: {}px", *auto_width); + let auto_width = if p.width_override.is_none() { + p.auto_width + } else { + 0.0 + }; + + let width_style = format!("min-width: 200px; width: {}px", auto_width); + let pin_button = p.on_toggle_pin.as_ref().map(|cb| { + let onclick = { + let cb = cb.clone(); + Callback::from(move |_: web_sys::MouseEvent| cb.emit(())) + }; + + let mut class = yew::classes!("sidebar_pin_button"); + if p.is_pinned { + class.push("is-pinned"); + } + + html! { + + } + }); + html! { <>