From 80f4303466380175e1f86b9e68ed42f552b16a42 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 8 Aug 2026 22:14:57 -0400 Subject: [PATCH] Add `STRUCT` and `LIST` support for Arrow/JSON Signed-off-by: Andrew Stein --- docs/md/how_to/javascript/events.md | 29 +- .../javascript/virtual_server/duckdb.md | 47 +- docs/md/how_to/python/table_data.md | 50 + .../md/how_to/python/virtual_server/duckdb.md | 13 + packages/react/README.md | 148 ++ packages/react/src/index.tsx | 53 +- packages/react/src/utils.tsx | 4 + packages/react/src/viewer.tsx | 103 +- .../test/js/superstore.spec.ts | 12 +- rust/perspective-client/build.rs | 4 + rust/perspective-client/perspective.proto | 42 +- rust/perspective-client/src/rust/client.rs | 14 +- .../src/rust/config/windows.rs | 70 +- rust/perspective-client/src/rust/table.rs | 13 + .../src/rust/virtual_server/data.rs | 213 ++- .../src/rust/virtual_server/features.rs | 56 +- .../generic_sql_model/table_make_view.rs | 87 +- .../virtual_server/generic_sql_model/tests.rs | 59 +- .../src/rust/virtual_server/mod.rs | 2 +- rust/perspective-js/src/rust/lib.rs | 2 +- rust/perspective-js/src/rust/typed_array.rs | 25 +- .../src/ts/virtual_servers/clickhouse.ts | 51 +- .../src/ts/virtual_servers/duckdb.ts | 144 +- .../test/js/constructors.spec.js | 47 +- .../test/js/constructors/arrow_nested.spec.ts | 356 ++++ .../test/js/constructors/json_nested.spec.ts | 442 +++++ .../js/constructors/nested_parity.spec.ts | 215 +++ .../test/js/duckdb/client.spec.js | 6 +- .../test/js/duckdb/coerce_types.spec.js | 150 ++ rust/perspective-js/test/js/duckdb/setup.js | 34 + .../test/js/duckdb/typed_arrays.spec.js | 45 + .../test/js/group_rollup_mode.spec.js | 96 + .../tests/table/test_table_arrow_nested.py | 535 ++++++ .../virtual_servers/test_coerce_types.py | 475 +++-- .../tests/virtual_servers/test_duckdb.py | 179 +- .../tests/virtual_servers/test_polars.py | 14 + .../perspective/virtual_servers/clickhouse.py | 45 +- .../perspective/virtual_servers/duckdb.py | 140 +- .../src/client/client_async.rs | 16 +- .../src/client/client_sync.rs | 13 +- .../src/server/virtual_server_sync.rs | 10 - .../cpp/perspective/CMakeLists.txt | 2 + .../cpp/perspective/src/cpp/arrow_loader.cpp | 506 ++++-- .../perspective/src/cpp/arrow_normalize.cpp | 591 +++++++ .../cpp/perspective/src/cpp/context_two.cpp | 6 +- .../cpp/perspective/src/cpp/json_loader.cpp | 1567 +++++++++++++++++ .../cpp/perspective/src/cpp/server.cpp | 197 ++- .../cpp/perspective/src/cpp/table.cpp | 1185 ++----------- .../src/include/perspective/arrow_loader.h | 48 +- .../src/include/perspective/arrow_normalize.h | 66 + .../src/include/perspective/flatten_mode.h | 23 + .../src/include/perspective/json_loader.h | 121 ++ .../src/include/perspective/table.h | 66 +- .../src/include/perspective/traversal.h | 4 + .../src/rust/components/window_editor.rs | 178 +- .../src/rust/custom_events.rs | 29 + .../src/rust/session/metadata.rs | 38 +- rust/perspective-viewer/src/rust/workspace.rs | 132 +- .../test/js/multi_panel/layout_events.spec.ts | 240 +++ 59 files changed, 7124 insertions(+), 1934 deletions(-) create mode 100644 packages/react/README.md create mode 100644 rust/perspective-js/test/js/constructors/arrow_nested.spec.ts create mode 100644 rust/perspective-js/test/js/constructors/json_nested.spec.ts create mode 100644 rust/perspective-js/test/js/constructors/nested_parity.spec.ts create mode 100644 rust/perspective-js/test/js/duckdb/coerce_types.spec.js create mode 100644 rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py create mode 100644 rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp create mode 100644 rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp create mode 100644 rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h create mode 100644 rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h create mode 100644 rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h create mode 100644 rust/perspective-viewer/test/js/multi_panel/layout_events.spec.ts diff --git a/docs/md/how_to/javascript/events.md b/docs/md/how_to/javascript/events.md index 6bbfae084e..c0fa61a1a0 100644 --- a/docs/md/how_to/javascript/events.md +++ b/docs/md/how_to/javascript/events.md @@ -88,8 +88,31 @@ elem.addEventListener("perspective-global-filter-update", function (event) { }); ``` +## Layout events + +A multi-panel `` reports changes to its panel _collection_ +on two separate channels. They are distinct facts — which panels exist, and +which one is selected — so neither event implies the other. + +- `perspective-layout-update` fires when a panel is added to or removed from + the layout. Its `detail.panels` is the placed panel ids in insertion order, + identical to what [`getPanelNames()`](#) returns. +- `perspective-active-panel-update` fires when the active panel changes, with + a `detail.panel` of the new panel's id — or `null` at zero panels. + +```javascript +elem.addEventListener("perspective-layout-update", function (event) { + console.log("Panels are now", event.detail.panels); +}); +``` + +Geometry changes — dragging a split divider, reordering tabs — do **not** fire +these events, because they change the layout tree without changing the panel +set. Use `saveWorkspace()` to read the current geometry. +
The workspace-layout-update and workspace-new-view events from the removed -@perspective-dev/workspace package no longer exist. Use -perspective-config-update and -perspective-global-filter-update.
+@perspective-dev/workspace package no longer exist. +perspective-layout-update is the closest replacement for the +former; for per-panel config changes use +perspective-config-update. diff --git a/docs/md/how_to/javascript/virtual_server/duckdb.md b/docs/md/how_to/javascript/virtual_server/duckdb.md index 23bfbe219e..3ad5cba89a 100644 --- a/docs/md/how_to/javascript/virtual_server/duckdb.md +++ b/docs/md/how_to/javascript/virtual_server/duckdb.md @@ -17,31 +17,60 @@ npm install @perspective-dev/client @perspective-dev/viewer @duckdb/duckdb-wasm Initialize DuckDB-WASM, load data, and connect it to a Perspective viewer: +`DuckDBHandler` is an optional submodule and is _not_ exported from the package +root, so it must be imported by path. It takes an `AsyncDuckDBConnection` — the +result of `db.connect()` — not the `AsyncDuckDB` itself. + ```javascript -import perspective from "@perspective-dev/client"; +import perspective, { createMessageHandler } from "@perspective-dev/client"; import "@perspective-dev/viewer"; import * as duckdb from "@duckdb/duckdb-wasm"; +import { DuckDBHandler } from "@perspective-dev/client/dist/esm/virtual_servers/duckdb.js"; // Initialize DuckDB-WASM const DUCKDB_BUNDLES = duckdb.getJsDelivrBundles(); const bundle = await duckdb.selectBundle(DUCKDB_BUNDLES); -const worker = await duckdb.createWorker(bundle.mainWorker); +const worker_url = URL.createObjectURL( + new Blob([`importScripts("${bundle.mainWorker}");`], { + type: "text/javascript", + }), +); + +const worker = new Worker(worker_url); const logger = new duckdb.ConsoleLogger(); const db = new duckdb.AsyncDuckDB(logger, worker); -await db.instantiate(bundle.mainModule); +await db.instantiate(bundle.mainModule, bundle.pthreadWorker); +URL.revokeObjectURL(worker_url); -// Load data into DuckDB +// Load data into DuckDB. This pragma is required to match Perspective's +// sort-null semantics. const conn = await db.connect(); +await conn.query(`SET default_null_order=NULLS_FIRST_ON_ASC_LAST_ON_DESC;`); await conn.query(`CREATE TABLE my_table AS SELECT * FROM 'data.parquet'`); // Create a Perspective virtual server backed by DuckDB -const handler = perspective.DuckDBHandler(db); -const messageHandler = perspective.createMessageHandler(handler); +const messageHandler = await createMessageHandler(new DuckDBHandler(conn)); -// Connect a viewer +// Connect a viewer. Table ids are database-qualified, so a table created as +// `my_table` is hosted as `memory.my_table`. const client = await perspective.worker(messageHandler); -const table = await client.open_table("my_table"); -document.getElementById("viewer").load(table); +const viewer = document.getElementById("viewer"); +viewer.load(client); +viewer.restore({ table: "memory.my_table" }); +``` + +
In the browser, DuckDBHandler resolves +Perspective's WASM module from the registered +<perspective-viewer> custom element, so it cannot be +constructed until that element has been defined. Off-browser, pass the module +explicitly as the second constructor argument.
+ +Perspective never intercepts your SQL — it only discovers what `SHOW ALL +TABLES` reports — so DuckDB's own remote-data features are available directly: + +```javascript +await conn.query(`CREATE SECRET (TYPE s3, KEY_ID '...', SECRET '...', REGION 'us-east-1')`); +await conn.query(`CREATE TABLE trades AS SELECT * FROM read_parquet('s3://bucket/trades/*.parquet')`); ``` ## Examples diff --git a/docs/md/how_to/python/table_data.md b/docs/md/how_to/python/table_data.md index 8c095cf706..5eb78aa2f9 100644 --- a/docs/md/how_to/python/table_data.md +++ b/docs/md/how_to/python/table_data.md @@ -36,6 +36,56 @@ with open("data.arrow", "rb") as f: table = perspective.table(f.read()) ``` +### Nested columns + +Perspective's data model is flat, so Arrow `struct` and `list` columns are +normalized on ingest. + +A `struct` column is hoisted into one dotted column per leaf, recursively. A +null parent nulls every descendant leaf: + +```python +arrow_table = pa.table({ + "id": pa.array([1, 2], type=pa.int64()), + "s": pa.array([{"a": 10}, {"a": 20}], type=pa.struct([("a", pa.int64())])), +}) + +# Schema is `{"id": "integer", "s.a": "integer"}` +table = perspective.table(arrow_table) +``` + +Because the flattened names are ordinary columns, a `Table` created from an +explicit schema accepts nested updates with no further configuration: + +```python +table = perspective.table({"id": "integer", "s.a": "integer"}) +table.update(arrow_table) +``` + +A `list` column is controlled by the `list_flatten` argument: + +- `"zip"` (default) expands a row into one row per list element, repeating + its non-list siblings. An empty or null list yields a single row with a + null in that column, rather than dropping the row. When a row has more than + one list column, their non-empty lengths must match. +- `"cartesian"` expands a row into the product of its list columns' lengths, + with an empty or null list counting as a single null element. +- `"stringify"` encodes each list as a JSON array in a single string column, + leaving the row count unchanged. + +```python +arrow_table = pa.table({ + "x": pa.array([1, 2], type=pa.int64()), + "y": pa.array([[10, 20], [30]], type=pa.list_(pa.int64())), +}) + +# `{"x": [1, 1, 2], "y": [10, 20, 30]}` +perspective.table(arrow_table) + +# `{"x": [1, 2], "y": ["[10,20]", "[30]"]}` +perspective.table(arrow_table, list_flatten="stringify") +``` + ## Polars ```python diff --git a/docs/md/how_to/python/virtual_server/duckdb.md b/docs/md/how_to/python/virtual_server/duckdb.md index dd8a08f75e..4e80fab695 100644 --- a/docs/md/how_to/python/virtual_server/duckdb.md +++ b/docs/md/how_to/python/virtual_server/duckdb.md @@ -48,6 +48,19 @@ const table = await websocket.open_table("my_table"); document.getElementById("viewer").load(table); ``` +## Window functions + +Window columns are DuckDB's own functions, under their DuckDB names — the +advertised name is emitted into the `OVER` clause verbatim. + +| | | +| --- | --- | +| Aggregating | `sum` `avg` `count` `min` `max` `product` `median` | +| Deviation / variance | `stddev_samp` `stddev_pop` `var_samp` `var_pop` | +| Navigation | `first_value` `last_value` `nth_value` `lag` `lead` | +| Ranking | `row_number` `rank` `dense_rank` `percent_rank` `cume_dist` `ntile` | +| Perspective's own | `diff` `rate` | + ## Examples - [Python DuckDB example](https://github.com/perspective-dev/perspective/tree/master/examples/python-duckdb-virtual) diff --git a/packages/react/README.md b/packages/react/README.md new file mode 100644 index 0000000000..2ff907420e --- /dev/null +++ b/packages/react/README.md @@ -0,0 +1,148 @@ +# `@perspective-dev/react` + +[![npm](https://img.shields.io/npm/v/@perspective-dev/react.svg?style=for-the-badge)](https://www.npmjs.com/package/@perspective-dev/react) + +React bindings for [Perspective](https://perspective-dev.github.io/), an +interactive analytics and data visualization component for large, real-time +and streaming datasets. This package wraps the +[``](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html) +Custom Element in an idiomatic, declarative React component, +``, which manages the element's imperative +`load()`/`restore()`/`delete()` lifecycle for you. + +## Installation + +```bash +npm install @perspective-dev/react +``` + +`@perspective-dev/client` and `@perspective-dev/viewer` are installed as +dependencies, but you'll also want at least one plugin package for the +visualizations themselves: + +```bash +npm install @perspective-dev/viewer-datagrid @perspective-dev/viewer-charts +``` + +## Setup + +Perspective's engine and UI are WebAssembly binaries which must be initialized +once, before the first `` renders. Plugins register +themselves via import side effects. See the +[User Guide's bundling section](https://perspective-dev.github.io/guide/how_to/javascript/importing.html) +for bundler configuration details. + +```tsx +import perspective from "@perspective-dev/client"; +import perspective_viewer from "@perspective-dev/viewer"; +import "@perspective-dev/viewer-datagrid"; +import "@perspective-dev/viewer-charts"; +import "@perspective-dev/viewer/dist/css/themes.css"; + +import SERVER_WASM from "@perspective-dev/server/dist/wasm/perspective-server.wasm"; +import CLIENT_WASM from "@perspective-dev/viewer/dist/wasm/perspective-viewer.wasm"; + +await Promise.all([ + perspective.init_server(fetch(SERVER_WASM)), + perspective_viewer.init_client(fetch(CLIENT_WASM)), +]); +``` + +## Usage + +Create a `Table` (here in a Web Worker `Client`) and pass it — or a `Promise` +of it — to ``: + +```tsx +import * as React from "react"; +import { PerspectiveViewer } from "@perspective-dev/react"; + +const WORKER = await perspective.worker(); + +const TABLE = WORKER.table( + fetch("superstore.lz4.arrow").then((resp) => resp.arrayBuffer()), + { name: "superstore" }, +); + +const App: React.FC = () => ( + +); +``` + +## Props + +| Prop | Type | Description | +| :--------------- | :------------------------------------------------------------ | :-------------------------------------------------------------- | +| `client` | `Client \| Table \| Promise \| Promise` | Data source. When `undefined`, the viewer `eject()`s. | +| `config` | `ViewerConfigUpdate \| WorkspaceConfigUpdate` | Declarative viewer state, applied via `restore()`. | +| `onConfigUpdate` | `(config: ViewerConfigUpdate) => void` | Called when the user reconfigures the viewer through its UI. | +| `onClick` | `(detail: PerspectiveClickEventDetail) => void` | Called when the user clicks a datapoint. | +| `onSelect` | `(detail: PerspectiveSelectEventDetail) => void` | Called when the user selects (or deselects) a datapoint or row. | + +A subset of standard HTML attributes — `className`, `id`, `style`, `hidden`, +`slot`, `tabIndex` and `title` — is forwarded to the underlying element. + +### `client` + +The viewer's data source, forwarded to +[`viewer.load()`](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html) +whenever it changes: + +- A `Table` (or `Promise
`) displays that table directly. +- A `Client` (e.g. from `perspective.worker()` or a WebSocket connection to a + remote server) connects the viewer to every table hosted by that client; + the table each panel displays is chosen by `config` or interactively by + the user. +- `undefined` ejects the viewer, returning it to an unloaded state without + unmounting it. + +The component does not take ownership of the `Table` — delete it yourself +when it is no longer needed (e.g. `table.delete({ lazy: true })`). + +### `config` + +Declarative viewer state — group-bys, splits, filters, sorts, expressions, +plugin and plugin config — applied with `restore()` whenever it (or `client`) +changes. A config with a `panels` property is treated as a multi-panel +workspace layout and applied with `restoreWorkspace()` instead. Configs are +compared structurally, so passing a fresh-but-equal object literal on each +render does not re-apply. + +Combine `config` with `onConfigUpdate` to make the viewer a controlled +component — store the user's latest configuration in state (or persist it) and +pass it back down: + +```tsx +const App: React.FC = () => { + const [config, setConfig] = React.useState({ + group_by: ["Category"], + }); + + return ( + + ); +}; +``` + +## Lifecycle + +On unmount, the component calls the element's `delete()` method, freeing the +viewer's WebAssembly resources. Tables and clients are created outside the +component and are yours to manage; a `Table` passed as `client` survives +unmount and can be shown again by a later mount. + +## See also + +- [`react-example`](https://github.com/perspective-dev/perspective/tree/master/examples/react-example) + — a complete bundler-configured project using this package, including a + multi-panel workspace config. +- [Perspective User Guide](https://perspective-dev.github.io/guide/) +- [`` API documentation](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html) +- [`@perspective-dev/client` API documentation](https://perspective-dev.github.io/browser/modules/src_ts_perspective.browser.ts.html) diff --git a/packages/react/src/index.tsx b/packages/react/src/index.tsx index 5b89cc2616..28dee2e02f 100644 --- a/packages/react/src/index.tsx +++ b/packages/react/src/index.tsx @@ -11,11 +11,60 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ /** + * React bindings for [Perspective](https://perspective-dev.github.io/). + * + * This module exports {@link PerspectiveViewer}, a declarative React wrapper + * for the `` Custom Element. The component manages the + * element's imperative lifecycle — `load()` and `restore()` in response to + * prop changes, `delete()` on unmount — and exposes the element's Custom + * Events as React-style callback props. + * + * Perspective's WebAssembly engine and UI must be initialized (and at least + * one plugin package imported) before the first `` + * renders: + * + * ```tsx + * import perspective from "@perspective-dev/client"; + * import perspective_viewer from "@perspective-dev/viewer"; + * import "@perspective-dev/viewer-datagrid"; + * import "@perspective-dev/viewer-charts"; + * + * import SERVER_WASM from "@perspective-dev/server/dist/wasm/perspective-server.wasm"; + * import CLIENT_WASM from "@perspective-dev/viewer/dist/wasm/perspective-viewer.wasm"; + * + * await Promise.all([ + * perspective.init_server(fetch(SERVER_WASM)), + * perspective_viewer.init_client(fetch(CLIENT_WASM)), + * ]); + * ``` + * + * Then pass a `Table` (or `Client`, or a `Promise` of either) and an optional + * config: + * + * ```tsx + * import { PerspectiveViewer } from "@perspective-dev/react"; + * + * const WORKER = await perspective.worker(); + * const TABLE = WORKER.table( + * fetch("superstore.lz4.arrow").then((resp) => resp.arrayBuffer()), + * { name: "superstore" }, + * ); + * + * const App: React.FC = () => ( + * + * ); + * ``` * * # See Also * - * [`react-example`](https://github.com/perspective-dev/perspective/tree/master/examples/react-example) - * project from the Perspective GitHub repo. + * - [`react-example`](https://github.com/perspective-dev/perspective/tree/master/examples/react-example) + * project from the Perspective GitHub repo, a complete bundler-configured + * application including a multi-panel workspace config. + * - [Perspective User Guide](https://perspective-dev.github.io/guide/) + * - [`` API documentation](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html) * * @module */ diff --git a/packages/react/src/utils.tsx b/packages/react/src/utils.tsx index db4c20ee7c..1b74ccdb67 100644 --- a/packages/react/src/utils.tsx +++ b/packages/react/src/utils.tsx @@ -12,6 +12,10 @@ import * as React from "react"; +/** + * Subscribe `cb` to a Custom Event on `el` for the lifetime of the component, + * projecting the event through `map` (by default, `e.detail`). + */ export function usePspListener( el: HTMLElement | undefined | null, event: string, diff --git a/packages/react/src/viewer.tsx b/packages/react/src/viewer.tsx index 9010ecbd25..ba8607d849 100644 --- a/packages/react/src/viewer.tsx +++ b/packages/react/src/viewer.tsx @@ -78,28 +78,127 @@ function PerspectiveViewerImpl(props: PerspectiveViewerProps) { } /** - * Props for the `` component. + * Props for the {@link PerspectiveViewer} component. */ export interface PerspectiveViewerProps { + /** + * The viewer's data source, forwarded to the element's `load()` method + * whenever it changes. + * + * - A `Table` (or `Promise
`) displays that table directly. + * - A `Client` (e.g. from `perspective.worker()` or a WebSocket + * connection to a remote server) connects the viewer to every table + * hosted by that client; the table each panel displays is chosen by + * {@link PerspectiveViewerProps.config} or interactively by the user. + * - `undefined` calls `eject()`, returning the viewer to an unloaded + * state without unmounting it. + * + * The component does not take ownership of the `Table` — delete it + * yourself when it is no longer needed, e.g. + * `table.delete({ lazy: true })`. + */ client?: psp.Client | Promise | psp.Table | Promise; + + /** + * Declarative viewer state — group-bys, splits, filters, sorts, + * expressions, plugin and plugin config — applied with the element's + * `restore()` method whenever it (or + * {@link PerspectiveViewerProps.client}) changes. A config with a + * `panels` property is treated as a multi-panel workspace layout and + * applied with `restoreWorkspace()` instead. + * + * Configs are compared structurally, so passing a fresh-but-equal object + * literal on each render does not re-apply. Combine with + * {@link PerspectiveViewerProps.onConfigUpdate} to use the viewer as a + * controlled component. + */ config?: pspViewer.ViewerConfigUpdate | pspViewer.WorkspaceConfigUpdate; + + /** + * Called with the viewer's complete configuration whenever the user + * reconfigures it through its UI (the element's + * `"perspective-config-update"` Custom Event). Use this to persist the + * user's view or reflect it back through + * {@link PerspectiveViewerProps.config}. + */ onConfigUpdate?: (config: pspViewer.ViewerConfigUpdate) => void; + + /** + * Called when the user clicks a datapoint (the element's + * `"perspective-click"` Custom Event), with the clicked row's data and a + * filter config which would select it. + */ onClick?: (data: pspViewer.PerspectiveClickEventDetail) => void; + + /** + * Called when the user selects or deselects a datapoint or row (the + * element's `"perspective-select"` Custom Event). + */ onSelect?: (data: pspViewer.PerspectiveSelectEventDetail) => void; // Applicable props from `React.HTMLAttributes`, which we cannot extend // directly because Perspective changes the signature of `onClick`. + + /** Forwarded to the element's `class` attribute. */ className?: string | undefined; + + /** Forwarded to the element's `hidden` attribute. */ hidden?: boolean | undefined; + + /** Forwarded to the element's `id` attribute. */ id?: string | undefined; + + /** Forwarded to the element's `slot` attribute. */ slot?: string | undefined; + + /** Forwarded to the element's inline `style`. */ style?: React.CSSProperties | undefined; + + /** Forwarded to the element's `tabindex` attribute. */ tabIndex?: number | undefined; + + /** Forwarded to the element's `title` attribute. */ title?: string | undefined; } /** - * A React wrapper component for `` Custom Element. + * A declarative React wrapper for the `` Custom Element. + * + * `` manages the element's imperative lifecycle: it + * `load()`s the {@link PerspectiveViewerProps.client}, `restore()`s the + * {@link PerspectiveViewerProps.config} whenever either changes, subscribes + * the `on*` callback props to the element's Custom Events, and `delete()`s + * the viewer (freeing its WebAssembly resources) on unmount. Tables and + * clients are created outside the component and are yours to manage — a + * `Table` passed as `client` survives unmount and can be shown again by a + * later mount. + * + * The component is memoized, so re-rendering a parent with unchanged props + * does not touch the viewer. + * + * @example + * ```tsx + * const WORKER = await perspective.worker(); + * const TABLE = WORKER.table( + * fetch("superstore.lz4.arrow").then((resp) => resp.arrayBuffer()), + * { name: "superstore" }, + * ); + * + * const App: React.FC = () => { + * const [config, setConfig] = React.useState({ + * group_by: ["State"], + * plugin: "Y Bar", + * }); + * + * return ( + * + * ); + * }; + * ``` */ export const PerspectiveViewer: React.FC = React.memo( PerspectiveViewerImpl, diff --git a/packages/viewer-datagrid/test/js/superstore.spec.ts b/packages/viewer-datagrid/test/js/superstore.spec.ts index 8525297537..7dd440b6a3 100644 --- a/packages/viewer-datagrid/test/js/superstore.spec.ts +++ b/packages/viewer-datagrid/test/js/superstore.spec.ts @@ -188,6 +188,16 @@ test.describe("Datagrid with superstore data set", () => { columns: ["State", "City", "Customer ID"], }); + const { resolve, promise } = Promise.withResolvers(); + window.__promise__ = promise; + const view = await document + .querySelector("perspective-viewer")! + .getView(); + + await view.on_update(() => { + resolve(true); + }); + return ( document.querySelector("perspective-viewer-datagrid") as any ).shadowRoot.querySelector("table tbody tr td"); @@ -199,7 +209,7 @@ test.describe("Datagrid with superstore data set", () => { (document.activeElement as HTMLElement | null)?.blur(), ); const result = await page.evaluate(async () => { - await document.querySelector("perspective-viewer")!.flush(); + await window.__promise__; const view = await document .querySelector("perspective-viewer")! .getView(); diff --git a/rust/perspective-client/build.rs b/rust/perspective-client/build.rs index 3fa3f6b08f..b1674b6504 100644 --- a/rust/perspective-client/build.rs +++ b/rust/perspective-client/build.rs @@ -61,6 +61,10 @@ fn prost_build() -> Result<()> { "JoinType", "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]", ) + .type_attribute( + "ListFlatten", + "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]", + ) .field_attribute("ViewToArrowResp.arrow", "#[serde(skip)]") .field_attribute("from_arrow", "#[serde(skip)]") .type_attribute(".", "#[derive(serde::Serialize)]") diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index 35a1e36ec4..7cd08cdd4c 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -234,7 +234,7 @@ message GetFeaturesResp { bool unordered = 10; message WindowAggregateOptions { - repeated WindowAggregate options = 1; + repeated WindowAggregateArgs options = 1; } message ColumnTypeOptions { @@ -251,6 +251,14 @@ message GetFeaturesResp { } } +message WindowAggregateArgs { + string name = 1; + repeated string frames = 2; + bool offset = 3; + bool alpha = 4; + optional ColumnType result_type = 5; +} + // `Client::get_hosted_tables` message GetHostedTablesReq { bool subscribe = 1; @@ -346,8 +354,16 @@ message MakeTableReq { // (memory-mapped file on native; OPFS on WASM) instead of memory. // Orthogonal to `make_table_type`, so it is a standalone field. optional bool page_to_disk = 3; + + optional ListFlatten list_flatten = 4; } } + +enum ListFlatten { + LIST_FLATTEN_ZIP = 0; + LIST_FLATTEN_CARTESIAN = 1; + LIST_FLATTEN_STRINGIFY = 2; +} message MakeTableResp {} enum JoinType { @@ -546,26 +562,14 @@ message ServerSystemInfoResp { } -enum WindowAggregate { - WINDOW_AGGREGATE_SUM = 0; - WINDOW_AGGREGATE_AVG = 1; - WINDOW_AGGREGATE_COUNT = 2; - WINDOW_AGGREGATE_MIN = 3; - WINDOW_AGGREGATE_MAX = 4; - WINDOW_AGGREGATE_STDDEV = 5; - WINDOW_AGGREGATE_VAR = 6; - WINDOW_AGGREGATE_FIRST = 7; - WINDOW_AGGREGATE_LAST = 8; - WINDOW_AGGREGATE_LAG = 9; - WINDOW_AGGREGATE_LEAD = 10; - WINDOW_AGGREGATE_DIFF = 11; - WINDOW_AGGREGATE_RATE = 12; - WINDOW_AGGREGATE_EMA = 13; -} - message WindowSpec { string source = 2; - WindowAggregate op = 3; + + // The aggregate, in the data model's own vocabulary - see + // `GetFeaturesResp.WindowAggregateArgs`. A string rather than an enum so + // that a `View` can carry an op the built-in engine has never heard of, + // e.g. kdb+'s `mdev`. + string op = 3; repeated string partition_by = 4; Order order_by = 5; oneof frame { diff --git a/rust/perspective-client/src/rust/client.rs b/rust/perspective-client/src/rust/client.rs index 8f07eb4f2a..3268dc090a 100644 --- a/rust/perspective-client/src/rust/client.rs +++ b/rust/perspective-client/src/rust/client.rs @@ -127,16 +127,10 @@ impl GetFeaturesResp { pub fn get_window_aggregates( &self, col_type: ColumnType, - ) -> Vec { + ) -> Vec { self.window_aggregates .get(&(col_type as u32)) - .map(|x| { - x.options - .iter() - .filter_map(|x| crate::proto::WindowAggregate::try_from(*x).ok()) - .map(|x| x.into()) - .collect() - }) + .map(|x| x.options.clone()) .unwrap_or_default() } @@ -645,6 +639,7 @@ impl Client { index: Some(on.to_owned()), limit: None, page_to_disk: None, + list_flatten: None, })), resp => Err(resp.into()), } @@ -689,9 +684,8 @@ impl Client { let options = TableOptions { index: info.index, limit: info.limit, - // `page_to_disk` is a server-side property not surfaced in table - // info; it does not affect client-side behavior. page_to_disk: None, + list_flatten: None, }; let client = self.clone(); diff --git a/rust/perspective-client/src/rust/config/windows.rs b/rust/perspective-client/src/rust/config/windows.rs index 464470eed7..f5c754219c 100644 --- a/rust/perspective-client/src/rust/config/windows.rs +++ b/rust/perspective-client/src/rust/config/windows.rs @@ -27,24 +27,8 @@ use ts_rs::TS; use crate::proto; -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, TS)] -#[serde(rename_all = "snake_case")] -pub enum WindowAggregate { - Sum, - Avg, - Count, - Min, - Max, - Stddev, - Var, - First, - Last, - Lag, - Lead, - Diff, - Rate, - Ema, -} +/// A window aggregate, named in the data model's own vocabulary. +pub type WindowAggregate = String; #[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] @@ -221,48 +205,6 @@ impl TryFrom for WindowSpec { } } -impl From for proto::WindowAggregate { - fn from(value: WindowAggregate) -> Self { - match value { - WindowAggregate::Sum => Self::Sum, - WindowAggregate::Avg => Self::Avg, - WindowAggregate::Count => Self::Count, - WindowAggregate::Min => Self::Min, - WindowAggregate::Max => Self::Max, - WindowAggregate::Stddev => Self::Stddev, - WindowAggregate::Var => Self::Var, - WindowAggregate::First => Self::First, - WindowAggregate::Last => Self::Last, - WindowAggregate::Lag => Self::Lag, - WindowAggregate::Lead => Self::Lead, - WindowAggregate::Diff => Self::Diff, - WindowAggregate::Rate => Self::Rate, - WindowAggregate::Ema => Self::Ema, - } - } -} - -impl From for WindowAggregate { - fn from(value: proto::WindowAggregate) -> Self { - match value { - proto::WindowAggregate::Sum => Self::Sum, - proto::WindowAggregate::Avg => Self::Avg, - proto::WindowAggregate::Count => Self::Count, - proto::WindowAggregate::Min => Self::Min, - proto::WindowAggregate::Max => Self::Max, - proto::WindowAggregate::Stddev => Self::Stddev, - proto::WindowAggregate::Var => Self::Var, - proto::WindowAggregate::First => Self::First, - proto::WindowAggregate::Last => Self::Last, - proto::WindowAggregate::Lag => Self::Lag, - proto::WindowAggregate::Lead => Self::Lead, - proto::WindowAggregate::Diff => Self::Diff, - proto::WindowAggregate::Rate => Self::Rate, - proto::WindowAggregate::Ema => Self::Ema, - } - } -} - impl From for proto::window_spec::Frame { fn from(value: WindowFrame) -> Self { match value { @@ -309,7 +251,7 @@ impl From for proto::WindowSpec { fn from(value: WindowSpec) -> Self { proto::WindowSpec { source: value.column, - op: proto::WindowAggregate::from(value.aggregate) as i32, + op: value.aggregate, partition_by: value.partition_by, order_by: value.order_by.map(|x| x.into()), frame: value.frame.map(|x| x.into()), @@ -323,9 +265,7 @@ impl From for WindowSpec { fn from(value: proto::WindowSpec) -> Self { WindowSpec { column: value.source, - aggregate: proto::WindowAggregate::try_from(value.op) - .unwrap_or(proto::WindowAggregate::Sum) - .into(), + aggregate: value.op, partition_by: value.partition_by, order_by: value.order_by.map(WindowSort::from), frame: value.frame.map(|x| x.into()), @@ -342,7 +282,7 @@ mod tests { fn spec(frame: Option) -> WindowSpec { WindowSpec { column: "price".to_string(), - aggregate: WindowAggregate::Sum, + aggregate: "sum".to_string(), partition_by: vec![], order_by: None, frame, diff --git a/rust/perspective-client/src/rust/table.rs b/rust/perspective-client/src/rust/table.rs index 6ecc5117f6..805aeabc85 100644 --- a/rust/perspective-client/src/rust/table.rs +++ b/rust/perspective-client/src/rust/table.rs @@ -99,6 +99,15 @@ pub struct TableInitOptions { #[serde(default)] #[ts(optional)] pub page_to_disk: Option, + + /// How Arrow `LIST` and JSON `Array` columns are ingested. `zip` (the + /// default) and `cartesian` expand a row into one row per list element, + /// and are incompatible with `index`, as the rows of an expansion + /// repeat their index. `stringify` encodes each list as a JSON array in + /// a single string column instead. + #[serde(default)] + #[ts(optional)] + pub list_flatten: Option, } impl TableInitOptions { @@ -112,8 +121,10 @@ impl TryFrom for MakeTableOptions { fn try_from(value: TableOptions) -> Result { let page_to_disk = value.page_to_disk; + let list_flatten = value.list_flatten.map(|x| x as i32); Ok(MakeTableOptions { page_to_disk, + list_flatten, make_table_type: match value { TableOptions { index: Some(_), @@ -137,6 +148,7 @@ pub(crate) struct TableOptions { pub index: Option, pub limit: Option, pub page_to_disk: Option, + pub list_flatten: Option, } impl From for TableOptions { @@ -145,6 +157,7 @@ impl From for TableOptions { index: value.index, limit: value.limit, page_to_disk: value.page_to_disk, + list_flatten: value.list_flatten, } } } diff --git a/rust/perspective-client/src/rust/virtual_server/data.rs b/rust/perspective-client/src/rust/virtual_server/data.rs index 47b5b3924c..900ba84205 100644 --- a/rust/perspective-client/src/rust/virtual_server/data.rs +++ b/rust/perspective-client/src/rust/virtual_server/data.rs @@ -20,12 +20,12 @@ use arrow_array::builder::{ use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; use arrow_array::{ - Array, ArrayAccessor, ArrayRef, BooleanArray, Date32Array, Date64Array, Decimal128Array, - Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeStringArray, - RecordBatch, StringArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray, - Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, - TimestampNanosecondArray, TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, - UInt64Array, + Array, ArrayRef, BooleanArray, Date32Array, Date64Array, Decimal128Array, DictionaryArray, + Float16Array, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, + LargeStringArray, RecordBatch, RecordBatchOptions, StringArray, Time32MillisecondArray, + Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray, + TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt8Array, + UInt16Array, UInt32Array, UInt64Array, }; use arrow_ipc::reader::{FileReader, StreamReader}; use arrow_ipc::writer::StreamWriter; @@ -49,6 +49,33 @@ fn dict_data_type() -> DataType { DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)) } +/// Reads a cell from a canonical `Dictionary(Int32, Utf8)` column, or +/// `None` for a null slot. +fn dict_str_value(col: &ArrayRef, row_idx: usize) -> Option<&str> { + if col.is_null(row_idx) { + return None; + } + + let typed = col + .as_any() + .downcast_ref::>() + .and_then(|dict| { + let values = dict.values().as_any().downcast_ref::()?; + Some((dict, values)) + }); + + match typed { + Some((dict, values)) => { + let key = dict.keys().value(row_idx) as usize; + (key < values.len() && !values.is_null(key)).then(|| values.value(key)) + }, + None => { + tracing::error!("Non-canonical dictionary column {}", col.data_type()); + None + }, + } +} + /// A single cell value in a row-oriented data representation. /// /// Used when converting [`VirtualDataSlice`] to row format for JSON @@ -289,21 +316,15 @@ fn cast_to_int64(array: &ArrayRef) -> Result, Box> { Ok(result) } -/// Extracts a single cell from an Arrow array as a [`Scalar`]. +/// Extracts a single cell from a *coerced* Arrow array as a [`Scalar`]. fn extract_scalar(array: &ArrayRef, row_idx: usize) -> Scalar { if array.is_null(row_idx) { return Scalar::Null; } match array.data_type() { - DataType::Utf8 => { - let arr = array.as_any().downcast_ref::().unwrap(); - Scalar::String(arr.value(row_idx).to_string()) - }, - DataType::Dictionary(..) => { - let dict = array.as_dictionary::(); - let values = dict.downcast_dict::().unwrap(); - Scalar::String(values.value(row_idx).to_string()) - }, + DataType::Dictionary(..) => dict_str_value(array, row_idx) + .map(|x| Scalar::String(x.to_string())) + .unwrap_or(Scalar::Null), DataType::Float64 => { let arr = array.as_any().downcast_ref::().unwrap(); Scalar::Float(arr.value(row_idx)) @@ -312,10 +333,6 @@ fn extract_scalar(array: &ArrayRef, row_idx: usize) -> Scalar { let arr = array.as_any().downcast_ref::().unwrap(); Scalar::Float(arr.value(row_idx) as f64) }, - DataType::Int64 => { - let arr = array.as_any().downcast_ref::().unwrap(); - Scalar::Float(arr.value(row_idx) as f64) - }, DataType::Boolean => { let arr = array.as_any().downcast_ref::().unwrap(); Scalar::Bool(arr.value(row_idx)) @@ -331,9 +348,9 @@ fn extract_scalar(array: &ArrayRef, row_idx: usize) -> Scalar { let arr = array.as_any().downcast_ref::().unwrap(); Scalar::Float(arr.value(row_idx) as f64 * 86_400_000.0) }, - _ => { - let scalar_arr = array.slice(row_idx, 1); - Scalar::String(format!("{:?}", scalar_arr)) + dt => { + tracing::error!("Non-canonical row path type {}", dt); + Scalar::Null }, } } @@ -365,7 +382,12 @@ fn timestamp_to_millis(array: &ArrayRef, unit: &TimeUnit) -> ArrayRef { arr.iter().map(|v| v.map(|v| v / 1_000_000)).collect() }, TimeUnit::Millisecond => { - return array.clone(); + let arr = array + .as_any() + .downcast_ref::() + .unwrap(); + + return Arc::new(arr.clone().with_timezone_opt(None::>)) as ArrayRef; }, }; Arc::new(millis) as ArrayRef @@ -381,7 +403,19 @@ fn coerce_column( Field::new(name, field.data_type().clone(), true), array.clone(), )), - DataType::Dictionary(..) => Ok((Field::new(name, dict_data_type(), true), array.clone())), + DataType::Dictionary(key, value) => { + if key.as_ref() == &DataType::Int32 && value.as_ref() == &DataType::Utf8 { + return Ok((Field::new(name, dict_data_type(), true), array.clone())); + } + + let dict = array + .as_any_dictionary_opt() + .ok_or_else(|| format!("Column '{}' is not a dictionary array", name))?; + + let values = arrow_select::take::take(dict.values(), dict.keys(), None)?; + let field = Field::new(name, values.data_type().clone(), true); + coerce_column(name, &field, &values) + }, DataType::Utf8 => { let arr = array.as_any().downcast_ref::().unwrap(); let mut builder = StringDictionaryBuilder::::new(); @@ -397,7 +431,7 @@ fn coerce_column( Arc::new(builder.finish()) as ArrayRef, )) }, - DataType::Timestamp(TimeUnit::Millisecond, _) => Ok(( + DataType::Timestamp(TimeUnit::Millisecond, None) => Ok(( Field::new(name, DataType::Timestamp(TimeUnit::Millisecond, None), true), array.clone(), )), @@ -466,6 +500,14 @@ fn coerce_column( Arc::new(result) as ArrayRef, )) }, + DataType::Float16 => { + let arr = array.as_any().downcast_ref::().unwrap(); + let result: Float64Array = arr.iter().map(|v| v.map(|v| v.to_f64())).collect(); + Ok(( + Field::new(name, DataType::Float64, true), + Arc::new(result) as ArrayRef, + )) + }, DataType::Decimal128(_, scale) => { let scale = *scale; let arr = array.as_any().downcast_ref::().unwrap(); @@ -600,37 +642,35 @@ impl VirtualDataSlice { /// and tree-hierarchy walkers) see them inline — matching the /// native `perspective-server`'s `to_arrow` output when /// `emit_legacy_row_path_names: false`. - /// - /// Also coerces non-standard Arrow types (e.g. `Decimal128`, `Int64`) - /// to Perspective-compatible types. Data column names are passed - /// through verbatim — pivoted views already name columns with - /// Perspective's column-path separator. pub fn from_arrow_ipc(&mut self, ipc: &[u8]) -> Result<(), Box> { let cursor = std::io::Cursor::new(ipc); - let batches: Vec = if &ipc[0..6] == "ARROW1".as_bytes() { - FileReader::try_new(cursor, None)?.collect::, _>>()? + let (ipc_schema, batches) = if &ipc[0..6] == "ARROW1".as_bytes() { + let reader = FileReader::try_new(cursor, None)?; + let schema = reader.schema(); + (schema, reader.collect::, _>>()?) } else { - StreamReader::try_new(cursor, None)?.collect::, _>>()? + let reader = StreamReader::try_new(cursor, None)?; + let schema = reader.schema(); + (schema, reader.collect::, _>>()?) }; let batch = match batches.len() { - 0 => return Err("Arrow IPC stream contained no record batches".into()), + 0 => RecordBatch::new_empty(ipc_schema), 1 => batches.into_iter().next().unwrap(), _ => arrow_select::concat::concat_batches(&batches[0].schema(), &batches)?, }; let has_group_by = !self.config.group_by.is_empty(); - let has_split_by = !self.config.split_by.is_empty(); - let is_total = self.config.group_rollup_mode == GroupRollupMode::Total; - - if !has_group_by && !has_split_by && !is_total { - self.frozen = Some(batch); - return Ok(()); - } - let num_rows = batch.num_rows(); let schema = batch.schema(); + let coerced = schema + .fields() + .iter() + .enumerate() + .map(|(col_idx, field)| coerce_column(field.name(), field, batch.column(col_idx))) + .collect::, _>>()?; + // Phase A: Extract row_path from __GROUPING_ID__ and __ROW_PATH_N__ if has_group_by { let group_by_len = self.config.group_by.len(); @@ -641,7 +681,7 @@ impl VirtualDataSlice { let grouping_id_idx = schema .index_of("__GROUPING_ID__") .map_err(|_| "Missing __GROUPING_ID__ column")?; - Some(cast_to_int64(batch.column(grouping_id_idx))?) + Some(cast_to_int64(&coerced[grouping_id_idx].1)?) }; let mut row_paths: Vec> = (0..num_rows).map(|_| Vec::new()).collect(); @@ -651,7 +691,7 @@ impl VirtualDataSlice { .index_of(&col_name) .map_err(|_| format!("Missing {} column", col_name))?; - let col = batch.column(col_idx); + let col = &coerced[col_idx].1; // In flat mode, all rows are leaf rows if is_flat { @@ -676,11 +716,9 @@ impl VirtualDataSlice { self.row_path = Some(row_paths); } - // Phase B: Rebuild RecordBatch without metadata columns, with - // column renames and type coercion. let mut new_fields = Vec::new(); let mut new_arrays: Vec = Vec::new(); - for (col_idx, field) in schema.fields().iter().enumerate() { + for (field, array) in coerced { let name = field.name(); // `__GROUPING_ID__` is an internal SQL-rollup discriminator // (used in Phase A above to decide which row-path levels @@ -701,17 +739,18 @@ impl VirtualDataSlice { continue; } - let (coerced_field, coerced_array) = coerce_column(name, field, batch.column(col_idx))?; - new_fields.push(coerced_field); - new_arrays.push(coerced_array); + new_fields.push(field); + new_arrays.push(array); } let new_schema = Arc::new(Schema::new(new_fields)); - self.frozen = if new_arrays.is_empty() { - Some(RecordBatch::new_empty(new_schema)) + self.frozen = Some(if new_arrays.is_empty() { + let options = RecordBatchOptions::new().with_row_count(Some(num_rows)); + RecordBatch::try_new_with_options(new_schema, new_arrays, &options)? } else { - Some(RecordBatch::try_new(new_schema, new_arrays)?) - }; + RecordBatch::try_new(new_schema, new_arrays)? + }); + Ok(()) } @@ -750,11 +789,19 @@ impl VirtualDataSlice { } let schema = Arc::new(Schema::new(fields)); - self.frozen = Some( + let batch = if arrays.is_empty() { + let num_rows = self.row_path.as_ref().map(|x| x.len()).unwrap_or(0); + let options = RecordBatchOptions::new().with_row_count(Some(num_rows)); + RecordBatch::try_new_with_options(schema, arrays, &options) + } else { RecordBatch::try_new(schema, arrays) - .expect("RecordBatch construction should not fail for well-formed builders"), + }; + + self.frozen = Some( + batch.expect("RecordBatch construction should not fail for well-formed builders"), ); } + self.frozen.as_ref().unwrap() } @@ -786,6 +833,13 @@ impl VirtualDataSlice { let num_rows = batch.num_rows(); let schema = batch.schema(); + let synthesize_row_path = style == RowPathStyle::PerLevel + && self.row_path.is_some() + && !schema + .fields() + .iter() + .any(|x| x.name().starts_with("__ROW_PATH_")); + (0..num_rows) .map(|row_idx| { let mut row = IndexMap::new(); @@ -799,6 +853,23 @@ impl VirtualDataSlice { ); } + if synthesize_row_path + && let Some(ref rp) = self.row_path + && row_idx < rp.len() + { + for level in 0..self.config.group_by.len() { + row.insert( + format!("__ROW_PATH_{}__", level), + match rp[row_idx].get(level) { + Some(Scalar::String(x)) => VirtualDataCell::String(Some(x.clone())), + Some(Scalar::Float(x)) => VirtualDataCell::Float(Some(*x)), + Some(Scalar::Bool(x)) => VirtualDataCell::Boolean(Some(*x)), + Some(Scalar::Null) | None => VirtualDataCell::String(None), + }, + ); + } + } + for (col_idx, field) in schema.fields().iter().enumerate() { if style == RowPathStyle::Sidecar && field.name().starts_with("__ROW_PATH_") { continue; @@ -828,11 +899,9 @@ impl VirtualDataSlice { let arr = col.as_any().downcast_ref::().unwrap(); VirtualDataCell::String(Some(arr.value(row_idx).to_string())) }, - DataType::Dictionary(..) => { - let dict = col.as_dictionary::(); - let values = dict.downcast_dict::().unwrap(); - VirtualDataCell::String(Some(values.value(row_idx).to_string())) - }, + DataType::Dictionary(..) => VirtualDataCell::String( + dict_str_value(col, row_idx).map(|x| x.to_string()), + ), DataType::Float64 => { let arr = col.as_any().downcast_ref::().unwrap(); VirtualDataCell::Float(Some(arr.value(row_idx))) @@ -955,21 +1024,11 @@ impl VirtualDataSlice { .collect::>(), )? }, - DataType::Dictionary(..) => { - let dict = col.as_dictionary::(); - let values = dict.downcast_dict::().unwrap(); - serde_json::to_value( - (0..num_rows) - .map(|i| { - if col.is_null(i) { - None - } else { - Some(values.value(i)) - } - }) - .collect::>(), - )? - }, + DataType::Dictionary(..) => serde_json::to_value( + (0..num_rows) + .map(|i| dict_str_value(col, i)) + .collect::>(), + )?, DataType::Float64 => { let arr = col.as_any().downcast_ref::().unwrap(); serde_json::to_value( diff --git a/rust/perspective-client/src/rust/virtual_server/features.rs b/rust/perspective-client/src/rust/virtual_server/features.rs index c361fc276c..b94e96e0b7 100644 --- a/rust/perspective-client/src/rust/virtual_server/features.rs +++ b/rust/perspective-client/src/rust/virtual_server/features.rs @@ -18,7 +18,7 @@ use ts_rs::TS; use crate::config::GroupRollupMode; use crate::proto::get_features_resp::{AggregateArgs, AggregateOptions, ColumnTypeOptions}; -use crate::proto::{ColumnType, GetFeaturesResp}; +use crate::proto::{ColumnType, GetFeaturesResp, WindowAggregateArgs}; /// Describes the capabilities supported by a virtual server handler. /// @@ -62,10 +62,10 @@ pub struct Features<'a> { #[ts(optional, as = "Option<_>")] pub expressions: bool, - /// Available window aggregates. + /// Available window aggregates per column type. #[serde(default)] #[ts(optional, as = "Option<_>")] - pub window_aggregates: IndexMap>, + pub window_aggregates: IndexMap>>, /// Whether update callbacks are supported. #[serde(default)] @@ -78,6 +78,48 @@ pub struct Features<'a> { pub unordered: bool, } +/// Specification for a window aggregate. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +pub struct WindowAggSpec<'a> { + pub name: Cow<'a, str>, + + /// Frame kinds this aggregate accepts (`rows`, `range`, `cumulative`). + /// Empty means it takes no frame at all. + #[serde(default)] + #[ts(optional, as = "Option<_>")] + pub frames: Vec>, + + /// Takes a row offset, as `lag` / `lead` / `diff` do. + #[serde(default)] + #[ts(optional, as = "Option<_>")] + pub offset: bool, + + /// Takes a smoothing factor, as `ema` does. + #[serde(default)] + #[ts(optional, as = "Option<_>")] + pub alpha: bool, + + /// The output column type. `None` means the source column's type, which + /// is what `min` / `max` / `lag` do. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub result_type: Option, +} + +impl<'a> From<&'a str> for WindowAggSpec<'a> { + /// A bare name, taking no frame and no arguments. + fn from(name: &'a str) -> Self { + WindowAggSpec { + name: Cow::Borrowed(name), + frames: vec![], + offset: false, + alpha: false, + result_type: None, + } + } +} + /// Specification for an aggregate function. /// /// Aggregates can either take no additional arguments ([`AggSpec::Single`]) @@ -114,7 +156,13 @@ impl<'a> From> for GetFeaturesResp { crate::proto::get_features_resp::WindowAggregateOptions { options: aggs .iter() - .map(|x| crate::proto::WindowAggregate::from(*x) as i32) + .map(|x| WindowAggregateArgs { + name: x.name.to_string(), + frames: x.frames.iter().map(|f| f.to_string()).collect(), + offset: x.offset, + alpha: x.alpha, + result_type: x.result_type.map(|t| t as i32), + }) .collect(), }, ) diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs index e1281c9f4e..7b1c90d80b 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs @@ -12,8 +12,8 @@ use super::GenericSQLError; use crate::config::{ - Aggregate, Filter, FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, ViewConfig, - WindowAggregate, WindowFrame, WindowSortDir, WindowSpec, + Aggregate, Filter, FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, ViewConfig, WindowFrame, + WindowSortDir, WindowSpec, }; fn aggregate_to_string(agg: &Aggregate) -> String { @@ -124,47 +124,72 @@ fn window_sql(w: &WindowSpec, resolve: &dyn Fn(&str) -> String) -> Result Some("SUM"), - WindowAggregate::Avg => Some("AVG"), - WindowAggregate::Count => Some("COUNT"), - WindowAggregate::Min => Some("MIN"), - WindowAggregate::Max => Some("MAX"), - WindowAggregate::Stddev => Some("STDDEV_SAMP"), - WindowAggregate::Var => Some("VAR_SAMP"), - _ => None, - }; - - if let Some(agg_fn) = agg_fn { + let op = w.aggregate.as_str(); + if matches!( + op, + "sum" + | "avg" + | "count" + | "min" + | "max" + | "product" + | "median" + | "stddev_samp" + | "stddev_pop" + | "var_samp" + | "var_pop" + | "first_value" + | "last_value" + ) { let frame = window_frame_sql(w.frame.as_ref()); return Ok(format!( "{}({}) OVER ({})", - agg_fn, + op, src, window_over_clause(w, Some(&frame)) )); } - match w.aggregate { - WindowAggregate::Lag | WindowAggregate::Lead => Ok(format!( + if matches!( + op, + "row_number" | "rank" | "dense_rank" | "percent_rank" | "cume_dist" + ) { + return Ok(format!("{}() OVER ({})", op, window_over_clause(w, None))); + } + + match op { + "lag" | "lead" => Ok(format!( "{}({}, {}) OVER ({})", - if w.aggregate == WindowAggregate::Lag { - "LAG" - } else { - "LEAD" - }, + op, src, w.offset.unwrap_or(1), window_over_clause(w, None) )), - WindowAggregate::Diff => Ok(format!( - "({} - LAG({}, {}) OVER ({}))", + "nth_value" => { + let frame = window_frame_sql(w.frame.as_ref()); + Ok(format!( + "nth_value({}, {}) OVER ({})", + src, + w.offset.unwrap_or(1), + window_over_clause(w, Some(&frame)) + )) + }, + "ntile" => Ok(format!( + "ntile({}) OVER ({})", + w.offset.unwrap_or(1), + window_over_clause(w, None) + )), + // `diff` and `rate` are Perspective's, not any SQL dialect's - they + // are synthesized here so a config authored against the engine keeps + // working against a SQL virtual server. + "diff" => Ok(format!( + "({} - lag({}, {}) OVER ({}))", src, src, w.offset.unwrap_or(1), window_over_clause(w, None) )), - WindowAggregate::Rate => { + "rate" => { let Some(order_by) = &w.order_by else { return Err(GenericSQLError::UnsupportedOperation( "window `rate` requires an explicit `order_by`".to_string(), @@ -184,19 +209,19 @@ fn window_sql(w: &WindowSpec, resolve: &dyn Fn(&str) -> String) -> Result Err(GenericSQLError::UnsupportedOperation( + "ema" => Err(GenericSQLError::UnsupportedOperation( "`ema` windows cannot be translated to a SQL window function (recursive); compute it \ in the Perspective engine instead" .to_string(), )), - _ => Err(GenericSQLError::UnsupportedOperation(format!( - "window op {:?} is not supported by the SQL translation", - w.aggregate + op => Err(GenericSQLError::UnsupportedOperation(format!( + "window op `{}` is not supported by the SQL translation", + op ))), } } diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs index 974d3556eb..7a4bdef98f 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs @@ -14,8 +14,7 @@ use std::collections::HashMap; use super::*; use crate::config::{ - Aggregate, GroupRollupMode, WindowAggregate, WindowFrame, WindowSort, WindowSortDir, - WindowSpec, Windows, + Aggregate, GroupRollupMode, WindowFrame, WindowSort, WindowSortDir, WindowSpec, Windows, }; #[test] @@ -863,14 +862,10 @@ fn test_table_make_view_total_with_split_by() { ); } -fn window_spec( - name: &str, - op: WindowAggregate, - frame: Option, -) -> (String, WindowSpec) { +fn window_spec(name: &str, op: &str, frame: Option) -> (String, WindowSpec) { (name.to_string(), WindowSpec { column: "price".to_string(), - aggregate: op, + aggregate: op.to_string(), partition_by: vec!["sym".to_string()], order_by: Some(WindowSort("t".to_string(), WindowSortDir::Asc)), frame, @@ -884,11 +879,7 @@ fn test_table_make_view_window_natural_order() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); let mut config = ViewConfig::default(); config.columns = vec![Some("t".to_string()), Some("cumsum".to_string())]; - let (name, mut spec) = window_spec( - "cumsum", - WindowAggregate::Sum, - Some(WindowFrame::Cumulative), - ); + let (name, mut spec) = window_spec("cumsum", "sum", Some(WindowFrame::Cumulative)); spec.order_by = None; config.windows = Windows(HashMap::from([(name, spec)])); let sql = builder @@ -909,7 +900,7 @@ fn test_table_make_view_window_range_requires_order_by() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); let mut config = ViewConfig::default(); config.columns = vec![Some("rs".to_string())]; - let (name, mut spec) = window_spec("rs", WindowAggregate::Sum, Some(WindowFrame::Range(10.0))); + let (name, mut spec) = window_spec("rs", "sum", Some(WindowFrame::Range(10.0))); spec.order_by = None; config.windows = Windows(HashMap::from([(name, spec)])); let result = builder.table_make_view("source_table", "dest_view", &config); @@ -924,11 +915,7 @@ fn test_table_make_view_window_order_desc() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); let mut config = ViewConfig::default(); config.columns = vec![Some("t".to_string()), Some("cumsum".to_string())]; - let (name, mut spec) = window_spec( - "cumsum", - WindowAggregate::Sum, - Some(WindowFrame::Cumulative), - ); + let (name, mut spec) = window_spec("cumsum", "sum", Some(WindowFrame::Cumulative)); spec.order_by.as_mut().unwrap().1 = WindowSortDir::Desc; config.windows = Windows(HashMap::from([(name, spec)])); let sql = builder @@ -949,7 +936,7 @@ fn test_table_make_view_window_cumulative_sum() { config.columns = vec![Some("t".to_string()), Some("cumsum".to_string())]; config.windows = Windows(HashMap::from([window_spec( "cumsum", - WindowAggregate::Sum, + "sum", Some(WindowFrame::Cumulative), )])); let sql = builder @@ -957,7 +944,7 @@ fn test_table_make_view_window_cumulative_sum() { .unwrap(); assert!(sql.contains( - "SUM(\"price\") OVER (PARTITION BY \"sym\" ORDER BY \"t\" ASC NULLS FIRST ROWS BETWEEN \ + "sum(\"price\") OVER (PARTITION BY \"sym\" ORDER BY \"t\" ASC NULLS FIRST ROWS BETWEEN \ UNBOUNDED PRECEDING AND CURRENT ROW) AS \"cumsum\"" )); assert!(sql.contains("FROM (SELECT *,")); @@ -970,18 +957,14 @@ fn test_table_make_view_window_rows_and_range_frames() { let mut config = ViewConfig::default(); config.columns = vec![Some("sma".to_string()), Some("rsum".to_string())]; config.windows = Windows(HashMap::from([ - window_spec("sma", WindowAggregate::Avg, Some(WindowFrame::Rows(20))), - window_spec( - "rsum", - WindowAggregate::Sum, - Some(WindowFrame::Range(100.0)), - ), + window_spec("sma", "avg", Some(WindowFrame::Rows(20))), + window_spec("rsum", "sum", Some(WindowFrame::Range(100.0))), ])); let sql = builder .table_make_view("source_table", "dest_view", &config) .unwrap(); - assert!(sql.contains("AVG(\"price\") OVER")); + assert!(sql.contains("avg(\"price\") OVER")); assert!(sql.contains("ROWS BETWEEN 20 PRECEDING AND CURRENT ROW")); assert!(sql.contains("RANGE BETWEEN 100 PRECEDING AND CURRENT ROW")); } @@ -991,20 +974,20 @@ fn test_table_make_view_window_lag_diff() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); let mut config = ViewConfig::default(); config.columns = vec![Some("lg".to_string()), Some("df".to_string())]; - let (lag_name, mut lag) = window_spec("lg", WindowAggregate::Lag, None); + let (lag_name, mut lag) = window_spec("lg", "lag", None); lag.offset = Some(2); config.windows = Windows(HashMap::from([ (lag_name, lag), - window_spec("df", WindowAggregate::Diff, None), + window_spec("df", "diff", None), ])); let sql = builder .table_make_view("source_table", "dest_view", &config) .unwrap(); assert!(sql.contains( - "LAG(\"price\", 2) OVER (PARTITION BY \"sym\" ORDER BY \"t\" ASC NULLS FIRST) AS \"lg\"" + "lag(\"price\", 2) OVER (PARTITION BY \"sym\" ORDER BY \"t\" ASC NULLS FIRST) AS \"lg\"" )); - assert!(sql.contains("(\"price\" - LAG(\"price\", 1) OVER")); + assert!(sql.contains("(\"price\" - lag(\"price\", 1) OVER")); } #[test] @@ -1014,14 +997,14 @@ fn test_table_make_view_window_rate() { config.columns = vec![Some("rt".to_string())]; config.windows = Windows(HashMap::from([window_spec( "rt", - WindowAggregate::Rate, + "rate", Some(WindowFrame::Range(10.0)), )])); let sql = builder .table_make_view("source_table", "dest_view", &config) .unwrap(); - assert!(sql.contains("FIRST_VALUE(\"price\") OVER")); + assert!(sql.contains("first_value(\"price\") OVER")); assert!(sql.contains("NULLIF(CAST(\"t\" AS DOUBLE)")); assert!(sql.contains("RANGE BETWEEN 10 PRECEDING AND CURRENT ROW")); } @@ -1035,14 +1018,14 @@ fn test_table_make_view_window_over_expression_source() { "double_price".to_string(), "\"price\" * 2".to_string(), )])); - let (w_name, mut w) = window_spec("w", WindowAggregate::Sum, Some(WindowFrame::Cumulative)); + let (w_name, mut w) = window_spec("w", "sum", Some(WindowFrame::Cumulative)); w.column = "double_price".to_string(); config.windows = Windows(HashMap::from([(w_name, w)])); let sql = builder .table_make_view("source_table", "dest_view", &config) .unwrap(); - assert!(sql.contains("SUM(\"price\" * 2) OVER")); + assert!(sql.contains("sum(\"price\" * 2) OVER")); } #[test] @@ -1057,7 +1040,7 @@ fn test_table_make_view_window_group_by_over_window_column() { )]); config.windows = Windows(HashMap::from([window_spec( "cumsum", - WindowAggregate::Sum, + "sum", Some(WindowFrame::Cumulative), )])); let sql = builder @@ -1074,7 +1057,7 @@ fn test_table_make_view_window_ema_unsupported() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); let mut config = ViewConfig::default(); config.columns = vec![Some("e".to_string())]; - let (w_name, mut w) = window_spec("e", WindowAggregate::Ema, None); + let (w_name, mut w) = window_spec("e", "ema", None); w.alpha = Some(0.5); config.windows = Windows(HashMap::from([(w_name, w)])); let result = builder.table_make_view("source_table", "dest_view", &config); diff --git a/rust/perspective-client/src/rust/virtual_server/mod.rs b/rust/perspective-client/src/rust/virtual_server/mod.rs index c35bd51c3f..88208d2cf6 100644 --- a/rust/perspective-client/src/rust/virtual_server/mod.rs +++ b/rust/perspective-client/src/rust/virtual_server/mod.rs @@ -24,7 +24,7 @@ mod server; pub use data::{RowPathStyle, SetVirtualDataColumn, VirtualDataCell, VirtualDataSlice}; pub use error::{ResultExt, VirtualServerError}; -pub use features::{AggSpec, Features}; +pub use features::{AggSpec, Features, WindowAggSpec}; pub use generic_sql_model::{ GenericSQLError, GenericSQLResult, GenericSQLVirtualServerModel, GenericSQLVirtualServerModelArgs, diff --git a/rust/perspective-js/src/rust/lib.rs b/rust/perspective-js/src/rust/lib.rs index a7aa0df477..21cf7a4e47 100644 --- a/rust/perspective-js/src/rust/lib.rs +++ b/rust/perspective-js/src/rust/lib.rs @@ -66,7 +66,7 @@ export type * from "../../src/ts/ts-rs/JoinType.ts"; export type * from "../../src/ts/ts-rs/TypedArrayWindow.ts"; export type * from "../../src/ts/ts-rs/Features.ts"; export type * from "../../src/ts/ts-rs/AggSpec.ts"; -export type * from "../../src/ts/ts-rs/WindowAggregate.ts"; +export type * from "../../src/ts/ts-rs/WindowAggSpec.ts"; import type {ColumnWindow} from "../../src/ts/ts-rs/ColumnWindow.d.ts"; import type {ColumnType} from "../../src/ts/ts-rs/ColumnType.d.ts"; diff --git a/rust/perspective-js/src/rust/typed_array.rs b/rust/perspective-js/src/rust/typed_array.rs index a14422ab6b..cd3e7aaa2c 100644 --- a/rust/perspective-js/src/rust/typed_array.rs +++ b/rust/perspective-js/src/rust/typed_array.rs @@ -14,7 +14,7 @@ use std::io::Cursor; use arrow_array::cast::AsArray; use arrow_array::types::*; -use arrow_array::{Array as _, ArrowPrimitiveType, PrimitiveArray}; +use arrow_array::{Array as _, ArrowPrimitiveType, DictionaryArray, PrimitiveArray, StringArray}; use arrow_ipc::reader::StreamReader; use arrow_schema::{DataType, TimeUnit}; use js_sys::{Array, Function, JsString, Uint8Array}; @@ -219,13 +219,32 @@ pub(crate) async fn decode_and_call( js_dicts.set(col_idx as u32, JsValue::NULL); }, DataType::Dictionary(..) => { - let dict = col.as_dictionary::(); + let dict = col + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + JsValue::from_str(&format!( + "Unsupported dictionary key type for typed array: {}", + col.data_type() + )) + })?; + let keys = dict.keys(); zero_invalid_slots(keys); let arr = unsafe { js_sys::Int32Array::view(keys.values().as_ref()) }; js_values.set(col_idx as u32, arr.into()); - let values = dict.values().as_string::(); + let values = dict + .values() + .as_any() + .downcast_ref::() + .ok_or_else(|| { + JsValue::from_str(&format!( + "Unsupported dictionary value type for typed array: {}", + dict.values().data_type() + )) + })?; + let js_dict = Array::new_with_length(values.len() as u32); for i in 0..values.len() { js_dict.set(i as u32, JsValue::from_str(values.value(i))); diff --git a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts index f10be40025..d5e7654120 100644 --- a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts +++ b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts @@ -26,7 +26,7 @@ import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnTy import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; import type { ViewConfigUpdate } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfigUpdate.d.ts"; import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; -import type { WindowAggregate } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggregate.d.ts"; +import type { WindowAggSpec } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggSpec.d.ts"; import type * as clickhouse from "@clickhouse/client-web"; const NUMBER_AGGS = [ @@ -65,28 +65,37 @@ const STRING_AGGS = [ "string_agg", ]; -// The window aggregates the SQL translation supports, per source -// column type (`ema` is recursive - no SQL window equivalent). -const WINDOW_AGGREGATES: WindowAggregate[] = [ - "sum", - "avg", - "count", - "min", - "max", - "stddev", - "var", - "lag", - "lead", - "diff", - "rate", +// Window functions. Renamed from Perspective's `stddev`/`var` to the SQL +// standard spellings DuckDB and ClickHouse both accept, since the advertised +// name is now emitted verbatim. +// +// NOTE: this set is inherited from the DuckDB handler and has NOT been audited +// against a live ClickHouse - see the aggregate lists below, which have the +// same problem. ClickHouse's own navigation functions are `lagInFrame` / +// `leadInFrame`, and its ranking set differs; both need verifying before being +// advertised here. +const FRAMES = ["rows", "range", "cumulative"]; + +const WINDOW_AGGREGATES: WindowAggSpec[] = [ + { name: "sum", frames: FRAMES, result_type: "float" }, + { name: "avg", frames: FRAMES, result_type: "float" }, + { name: "count", frames: FRAMES, result_type: "float" }, + { name: "min", frames: FRAMES }, + { name: "max", frames: FRAMES }, + { name: "stddev_samp", frames: FRAMES, result_type: "float" }, + { name: "var_samp", frames: FRAMES, result_type: "float" }, + { name: "lag", offset: true }, + { name: "lead", offset: true }, + { name: "diff", offset: true, result_type: "float" }, + { name: "rate", frames: ["range"], result_type: "float" }, ]; -const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [ - "count", - "min", - "max", - "lag", - "lead", +const WINDOW_AGGREGATES_ANY: WindowAggSpec[] = [ + { name: "count", frames: FRAMES, result_type: "float" }, + { name: "min", frames: FRAMES }, + { name: "max", frames: FRAMES }, + { name: "lag", offset: true }, + { name: "lead", offset: true }, ]; const FILTER_OPS = [ diff --git a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts index 82d3bfdbfe..b41a0855a5 100644 --- a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts +++ b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts @@ -26,7 +26,7 @@ import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnTy import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; import type { ViewConfigUpdate } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfigUpdate.d.ts"; import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; -import type { WindowAggregate } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggregate.d.ts"; +import type { WindowAggSpec } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggSpec.d.ts"; import type { Scalar } from "@perspective-dev/client/dist/esm/ts-rs/Scalar.d.ts"; import type * as duckdb from "@duckdb/duckdb-wasm"; @@ -66,28 +66,61 @@ const STRING_AGGS = [ "string_agg", ]; -// The window aggregates the SQL translation supports, per source -// column type (`ema` is recursive - no SQL window equivalent). -const WINDOW_AGGREGATES: WindowAggregate[] = [ - "sum", - "avg", - "count", - "min", - "max", - "stddev", - "var", - "lag", - "lead", - "diff", - "rate", +const FRAMES = ["rows", "range", "cumulative"]; + +const WINDOW_AGGREGATES: WindowAggSpec[] = [ + { name: "sum", frames: FRAMES, result_type: "float" }, + { name: "avg", frames: FRAMES, result_type: "float" }, + { name: "count", frames: FRAMES, result_type: "float" }, + { name: "min", frames: FRAMES }, + { name: "max", frames: FRAMES }, + { name: "product", frames: FRAMES, result_type: "float" }, + { name: "median", frames: FRAMES, result_type: "float" }, + // DuckDB spells sample and population variants separately, so both are + // offered rather than one being picked on the user's behalf. + { name: "stddev_samp", frames: FRAMES, result_type: "float" }, + { name: "stddev_pop", frames: FRAMES, result_type: "float" }, + { name: "var_samp", frames: FRAMES, result_type: "float" }, + { name: "var_pop", frames: FRAMES, result_type: "float" }, + // Navigation. + { name: "first_value", frames: FRAMES }, + { name: "last_value", frames: FRAMES }, + { name: "nth_value", frames: FRAMES, offset: true }, + { name: "lag", offset: true }, + { name: "lead", offset: true }, + // Ranking. These take no source column - the window's `order_by` is their + // input - but Perspective requires one, so the choice of source is + // immaterial for them. + { name: "row_number", result_type: "float" }, + { name: "rank", result_type: "float" }, + { name: "dense_rank", result_type: "float" }, + { name: "percent_rank", result_type: "float" }, + { name: "cume_dist", result_type: "float" }, + // `ntile`'s argument is a bucket count rather than a row offset. + { name: "ntile", offset: true, result_type: "float" }, + // Perspective's own, with no DuckDB equivalent - the SQL translation + // synthesizes them from `lag` and `first_value`. + { name: "diff", offset: true, result_type: "float" }, + { name: "rate", frames: ["range"], result_type: "float" }, ]; -const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [ - "count", - "min", - "max", - "lag", - "lead", +// Arithmetic is undefined for the non-numeric types; ordering and navigation +// are not. +const WINDOW_AGGREGATES_ANY: WindowAggSpec[] = [ + { name: "count", frames: FRAMES, result_type: "float" }, + { name: "min", frames: FRAMES }, + { name: "max", frames: FRAMES }, + { name: "first_value", frames: FRAMES }, + { name: "last_value", frames: FRAMES }, + { name: "nth_value", frames: FRAMES, offset: true }, + { name: "lag", offset: true }, + { name: "lead", offset: true }, + { name: "row_number", result_type: "float" }, + { name: "rank", result_type: "float" }, + { name: "dense_rank", result_type: "float" }, + { name: "percent_rank", result_type: "float" }, + { name: "cume_dist", result_type: "float" }, + { name: "ntile", offset: true, result_type: "float" }, ]; const FILTER_OPS = [ @@ -123,51 +156,68 @@ const STRING_FILTER_OPS = [ "NOT ILIKE", ]; +/** + * Convert a DuckDB `dtype` to a Perspective `ColumnType`. + */ function duckdbTypeToPsp(name: string): ColumnType { name = name.toLowerCase(); - if (name === "varchar" || name == "utf8") { - return "string"; + + if (name.startsWith("bool")) { + return "boolean"; } + // 32-bit and narrower - `coerce_column` widens these to `Int32`. if ( - name === "double" || - name === "bigint" || - name === "hugeint" || - name === "float64" || - name.startsWith("decimal") + ["tinyint", "smallint", "integer", "utinyint", "usmallint"].includes( + name, + ) || + ["int8", "int16", "int32", "uint8", "uint16"].includes(name) ) { - return "float"; + return "integer"; } - if (name.startsWith("int")) { - return "integer"; + // Wider than `Int32`, or fractional - all coerce to `Float64`. + if ( + ["bigint", "hugeint", "uhugeint", "uinteger", "ubigint"].includes( + name, + ) || + ["float", "real", "double", "varint"].includes(name) || + ["int64", "uint32", "uint64", "float32", "float64"].includes(name) || + name.startsWith("decimal") || + name.startsWith("numeric") + ) { + return "float"; } if (name.startsWith("date")) { return "date"; } - if (name.startsWith("bool")) { - return "boolean"; - } - - if (name.startsWith("timestamp")) { + // `timestamp`, `timestamptz`, `timestamp_ns`, and `time`/`timetz`, + // which coerce to `Timestamp(Millisecond)` rather than to a number. + if (name.startsWith("time")) { return "datetime"; } - if (name.startsWith("json")) { - return "string"; - } - - if (name.startsWith("struct")) { - return "string"; - } - - if (name.startsWith("time")) { - return "float"; + // Everything else renders as text: `varchar`, `enum(...)` (which + // arrives dictionary-encoded), `json`, `uuid`, `blob`, `interval`, + // and the nested types. + if ( + !( + name.startsWith("varchar") || + name === "utf8" || + name.startsWith("enum") || + ["json", "uuid", "blob", "bit", "interval"].includes(name) || + name.startsWith("struct") || + name.startsWith("map") || + name.startsWith("union") || + name.endsWith("[]") + ) + ) { + // Unknown, not fatal - the column still renders, as text. + console.warn(`Unknown type '${name}'`); } - console.warn(`Unknown type '${name}'`); return "string"; } diff --git a/rust/perspective-js/test/js/constructors.spec.js b/rust/perspective-js/test/js/constructors.spec.js index 5dec9b300e..ad4ca853ff 100644 --- a/rust/perspective-js/test/js/constructors.spec.js +++ b/rust/perspective-js/test/js/constructors.spec.js @@ -960,6 +960,42 @@ function validate_typed_array(typed_array, column_data) { table.delete(); }); + test("column introduced by a later record", async function () { + var table = await perspective.table(`{"a":1}\n{"a":2,"b":3}`, { + format: "ndjson", + }); + + expect(await table.schema()).toEqual({ + a: "integer", + b: "integer", + }); + + var view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 2], + b: [null, 3], + }); + + view.delete(); + table.delete(); + }); + + test("column introduced as null then typed", async function () { + var table = await perspective.table( + `{"a":1,"b":null}\n{"a":2,"b":"x"}`, + { format: "ndjson" }, + ); + + var view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 2], + b: [null, "x"], + }); + + view.delete(); + table.delete(); + }); + test("date types", async function () { const ndjson = []; for (const row of data_4) { @@ -1075,7 +1111,10 @@ function validate_typed_array(typed_array, column_data) { }); test("Arrow Lists constructor", async function () { - const table = await perspective.table(arrows.lists_arrow.slice()); + const table = await perspective.table(arrows.lists_arrow.slice(), { + list_flatten: "stringify", + }); + const view = await table.view(); const result = await view.to_columns(); expect(result).toEqual(arrow_lists_data); @@ -1083,6 +1122,12 @@ function validate_typed_array(typed_array, column_data) { table.delete(); }); + test("Arrow Lists constructor rejects a ragged zip", async function () { + await expect( + perspective.table(arrows.lists_arrow.slice()), + ).rejects.toThrow(/Cannot zip list columns/); + }); + test("Arrow dictionary constructor", async function () { const table = await perspective.table(arrows.dict_arrow.slice()); const view = await table.view(); diff --git a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts new file mode 100644 index 0000000000..b163ec5661 --- /dev/null +++ b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts @@ -0,0 +1,356 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import * as arrow from "apache-arrow"; +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client"; + +const field = (name: string, type: arrow.DataType) => + arrow.Field.new({ name, type }); + +const int32 = () => new arrow.Int32(); +const float64 = () => new arrow.Float64(); +const utf8 = () => new arrow.Utf8(); + +const list = (type: arrow.DataType) => new arrow.List(field("item", type)); + +const struct = (fields: [string, arrow.DataType][]) => + new arrow.Struct(fields.map(([n, t]) => field(n, t))); + +const ipc = (columns: Record) => + arrow.tableToIPC(arrow.tableFromArrays(columns as any)); + +test.describe("Arrow nested columns", function () { + test.describe("Struct", function () { + test("Flattens a struct into dotted columns", async function () { + const table = await perspective.table( + ipc({ + id: arrow.vectorFromArray([1, 2], int32()), + s: arrow.vectorFromArray( + [ + { a: 10, b: 1.5 }, + { a: 20, b: 2.5 }, + ], + struct([ + ["a", int32()], + ["b", float64()], + ]), + ), + }), + ); + + expect(await table.schema()).toEqual({ + id: "integer", + "s.a": "integer", + "s.b": "float", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 2], + "s.a": [10, 20], + "s.b": [1.5, 2.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("Recurses through a struct of struct", async function () { + const table = await perspective.table( + ipc({ + s: arrow.vectorFromArray( + [{ b: { c: 1 } }, { b: { c: 2 } }], + struct([["b", struct([["c", int32()]])]]), + ), + }), + ); + + expect(await table.schema()).toEqual({ "s.b.c": "integer" }); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ "s.b.c": [1, 2] }); + await view.delete(); + await table.delete(); + }); + + test("A null parent nulls every descendant leaf", async function () { + const table = await perspective.table( + ipc({ + s: arrow.vectorFromArray( + [{ a: 1, b: 2 }, null, { a: 3, b: 4 }], + struct([ + ["a", int32()], + ["b", int32()], + ]), + ), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "s.a": [1, null, 3], + "s.b": [2, null, 4], + }); + + await view.delete(); + await table.delete(); + }); + + test("Updates match a declared dotted schema", async function () { + const table = await perspective.table({ + id: "integer", + "s.a": "integer", + }); + + await table.update( + ipc({ + id: arrow.vectorFromArray([1], int32()), + s: arrow.vectorFromArray( + [{ a: 7 }], + struct([["a", int32()]]), + ), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ id: [1], "s.a": [7] }); + await view.delete(); + await table.delete(); + }); + }); + + test.describe("List", function () { + test("Expands one row per element by default", async function () { + const table = await perspective.table( + ipc({ + x: arrow.vectorFromArray([1, 2], int32()), + y: arrow.vectorFromArray( + [[10, 20, 30], [40]], + list(int32()), + ), + }), + ); + + expect(await table.schema()).toEqual({ + x: "integer", + y: "integer", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + x: [1, 1, 1, 2], + y: [10, 20, 30, 40], + }); + + await view.delete(); + await table.delete(); + }); + + test("An empty list yields one null row", async function () { + const table = await perspective.table( + ipc({ + x: arrow.vectorFromArray([1, 2], int32()), + y: arrow.vectorFromArray([[], [40]], list(int32())), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + x: [1, 2], + y: [null, 40], + }); + + await view.delete(); + await table.delete(); + }); + + test("Expands and flattens a list of struct", async function () { + const table = await perspective.table( + ipc({ + id: arrow.vectorFromArray([1], int32()), + orders: arrow.vectorFromArray( + [[{ price: 1.5 }, { price: 2.5 }]], + list(struct([["price", float64()]])), + ), + }), + ); + + expect(await table.schema()).toEqual({ + id: "integer", + "orders.price": "float", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 1], + "orders.price": [1.5, 2.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("Expands a list nested inside a struct", async function () { + const table = await perspective.table( + ipc({ + id: arrow.vectorFromArray([1, 2], int32()), + s: arrow.vectorFromArray( + [{ a: [10, 20] }, { a: [30] }], + struct([["a", list(int32())]]), + ), + }), + ); + + expect(await table.schema()).toEqual({ + id: "integer", + "s.a": "integer", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 1, 2], + "s.a": [10, 20, 30], + }); + + await view.delete(); + await table.delete(); + }); + + test("Gathers string siblings across an expansion", async function () { + const table = await perspective.table( + ipc({ + s: arrow.vectorFromArray(["a", "b"], utf8()), + y: arrow.vectorFromArray([[1, 2, 3], [4]], list(int32())), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + s: ["a", "a", "a", "b"], + y: [1, 2, 3, 4], + }); + + await view.delete(); + await table.delete(); + }); + + test("Expands by cartesian product when configured", async function () { + const table = await perspective.table( + ipc({ + a: arrow.vectorFromArray([[1, 2]], list(int32())), + b: arrow.vectorFromArray([[3, 4, 5]], list(int32())), + }), + { list_flatten: "cartesian" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 1, 1, 2, 2, 2], + b: [3, 4, 5, 3, 4, 5], + }); + + await view.delete(); + await table.delete(); + }); + + test("Encodes lists as JSON when configured", async function () { + const table = await perspective.table( + ipc({ + x: arrow.vectorFromArray([1, 2], int32()), + y: arrow.vectorFromArray([[10, 20], [30]], list(int32())), + }), + { list_flatten: "stringify" }, + ); + + expect(await table.schema()).toEqual({ x: "integer", y: "string" }); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + x: [1, 2], + y: ["[10,20]", "[30]"], + }); + + await view.delete(); + await table.delete(); + }); + }); + + test.describe("Expanded index", function () { + const orders = (ids: number[], prices: number[]) => + ipc({ + batch: arrow.vectorFromArray([1], int32()), + orders: arrow.vectorFromArray( + [ids.map((id, i) => ({ id, price: prices[i] }))], + list( + struct([ + ["id", int32()], + ["price", float64()], + ]), + ), + ), + }); + + test("Indexes on a column drawn from the list", async function () { + const table = await perspective.table( + orders([1, 2, 3], [1.5, 2.5, 3.5]), + { index: "orders.id" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + batch: [1, 1, 1], + "orders.id": [1, 2, 3], + "orders.price": [1.5, 2.5, 3.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("Updates a list-derived index by element", async function () { + const table = await perspective.table(orders([1, 2], [1.5, 2.5]), { + index: "orders.id", + }); + + await table.update(orders([2, 3], [9.5, 3.5])); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + batch: [1, 1, 1], + "orders.id": [1, 2, 3], + "orders.price": [1.5, 9.5, 3.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("Rejects an index on a repeated sibling", async function () { + await expect( + perspective.table(orders([1, 2], [1.5, 2.5]), { + index: "batch", + }), + ).rejects.toThrow(/`batch`/); + }); + + test("Rejects any index under a multi-list cartesian", async function () { + await expect( + perspective.table( + ipc({ + id: arrow.vectorFromArray([[1, 2]], list(int32())), + b: arrow.vectorFromArray([[3, 4, 5]], list(int32())), + }), + { index: "id", list_flatten: "cartesian" }, + ), + ).rejects.toThrow(/`id`/); + }); + }); +}); diff --git a/rust/perspective-js/test/js/constructors/json_nested.spec.ts b/rust/perspective-js/test/js/constructors/json_nested.spec.ts new file mode 100644 index 0000000000..875f77fd9e --- /dev/null +++ b/rust/perspective-js/test/js/constructors/json_nested.spec.ts @@ -0,0 +1,442 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client"; + +/** + * The same logical records in all three JSON shapes. Object flattening must not + * depend on which one the caller used. + */ +const FORMATS: Record { data: any; options: any }> = { + rows: (records) => ({ data: records, options: {} }), + columns: (records) => ({ + data: Object.fromEntries( + [...new Set(records.flatMap((r) => Object.keys(r)))].map((k) => [ + k, + records.map((r) => (k in r ? r[k] : null)), + ]), + ), + options: {}, + }), + ndjson: (records) => ({ + data: records.map((r) => JSON.stringify(r)).join("\n"), + options: { format: "ndjson" }, + }), +}; + +test.describe("JSON nested columns", function () { + for (const [name, shape] of Object.entries(FORMATS)) { + test.describe(name, function () { + test("flattens an object into dotted columns", async function () { + const { data, options } = shape([ + { id: 1, s: { a: 10, b: "x" } }, + { id: 2, s: { a: 20, b: "y" } }, + ]); + + const table = await perspective.table(data, options); + expect(await table.schema()).toEqual({ + id: "integer", + "s.a": "integer", + "s.b": "string", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 2], + "s.a": [10, 20], + "s.b": ["x", "y"], + }); + + await view.delete(); + await table.delete(); + }); + + test("recurses through nested objects", async function () { + const { data, options } = shape([ + { s: { b: { c: 1 } } }, + { s: { b: { c: 2 } } }, + ]); + + const table = await perspective.table(data, options); + expect(await table.schema()).toEqual({ "s.b.c": "integer" }); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ "s.b.c": [1, 2] }); + await view.delete(); + await table.delete(); + }); + + test("ragged objects across records", async function () { + const { data, options } = shape([ + { s: { a: 1 } }, + { s: { b: 2 } }, + ]); + + const table = await perspective.table(data, options); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "s.a": [1, null], + "s.b": [null, 2], + }); + + await view.delete(); + await table.delete(); + }); + + test("a path seen as both scalar and object", async function () { + const { data, options } = shape([{ s: 1 }, { s: { a: 2 } }]); + const table = await perspective.table(data, options); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + s: [1, null], + "s.a": [null, 2], + }); + + await view.delete(); + await table.delete(); + }); + + test("a flat key before a nested one", async function () { + // The flat fast path fills optimistically and abandons on the + // first value needing descent, so `a` is written twice -- + // once flat, once at slot 0 of the expansion. + const { data, options } = shape([ + { a: 1, s: { b: 2 }, y: [10, 20] }, + { a: 3, s: { b: 4 }, y: [30] }, + ]); + + const table = await perspective.table(data, options); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 1, 3], + "s.b": [2, 2, 4], + y: [10, 20, 30], + }); + + await view.delete(); + await table.delete(); + }); + + test("an empty object contributes no column", async function () { + const { data, options } = shape([{ id: 1, s: {} }]); + const table = await perspective.table(data, options); + expect(await table.schema()).toEqual({ id: "integer" }); + await table.delete(); + }); + }); + } + + for (const [name, shape] of Object.entries(FORMATS)) { + test.describe(`${name} arrays`, function () { + test("expands one row per element by default", async function () { + const { data, options } = shape([ + { x: 1, y: [10, 20, 30] }, + { x: 2, y: [40] }, + ]); + + const table = await perspective.table(data, options); + expect(await table.schema()).toEqual({ + x: "integer", + y: "integer", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + x: [1, 1, 1, 2], + y: [10, 20, 30, 40], + }); + + await view.delete(); + await table.delete(); + }); + + test("an empty array yields one null row", async function () { + const { data, options } = shape([ + { x: 1, y: [] }, + { x: 2, y: [40] }, + ]); + + const table = await perspective.table(data, options); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + x: [1, 2], + y: [null, 40], + }); + + await view.delete(); + await table.delete(); + }); + + test("expands and flattens an array of objects", async function () { + const { data, options } = shape([ + { id: 1, orders: [{ price: 1.5 }, { price: 2.5 }] }, + ]); + + const table = await perspective.table(data, options); + expect(await table.schema()).toEqual({ + id: "integer", + "orders.price": "float", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 1], + "orders.price": [1.5, 2.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("expands an array nested inside an object", async function () { + const { data, options } = shape([ + { id: 1, s: { a: [10, 20] } }, + { id: 2, s: { a: [30] } }, + ]); + + const table = await perspective.table(data, options); + expect(await table.schema()).toEqual({ + id: "integer", + "s.a": "integer", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 1, 2], + "s.a": [10, 20, 30], + }); + + await view.delete(); + await table.delete(); + }); + + test("zips arrays of equal length", async function () { + const { data, options } = shape([{ a: [1, 2], b: [3, 4] }]); + const table = await perspective.table(data, options); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 2], + b: [3, 4], + }); + + await view.delete(); + await table.delete(); + }); + + test("rejects a ragged zip", async function () { + const { data, options } = shape([{ a: [1, 2], b: [3, 4, 5] }]); + await expect(perspective.table(data, options)).rejects.toThrow( + /Cannot zip/, + ); + }); + + test("recurses through nested arrays", async function () { + const { data, options } = shape([{ y: [[1, 2], [3]] }]); + const table = await perspective.table(data, options); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ y: [1, 2, 3] }); + await view.delete(); + await table.delete(); + }); + + test("expands by cartesian product when configured", async function () { + const { data, options } = shape([{ a: [1, 2], b: [3, 4, 5] }]); + const table = await perspective.table(data, { + ...options, + list_flatten: "cartesian", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 1, 1, 2, 2, 2], + b: [3, 4, 5, 3, 4, 5], + }); + + await view.delete(); + await table.delete(); + }); + + test("encodes arrays as JSON when configured", async function () { + const { data, options } = shape([ + { x: 1, y: [10, 20] }, + { x: 2, y: [30] }, + ]); + + const table = await perspective.table(data, { + ...options, + list_flatten: "stringify", + }); + + expect(await table.schema()).toEqual({ + x: "integer", + y: "string", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + x: [1, 2], + y: ["[10,20]", "[30]"], + }); + + await view.delete(); + await table.delete(); + }); + }); + } + + test.describe("expanded index", function () { + const orders = (ids: number[], prices: number[]) => [ + { + batch: 1, + orders: ids.map((id, i) => ({ id, price: prices[i] })), + }, + ]; + + test("indexes on a column drawn from the array", async function () { + const table = await perspective.table( + orders([1, 2, 3], [1.5, 2.5, 3.5]) as any, + { index: "orders.id" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + batch: [1, 1, 1], + "orders.id": [1, 2, 3], + "orders.price": [1.5, 2.5, 3.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("updates a array-derived index by element", async function () { + const table = await perspective.table( + orders([1, 2], [1.5, 2.5]) as any, + { index: "orders.id" }, + ); + + await table.update(orders([2, 3], [9.5, 3.5]) as any); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + batch: [1, 1, 1], + "orders.id": [1, 2, 3], + "orders.price": [1.5, 9.5, 3.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("rejects an index on a repeated sibling", async function () { + await expect( + perspective.table(orders([1, 2], [1.5, 2.5]) as any, { + index: "batch", + }), + ).rejects.toThrow(/`batch`/); + }); + + test("rejects any index under a multi-array cartesian", async function () { + await expect( + perspective.table([{ id: [1, 2], b: [3, 4, 5] }] as any, { + index: "id", + list_flatten: "cartesian", + }), + ).rejects.toThrow(/`id`/); + }); + + test("stringify keeps a sibling index usable", async function () { + const table = await perspective.table( + [ + { id: 1, y: [10, 20] }, + { id: 2, y: [30] }, + ] as any, + { index: "id", list_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 2], + y: ["[10,20]", "[30]"], + }); + + await view.delete(); + await table.delete(); + }); + }); + + test("updates against a declared dotted schema", async function () { + const table = await perspective.table({ + id: "integer", + "s.a": "integer", + }); + + await table.update([{ id: 1, s: { a: 7 } }] as any); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ id: [1], "s.a": [7] }); + await view.delete(); + await table.delete(); + }); + + test("update ignores a leaf the schema lacks", async function () { + const table = await perspective.table({ + id: "integer", + "s.a": "integer", + }); + + await table.update([{ id: 1, s: { a: 7, zzz: 9 } }] as any); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ id: [1], "s.a": [7] }); + await view.delete(); + await table.delete(); + }); + + test("indexes on a flattened column", async function () { + const table = await perspective.table( + [ + { s: { id: 1 }, v: "a" }, + { s: { id: 2 }, v: "b" }, + ] as any, + { index: "s.id" }, + ); + + await table.update([{ s: { id: 2 }, v: "z" }] as any); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "s.id": [1, 2], + v: ["a", "z"], + }); + + await view.delete(); + await table.delete(); + }); + + test("rejects a key colliding with a flattened path", async function () { + await expect( + perspective.table([{ s: { a: 1 }, "s.a": 2 }] as any), + ).rejects.toThrow(/both a key and the flattened path/); + }); + + test("ndjson grows into an object column from a later record", async function () { + const table = await perspective.table(`{"a":1}\n{"a":2,"s":{"b":3}}`, { + format: "ndjson", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 2], + "s.b": [null, 3], + }); + + await view.delete(); + await table.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/constructors/nested_parity.spec.ts b/rust/perspective-js/test/js/constructors/nested_parity.spec.ts new file mode 100644 index 0000000000..a08ff91f4b --- /dev/null +++ b/rust/perspective-js/test/js/constructors/nested_parity.spec.ts @@ -0,0 +1,215 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import * as arrow from "apache-arrow"; +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client"; + +const field = (name: string, type: arrow.DataType) => + arrow.Field.new({ name, type }); + +const int32 = () => new arrow.Int32(); +const float64 = () => new arrow.Float64(); +const list = (type: arrow.DataType) => new arrow.List(field("item", type)); +const struct = (fields: [string, arrow.DataType][]) => + new arrow.Struct(fields.map(([n, t]) => field(n, t))); + +const ipc = (columns: Record) => + arrow.tableToIPC(arrow.tableFromArrays(columns as any)); + +async function parity( + name: string, + records: any[], + arrowColumns: Record, + options: any = {}, +) { + const from_json = await perspective.table(records as any, options); + const from_arrow = await perspective.table(ipc(arrowColumns), options); + + const json_view = await from_json.view(); + const arrow_view = await from_arrow.view(); + + expect(await from_json.schema()).toEqual(await from_arrow.schema()); + expect(await json_view.to_columns()).toEqual(await arrow_view.to_columns()); + + await json_view.delete(); + await arrow_view.delete(); + await from_json.delete(); + await from_arrow.delete(); +} + +test.describe("Nested ingest parity", function () { + test("struct flattening", async function () { + await parity( + "struct", + [ + { id: 1, s: { a: 10, b: 1.5 } }, + { id: 2, s: { a: 20, b: 2.5 } }, + ], + { + id: arrow.vectorFromArray([1, 2], int32()), + s: arrow.vectorFromArray( + [ + { a: 10, b: 1.5 }, + { a: 20, b: 2.5 }, + ], + struct([ + ["a", int32()], + ["b", float64()], + ]), + ), + }, + ); + }); + + test("nested struct", async function () { + await parity( + "nested", + [{ s: { b: { c: 1 } } }, { s: { b: { c: 2 } } }], + { + s: arrow.vectorFromArray( + [{ b: { c: 1 } }, { b: { c: 2 } }], + struct([["b", struct([["c", int32()]])]]), + ), + }, + ); + }); + + test("list expansion", async function () { + await parity( + "zip", + [ + { x: 1, y: [10, 20, 30] }, + { x: 2, y: [40] }, + ], + { + x: arrow.vectorFromArray([1, 2], int32()), + y: arrow.vectorFromArray([[10, 20, 30], [40]], list(int32())), + }, + ); + }); + + test("empty list yields one null row", async function () { + await parity( + "empty", + [ + { x: 1, y: [] }, + { x: 2, y: [40] }, + ], + { + x: arrow.vectorFromArray([1, 2], int32()), + y: arrow.vectorFromArray([[], [40]], list(int32())), + }, + ); + }); + + test("list of struct", async function () { + await parity( + "list_of_struct", + [{ id: 1, orders: [{ price: 1.5 }, { price: 2.5 }] }], + { + id: arrow.vectorFromArray([1], int32()), + orders: arrow.vectorFromArray( + [[{ price: 1.5 }, { price: 2.5 }]], + list(struct([["price", float64()]])), + ), + }, + ); + }); + + test("struct containing a list", async function () { + // The mirror of `list of struct`: Arrow reaches it by hoisting then + // exploding, JSON by an object combining a child array's width. The + // two arrive at the same flat shape by different routes. + await parity( + "struct_of_list", + [ + { id: 1, s: { a: [10, 20] } }, + { id: 2, s: { a: [30] } }, + ], + { + id: arrow.vectorFromArray([1, 2], int32()), + s: arrow.vectorFromArray( + [{ a: [10, 20] }, { a: [30] }], + struct([["a", list(int32())]]), + ), + }, + ); + }); + + test("nested lists", async function () { + await parity("nested_list", [{ y: [[1, 2], [3]] }], { + y: arrow.vectorFromArray([[[1, 2], [3]]], list(list(int32()))), + }); + }); + + test("cartesian product", async function () { + await parity( + "cartesian", + [{ a: [1, 2], b: [3, 4, 5] }], + { + a: arrow.vectorFromArray([[1, 2]], list(int32())), + b: arrow.vectorFromArray([[3, 4, 5]], list(int32())), + }, + { list_flatten: "cartesian" }, + ); + }); + + test("index drawn from the expanded array", async function () { + await parity( + "indexed", + [{ batch: 1, orders: [{ id: 1 }, { id: 2 }] }], + { + batch: arrow.vectorFromArray([1], int32()), + orders: arrow.vectorFromArray( + [[{ id: 1 }, { id: 2 }]], + list(struct([["id", int32()]])), + ), + }, + { index: "orders.id" }, + ); + }); + + test("both reject a repeated index", async function () { + const records = [{ batch: 1, orders: [{ id: 1 }, { id: 2 }] }]; + const columns = { + batch: arrow.vectorFromArray([1], int32()), + orders: arrow.vectorFromArray( + [[{ id: 1 }, { id: 2 }]], + list(struct([["id", int32()]])), + ), + }; + + await expect( + perspective.table(records as any, { index: "batch" }), + ).rejects.toThrow(/`batch`/); + + await expect( + perspective.table(ipc(columns), { index: "batch" }), + ).rejects.toThrow(/`batch`/); + }); + + test("both reject a ragged zip", async function () { + const columns = { + a: arrow.vectorFromArray([[1, 2]], list(int32())), + b: arrow.vectorFromArray([[3, 4, 5]], list(int32())), + }; + + await expect( + perspective.table([{ a: [1, 2], b: [3, 4, 5] }] as any), + ).rejects.toThrow(/[Cc]annot zip/); + + await expect(perspective.table(ipc(columns))).rejects.toThrow( + /[Cc]annot zip/, + ); + }); +}); diff --git a/rust/perspective-js/test/js/duckdb/client.spec.js b/rust/perspective-js/test/js/duckdb/client.spec.js index d1ae2eff8a..4e18eccbd0 100644 --- a/rust/perspective-js/test/js/duckdb/client.spec.js +++ b/rust/perspective-js/test/js/duckdb/client.spec.js @@ -17,6 +17,10 @@ describeDuckDB("client", (getClient) => { test("get_hosted_table_names()", async function () { const client = getClient(); const tables = await client.get_hosted_table_names(); - expect(tables).toEqual(["memory.superstore", "memory.underscore_test"]); + expect(tables).toEqual([ + "memory.coerce_types", + "memory.superstore", + "memory.underscore_test", + ]); }); }); diff --git a/rust/perspective-js/test/js/duckdb/coerce_types.spec.js b/rust/perspective-js/test/js/duckdb/coerce_types.spec.js new file mode 100644 index 0000000000..4c4125df17 --- /dev/null +++ b/rust/perspective-js/test/js/duckdb/coerce_types.spec.js @@ -0,0 +1,150 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import { describeDuckDB } from "./setup.js"; + +describeDuckDB("coerce_types", (getClient) => { + test("schema maps every type", async function () { + const table = await getClient().open_table("memory.coerce_types"); + expect(await table.schema()).toEqual({ + tiny: "integer", + small: "integer", + utiny: "integer", + usmall: "integer", + uint: "float", + ubig: "float", + big: "float", + float: "float", + decimal: "float", + time: "datetime", + timestamp: "datetime", + date: "date", + enum: "string", + string: "string", + }); + }); + + test("narrow integers, flat", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ + columns: ["tiny", "small", "utiny", "usmall"], + }); + + expect(await view.to_json()).toEqual([ + { tiny: -1, small: -300, utiny: 255, usmall: 65535 }, + { tiny: 1, small: 300, utiny: 0, usmall: 0 }, + ]); + + await view.delete(); + }); + + test("wide and fractional numbers, flat", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ + columns: ["uint", "ubig", "big", "float", "decimal"], + }); + + const json = await view.to_json(); + expect(json[0].uint).toEqual(4294967295); + expect(json[0].ubig).toEqual(9007199254740992); + expect(json[0].big).toEqual(9007199254740992); + expect(json[0].float).toEqual(1.5); + expect(json[0].decimal).toBeCloseTo(1.234, 6); + expect(json[1].big).toEqual(-9007199254740992); + expect(json[1].decimal).toBeCloseTo(-5.678, 6); + await view.delete(); + }); + + test("temporal types, flat", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ + columns: ["time", "timestamp", "date"], + }); + + expect(await view.to_json()).toEqual([ + { time: 3661000, timestamp: 1672531200000, date: 1672531200000 }, + { time: 1000, timestamp: 1672617600000, date: 1672617600000 }, + ]); + + await view.delete(); + }); + + test("dictionary-encoded ENUM, flat", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ columns: ["enum", "string"] }); + expect(await view.to_json()).toEqual([ + { enum: "happy", string: "a" }, + { enum: "sad", string: "b" }, + ]); + + await view.delete(); + }); + + test("dictionary-encoded ENUM, grouped", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ + group_by: ["enum"], + columns: ["tiny"], + aggregates: { tiny: "sum" }, + }); + + expect(await view.to_json()).toEqual([ + { __ROW_PATH__: [], tiny: 0 }, + { __ROW_PATH__: ["happy"], tiny: -1 }, + { __ROW_PATH__: ["sad"], tiny: 1 }, + ]); + + await view.delete(); + }); + + test("DECIMAL row path is a number, not a debug string", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ + group_by: ["decimal"], + columns: ["tiny"], + aggregates: { tiny: "sum" }, + }); + + const json = await view.to_json(); + expect(json[0].__ROW_PATH__).toEqual([]); + expect(json[1].__ROW_PATH__[0]).toBeCloseTo(-5.678, 6); + expect(json[2].__ROW_PATH__[0]).toBeCloseTo(1.234, 6); + await view.delete(); + }); + + test("filter matching nothing is an empty view", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ + columns: ["tiny"], + filter: [["string", "==", "no such value"]], + }); + + expect(await view.to_json()).toEqual([]); + expect(await view.to_columns()).toEqual({ tiny: [] }); + await view.delete(); + }); + + test("column values query, no columns selected", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ group_by: ["enum"], columns: [] }); + const csv = await view.to_csv(); + expect(csv.split("\n").filter((x) => x.length > 0)).toEqual([ + "__ROW_PATH_0__", + "null", + '"happy"', + '"sad"', + ]); + + await view.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/duckdb/setup.js b/rust/perspective-js/test/js/duckdb/setup.js index 1e06eeed03..4150207469 100644 --- a/rust/perspective-js/test/js/duckdb/setup.js +++ b/rust/perspective-js/test/js/duckdb/setup.js @@ -92,6 +92,39 @@ async function loadUnderscoreData(db) { `); } +async function loadCoerceTypesData(db) { + await db.query(`CREATE TYPE mood AS ENUM ('happy', 'sad')`); + await db.query(` + CREATE TABLE coerce_types ( + "tiny" TINYINT, + "small" SMALLINT, + "utiny" UTINYINT, + "usmall" USMALLINT, + "uint" UINTEGER, + "ubig" UBIGINT, + "big" BIGINT, + "float" REAL, + "decimal" DECIMAL(18, 3), + "time" TIME, + "timestamp" TIMESTAMP, + "date" DATE, + "enum" mood, + "string" VARCHAR + ); + `); + + await db.query(` + INSERT INTO coerce_types VALUES + (-1, -300, 255, 65535, 4294967295, 9007199254740992, + 9007199254740992, 1.5, 1.234, TIME '01:01:01', + TIMESTAMP '2023-01-01 00:00:00', DATE '2023-01-01', + 'happy', 'a'), + (1, 300, 0, 0, 0, 0, -9007199254740992, -1.5, -5.678, + TIME '00:00:01', TIMESTAMP '2023-01-02 00:00:00', + DATE '2023-01-02', 'sad', 'b'); + `); +} + export function describeDuckDB(name, fn) { test.describe("DuckDB Virtual Server " + name, function () { let db; @@ -105,6 +138,7 @@ export function describeDuckDB(name, fn) { client = await perspective.worker(server); await loadSuperstoreData(db); await loadUnderscoreData(db); + await loadCoerceTypesData(db); }); fn(() => client); diff --git a/rust/perspective-js/test/js/duckdb/typed_arrays.spec.js b/rust/perspective-js/test/js/duckdb/typed_arrays.spec.js index 9862497e31..058bb3b099 100644 --- a/rust/perspective-js/test/js/duckdb/typed_arrays.spec.js +++ b/rust/perspective-js/test/js/duckdb/typed_arrays.spec.js @@ -88,4 +88,49 @@ describeDuckDB("typed_arrays", (getClient) => { await view.delete(); }); + + test("string columns of a flat view are dictionaries", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ columns: ["Region"] }); + let seen = 0; + await view.with_typed_arrays( + { start_row: 0, end_row: 5 }, + (names, values, validities, dictionaries) => { + for (let c = 0; c < names.length; c++) { + if (names[c] !== "Region") { + continue; + } + + seen++; + const dict = dictionaries[c]; + expect(dict).not.toBeNull(); + expect(dict).toContain("South"); + expect(dict[values[c][0]]).toEqual("South"); + } + }, + ); + + expect(seen).toBe(1); + await view.delete(); + }); + + test("a DECIMAL column of a flat view reads as Float64", async function () { + const table = await getClient().open_table("memory.coerce_types"); + const view = await table.view({ columns: ["decimal"] }); + let seen = 0; + await view.with_typed_arrays({}, (names, values) => { + for (let c = 0; c < names.length; c++) { + if (names[c] !== "decimal") { + continue; + } + + seen++; + expect(values[c]).toBeInstanceOf(Float64Array); + expect(values[c][0]).toBeCloseTo(1.234, 6); + } + }); + + expect(seen).toBe(1); + await view.delete(); + }); }); diff --git a/rust/perspective-js/test/js/group_rollup_mode.spec.js b/rust/perspective-js/test/js/group_rollup_mode.spec.js index 7b34ad0234..87f2d6655f 100644 --- a/rust/perspective-js/test/js/group_rollup_mode.spec.js +++ b/rust/perspective-js/test/js/group_rollup_mode.spec.js @@ -446,6 +446,102 @@ const data = { table.delete(); }); + test.describe("empty result with sort", function () { + test("filter rejects every row", async function () { + const table = await perspective.table(data); + const view = await table.view({ + group_by: ["y"], + split_by: ["z"], + group_rollup_mode: "flat", + sort: [["w", "desc"]], + filter: [["x", "<", 0]], + }); + expect(await view.to_columns()).toStrictEqual({ + __ROW_PATH__: [], + }); + expect(await view.num_rows()).toEqual(0); + view.delete(); + table.delete(); + }); + + test("null filter operand rejects every row", async function () { + const table = await perspective.table(data); + const view = await table.view({ + group_by: ["y"], + split_by: ["z"], + group_rollup_mode: "flat", + sort: [["w", "desc"]], + filter: [["x", "<", null]], + }); + expect(await view.to_columns()).toStrictEqual({ + __ROW_PATH__: [], + }); + view.delete(); + table.delete(); + }); + + test("update into an all-filtered view", async function () { + const table = await perspective.table({ + w: "float", + x: "integer", + y: "string", + z: "boolean", + }); + const view = await table.view({ + group_by: ["y"], + split_by: ["z"], + group_rollup_mode: "flat", + sort: [["w", "desc"]], + filter: [["x", "<", 0]], + }); + await table.update(data); + expect(await view.to_columns()).toStrictEqual({ + __ROW_PATH__: [], + }); + view.delete(); + table.delete(); + }); + + test("remove every row", async function () { + const table = await perspective.table(data, { index: "w" }); + const view = await table.view({ + group_by: ["y"], + split_by: ["z"], + group_rollup_mode: "flat", + sort: [["w", "desc"]], + }); + expect(await view.num_rows()).toEqual(4); + await table.remove(data.w); + expect(await view.to_columns()).toStrictEqual({ + __ROW_PATH__: [], + }); + expect(await view.num_rows()).toEqual(0); + view.delete(); + table.delete(); + }); + + test("recovers when rows return", async function () { + const table = await perspective.table(data, { index: "w" }); + const view = await table.view({ + group_by: ["y"], + group_rollup_mode: "flat", + sort: [["w", "desc"]], + }); + await table.remove(data.w); + expect(await view.num_rows()).toEqual(0); + await table.update([{ w: 1.5, x: 1, y: "a", z: true }]); + expect(await view.to_columns()).toStrictEqual({ + __ROW_PATH__: [["a"]], + w: [1.5], + x: [1], + y: [1], + z: [1], + }); + view.delete(); + table.delete(); + }); + }); + test("viewport pagination", async function () { const table = await perspective.table(data); const view = await table.view({ diff --git a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py new file mode 100644 index 0000000000..8ab0d19662 --- /dev/null +++ b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py @@ -0,0 +1,535 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import pyarrow as pa +import pytest +import perspective as psp + + +client = psp.Server().new_local_client() +Table = client.table + + +def arrow_bytes(table): + sink = pa.BufferOutputStream() + with pa.RecordBatchFileWriter(sink, table.schema) as writer: + writer.write_table(table) + + return sink.getvalue().to_pybytes() + + +class TestTableArrowStruct(object): + def test_struct_flattens_to_dotted_columns(self): + data = pa.table( + { + "id": pa.array([1, 2], type=pa.int64()), + "s": pa.array( + [{"a": 10, "b": 1.5}, {"a": 20, "b": 2.5}], + type=pa.struct([("a", pa.int64()), ("b", pa.float64())]), + ), + } + ) + + tbl = Table(data) + assert tbl.schema() == { + "id": "integer", + "s.a": "integer", + "s.b": "float", + } + + assert tbl.view().to_columns() == { + "id": [1, 2], + "s.a": [10, 20], + "s.b": [1.5, 2.5], + } + + def test_struct_of_struct_recurses(self): + inner = pa.struct([("c", pa.int64())]) + data = pa.table( + { + "s": pa.array( + [{"b": {"c": 1}}, {"b": {"c": 2}}], + type=pa.struct([("b", inner)]), + ) + } + ) + + tbl = Table(data) + assert tbl.schema() == {"s.b.c": "integer"} + assert tbl.view().to_columns() == {"s.b.c": [1, 2]} + + def test_struct_null_parent_nulls_all_leaves(self): + data = pa.table( + { + "s": pa.array( + [{"a": 1, "b": 2}, None, {"a": 3, "b": 4}], + type=pa.struct([("a", pa.int64()), ("b", pa.int64())]), + ) + } + ) + + tbl = Table(data) + assert tbl.view().to_columns() == { + "s.a": [1, None, 3], + "s.b": [2, None, 4], + } + + def test_struct_null_child_and_null_parent(self): + data = pa.table( + { + "s": pa.array( + [{"a": 1}, {"a": None}, None], + type=pa.struct([("a", pa.int64())]), + ) + } + ) + + tbl = Table(data) + assert tbl.view().to_columns() == {"s.a": [1, None, None]} + + def test_struct_sliced_input(self): + data = pa.table( + { + "s": pa.array( + [{"a": 1}, {"a": 2}, {"a": 3}, {"a": 4}], + type=pa.struct([("a", pa.int64())]), + ) + } + ).slice(1, 2) + + tbl = Table(data) + assert tbl.view().to_columns() == {"s.a": [2, 3]} + + def test_struct_multi_chunk(self): + chunk = pa.array([{"a": 1}], type=pa.struct([("a", pa.int64())])) + data = pa.table({"s": pa.chunked_array([chunk, chunk, chunk])}) + tbl = Table(data) + assert tbl.view().to_columns() == {"s.a": [1, 1, 1]} + + def test_struct_update_matches_declared_dotted_schema(self): + tbl = Table({"id": "integer", "s.a": "integer"}) + tbl.update( + arrow_bytes( + pa.table( + { + "id": pa.array([1], type=pa.int64()), + "s": pa.array( + [{"a": 7}], type=pa.struct([("a", pa.int64())]) + ), + } + ) + ) + ) + + assert tbl.view().to_columns() == {"id": [1], "s.a": [7]} + + +class TestTableArrowList(object): + def test_list_zip_is_the_default(self): + data = pa.table( + { + "x": pa.array([1, 2], type=pa.int64()), + "y": pa.array([[10, 20, 30], [40]], type=pa.list_(pa.int64())), + } + ) + + tbl = Table(data) + assert tbl.schema() == {"x": "integer", "y": "integer"} + assert tbl.view().to_columns() == { + "x": [1, 1, 1, 2], + "y": [10, 20, 30, 40], + } + + def test_list_empty_yields_one_null_row(self): + data = pa.table( + { + "x": pa.array([1, 2], type=pa.int64()), + "y": pa.array([[], [40]], type=pa.list_(pa.int64())), + } + ) + + assert Table(data).view().to_columns() == { + "x": [1, 2], + "y": [None, 40], + } + + def test_list_null_yields_one_null_row(self): + data = pa.table( + { + "x": pa.array([1, 2], type=pa.int64()), + "y": pa.array([None, [40]], type=pa.list_(pa.int64())), + } + ) + + assert Table(data).view().to_columns() == { + "x": [1, 2], + "y": [None, 40], + } + + def test_list_of_struct_composes_both_passes(self): + data = pa.table( + { + "id": pa.array([1], type=pa.int64()), + "orders": pa.array( + [[{"price": 1.5}, {"price": 2.5}]], + type=pa.list_(pa.struct([("price", pa.float64())])), + ), + } + ) + + tbl = Table(data) + assert tbl.schema() == {"id": "integer", "orders.price": "float"} + assert tbl.view().to_columns() == { + "id": [1, 1], + "orders.price": [1.5, 2.5], + } + + def test_list_nested_inside_a_struct(self): + data = pa.table( + { + "id": pa.array([1, 2], type=pa.int64()), + "s": pa.array( + [{"a": [10, 20]}, {"a": [30]}], + type=pa.struct([("a", pa.list_(pa.int64()))]), + ), + } + ) + + tbl = Table(data) + assert tbl.schema() == {"id": "integer", "s.a": "integer"} + assert tbl.view().to_columns() == { + "id": [1, 1, 2], + "s.a": [10, 20, 30], + } + + def test_nested_list_recurses(self): + data = pa.table( + { + "y": pa.array( + [[[1, 2], [3]]], type=pa.list_(pa.list_(pa.int64())) + ) + } + ) + + assert Table(data).view().to_columns() == {"y": [1, 2, 3]} + + def test_list_zip_equal_lengths(self): + data = pa.table( + { + "a": pa.array([[1, 2]], type=pa.list_(pa.int64())), + "b": pa.array([[3, 4]], type=pa.list_(pa.int64())), + } + ) + + assert Table(data).view().to_columns() == {"a": [1, 2], "b": [3, 4]} + + def test_list_multi_chunk(self): + chunk = pa.array([[1, 2]], type=pa.list_(pa.int64())) + data = pa.table({"y": pa.chunked_array([chunk, chunk])}) + assert Table(data).view().to_columns() == {"y": [1, 2, 1, 2]} + + def test_list_cartesian(self): + data = pa.table( + { + "a": pa.array([[1, 2]], type=pa.list_(pa.int64())), + "b": pa.array([[3, 4, 5]], type=pa.list_(pa.int64())), + } + ) + + tbl = Table(data, list_flatten="cartesian") + assert tbl.view().to_columns() == { + "a": [1, 1, 1, 2, 2, 2], + "b": [3, 4, 5, 3, 4, 5], + } + + def test_list_cartesian_empty_counts_as_one(self): + data = pa.table( + { + "a": pa.array([[1, 2]], type=pa.list_(pa.int64())), + "b": pa.array([[]], type=pa.list_(pa.int64())), + } + ) + + tbl = Table(data, list_flatten="cartesian") + assert tbl.view().to_columns() == {"a": [1, 2], "b": [None, None]} + + def test_list_stringify_preserves_legacy_behavior(self): + data = pa.table( + { + "x": pa.array([1, 2], type=pa.int64()), + "y": pa.array([[10, 20], [30]], type=pa.list_(pa.int64())), + } + ) + + tbl = Table(data, list_flatten="stringify") + assert tbl.schema() == {"x": "integer", "y": "string"} + assert tbl.view().to_columns() == { + "x": [1, 2], + "y": ["[10,20]", "[30]"], + } + + def test_list_stringify_integer_widths(self): + data = pa.table( + { + "i8": pa.array([[-1, 2]], type=pa.list_(pa.int8())), + "i16": pa.array([[-300, 300]], type=pa.list_(pa.int16())), + "i32": pa.array([[-70000, 70000]], type=pa.list_(pa.int32())), + "i64": pa.array( + [[-(2**40), 2**40]], type=pa.list_(pa.int64()) + ), + "u8": pa.array([[255]], type=pa.list_(pa.uint8())), + "u16": pa.array([[65535]], type=pa.list_(pa.uint16())), + "u32": pa.array([[4294967295]], type=pa.list_(pa.uint32())), + # Above INT64_MAX, so a signed writer would emit a negative. + "u64": pa.array( + [[18446744073709551615]], type=pa.list_(pa.uint64()) + ), + } + ) + + assert Table(data, list_flatten="stringify").view().to_columns() == { + "i8": ["[-1,2]"], + "i16": ["[-300,300]"], + "i32": ["[-70000,70000]"], + "i64": ["[-1099511627776,1099511627776]"], + "u8": ["[255]"], + "u16": ["[65535]"], + "u32": ["[4294967295]"], + "u64": ["[18446744073709551615]"], + } + + def test_list_flatten_mode_persists_across_update(self): + schema = pa.schema( + [("x", pa.int64()), ("y", pa.list_(pa.int64()))] + ) + + tbl = Table( + arrow_bytes(pa.table({"x": [1], "y": [[10, 20]]}, schema=schema)), + list_flatten="cartesian", + ) + + tbl.update( + arrow_bytes(pa.table({"x": [2], "y": [[30, 40]]}, schema=schema)) + ) + + assert tbl.view().to_columns() == { + "x": [1, 1, 2, 2], + "y": [10, 20, 30, 40], + } + + def test_list_limit_counts_expanded_rows(self): + data = pa.table( + {"y": pa.array([[1, 2, 3, 4]], type=pa.list_(pa.int64()))} + ) + + assert Table(data, limit=2).view().to_columns() == {"y": [3, 4]} + + +class TestTableArrowExpandedIndex(object): + """An index is only rejected when expansion would REPEAT it. A column drawn + from the exploded list takes a distinct element per row, so it is a + legitimate key.""" + + def orders(self, ids, prices): + return pa.table( + { + "batch": pa.array([1], type=pa.int64()), + "orders": pa.array( + [[{"id": i, "price": p} for i, p in zip(ids, prices)]], + type=pa.list_( + pa.struct([("id", pa.int64()), ("price", pa.float64())]) + ), + ), + } + ) + + def test_index_on_a_column_from_the_list_is_allowed(self): + tbl = Table(self.orders([1, 2, 3], [1.5, 2.5, 3.5]), index="orders.id") + assert tbl.view().to_columns() == { + "batch": [1, 1, 1], + "orders.id": [1, 2, 3], + "orders.price": [1.5, 2.5, 3.5], + } + + def test_index_from_the_list_updates_by_element(self): + tbl = Table(self.orders([1, 2], [1.5, 2.5]), index="orders.id") + tbl.update(arrow_bytes(self.orders([2, 3], [9.5, 3.5]))) + assert tbl.view().to_columns() == { + "batch": [1, 1, 1], + "orders.id": [1, 2, 3], + "orders.price": [1.5, 9.5, 3.5], + } + + def test_index_on_a_sibling_is_rejected(self): + with pytest.raises(psp.PerspectiveError, match=r"`batch`"): + Table(self.orders([1, 2], [1.5, 2.5]), index="batch") + + def test_index_on_a_list_column_itself_is_allowed(self): + data = pa.table( + { + "x": pa.array([1], type=pa.int64()), + "id": pa.array([[10, 20]], type=pa.list_(pa.int64())), + } + ) + + assert Table(data, index="id").view().to_columns() == { + "x": [1, 1], + "id": [10, 20], + } + + def test_cartesian_with_two_lists_repeats_every_column(self): + data = pa.table( + { + "id": pa.array([[1, 2]], type=pa.list_(pa.int64())), + "b": pa.array([[3, 4, 5]], type=pa.list_(pa.int64())), + } + ) + + with pytest.raises(psp.PerspectiveError, match=r"`id`"): + Table(data, index="id", list_flatten="cartesian") + + def test_cartesian_with_one_list_is_allowed(self): + data = pa.table( + { + "x": pa.array([1], type=pa.int64()), + "id": pa.array([[10, 20]], type=pa.list_(pa.int64())), + } + ) + + tbl = Table(data, index="id", list_flatten="cartesian") + assert tbl.view().to_columns() == {"x": [1, 1], "id": [10, 20]} + + def test_stringify_keeps_a_sibling_index_usable(self): + data = pa.table( + { + "id": pa.array([1, 2], type=pa.int64()), + "y": pa.array([[10, 20], [30]], type=pa.list_(pa.int64())), + } + ) + + tbl = Table(data, index="id", list_flatten="stringify") + assert tbl.view().to_columns() == { + "id": [1, 2], + "y": ["[10,20]", "[30]"], + } + + +class TestTableArrowGather(object): + """The expansion is deferred into the `t_column` write rather than + materialized in Arrow, so these stress the gather paths specifically.""" + + def test_expansion_gathers_across_chunks(self): + x = pa.chunked_array( + [pa.array([1], type=pa.int64()), pa.array([2], type=pa.int64())] + ) + + y = pa.chunked_array( + [ + pa.array([[10, 20]], type=pa.list_(pa.int64())), + pa.array([[30]], type=pa.list_(pa.int64())), + ] + ) + + assert Table(pa.table({"x": x, "y": y})).view().to_columns() == { + "x": [1, 1, 2], + "y": [10, 20, 30], + } + + def test_expansion_gathers_strings(self): + data = pa.table( + { + "s": pa.array(["a", "b"], type=pa.string()), + "y": pa.array([[1, 2, 3], [4]], type=pa.list_(pa.int64())), + } + ) + + assert Table(data).view().to_columns() == { + "s": ["a", "a", "a", "b"], + "y": [1, 2, 3, 4], + } + + def test_expansion_gathers_nulls_in_siblings(self): + data = pa.table( + { + "x": pa.array([None, 2], type=pa.int64()), + "y": pa.array([[10, 20], [30]], type=pa.list_(pa.int64())), + } + ) + + assert Table(data).view().to_columns() == { + "x": [None, None, 2], + "y": [10, 20, 30], + } + + def test_expansion_of_all_empty_lists(self): + data = pa.table( + { + "x": pa.array([1, 2], type=pa.int64()), + "y": pa.array([[], []], type=pa.list_(pa.int64())), + } + ) + + assert Table(data).view().to_columns() == { + "x": [1, 2], + "y": [None, None], + } + + def test_sliced_input_with_nulls(self): + data = pa.table( + {"a": pa.array([1, None, 3, None, 5], type=pa.int64())} + ).slice(1, 3) + + assert Table(data).view().to_columns() == {"a": [None, 3, None]} + + +class TestTableArrowFlatUnaffected(object): + """A table with neither struct nor list columns must be untouched by + normalization; these lock the flat path against regressions.""" + + def test_flat_arrow_roundtrip(self): + data = pa.table( + { + "a": pa.array([1, 2, 3], type=pa.int64()), + "b": pa.array(["x", "y", "z"], type=pa.string()), + "c": pa.array([1.5, 2.5, 3.5], type=pa.float64()), + } + ) + + tbl = Table(data) + assert tbl.schema() == {"a": "integer", "b": "string", "c": "float"} + assert tbl.view().to_columns() == { + "a": [1, 2, 3], + "b": ["x", "y", "z"], + "c": [1.5, 2.5, 3.5], + } + + def test_flat_arrow_indexed_update(self): + data = pa.table( + { + "a": pa.array([1, 2], type=pa.int64()), + "b": pa.array([10, 20], type=pa.int64()), + } + ) + + tbl = Table(data, index="a") + tbl.update( + arrow_bytes( + pa.table( + { + "a": pa.array([2], type=pa.int64()), + "b": pa.array([99], type=pa.int64()), + } + ) + ) + ) + + assert tbl.view().to_columns() == {"a": [1, 2], "b": [10, 99]} diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_coerce_types.py b/rust/perspective-python/perspective/tests/virtual_servers/test_coerce_types.py index 5280e25e78..3d2cac9fed 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_coerce_types.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_coerce_types.py @@ -11,199 +11,454 @@ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ import io -import json from decimal import Decimal import pyarrow as pa import pyarrow.ipc as ipc +import pytest + +import perspective +from perspective import Client +from perspective.virtual_servers import VirtualServerHandler + +FEATURES = { + "group_by": True, + "split_by": True, + "sort": True, + "group_rollup_mode": ["rollup", "flat", "total"], + "filter_ops": { + "integer": ["=="], + "float": ["=="], + "string": ["=="], + "boolean": ["=="], + "date": ["=="], + "datetime": ["=="], + }, + "aggregates": { + "integer": ["sum"], + "float": ["sum"], + "string": ["count"], + }, +} + + +class ArrowFixtureHandler(VirtualServerHandler): + def __init__(self, arrow_table, schema, ipc_bytes=None): + self.arrow_table = arrow_table + self.schema = schema + # `write_table` *drops* zero-row batches, so the only way to serve + # a stream that carries one is to hand over the bytes. + self.ipc_bytes = ipc_bytes + + def get_features(self): + return FEATURES + + def get_hosted_tables(self): + return ["fixture"] + + def table_schema(self, table_name): + return self.schema + + def table_size(self, table_name): + return self.arrow_table.num_rows + + def table_make_view(self, table_name, view_name, config): + pass + + def view_delete(self, view_name): + pass + + def view_get_data(self, view_name, config, schema, viewport, data): + if self.ipc_bytes is not None: + data.from_arrow_ipc(self.ipc_bytes) + return + + buf = io.BytesIO() + with ipc.new_stream(buf, self.arrow_table.schema) as writer: + writer.write_table(self.arrow_table) + + data.from_arrow_ipc(buf.getvalue()) + + +class SetColFixtureHandler(VirtualServerHandler): + def __init__(self, row_path, schema): + self.row_path = row_path + self.schema = schema + + def get_features(self): + return FEATURES + + def get_hosted_tables(self): + return ["fixture"] + + def table_schema(self, table_name): + return self.schema + + def table_size(self, table_name): + return len(self.row_path) + + def table_make_view(self, table_name, view_name, config): + pass + + def view_delete(self, view_name): + pass + + def view_get_data(self, view_name, config, schema, viewport, data): + for row_idx, value in enumerate(self.row_path): + data.set_col("string", "__ROW_PATH_0__", row_idx, value, 0) -from perspective import VirtualDataSlice +def make_client(handler): + session = perspective.VirtualServer(handler) -def round_trip(arrow_table): - """Serialize a PyArrow table to IPC, feed through VirtualDataSlice, read back as JSON.""" - buf = io.BytesIO() - with ipc.new_stream(buf, arrow_table.schema) as writer: - writer.write_table(arrow_table) - ds = VirtualDataSlice() - ds.from_arrow_ipc(buf.getvalue()) - return json.loads(ds.render_to_columns_json()) + def handle_request(msg): + handle_response(session.handle_request(msg)) + + def handle_response(msg): + client.handle_response(msg) + + client = Client(handle_request) + return client + + +def arrow_client(arrow_table, schema=None, ipc_bytes=None): + if schema is None: + schema = { + name: "string" + for name in arrow_table.column_names + if not name.startswith("__") + } + + return make_client(ArrowFixtureHandler(arrow_table, schema, ipc_bytes)) + + +def round_trip(arrow_table, schema=None, **config): + table = arrow_client(arrow_table, schema).open_table("fixture") + view = table.view(**config) + result = view.to_columns() + view.delete() + return result + + +def dictionary(values, indices, index_type, value_type=pa.utf8()): + return pa.DictionaryArray.from_arrays( + pa.array(indices, type=index_type), + pa.array(values, type=value_type), + ) class TestCoerceSmallIntegers: def test_coerce_int8(self): table = pa.table({"col": pa.array([-1, 127, None], type=pa.int8())}) - result = round_trip(table) + result = round_trip(table, {"col": "integer"}) assert result["col"] == [-1, 127, None] def test_coerce_int16(self): table = pa.table({"col": pa.array([-300, 32000, None], type=pa.int16())}) - result = round_trip(table) + result = round_trip(table, {"col": "integer"}) assert result["col"] == [-300, 32000, None] class TestCoerceUnsignedIntegers: def test_coerce_uint8(self): table = pa.table({"col": pa.array([0, 255, None], type=pa.uint8())}) - result = round_trip(table) + result = round_trip(table, {"col": "integer"}) assert result["col"] == [0, 255, None] def test_coerce_uint16(self): table = pa.table({"col": pa.array([0, 65535, None], type=pa.uint16())}) - result = round_trip(table) + result = round_trip(table, {"col": "integer"}) assert result["col"] == [0, 65535, None] def test_coerce_uint32(self): table = pa.table({"col": pa.array([0, 4_294_967_295, None], type=pa.uint32())}) - result = round_trip(table) + result = round_trip(table, {"col": "float"}) assert result["col"] == [0.0, 4_294_967_295.0, None] def test_coerce_uint64(self): table = pa.table({"col": pa.array([0, 1 << 53, None], type=pa.uint64())}) - result = round_trip(table) + result = round_trip(table, {"col": "float"}) assert result["col"] == [0.0, 9_007_199_254_740_992.0, None] -class TestCoerceFloat32: +class TestCoerceFloats: def test_coerce_float32(self): table = pa.table({"col": pa.array([3.14, -0.0, None], type=pa.float32())}) - result = round_trip(table) - assert abs(result["col"][0] - 3.14) < 0.001 + result = round_trip(table, {"col": "float"}) + assert result["col"][0] == pytest.approx(3.14, abs=0.001) assert result["col"][1] == 0.0 assert result["col"][2] is None + def test_coerce_float16(self): + table = pa.table({"col": pa.array([1.5, -2.0, None], type=pa.float16())}) + result = round_trip(table, {"col": "float"}) + assert result["col"] == [1.5, -2.0, None] + + +class TestCoerceDecimal: + def test_coerce_decimal128(self): + table = pa.table( + { + "col": pa.array( + [Decimal("1.234"), Decimal("-5.678"), None], + type=pa.decimal128(18, 3), + ) + } + ) + result = round_trip(table, {"col": "float"}) + assert result["col"][0] == pytest.approx(1.234) + assert result["col"][1] == pytest.approx(-5.678) + assert result["col"][2] is None + -class TestCoerceDate64: +class TestCoerceDates: def test_coerce_date64(self): day = 19738 table = pa.table({"col": pa.array([day * 86_400_000, None], type=pa.date64())}) - result = round_trip(table) + result = round_trip(table, {"col": "date"}) + assert result["col"] == [day * 86_400_000, None] + + def test_coerce_date32(self): + day = 19738 + table = pa.table({"col": pa.array([day, None], type=pa.date32())}) + result = round_trip(table, {"col": "date"}) assert result["col"] == [day * 86_400_000, None] -class TestCoerceTime: - def test_coerce_time32_second(self): - table = pa.table({"col": pa.array([49530, None], type=pa.time32("s"))}) - result = round_trip(table) - assert result["col"] == [49_530_000, None] +class TestCoerceTimes: + def test_coerce_time32_seconds(self): + table = pa.table({"col": pa.array([3661, None], type=pa.time32("s"))}) + result = round_trip(table, {"col": "datetime"}) + assert result["col"] == [3_661_000, None] - def test_coerce_time32_millisecond(self): - table = pa.table({"col": pa.array([49_530_000, None], type=pa.time32("ms"))}) - result = round_trip(table) - assert result["col"] == [49_530_000, None] + def test_coerce_time32_millis(self): + table = pa.table({"col": pa.array([3_661_000, None], type=pa.time32("ms"))}) + result = round_trip(table, {"col": "datetime"}) + assert result["col"] == [3_661_000, None] - def test_coerce_time64_microsecond(self): + def test_coerce_time64_micros(self): + table = pa.table({"col": pa.array([3_661_000_000, None], type=pa.time64("us"))}) + result = round_trip(table, {"col": "datetime"}) + assert result["col"] == [3_661_000, None] + + def test_coerce_time64_nanos(self): table = pa.table( - {"col": pa.array([49_530_000_000, None], type=pa.time64("us"))} + {"col": pa.array([3_661_000_000_000, None], type=pa.time64("ns"))} ) - result = round_trip(table) - assert result["col"] == [49_530_000, None] + result = round_trip(table, {"col": "datetime"}) + assert result["col"] == [3_661_000, None] + + @pytest.mark.parametrize( + "unit,value", + [ + ("s", 1_700_000_000), + ("ms", 1_700_000_000_000), + ("us", 1_700_000_000_000_000), + ("ns", 1_700_000_000_000_000_000), + ], + ) + def test_coerce_timestamp_with_timezone(self, unit, value): + table = pa.table( + {"col": pa.array([value, None], type=pa.timestamp(unit, tz="UTC"))} + ) + result = round_trip(table, {"col": "datetime"}) + assert result["col"] == [1_700_000_000_000, None] - def test_coerce_time64_nanosecond(self): + def test_coerce_timestamp_seconds(self): table = pa.table( - {"col": pa.array([49_530_000_000_000, None], type=pa.time64("ns"))} + {"col": pa.array([1_700_000_000, None], type=pa.timestamp("s"))} ) - result = round_trip(table) - assert result["col"] == [49_530_000, None] + result = round_trip(table, {"col": "datetime"}) + assert result["col"] == [1_700_000_000_000, None] -class TestCoerceLargeUtf8: - def test_coerce_large_utf8(self): - table = pa.table({"col": pa.array(["hello", "", None], type=pa.large_utf8())}) +class TestCoerceStrings: + def test_coerce_utf8(self): + table = pa.table({"col": pa.array(["a", None, "c"], type=pa.utf8())}) result = round_trip(table) - assert result["col"] == ["hello", "", None] - + assert result["col"] == ["a", None, "c"] -class TestCoerceDecimal128: - def test_coerce_decimal128(self): - arr = pa.array([Decimal("1234.5678"), None], type=pa.decimal128(10, 4)) - table = pa.table({"col": arr}) + def test_coerce_large_utf8(self): + table = pa.table({"col": pa.array(["a", None, "c"], type=pa.large_utf8())}) result = round_trip(table) - assert result["col"] == [1234.5678, None] + assert result["col"] == ["a", None, "c"] -class TestCoerceInt64: - def test_coerce_int64(self): - table = pa.table({"col": pa.array([1, -1, None], type=pa.int64())}) +class TestFallback: + def test_fallback_fixed_size_binary(self): + # No canonical mapping - lossy but total, and warned about. + table = pa.table( + {"col": pa.array([b"ab", b"cd", None], type=pa.binary(2))} + ) result = round_trip(table) - assert result["col"] == [1.0, -1.0, None] - + assert isinstance(result["col"][0], str) + assert isinstance(result["col"][1], str) + assert result["col"][0] != result["col"][1] + assert result["col"][2] is None -class TestCoerceTimestamp: - def test_coerce_timestamp_second(self): - table = pa.table({"col": pa.array([1000, None], type=pa.timestamp("s"))}) - result = round_trip(table) - assert result["col"] == [1_000_000, None] - def test_coerce_timestamp_microsecond(self): - table = pa.table({"col": pa.array([1_000_000, None], type=pa.timestamp("us"))}) - result = round_trip(table) - assert result["col"] == [1000, None] +class TestDictionary: + @pytest.mark.parametrize( + "index_type", + [ + pa.int8(), + pa.int16(), + pa.int32(), + pa.int64(), + pa.uint8(), + pa.uint16(), + pa.uint32(), + pa.uint64(), + ], + ) + @pytest.mark.parametrize("value_type", [pa.utf8(), pa.large_utf8()]) + def test_dictionary_keys_and_values(self, index_type, value_type): + col = dictionary( + ["alpha", "beta"], [0, 1, 0, None], index_type, value_type + ) + result = round_trip(pa.table({"col": col})) + assert result["col"] == ["alpha", "beta", "alpha", None] - def test_coerce_timestamp_nanosecond(self): - table = pa.table( - {"col": pa.array([1_000_000_000, None], type=pa.timestamp("ns"))} + def test_dictionary_null_value_slot(self): + col = pa.DictionaryArray.from_arrays( + pa.array([0, 1], type=pa.int8()), + pa.array(["alpha", None], type=pa.utf8()), ) - result = round_trip(table) - assert result["col"] == [1000, None] + result = round_trip(pa.table({"col": col})) + assert result["col"] == ["alpha", None] + def test_dictionary_of_integers(self): + col = pa.DictionaryArray.from_arrays( + pa.array([0, 1, 0], type=pa.uint8()), + pa.array([-7, 42], type=pa.int16()), + ) + result = round_trip(pa.table({"col": col}), {"col": "integer"}) + assert result["col"] == [-7, 42, -7] -class TestPassthrough: - def test_passthrough_bool(self): - table = pa.table({"col": pa.array([True, False, None], type=pa.bool_())}) - result = round_trip(table) - assert result["col"] == [True, False, None] + def test_dictionary_group_by(self): + table = pa.table( + { + "__GROUPING_ID__": pa.array([1, 0, 0], type=pa.uint64()), + "__ROW_PATH_0__": dictionary( + ["alpha", "beta"], [None, 0, 1], pa.uint8() + ), + "Sales": pa.array([3.0, 1.0, 2.0], type=pa.float64()), + } + ) - def test_passthrough_utf8(self): - table = pa.table({"col": pa.array(["a", "b", None], type=pa.utf8())}) - result = round_trip(table) - assert result["col"] == ["a", "b", None] + result = round_trip( + table, + {"Region": "string", "Sales": "float"}, + group_by=["Region"], + columns=["Sales"], + ) - def test_passthrough_float64(self): - table = pa.table({"col": pa.array([1.5, -2.5, None], type=pa.float64())}) - result = round_trip(table) - assert result["col"] == [1.5, -2.5, None] + assert result["__ROW_PATH__"] == [[], ["alpha"], ["beta"]] + assert result["Sales"] == [3.0, 1.0, 2.0] - def test_passthrough_int32(self): - table = pa.table({"col": pa.array([1, -1, None], type=pa.int32())}) - result = round_trip(table) - assert result["col"] == [1, -1, None] - def test_passthrough_date32(self): - table = pa.table({"col": pa.array([19738, None], type=pa.date32())}) - result = round_trip(table) - assert result["col"] == [19738 * 86_400_000, None] +class TestGroupingId: + def test_unsigned_grouping_id(self): + table = pa.table( + { + "__GROUPING_ID__": pa.array([1, 0, 0], type=pa.uint64()), + "__ROW_PATH_0__": pa.array([None, "alpha", "beta"], type=pa.utf8()), + "Sales": pa.array([3.0, 1.0, 2.0], type=pa.float64()), + } + ) - def test_passthrough_timestamp_millisecond(self): - table = pa.table({"col": pa.array([1000, None], type=pa.timestamp("ms"))}) - result = round_trip(table) - assert result["col"] == [1000, None] + result = round_trip( + table, + {"Region": "string", "Sales": "float"}, + group_by=["Region"], + columns=["Sales"], + ) + assert result["__ROW_PATH__"] == [[], ["alpha"], ["beta"]] -class TestFallback: - def test_fallback_fixed_size_binary(self): + def test_decimal_row_path(self): table = pa.table( - {"col": pa.array([b"\x01\x02", b"\x03\x04", None], type=pa.binary(2))} + { + "__GROUPING_ID__": pa.array([1, 0], type=pa.int64()), + "__ROW_PATH_0__": pa.array( + [None, Decimal("1.500")], type=pa.decimal128(18, 3) + ), + "Sales": pa.array([3.0, 3.0], type=pa.float64()), + } ) - result = round_trip(table) - col = result["col"] - assert isinstance(col[0], str) - assert isinstance(col[1], str) - assert col[0] != col[1] - assert col[2] is None + + result = round_trip( + table, + {"Price": "float", "Sales": "float"}, + group_by=["Price"], + columns=["Sales"], + ) + + assert result["__ROW_PATH__"] == [[], [1.5]] class TestEmpty: - def test_empty_batch(self): - # PyArrow's new_stream with an empty table writes no record batches, - # so we use RecordBatchStreamWriter directly to produce a valid empty batch. - schema = pa.schema([("col", pa.int8())]) - batch = pa.record_batch({"col": pa.array([], type=pa.int8())}) + def test_zero_row_batch(self): + # One batch, no rows - written explicitly, since `write_table` + # would drop it. + schema = pa.schema([("col", pa.int32())]) buf = io.BytesIO() writer = ipc.RecordBatchStreamWriter(buf, schema) - writer.write_batch(batch) + writer.write_batch(pa.record_batch([[]], schema=schema)) writer.close() - ds = VirtualDataSlice() - ds.from_arrow_ipc(buf.getvalue()) - result = json.loads(ds.render_to_columns_json()) + + table = pa.Table.from_batches([], schema) + client = arrow_client(table, {"col": "integer"}, ipc_bytes=buf.getvalue()) + view = client.open_table("fixture").view() + assert view.to_columns()["col"] == [] + view.delete() + + def test_no_batches_at_all(self): + schema = pa.schema([("col", pa.int32())]) + table = pa.Table.from_batches([], schema) + result = round_trip(table, {"col": "integer"}) assert result["col"] == [] + + def test_row_path_only_batch(self): + table = pa.table( + { + "__GROUPING_ID__": pa.array([1, 0, 0], type=pa.int64()), + "__ROW_PATH_0__": pa.array([None, "alpha", "beta"], type=pa.utf8()), + } + ) + + client = arrow_client(table, {"Region": "string"}) + view = client.open_table("fixture").view(group_by=["Region"], columns=[]) + assert view.to_json() == [ + {"__ROW_PATH__": []}, + {"__ROW_PATH__": ["alpha"]}, + {"__ROW_PATH__": ["beta"]}, + ] + + view.delete() + + def test_metadata_only_batch_preserves_row_count(self): + table = pa.table({"__GROUPING_ID__": pa.array([0, 0, 0], type=pa.int64())}) + client = arrow_client(table, {"Region": "string"}) + view = client.open_table("fixture").view(columns=[]) + assert view.to_json() == [{}, {}, {}] + view.delete() + + def test_set_col_row_path_only(self): + handler = SetColFixtureHandler(["alpha", "beta"], {"Region": "string"}) + view = make_client(handler).open_table("fixture").view( + group_by=["Region"], columns=[] + ) + + assert view.to_json() == [ + {"__ROW_PATH__": ["alpha"]}, + {"__ROW_PATH__": ["beta"]}, + ] + + view.delete() + + +class TestNotConstructible: + def test_virtual_data_slice_is_not_constructible(self): + with pytest.raises(TypeError): + perspective.VirtualDataSlice() diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py b/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py index c97697909b..123a068412 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py @@ -47,6 +47,45 @@ def _get_superstore_parquet(): SUPERSTORE_PARQUET = _get_superstore_parquet() +def _load_coerce_types(db): + """A column of each DuckDB type whose Arrow needs coercing. + + `ENUM` is the interesting one: DuckDB dictionary-encodes it and picks + the key width from cardinality, so it arrives as a dictionary with + unsigned keys rather than the `Int32` Perspective uses. + """ + db.execute("CREATE TYPE mood AS ENUM ('happy', 'sad')") + db.execute(""" + CREATE TABLE coerce_types ( + "tiny" TINYINT, + "small" SMALLINT, + "utiny" UTINYINT, + "usmall" USMALLINT, + "uint" UINTEGER, + "ubig" UBIGINT, + "big" BIGINT, + "float" REAL, + "decimal" DECIMAL(18, 3), + "time" TIME, + "timestamp" TIMESTAMP, + "date" DATE, + "enum" mood, + "string" VARCHAR + ) + """) + + db.execute(""" + INSERT INTO coerce_types VALUES + (-1, -300, 255, 65535, 4294967295, 9007199254740992, + 9007199254740992, 1.5, 1.234, TIME '01:01:01', + TIMESTAMP '2023-01-01 00:00:00', DATE '2023-01-01', + 'happy', 'a'), + (1, 300, 0, 0, 0, 0, -9007199254740992, -1.5, -5.678, + TIME '00:00:01', TIMESTAMP '2023-01-02 00:00:00', + DATE '2023-01-02', 'sad', 'b') + """) + + @pytest.fixture(scope="module") def client(): db = duckdb.connect() @@ -55,6 +94,7 @@ def client(): f"CREATE TABLE superstore AS SELECT * FROM read_parquet('{SUPERSTORE_PARQUET}')" ) + _load_coerce_types(db) server = DuckDBVirtualServer(db) def handle_request(msg): @@ -71,7 +111,7 @@ def handle_response(msg): class TestDuckDBClient: def test_get_hosted_table_names(self, client): tables = client.get_hosted_table_names() - assert tables == ["memory.superstore"] + assert sorted(tables) == ["memory.coerce_types", "memory.superstore"] class TestDuckDBTable: @@ -906,3 +946,140 @@ def test_min_max_with_filter(self, client): assert min_val >= 11 assert max_val == 14 view.delete() + + +class TestDuckDBCoerceTypes: + """The DuckDB types whose Arrow is not already one of Perspective's.""" + + def test_schema(self, client): + table = client.open_table("memory.coerce_types") + assert table.schema() == { + "tiny": "integer", + "small": "integer", + "utiny": "integer", + "usmall": "integer", + "uint": "float", + "ubig": "float", + "big": "float", + "float": "float", + "decimal": "float", + "time": "datetime", + "timestamp": "datetime", + "date": "date", + "enum": "string", + "string": "string", + } + + def test_narrow_integers_flat(self, client): + table = client.open_table("memory.coerce_types") + view = table.view(columns=["tiny", "small", "utiny", "usmall"]) + assert view.to_json() == [ + {"tiny": -1, "small": -300, "utiny": 255, "usmall": 65535}, + {"tiny": 1, "small": 300, "utiny": 0, "usmall": 0}, + ] + view.delete() + + def test_wide_and_fractional_numbers_flat(self, client): + table = client.open_table("memory.coerce_types") + view = table.view(columns=["uint", "ubig", "big", "float", "decimal"]) + assert view.to_json() == [ + { + "uint": 4294967295.0, + "ubig": 9007199254740992.0, + "big": 9007199254740992.0, + "float": 1.5, + "decimal": pytest.approx(1.234), + }, + { + "uint": 0.0, + "ubig": 0.0, + "big": -9007199254740992.0, + "float": -1.5, + "decimal": pytest.approx(-5.678), + }, + ] + view.delete() + + def test_temporal_flat(self, client): + table = client.open_table("memory.coerce_types") + view = table.view(columns=["time", "timestamp", "date"]) + assert view.to_json() == [ + { + "time": 3661000, + "timestamp": 1672531200000, + "date": 1672531200000, + }, + { + "time": 1000, + "timestamp": 1672617600000, + "date": 1672617600000, + }, + ] + view.delete() + + def test_enum_flat(self, client): + # Dictionary-encoded by DuckDB, with a key width chosen from the + # `ENUM`'s cardinality rather than the `Int32` Perspective uses. + table = client.open_table("memory.coerce_types") + view = table.view(columns=["enum", "string"]) + assert view.to_json() == [ + {"enum": "happy", "string": "a"}, + {"enum": "sad", "string": "b"}, + ] + view.delete() + + def test_enum_group_by(self, client): + # The #3149 repro: the row path is read out of that dictionary. + table = client.open_table("memory.coerce_types") + view = table.view( + group_by=["enum"], + columns=["tiny"], + aggregates={"tiny": "sum"}, + ) + assert view.to_json() == [ + {"__ROW_PATH__": [], "tiny": 0}, + {"__ROW_PATH__": ["happy"], "tiny": -1}, + {"__ROW_PATH__": ["sad"], "tiny": 1}, + ] + view.delete() + + def test_decimal_group_by(self, client): + table = client.open_table("memory.coerce_types") + view = table.view( + group_by=["decimal"], + columns=["tiny"], + aggregates={"tiny": "sum"}, + ) + assert view.to_json() == [ + {"__ROW_PATH__": [], "tiny": 0}, + {"__ROW_PATH__": [pytest.approx(-5.678)], "tiny": 1}, + {"__ROW_PATH__": [pytest.approx(1.234)], "tiny": -1}, + ] + view.delete() + + def test_column_values_view(self, client): + # The filter dropdown's query shape - group by the column, select + # no columns at all. + table = client.open_table("memory.coerce_types") + view = table.view(group_by=["enum"], columns=[]) + csv = view.to_csv() + assert [line for line in csv.splitlines() if line] == [ + "__ROW_PATH_0__", + "null", + '"happy"', + '"sad"', + ] + view.delete() + + def test_filter_matching_nothing(self, client): + # DuckDB returns a chunkless table for an empty result, which + # `write_table` serializes as a schema with no record batches at + # all. That is a view with no rows, not a broken stream. + table = client.open_table("memory.coerce_types") + view = table.view( + columns=["tiny"], + filter=[["string", "==", "no such value"]], + ) + assert view.to_json() == [] + assert view.to_columns() == {"tiny": []} + view.delete() diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py b/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py index 29b6363ebb..90127fbe7d 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py @@ -1030,3 +1030,17 @@ def test_expressions_group_by_sort(self, client): ] ) view.delete() + + def test_column_values_view(self, client): + table = client.open_table("superstore") + view = table.view(group_by=["Region"], columns=[]) + csv = view.to_csv() + assert [line for line in csv.splitlines() if line] == [ + "__ROW_PATH_0__", + "null", + '"Central"', + '"East"', + '"South"', + '"West"', + ] + view.delete() diff --git a/rust/perspective-python/perspective/virtual_servers/clickhouse.py b/rust/perspective-python/perspective/virtual_servers/clickhouse.py index 0265abc2a3..43b775c2f0 100644 --- a/rust/perspective-python/perspective/virtual_servers/clickhouse.py +++ b/rust/perspective-python/perspective/virtual_servers/clickhouse.py @@ -55,28 +55,37 @@ "string_agg", ] -# The window aggregates the SQL translation supports, per source column -# type (`ema` is recursive - no SQL window equivalent). +# Window functions. Renamed from Perspective's `stddev`/`var` to the SQL +# standard spellings DuckDB and ClickHouse both accept, since the advertised +# name is now emitted verbatim. +# +# NOTE: this set is inherited from the DuckDB handler and has NOT been audited +# against a live ClickHouse - see the aggregate lists below, which have the +# same problem. ClickHouse's own navigation functions are `lagInFrame` / +# `leadInFrame`, and its ranking set differs; both need verifying before being +# advertised here. +FRAMES = ["rows", "range", "cumulative"] + WINDOW_AGGREGATES = [ - "sum", - "avg", - "count", - "min", - "max", - "stddev", - "var", - "lag", - "lead", - "diff", - "rate", + {"name": "sum", "frames": FRAMES, "result_type": "float"}, + {"name": "avg", "frames": FRAMES, "result_type": "float"}, + {"name": "count", "frames": FRAMES, "result_type": "float"}, + {"name": "min", "frames": FRAMES}, + {"name": "max", "frames": FRAMES}, + {"name": "stddev_samp", "frames": FRAMES, "result_type": "float"}, + {"name": "var_samp", "frames": FRAMES, "result_type": "float"}, + {"name": "lag", "offset": True}, + {"name": "lead", "offset": True}, + {"name": "diff", "offset": True, "result_type": "float"}, + {"name": "rate", "frames": ["range"], "result_type": "float"}, ] WINDOW_AGGREGATES_ANY = [ - "count", - "min", - "max", - "lag", - "lead", + {"name": "count", "frames": FRAMES, "result_type": "float"}, + {"name": "min", "frames": FRAMES}, + {"name": "max", "frames": FRAMES}, + {"name": "lag", "offset": True}, + {"name": "lead", "offset": True}, ] FILTER_OPS = [ diff --git a/rust/perspective-python/perspective/virtual_servers/duckdb.py b/rust/perspective-python/perspective/virtual_servers/duckdb.py index 83a1469ee4..87b9e1bfad 100644 --- a/rust/perspective-python/perspective/virtual_servers/duckdb.py +++ b/rust/perspective-python/perspective/virtual_servers/duckdb.py @@ -68,28 +68,69 @@ "string_agg", ] -# The window aggregates the SQL translation supports, per source column -# type (`ema` is recursive - no SQL window equivalent). +# Window functions, in DuckDB's own vocabulary - the advertised name is the +# SQL function, emitted verbatim. `frames` are the frame kinds the function +# accepts (empty = none), and `result_type` is the output column type, omitted +# where it is the source column's. +# +# Result types follow `duckdb_type_to_psp`: DuckDB's counts and ranks are +# `BIGINT`, which Perspective's 32-bit `integer` cannot hold, so they are +# `float` - the same mapping the view's own schema will report. +FRAMES = ["rows", "range", "cumulative"] + WINDOW_AGGREGATES = [ - "sum", - "avg", - "count", - "min", - "max", - "stddev", - "var", - "lag", - "lead", - "diff", - "rate", + {"name": "sum", "frames": FRAMES, "result_type": "float"}, + {"name": "avg", "frames": FRAMES, "result_type": "float"}, + {"name": "count", "frames": FRAMES, "result_type": "float"}, + {"name": "min", "frames": FRAMES}, + {"name": "max", "frames": FRAMES}, + {"name": "product", "frames": FRAMES, "result_type": "float"}, + {"name": "median", "frames": FRAMES, "result_type": "float"}, + # DuckDB spells sample and population variants separately, so both are + # offered rather than one being picked on the user's behalf. + {"name": "stddev_samp", "frames": FRAMES, "result_type": "float"}, + {"name": "stddev_pop", "frames": FRAMES, "result_type": "float"}, + {"name": "var_samp", "frames": FRAMES, "result_type": "float"}, + {"name": "var_pop", "frames": FRAMES, "result_type": "float"}, + # Navigation. + {"name": "first_value", "frames": FRAMES}, + {"name": "last_value", "frames": FRAMES}, + {"name": "nth_value", "frames": FRAMES, "offset": True}, + {"name": "lag", "offset": True}, + {"name": "lead", "offset": True}, + # Ranking. These take no source column - the window's `order_by` is their + # input - but Perspective requires one, so the choice of source is + # immaterial for them. + {"name": "row_number", "result_type": "float"}, + {"name": "rank", "result_type": "float"}, + {"name": "dense_rank", "result_type": "float"}, + {"name": "percent_rank", "result_type": "float"}, + {"name": "cume_dist", "result_type": "float"}, + # `ntile`'s argument is a bucket count rather than a row offset. + {"name": "ntile", "offset": True, "result_type": "float"}, + # Perspective's own, with no DuckDB equivalent - the SQL translation + # synthesizes them from `lag` and `first_value`. + {"name": "diff", "offset": True, "result_type": "float"}, + {"name": "rate", "frames": ["range"], "result_type": "float"}, ] +# Arithmetic is undefined for the non-numeric types; ordering and navigation +# are not. WINDOW_AGGREGATES_ANY = [ - "count", - "min", - "max", - "lag", - "lead", + {"name": "count", "frames": FRAMES, "result_type": "float"}, + {"name": "min", "frames": FRAMES}, + {"name": "max", "frames": FRAMES}, + {"name": "first_value", "frames": FRAMES}, + {"name": "last_value", "frames": FRAMES}, + {"name": "nth_value", "frames": FRAMES, "offset": True}, + {"name": "lag", "offset": True}, + {"name": "lead", "offset": True}, + {"name": "row_number", "result_type": "float"}, + {"name": "rank", "result_type": "float"}, + {"name": "dense_rank", "result_type": "float"}, + {"name": "percent_rank", "result_type": "float"}, + {"name": "cume_dist", "result_type": "float"}, + {"name": "ntile", "offset": True, "result_type": "float"}, ] FILTER_OPS = [ @@ -228,22 +269,61 @@ def view_get_data(self, view_name, config, schema, viewport, data): def duckdb_type_to_psp(name): - """Convert a DuckDB `dtype` to a Perspective `ColumnType`.""" - if name == "VARCHAR": - return "string" - if name in ("DOUBLE", "BIGINT", "HUGEINT"): - return "float" - if name == "INTEGER": + """Convert a DuckDB `dtype` to a Perspective `ColumnType`. + + Must agree with `coerce_column` in `perspective-client`, which decides + the Arrow type the same column's data arrives as - a column declared + `integer` whose data coerces to `Float64` gets numeric filters the + engine then rejects. The mapping is duplicated in `duckdb.ts` for + DuckDB WASM; change both. + + `BIGINT` and wider go to `float` because Perspective's `integer` is + 32-bit, matching the `Int64 -> Float64` coercion. `TIME` goes to + `datetime` because that is what `Time32`/`Time64` coerce to. + """ + name = name.upper() + + if name.startswith("BOOL"): + return "boolean" + + # 32-bit and narrower - `coerce_column` widens these to `Int32`. + if name in ("TINYINT", "SMALLINT", "INTEGER", "UTINYINT", "USMALLINT"): return "integer" - if name == "DATE": + + # Wider than `Int32`, or fractional - all coerce to `Float64`. + if ( + name in ("BIGINT", "HUGEINT", "UHUGEINT", "UINTEGER", "UBIGINT") + or name in ("FLOAT", "REAL", "DOUBLE", "VARINT") + or name.startswith("DECIMAL") + or name.startswith("NUMERIC") + ): + return "float" + + if name.startswith("DATE"): return "date" - if name == "BOOLEAN": - return "boolean" - if name == "TIMESTAMP": + + # `TIMESTAMP`, `TIMESTAMPTZ`, `TIMESTAMP_NS`, and `TIME`/`TIMETZ`, + # which coerce to `Timestamp(Millisecond)` rather than to a number. + if name.startswith("TIME"): return "datetime" - msg = f"Unknown type '{name}'" - raise ValueError(msg) + # Everything else renders as text: `VARCHAR`, `ENUM(...)` (which + # arrives dictionary-encoded), `JSON`, `UUID`, `BLOB`, `INTERVAL`, + # and the nested types. + if not ( + name.startswith("VARCHAR") + or name.startswith("ENUM") + or name in ("JSON", "UUID", "BLOB", "BIT", "INTERVAL") + or name.startswith("STRUCT") + or name.startswith("MAP") + or name.startswith("UNION") + or name.endswith("[]") + ): + # Unknown, not fatal - the column still renders, as text. Raising + # here would take down the whole table for one odd column. + logger.warning(f"Unknown type '{name}'") + + return "string" def run_query(db, query, execute=False, columns=False): diff --git a/rust/perspective-python/src/client/client_async.rs b/rust/perspective-python/src/client/client_async.rs index d6f5eba281..d7cce0dbfb 100644 --- a/rust/perspective-python/src/client/client_async.rs +++ b/rust/perspective-python/src/client/client_async.rs @@ -16,6 +16,7 @@ use std::str::FromStr; use std::sync::Arc; use futures::FutureExt; +use perspective_client::proto::ListFlatten; use perspective_client::{ Client, ColumnWindow, DeleteOptions, OnUpdateData, OnUpdateMode, OnUpdateOptions, Table, TableData, TableInitOptions, TableReadFormat, TableRef, UpdateData, UpdateOptions, View, @@ -52,6 +53,16 @@ fn py_to_table_ref_from_owned(py: Python<'_>, val: &Py) -> PyResult
) -> PyResult> { + match value.as_deref() { + None => Ok(None), + Some("zip") => Ok(Some(ListFlatten::Zip)), + Some("cartesian") => Ok(Some(ListFlatten::Cartesian)), + Some("stringify") => Ok(Some(ListFlatten::Stringify)), + Some(x) => Err(PyValueError::new_err(format!("Unknown `list_flatten`"))), + } +} + /// `perspective_server::Server`, whether locally in-memory or remote over some /// transport like a WebSocket. /// @@ -177,7 +188,8 @@ impl AsyncClient { /// ```python /// table = await client.table("x,y\n1,2\n3,4") /// ``` - #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None))] pub async fn table( &self, input: Py, @@ -186,6 +198,7 @@ impl AsyncClient { name: Option>, format: Option>, page_to_disk: Option, + list_flatten: Option>, ) -> PyResult { let client = self.client.clone(); let py_client = Python::with_gil(|_| self.clone()); @@ -193,6 +206,7 @@ impl AsyncClient { let mut options = TableInitOptions { name: name.map(|x| x.extract::(py)).transpose()?, page_to_disk, + list_flatten: parse_list_flatten(list_flatten.map(|x| x.to_string()))?, ..TableInitOptions::default() }; diff --git a/rust/perspective-python/src/client/client_sync.rs b/rust/perspective-python/src/client/client_sync.rs index e125e3f629..b847d25a21 100644 --- a/rust/perspective-python/src/client/client_sync.rs +++ b/rust/perspective-python/src/client/client_sync.rs @@ -166,7 +166,7 @@ impl Client { /// table = client.table("x,y\n1,2\n3,4") /// ``` #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None))] + #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None))] pub fn table( &self, py: Python<'_>, @@ -176,10 +176,19 @@ impl Client { name: Option>, format: Option>, page_to_disk: Option, + list_flatten: Option>, ) -> PyResult
{ Ok(Table( self.0 - .table(input, limit, index, name, format, page_to_disk) + .table( + input, + limit, + index, + name, + format, + page_to_disk, + list_flatten, + ) .py_block_on(py)?, )) } diff --git a/rust/perspective-python/src/server/virtual_server_sync.rs b/rust/perspective-python/src/server/virtual_server_sync.rs index abb9b8e170..678476bdb3 100644 --- a/rust/perspective-python/src/server/virtual_server_sync.rs +++ b/rust/perspective-python/src/server/virtual_server_sync.rs @@ -393,16 +393,6 @@ pub struct PyVirtualDataSlice(Arc>); #[pymethods] impl PyVirtualDataSlice { - #[new] - pub fn py_new() -> Self { - use perspective_client::config::{GroupRollupMode, ViewConfig}; - let config = ViewConfig { - group_rollup_mode: GroupRollupMode::Total, - ..Default::default() - }; - PyVirtualDataSlice(Arc::new(Mutex::new(VirtualDataSlice::new(config)))) - } - #[allow(clippy::wrong_self_convention)] pub fn from_arrow_ipc(&self, ipc: &[u8]) -> PyResult<()> { self.0 diff --git a/rust/perspective-server/cpp/perspective/CMakeLists.txt b/rust/perspective-server/cpp/perspective/CMakeLists.txt index 67a35a1000..cabb6e9218 100644 --- a/rust/perspective-server/cpp/perspective/CMakeLists.txt +++ b/rust/perspective-server/cpp/perspective/CMakeLists.txt @@ -443,6 +443,7 @@ set(SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/aggspec.cpp ${PSP_CPP_SRC}/src/cpp/arg_sort.cpp ${PSP_CPP_SRC}/src/cpp/arrow_loader.cpp + ${PSP_CPP_SRC}/src/cpp/arrow_normalize.cpp ${PSP_CPP_SRC}/src/cpp/arrow_writer.cpp ${PSP_CPP_SRC}/src/cpp/base.cpp ${PSP_CPP_SRC}/src/cpp/base_impl_linux.cpp @@ -485,6 +486,7 @@ set(SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/filter.cpp ${PSP_CPP_SRC}/src/cpp/flat_traversal.cpp ${PSP_CPP_SRC}/src/cpp/get_data_extents.cpp + ${PSP_CPP_SRC}/src/cpp/json_loader.cpp ${PSP_CPP_SRC}/src/cpp/gnode.cpp ${PSP_CPP_SRC}/src/cpp/gnode_state.cpp ${PSP_CPP_SRC}/src/cpp/mask.cpp diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp index ea5a1a9f8b..07faefe07b 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include "perspective/exception.h" #include +#include #include #include @@ -205,7 +207,9 @@ convert_type(const std::string& src) { } void -ArrowLoader::initialize(const std::uint8_t* ptr, const uint32_t length) { +ArrowLoader::initialize( + const std::uint8_t* ptr, const uint32_t length, t_list_flatten mode +) { if (std::memcmp("ARROW1", (const void*)ptr, 6) == 0) { load_file(ptr, length, m_table); } else { @@ -217,15 +221,24 @@ ArrowLoader::initialize(const std::uint8_t* ptr, const uint32_t length) { PSP_COMPLAIN_AND_ABORT(validation.ToString()); } - std::shared_ptr schema = m_table->schema(); - std::vector> fields = schema->fields(); + if (!normalize_table_is_noop(*m_table, mode)) { + m_expanded = normalize_table_expands(*m_table, mode); + m_normalized = std::make_unique( + normalize_table(m_table, mode) + ); + } - for (const auto& field : fields) { + for (const auto& field : fields()) { m_names.push_back(field->name()); m_types.push_back(convert_type(field->type()->name())); } } +const std::vector>& +ArrowLoader::fields() const { + return m_normalized ? m_normalized->fields : m_table->schema()->fields(); +} + void ArrowLoader::init_csv( const std::string_view& csv, @@ -234,9 +247,7 @@ ArrowLoader::init_csv( psp_schema ) { m_table = deduplicate_table(csvToTable(csv, is_update, psp_schema)); - std::shared_ptr schema = m_table->schema(); - std::vector> fields = schema->fields(); - for (const auto& field : fields) { + for (const auto& field : fields()) { m_names.push_back(field->name()); m_types.push_back(convert_type(field->type()->name())); } @@ -251,8 +262,7 @@ ArrowLoader::fill_table( bool is_update ) { bool implicit_index = false; - std::shared_ptr schema = m_table->schema(); - std::vector> fields = schema->fields(); + const auto& arrow_fields = fields(); parallel_for(int(m_names.size()), [&](int cidx) { auto name = m_names[cidx]; @@ -262,7 +272,7 @@ ArrowLoader::fill_table( // Skip columns that are defined in the arrow but not // in the Table's input schema. - auto raw_type = fields[cidx]->type()->name(); + auto raw_type = arrow_fields[cidx]->type()->name(); if (name == "__INDEX__") { implicit_index = true; @@ -308,18 +318,62 @@ ArrowLoader::fill_table( } } -template +/** + * Read output row `i` straight through, i.e. no row expansion. + */ +struct t_identity_gather { + static constexpr bool is_identity = true; + + std::int64_t operator[](std::int64_t i) const { return i; } + + bool is_null(std::int64_t) const { return false; } +}; + +/** + * Read output row `i` from the source row an expansion assigned to it. + */ +struct t_index_gather { + static constexpr bool is_identity = false; + const std::int64_t* m_indices; + std::int64_t operator[](std::int64_t i) const { + return m_indices[i] < 0 ? 0 : m_indices[i]; + } + + bool is_null(std::int64_t i) const { return m_indices[i] < 0; } +}; + +#define COPY_COLUMN_PRIMITIVE(CTYPE, ARROW_TYPE) \ + { \ + auto scol = std::static_pointer_cast(src); \ + const auto* vals = scol->raw_values(); \ + if constexpr (GATHER::is_identity) { \ + std::memcpy( \ + dest->get_nth(offset), \ + (void*)vals, \ + len * sizeof(CTYPE) \ + ); \ + } else { \ + for (std::int64_t i = 0; i < len; ++i) { \ + dest->set_nth( \ + offset + i, static_cast(vals[gather[i]]) \ + ); \ + } \ + } \ + } + +template void iter_col_copy( const std::shared_ptr& dest, std::shared_ptr src, const int64_t offset, - const int64_t len + const int64_t len, + const GATHER& gather ) { std::shared_ptr scol = std::static_pointer_cast(src); const typename T::value_type* vals = scol->raw_values(); - for (uint32_t i = 0; i < len; i++) { - dest->set_nth(offset + i, static_cast(vals[i])); + for (int64_t i = 0; i < len; i++) { + dest->set_nth(offset + i, static_cast(vals[gather[i]])); } } @@ -376,7 +430,11 @@ copy_integer_list( for (uint32_t j = 0; j < row_array_length; j++) { const auto elem_location = array_offsets[i] + j; const auto elem = raw_values[elem_location]; - writer.Int64(elem); + if constexpr (std::is_unsigned_v) { + writer.Uint64(elem); + } else { + writer.Int64(elem); + } } writer.EndArray(); dest->set_nth(i, s.GetString()); @@ -432,15 +490,23 @@ copy_float_list( } } +template void -copy_array( +copy_array_impl( const std::shared_ptr& dest, const std::shared_ptr& src, const int64_t offset, - const int64_t len + const int64_t len, + const GATHER& gather ) { switch (src->type()->id()) { case arrow::ListType::type_id: { + if constexpr (!GATHER::is_identity) { + PSP_COMPLAIN_AND_ABORT( + "Cannot expand rows of a stringified list column\n" + ); + } + auto list = std::static_pointer_cast<::arrow::ListArray>(src); switch (list->value_type()->id()) { @@ -465,9 +531,21 @@ copy_array( case ::arrow::Int32Type::type_id: { copy_integer_list(list, dest, len); } break; + case ::arrow::Int64Type::type_id: { + copy_integer_list(list, dest, len); + } break; + case ::arrow::UInt8Type::type_id: { + copy_integer_list(list, dest, len); + } break; + case ::arrow::UInt16Type::type_id: { + copy_integer_list(list, dest, len); + } break; case ::arrow::UInt32Type::type_id: { copy_integer_list(list, dest, len); } break; + case ::arrow::UInt64Type::type_id: { + copy_integer_list(list, dest, len); + } break; case ::arrow::StringType::type_id: { copy_string_list(list, dest, len); } break; @@ -536,42 +614,42 @@ copy_array( switch (indices->type()->id()) { case arrow::Int8Type::type_id: { iter_col_copy<::arrow::Int8Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; case ::arrow::UInt8Type::type_id: { iter_col_copy<::arrow::UInt8Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; case ::arrow::Int16Type::type_id: { iter_col_copy<::arrow::Int16Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; case ::arrow::UInt16Type::type_id: { iter_col_copy<::arrow::UInt16Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; case ::arrow::Int32Type::type_id: { iter_col_copy<::arrow::Int32Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; case ::arrow::UInt32Type::type_id: { iter_col_copy<::arrow::UInt32Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; case ::arrow::Int64Type::type_id: { iter_col_copy<::arrow::Int64Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; case ::arrow::UInt64Type::type_id: { iter_col_copy<::arrow::UInt64Array, t_uindex>( - dest, indices, offset, len + dest, indices, offset, len, gather ); } break; default: { @@ -592,9 +670,10 @@ copy_array( std::string elem; - for (std::uint32_t i = 0; i < len; ++i) { - arrow::LargeStringArray::offset_type bidx = offsets[i]; - std::size_t es = offsets[i + 1] - bidx; + for (std::int64_t i = 0; i < len; ++i) { + const auto src_i = gather[i]; + arrow::LargeStringArray::offset_type bidx = offsets[src_i]; + std::size_t es = offsets[src_i + 1] - bidx; elem.assign(reinterpret_cast(values) + bidx, es); dest->set_nth(offset + i, elem); } @@ -608,76 +687,37 @@ copy_array( std::string elem; - for (std::uint32_t i = 0; i < len; ++i) { - std::int32_t bidx = offsets[i]; - std::size_t es = offsets[i + 1] - bidx; + for (std::int64_t i = 0; i < len; ++i) { + const auto src_i = gather[i]; + std::int32_t bidx = offsets[src_i]; + std::size_t es = offsets[src_i + 1] - bidx; elem.assign(reinterpret_cast(values) + bidx, es); dest->set_nth(offset + i, elem); } } break; case arrow::Int8Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len - ); + COPY_COLUMN_PRIMITIVE(std::int8_t, arrow::Int8Array); } break; case arrow::UInt8Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len - ); + COPY_COLUMN_PRIMITIVE(std::uint8_t, arrow::UInt8Array); } break; case arrow::Int16Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 2 - ); + COPY_COLUMN_PRIMITIVE(std::int16_t, arrow::Int16Array); } break; case arrow::UInt16Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 2 - ); + COPY_COLUMN_PRIMITIVE(std::uint16_t, arrow::UInt16Array); } break; case arrow::Int32Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 4 - ); + COPY_COLUMN_PRIMITIVE(std::int32_t, arrow::Int32Array); } break; case arrow::UInt32Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 4 - ); + COPY_COLUMN_PRIMITIVE(std::uint32_t, arrow::UInt32Array); } break; case arrow::Int64Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 8 - ); + COPY_COLUMN_PRIMITIVE(std::int64_t, arrow::Int64Array); } break; case arrow::UInt64Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 8 - ); + COPY_COLUMN_PRIMITIVE(std::uint64_t, arrow::UInt64Array); } break; case arrow::TimestampType::type_id: { std::shared_ptr tunit = @@ -685,28 +725,39 @@ copy_array( auto scol = std::static_pointer_cast(src); switch (tunit->unit()) { case arrow::TimeUnit::MILLI: { - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 8 - ); + const int64_t* vals = scol->raw_values(); + if constexpr (GATHER::is_identity) { + std::memcpy( + dest->get_nth(offset), (void*)vals, len * 8 + ); + } else { + for (int64_t i = 0; i < len; i++) { + dest->set_nth(offset + i, vals[gather[i]]); + } + } } break; case arrow::TimeUnit::NANO: { const int64_t* vals = scol->raw_values(); - for (uint32_t i = 0; i < len; i++) { - dest->set_nth(offset + i, vals[i] / 1000000); + for (int64_t i = 0; i < len; i++) { + dest->set_nth( + offset + i, vals[gather[i]] / 1000000 + ); } } break; case arrow::TimeUnit::MICRO: { const int64_t* vals = scol->raw_values(); - for (uint32_t i = 0; i < len; i++) { - dest->set_nth(offset + i, vals[i] / 1000); + for (int64_t i = 0; i < len; i++) { + dest->set_nth( + offset + i, vals[gather[i]] / 1000 + ); } } break; case arrow::TimeUnit::SECOND: { const int64_t* vals = scol->raw_values(); - for (uint32_t i = 0; i < len; i++) { - dest->set_nth(offset + i, vals[i] * 1000); + for (int64_t i = 0; i < len; i++) { + dest->set_nth( + offset + i, vals[gather[i]] * 1000 + ); } } break; } @@ -716,8 +767,8 @@ copy_array( std::static_pointer_cast(src->type()); auto scol = std::static_pointer_cast(src); const int64_t* vals = scol->raw_values(); - for (uint32_t i = 0; i < len; i++) { - std::chrono::milliseconds timestamp(vals[i]); + for (int64_t i = 0; i < len; i++) { + std::chrono::milliseconds timestamp(vals[gather[i]]); date::sys_days days(date::floor(timestamp)); auto ymd = date::year_month_day{days}; std::int32_t year = static_cast(ymd.year()); @@ -733,8 +784,8 @@ copy_array( std::static_pointer_cast(src->type()); auto scol = std::static_pointer_cast(src); const int32_t* vals = scol->raw_values(); - for (uint32_t i = 0; i < len; i++) { - date::days days{vals[i]}; + for (int64_t i = 0; i < len; i++) { + date::days days{vals[gather[i]]}; auto ymd = date::year_month_day{date::sys_days{days}}; // years are signed, month/day are unsigned std::int32_t year = static_cast(ymd.year()); @@ -746,18 +797,10 @@ copy_array( } } break; case arrow::FloatType::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), (void*)scol->raw_values(), len * 4 - ); + COPY_COLUMN_PRIMITIVE(float, arrow::FloatArray); } break; case arrow::DoubleType::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 8 - ); + COPY_COLUMN_PRIMITIVE(double, arrow::DoubleArray); } break; case arrow::Decimal128Type::type_id: case arrow::DecimalType::type_id: { @@ -767,16 +810,19 @@ copy_array( std::static_pointer_cast(src->type()); int32_t scale = decimal_type->scale(); auto* vals = (arrow::Decimal128*)scol->raw_values(); - for (uint32_t i = 0; i < len; ++i) { - dest->set_nth(offset + i, vals[i].ToDouble(scale)); + for (int64_t i = 0; i < len; ++i) { + dest->set_nth( + offset + i, vals[gather[i]].ToDouble(scale) + ); } } break; case arrow::BooleanType::type_id: { auto scol = std::static_pointer_cast(src); - const uint8_t* null_bitmap = scol->values()->data(); - for (uint32_t i = 0; i < len; ++i) { - std::uint8_t elem = null_bitmap[i / 8]; - bool v = (elem & (1 << (i % 8))) != 0; + const uint8_t* bitmap = scol->values()->data(); + for (int64_t i = 0; i < len; ++i) { + const auto src_i = gather[i] + scol->offset(); + std::uint8_t elem = bitmap[src_i / 8]; + bool v = (elem & (1 << (src_i % 8))) != 0; dest->set_nth(offset + i, v); } } break; @@ -786,12 +832,7 @@ copy_array( } } break; case arrow::Time32Type::type_id: { - auto scol = std::static_pointer_cast(src); - std::memcpy( - dest->get_nth(offset), - (void*)scol->raw_values(), - len * 4 - ); + COPY_COLUMN_PRIMITIVE(std::uint32_t, arrow::Time32Array); } break; // case arrow::Type { @@ -806,39 +847,49 @@ copy_array( } } +void +copy_array( + const std::shared_ptr& dest, + const std::shared_ptr& src, + const int64_t offset, + const int64_t len +) { + copy_array_impl(dest, src, offset, len, t_identity_gather{}); +} + // Defines the full matrix of type interactions between arrow arrays and // schema-defined tables. #define FILL_COLUMN_ITER(ARRAY_TYPE) \ switch (column_dtype) { \ case DTYPE_INT8: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_UINT8: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_INT16: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_UINT16: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_INT32: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_UINT32: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_INT64: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_UINT64: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_FLOAT32: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ case DTYPE_FLOAT64: { \ - iter_col_copy(col, array, offset, len); \ + iter_col_copy(col, array, offset, len, gather); \ } break; \ default: { \ std::stringstream ss; \ @@ -848,6 +899,71 @@ copy_array( } \ } +/** + * Mark `[offset, offset + len)` valid or invalid from `array`'s null bitmap, + * read through `gather`. + */ +template +static void +fill_validity( + const std::shared_ptr& col, + const std::shared_ptr& array, + const std::int64_t offset, + const std::int64_t len, + bool is_update, + const GATHER& gather +) { + const auto invalidate = [&](std::int64_t i) { + if (is_update) { + col->unset(offset + i); + } else { + col->clear(offset + i); + } + }; + + const std::int64_t null_count = array->null_count(); + if (null_count == 0 && GATHER::is_identity) { + col->set_valid_range(offset, len); + return; + } + + const uint8_t* null_bitmap = array->null_bitmap_data(); + if (null_count != 0 && null_bitmap == nullptr) { + for (std::int64_t i = 0; i < len; ++i) { + invalidate(i); + } + + return; + } + + const std::int64_t bit_base = array->offset(); + for (std::int64_t i = 0; i < len; ++i) { + bool valid = !gather.is_null(i); + if (valid && null_bitmap != nullptr) { + const std::int64_t bit = bit_base + gather[i]; + valid = (null_bitmap[bit / 8] & (1 << (bit % 8))) != 0; + } + + if (valid) { + col->set_valid(offset + i, true); + } else { + invalidate(i); + } + } +} + +template +void fill_column_chunk( + const std::shared_ptr& col, + const std::shared_ptr& array, + const std::string& name, + t_dtype type, + std::int64_t offset, + std::int64_t len, + bool is_update, + const GATHER& gather +); + void ArrowLoader::fill_column( t_data_table& tbl, @@ -858,9 +974,16 @@ ArrowLoader::fill_column( std::string& raw_type, bool is_update ) { - int64_t offset = 0; - std::shared_ptr carray = - m_table->GetColumnByName(name); + std::shared_ptr carray; + const std::vector* indices = nullptr; + if (m_normalized) { + carray = m_normalized->columns[cidx]; + if (!m_normalized->gathers[cidx].empty()) { + indices = &m_normalized->gathers[cidx]; + } + } else { + carray = m_table->GetColumnByName(name); + } if (carray == nullptr) { LOG_DEBUG( @@ -869,9 +992,59 @@ ArrowLoader::fill_column( ); return; } + + if (indices != nullptr) { + if (carray->num_chunks() == 0 || carray->chunk(0)->length() == 0) { + for (std::size_t i = 0; i < indices->size(); ++i) { + if (is_update) { + col->unset(i); + } else { + col->clear(i); + } + } + + return; + } + + fill_column_chunk( + col, + carray->chunk(0), + name, + type, + 0, + static_cast(indices->size()), + is_update, + t_index_gather{indices->data()} + ); + + return; + } + + int64_t offset = 0; for (auto i = 0; i < carray->num_chunks(); ++i) { std::shared_ptr array = carray->chunk(i); int64_t len = array->length(); + fill_column_chunk( + col, array, name, type, offset, len, is_update, t_identity_gather{} + ); + + offset += len; + } +} + +template +void +fill_column_chunk( + const std::shared_ptr& col, + const std::shared_ptr& array, + const std::string& name, + t_dtype type, + std::int64_t offset, + std::int64_t len, + bool is_update, + const GATHER& gather +) { + { // If the Arrow array schema is different from the data // table schema, iteratively fill. @@ -924,51 +1097,10 @@ ArrowLoader::fill_column( }; } } else { - copy_array(col, array, offset, len); + copy_array_impl(col, array, offset, len, gather); } - // Fill validity bitmap. Operate only on the current chunk's - // range [offset, offset+len); a whole-column fill here would - // clobber validity bits set by other chunks in a multi-batch - // ChunkedArray. - std::int64_t null_count = array->null_count(); - - if (null_count == 0) { - col->set_valid_range(offset, len); - } else { - const uint8_t* null_bitmap = array->null_bitmap_data(); - - // If the arrow column is of null type, the null - // bitmap is a nullptr - so just mark this chunk's rows - // as invalid and move on. - if (null_bitmap == nullptr) { - for (uint32_t i = 0; i < len; ++i) { - if (is_update) { - col->unset(offset + i); - } else { - col->clear(offset + i); - } - } - } else { - // Read the null bitmap and set the correct rows - // as valid - for (uint32_t i = 0; i < len; ++i) { - std::uint8_t elem = null_bitmap[i / 8]; - bool v = (elem & (1 << (i % 8))) != 0; - if (!v) { - if (is_update) { - col->unset(offset + i); - } else { - col->clear(offset + i); - } - } else { - col->set_valid(offset + i, v); - } - } - } - } - - offset += len; + fill_validity(col, array, offset, len, is_update, gather); } } @@ -976,7 +1108,8 @@ ArrowLoader::fill_column( std::uint32_t ArrowLoader::row_count() const { - const std::int64_t n = m_table->num_rows(); + const std::int64_t n = + m_normalized ? m_normalized->num_rows : m_table->num_rows(); if (n < 0 || static_cast(n) > std::numeric_limits::max()) { @@ -988,6 +1121,33 @@ ArrowLoader::row_count() const { return static_cast(n); } +std::optional +ArrowLoader::repeated_index(const std::string& index) const { + if (!m_expanded) { + return std::nullopt; + } + + const auto is_per_element = [&](const std::string& name) { + const auto it = std::find(m_names.begin(), m_names.end(), name); + return it != m_names.end() && m_normalized + && m_normalized->per_element[std::distance(m_names.begin(), it)]; + }; + + if (!index.empty() && !is_per_element(index)) { + return index; + } + + const std::string implicit{"__INDEX__"}; + const auto has_implicit = + std::find(m_names.begin(), m_names.end(), implicit) != m_names.end(); + + if (has_implicit && !is_per_element(implicit)) { + return implicit; + } + + return std::nullopt; +} + std::vector ArrowLoader::names() const { return m_names; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp new file mode 100644 index 0000000000..fb9cf11611 --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp @@ -0,0 +1,591 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#include +#include "perspective/base.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace perspective::apachearrow { + +const char* const FLATTEN_SEPARATOR = "."; + +namespace { + + bool + is_list_id(arrow::Type::type id) { + return id == arrow::Type::LIST || id == arrow::Type::LARGE_LIST; + } + + bool + type_expands(const arrow::DataType& type) { + if (is_list_id(type.id())) { + return true; + } + + if (type.id() == arrow::Type::STRUCT) { + for (const auto& child : type.fields()) { + if (type_expands(*child->type())) { + return true; + } + } + } + + return false; + } + + [[noreturn]] void + abort_with(const std::string& msg) { + PSP_COMPLAIN_AND_ABORT(msg); + std::abort(); + } + + template + T + unwrap(arrow::Result result, const char* what) { + if (!result.ok()) { + std::stringstream ss; + ss << what << ": " << result.status().ToString() << "\n"; + abort_with(ss.str()); + } + + return result.MoveValueUnsafe(); + } + + /** + * Project child `c` out of a `StructArray`, combining the parent's validity + * into the child's. + */ + std::shared_ptr + struct_child(const std::shared_ptr& parent, int c) { + auto child = parent->field(c); + if (parent->null_count() == 0) { + return child; + } + + const auto length = child->length(); + const auto c_offset = child->offset(); + const auto p_offset = parent->offset(); + const auto* p_bitmap = parent->null_bitmap_data(); + auto data = child->data()->Copy(); + if (child->null_count() == 0 && c_offset == p_offset) { + data->buffers[0] = parent->data()->buffers[0]; + data->null_count = parent->null_count(); + return arrow::MakeArray(data); + } + + auto buffer = unwrap( + arrow::AllocateEmptyBitmap( + c_offset + length, arrow::default_memory_pool() + ), + "Could not allocate struct validity bitmap" + ); + + if (child->null_count() == 0) { + arrow::internal::CopyBitmap( + p_bitmap, p_offset, length, buffer->mutable_data(), c_offset + ); + } else { + arrow::internal::BitmapAnd( + p_bitmap, + p_offset, + child->null_bitmap_data(), + c_offset, + length, + c_offset, + buffer->mutable_data() + ); + } + + data->buffers[0] = std::move(buffer); + data->null_count = arrow::kUnknownNullCount; + return arrow::MakeArray(data); + } + + /** + * Hoist every top-level struct column's children into dotted columns. + * Returns whether anything changed. + */ + bool + flatten_structs( + std::vector>& fields, + std::vector>& columns, + std::vector>& gathers, + std::vector& per_element + ) { + bool changed = false; + for (const auto& field : fields) { + if (field->type()->id() == arrow::Type::STRUCT) { + changed = true; + break; + } + } + + if (!changed) { + return false; + } + + std::vector> out_fields; + std::vector> out_columns; + std::vector> out_gathers; + std::vector out_per_element; + for (std::size_t i = 0; i < fields.size(); ++i) { + if (fields[i]->type()->id() != arrow::Type::STRUCT) { + out_fields.push_back(fields[i]); + out_columns.push_back(columns[i]); + out_gathers.push_back(gathers[i]); + out_per_element.push_back(per_element[i]); + continue; + } + + const auto& children = fields[i]->type()->fields(); + for (int c = 0; c < static_cast(children.size()); ++c) { + std::vector> chunks; + chunks.reserve(columns[i]->num_chunks()); + for (const auto& chunk : columns[i]->chunks()) { + chunks.push_back(struct_child( + std::static_pointer_cast(chunk), c + )); + } + + out_fields.push_back(arrow::field( + fields[i]->name() + FLATTEN_SEPARATOR + children[c]->name(), + children[c]->type() + )); + + out_columns.push_back(std::make_shared( + std::move(chunks), children[c]->type() + )); + + out_gathers.push_back(gathers[i]); + out_per_element.push_back(per_element[i]); + } + } + + fields = std::move(out_fields); + columns = std::move(out_columns); + gathers = std::move(out_gathers); + per_element = std::move(out_per_element); + return true; + } + + /** + * The per-row geometry of one list column, flattened across chunks so that + * rows can be addressed globally. Chunk layout is per-column in an + * `arrow::Table` and need not agree between columns. + */ + struct t_list_column { + std::size_t index; + std::vector start; + std::vector length; + std::shared_ptr values; + }; + + template + void + read_list_offsets( + const std::shared_ptr& column, t_list_column& out + ) { + std::vector> value_slices; + std::int64_t base = 0; + for (const auto& chunk : column->chunks()) { + auto list = std::static_pointer_cast(chunk); + const auto* offsets = list->raw_value_offsets(); + const auto len = list->length(); + const auto first = len > 0 ? offsets[0] : 0; + const auto last = len > 0 ? offsets[len] : 0; + for (std::int64_t i = 0; i < len; ++i) { + if (list->IsNull(i)) { + out.start.push_back(0); + out.length.push_back(0); + } else { + out.start.push_back(base + (offsets[i] - first)); + out.length.push_back(offsets[i + 1] - offsets[i]); + } + } + + value_slices.push_back(list->values()->Slice(first, last - first)); + base += last - first; + } + + out.values = std::make_shared( + std::move(value_slices), + std::static_pointer_cast(column->type()) + ->value_type() + ); + } + + std::shared_ptr + build_indices(const std::vector& indices, bool has_null) { + arrow::Int64Builder builder; + if (!builder.Reserve(static_cast(indices.size())).ok()) { + abort_with("Could not reserve list expansion indices\n"); + } + + for (auto index : indices) { + if (has_null && index < 0) { + builder.UnsafeAppendNull(); + } else { + builder.UnsafeAppend(index); + } + } + + std::shared_ptr out; + if (!builder.Finish(&out).ok()) { + abort_with("Could not build list expansion indices\n"); + } + + return out; + } + + std::shared_ptr + take( + const std::shared_ptr& values, + const std::shared_ptr& indices + ) { + auto result = unwrap( + arrow::compute::Take( + arrow::Datum(values), + arrow::Datum(indices), + arrow::compute::TakeOptions::NoBoundsCheck() + ), + "Could not expand list column" + ); + + return result.chunked_array(); + } + + /** + * Apply every pending gather, so a subsequent expansion pass can read + * offsets positionally again. + */ + void + materialize( + std::vector>& columns, + std::vector>& gathers + ) { + for (std::size_t i = 0; i < columns.size(); ++i) { + if (gathers[i].empty()) { + continue; + } + + bool has_null = false; + for (auto index : gathers[i]) { + if (index < 0) { + has_null = true; + break; + } + } + + columns[i] = take(columns[i], build_indices(gathers[i], has_null)); + gathers[i].clear(); + } + } + + /** + * Collapse to a single chunk, so a gather index needs no chunk resolution. + */ + std::shared_ptr + combine_chunks(const std::shared_ptr& column) { + if (column->num_chunks() <= 1) { + return column; + } + + auto combined = unwrap( + arrow::Concatenate(column->chunks(), arrow::default_memory_pool()), + "Could not combine chunks of an expanded column" + ); + + return std::make_shared( + arrow::ArrayVector{std::move(combined)}, column->type() + ); + } + + /** + * Expand every top-level list column into rows. Returns whether anything + * changed. + */ + bool + explode_lists( + std::vector>& fields, + std::vector>& columns, + std::vector>& gathers, + std::vector& per_element, + std::int64_t& num_rows, + t_list_flatten mode + ) { + bool has_list = false; + for (const auto& field : fields) { + if (is_list_id(field->type()->id())) { + has_list = true; + break; + } + } + + if (!has_list) { + return false; + } + + for (const auto& gather : gathers) { + if (!gather.empty()) { + materialize(columns, gathers); + break; + } + } + + std::vector lists; + for (std::size_t i = 0; i < fields.size(); ++i) { + const auto id = fields[i]->type()->id(); + if (!is_list_id(id)) { + continue; + } + + t_list_column list; + list.index = i; + list.start.reserve(num_rows); + list.length.reserve(num_rows); + if (id == arrow::Type::LIST) { + read_list_offsets(columns[i], list); + } else { + read_list_offsets(columns[i], list); + } + + lists.push_back(std::move(list)); + } + + std::vector parents; + std::vector> children(lists.size()); + std::vector has_null(lists.size(), false); + bool identity = mode == LIST_FLATTEN_ZIP; + + for (std::int64_t row = 0; row < num_rows; ++row) { + std::int64_t width = 1; + if (mode == LIST_FLATTEN_ZIP) { + std::int64_t zipped = -1; + std::size_t witness = 0; + for (std::size_t l = 0; l < lists.size(); ++l) { + const auto len = lists[l].length[row]; + if (len == 0) { + identity = false; + continue; + } + + if (zipped < 0) { + witness = l; + } else if (len != zipped) { + std::stringstream ss; + ss << "Cannot zip list columns `" + << fields[lists[witness].index]->name() << "` (" + << zipped << ") and `" + << fields[lists[l].index]->name() << "` (" << len + << ") of differing length in row " << row + << "; use the `cartesian` list flatten mode.\n"; + abort_with(ss.str()); + } + + zipped = len; + } + + if (zipped > 0) { + width = zipped; + } + } else { + for (const auto& list : lists) { + const auto len = std::max(list.length[row], 1); + if (width > std::numeric_limits::max() / len) { + std::stringstream ss; + ss << "Cartesian list expansion overflows in row " << row + << "\n"; + abort_with(ss.str()); + } + + width *= len; + } + } + + const auto base = static_cast(parents.size()); + parents.resize(base + width, row); + for (std::size_t l = 0; l < lists.size(); ++l) { + const auto len = lists[l].length[row]; + const auto start = lists[l].start[row]; + children[l].resize(base + width); + if (len == 0) { + has_null[l] = true; + for (std::int64_t k = 0; k < width; ++k) { + children[l][base + k] = -1; + } + + continue; + } + + if (mode == LIST_FLATTEN_ZIP) { + for (std::int64_t k = 0; k < width; ++k) { + children[l][base + k] = start + k; + } + } else { + std::int64_t stride = 1; + for (std::size_t r = l + 1; r < lists.size(); ++r) { + stride *= std::max(lists[r].length[row], 1); + } + + for (std::int64_t k = 0; k < width; ++k) { + children[l][base + k] = start + ((k / stride) % len); + } + } + } + } + + const auto expanded = static_cast(parents.size()); + if (expanded > std::numeric_limits::max()) { + std::stringstream ss; + ss << "List expansion produced " << expanded + << " rows, which exceeds the maximum supported size\n"; + abort_with(ss.str()); + } + + const bool row_aligned = expanded == num_rows; + const bool marks_per_element = + mode != LIST_FLATTEN_CARTESIAN || lists.size() == 1; + + std::vector> out_fields; + std::vector> out_columns; + std::vector> out_gathers; + std::vector out_per_element; + std::size_t l = 0; + for (std::size_t i = 0; i < fields.size(); ++i) { + if (l < lists.size() && lists[l].index == i) { + out_fields.push_back( + arrow::field(fields[i]->name(), lists[l].values->type()) + ); + + if (identity) { + out_columns.push_back(lists[l].values); + out_gathers.emplace_back(); + } else { + out_columns.push_back(combine_chunks(lists[l].values)); + out_gathers.push_back(std::move(children[l])); + } + + out_per_element.push_back(marks_per_element); + l += 1; + continue; + } + + out_fields.push_back(fields[i]); + if (row_aligned) { + out_columns.push_back(columns[i]); + out_gathers.emplace_back(); + } else { + out_columns.push_back(combine_chunks(columns[i])); + out_gathers.push_back(parents); + } + + out_per_element.push_back(false); + } + + fields = std::move(out_fields); + columns = std::move(out_columns); + gathers = std::move(out_gathers); + per_element = std::move(out_per_element); + num_rows = expanded; + return true; + } + +} // namespace + +bool +normalize_table_is_noop(const arrow::Table& input, t_list_flatten mode) { + for (const auto& field : input.schema()->fields()) { + const auto id = field->type()->id(); + if (id == arrow::Type::STRUCT) { + return false; + } + + if (mode != LIST_FLATTEN_STRINGIFY && is_list_id(id)) { + return false; + } + } + + return true; +} + +bool +normalize_table_expands(const arrow::Table& input, t_list_flatten mode) { + if (mode == LIST_FLATTEN_STRINGIFY) { + return false; + } + + for (const auto& field : input.schema()->fields()) { + if (type_expands(*field->type())) { + return true; + } + } + + return false; +} + +t_normalized_table +normalize_table(std::shared_ptr input, t_list_flatten mode) { + t_normalized_table out; + out.fields = input->schema()->fields(); + out.columns = input->columns(); + out.gathers.resize(out.fields.size()); + out.per_element.assign(out.fields.size(), false); + out.num_rows = input->num_rows(); + while (true) { + bool changed = flatten_structs( + out.fields, out.columns, out.gathers, out.per_element + ); + + if (mode != LIST_FLATTEN_STRINGIFY + && explode_lists( + out.fields, + out.columns, + out.gathers, + out.per_element, + out.num_rows, + mode + )) { + changed = true; + } + + if (!changed) { + break; + } + } + + std::set seen; + for (const auto& field : out.fields) { + if (!seen.insert(field->name()).second) { + std::stringstream ss; + ss << "Flattening produced duplicate column `" << field->name() + << "`\n"; + abort_with(ss.str()); + } + } + + return out; +} + +} // namespace perspective::apachearrow diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp index bd9f0229c5..38c04ce9e6 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp @@ -578,7 +578,8 @@ t_ctx2::notify(const t_data_table& flattened, bool /* is_registration */) { ); } } - if (!m_sortby.empty()) { + + if (!m_sortby.empty() && !m_leaves_only && !m_total_only) { sort_by(m_sortby); } @@ -656,7 +657,8 @@ t_ctx2::notify( } } - if (!m_sortby.empty()) { + // See the single-argument `notify` overload. + if (!m_sortby.empty() && !m_leaves_only && !m_total_only) { sort_by(m_sortby); } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp new file mode 100644 index 0000000000..7b95436c9a --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp @@ -0,0 +1,1567 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + + +#include +#include "perspective/base.h" +#include "perspective/raw_types.h" +#include "perspective/arrow_csv.h" +#include "rapidjson/document.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace perspective::json { + +static bool +ichar_equals(char a, char b) { + return std::tolower(static_cast(a)) + == std::tolower(static_cast(b)); +} + +static bool +istrequals(std::string_view a, std::string_view b) { + return a.size() == b.size() + && std::equal(a.begin(), a.end(), b.begin(), ichar_equals); +} + +t_dtype +rapidjson_type_to_dtype(const rapidjson::Value& value) { + switch (value.GetType()) { + case rapidjson::Type::kStringType: { + const auto& str = value.GetString(); + if (str[0] == '\0') { + return t_dtype::DTYPE_STR; + } + + if (istrequals(str, "true") || istrequals(str, "false")) { + return t_dtype::DTYPE_BOOL; + } + + // TODO JSON will no longer support date/datetime inference. The + // only way to load JSON data with these types will be with a + // Schema! + + char* endptr; + strtol(str, &endptr, 10); + if (*endptr == '\0') { + return t_dtype::DTYPE_INT32; + } + + strtof(str, &endptr); + if (*endptr == '\0') { + return t_dtype::DTYPE_FLOAT64; + } + + std::tm tm; + std::memset(&tm, 0, sizeof(tm)); + std::chrono::system_clock::time_point tp; + + if (parse_all_date_time(tm, tp, str)) { + if (tm.tm_hour == 0 && tm.tm_min == 0 && tm.tm_sec == 0) { + return t_dtype::DTYPE_DATE; + } + return t_dtype::DTYPE_TIME; + } + + auto datetime = apachearrow::parseAsArrowTimestamp(str); + if (datetime != std::nullopt) { + return t_dtype::DTYPE_TIME; + } + + return t_dtype::DTYPE_STR; + } + case rapidjson::Type::kNumberType: { + if (value.IsInt64()) { + if (value.GetInt64() + > std::numeric_limits::max()) { + return t_dtype::DTYPE_FLOAT64; + } + return t_dtype::DTYPE_INT32; + } + if (value.IsInt()) { + return t_dtype::DTYPE_INT32; + } + + return t_dtype::DTYPE_FLOAT64; + } + case rapidjson::Type::kTrueType: + case rapidjson::Type::kFalseType: + return t_dtype::DTYPE_BOOL; + case rapidjson::kNullType: + return t_dtype::DTYPE_NONE; + case rapidjson::kArrayType: + // Only reachable under `stringify`; the expanding modes descend. + return t_dtype::DTYPE_STR; + case rapidjson::kObjectType: + PSP_COMPLAIN_AND_ABORT("Unknown JSON type"); + return t_dtype::DTYPE_NONE; + default: + PSP_COMPLAIN_AND_ABORT("Unknown JSON type"); + return t_dtype::DTYPE_NONE; + } +} +template +struct promote { + constexpr static t_dtype dtype = DTYPE_NONE; +}; + +#define PROMOTE_IMPL(A, B, C) \ + template <> \ + struct promote { \ + constexpr static t_dtype dtype = C; \ + }; + +PROMOTE_IMPL(DTYPE_INT32, DTYPE_INT64, DTYPE_INT64) +// PROMOTE_IMPL(std::int32_t, std::float_t, DTYPE_FLOAT32) +// PROMOTE_IMPL(std::int32_t, std::double_t, DTYPE_FLOAT64) + +template +static A +json_into(const rapidjson::Value& value) { + if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { + if (value.IsInt()) { + return value.GetInt(); + } + if (value.IsInt64()) { + return value.GetInt64(); + } + if (value.IsDouble()) { + return value.GetDouble(); + } + if (value.IsFloat()) { + return value.GetFloat(); + } + if (value.IsString()) { + if constexpr (std::is_same_v) { + return std::atoi(value.GetString()); + } else if constexpr (std::is_same_v) { + return std::atoll(value.GetString()); + } else if constexpr (std::is_same_v || std::is_same_v) { + return std::atof(value.GetString()); + } else { + static_assert(!std::is_same_v, "No coercion for type"); + } + } + if (value.IsNull()) { + return 0; + } + + std::stringstream ss; + ss << "Could not coerce " << value.GetType() << " to " + << "a number"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } else if constexpr (std::is_same_v) { + switch (value.GetType()) { + case rapidjson::kNullType: + return ""; + case rapidjson::kFalseType: + return "false"; + case rapidjson::kTrueType: + return "true"; + case rapidjson::kObjectType: + PSP_COMPLAIN_AND_ABORT("Cannot coerce object to string"); + case rapidjson::kArrayType: { + // `stringify` keeps the array as its JSON text. + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return buffer.GetString(); + } + case rapidjson::kStringType: + return value.GetString(); + case rapidjson::kNumberType: + if (value.IsInt()) { + return std::to_string(value.GetInt()); + } + if (value.IsInt64()) { + return std::to_string(value.GetInt64()); + } + if (value.IsDouble()) { + return std::to_string(value.GetDouble()); + } + if (value.IsFloat()) { + return std::to_string(value.GetFloat()); + } + } + + std::stringstream ss; + ss << "Could not coerce " << value.GetType() << " to " + << "a string"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } else if constexpr (std::is_same_v) { + std::tm tm; + if (value.IsString()) { + if (!parse_all_date_time(tm, value.GetString())) { + PSP_COMPLAIN_AND_ABORT("Could not coerce to date"); + } + } else if (value.IsInt64()) { + return t_date::from_epoch_ms(value.GetInt64()); + } else { + PSP_COMPLAIN_AND_ABORT("Could not coerce to date"); + } + + return t_date(tm.tm_year + 1900, tm.tm_mon, tm.tm_mday); + } else if constexpr (std::is_same_v) { + if (value.IsString()) { + std::chrono::system_clock::time_point tp; + if (!parse_all_date_time(tp, value.GetString())) { + PSP_COMPLAIN_AND_ABORT("Could not coerce to time"); + } + + return t_time(std::chrono::duration_cast( + tp.time_since_epoch() + ) + .count()); + } + if (value.IsDouble()) { + return t_time(value.GetDouble()); + } + if (value.IsInt64()) { + return t_time(value.GetInt64()); + } + if (value.IsInt()) { + return t_time(value.GetInt()); + } + PSP_COMPLAIN_AND_ABORT( + "Could not coerce " + std::to_string(value.GetType()) + + " to a time." + ); + } else { + static_assert(!std::is_same_v, "No coercion for type"); + } +} + +std::optional +fill_column_json( + const std::shared_ptr& col, + const t_uindex i, + const rapidjson::Value& value, + const bool is_update +) { + if (value.IsNull()) { + if (is_update) { + col->unset(i); + } else { + col->clear(i); + } + return std::nullopt; + } + + switch (col->get_dtype()) { + case t_dtype::DTYPE_STR: { + if (!value.IsString()) { + auto v = json_into(value); + col->set_nth(i, v); + } else { + col->set_nth(i, value.GetString()); + } + return std::nullopt; + } + case t_dtype::DTYPE_INT32: { + if (value.IsInt()) { + col->set_nth(i, value.GetInt()); + return std::nullopt; + } + + if (value.IsInt64()) { + if (value.GetInt64() > std::numeric_limits::max()) + [[likely]] { + if (!is_update) { + LOG_DEBUG("Promoting due to int32 overflow"); + return {DTYPE_FLOAT64}; + } + } + + // Coerce in update mode + col->set_nth( + i, static_cast(value.GetInt64()) + ); + + return std::nullopt; + } + + if (value.IsDouble()) { + if (is_update) { + col->set_nth( + i, static_cast(value.GetDouble()) + ); + return std::nullopt; + } + + return {DTYPE_FLOAT64}; + } + + if (value.IsString()) { + const auto& str = value.GetString(); + if (str[0] == '\0') { + if (is_update) { + col->set_valid(i, false); + return std::nullopt; + } + + return {t_dtype::DTYPE_STR}; + } + + char* endptr; + std::int32_t result = strtol(str, &endptr, 10); + if (*endptr == '\0') { + col->set_nth(i, result); + return std::nullopt; + } + + float result2 = strtof(str, &endptr); + if (*endptr == '\0') { + if (is_update) { + col->set_nth( + i, static_cast(result2) + ); + return std::nullopt; + } + + return {t_dtype::DTYPE_FLOAT64}; + } + + return {t_dtype::DTYPE_STR}; + } + + std::stringstream ss; + ss << "Expected int, found " << value.GetType(); + PSP_COMPLAIN_AND_ABORT(ss.str()); + return std::nullopt; + } + case t_dtype::DTYPE_INT64: { + if (value.IsInt64()) [[likely]] { + col->set_nth(i, value.GetInt()); + } else if (value.IsDouble()) { + return {DTYPE_FLOAT64}; + } else if (value.IsString()) { + col->set_nth(i, std::atoll(value.GetString())); + } else { + std::stringstream ss; + ss << "Expected int64, found " << value.GetType(); + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + return std::nullopt; + } + case t_dtype::DTYPE_FLOAT64: { + if (value.IsDouble()) [[likely]] { + col->set_nth(i, value.GetDouble()); + } else if (value.IsInt64()) { + col->set_nth(i, static_cast(value.GetInt64())); + } else if (value.IsInt()) { + col->set_nth(i, value.GetInt()); + } else if (value.IsString()) { + col->set_nth(i, std::atof(value.GetString())); + } else { + std::stringstream ss; + ss << "Expected double, found " << value.GetType(); + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + return std::nullopt; + } + case t_dtype::DTYPE_BOOL: { + if (value.IsBool()) [[likely]] { + col->set_nth(i, value.GetBool()); + } else if (value.IsString() && istrequals(value.GetString(), "true")) { + col->set_nth(i, true); + } else if (value.IsString() && istrequals(value.GetString(), "false")) { + col->set_nth(i, false); + } else if (value.IsInt()) { + col->set_nth(i, value.GetInt() != 0); + } else { + std::stringstream ss; + ss << "Expected bool, found " << value.GetType(); + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + return std::nullopt; + } + case t_dtype::DTYPE_TIME: { + col->set_nth(i, json_into(value)); + return std::nullopt; + } + case t_dtype::DTYPE_DATE: { + col->set_nth(i, json_into(value)); + return std::nullopt; + } + default: + PSP_COMPLAIN_AND_ABORT("JSON field not yet implemented"); + return std::nullopt; + } +} + +/** + * The separator joining an object's key to its childrens'. MUST match + * `apachearrow::FLATTEN_SEPARATOR`, or the same logical record would land in + * different columns depending on whether it arrived as JSON or as Arrow. + */ +static const char FLATTEN_SEPARATOR = '.'; + +/** + * Visit every scalar beneath `value`, naming it by its dotted path from + * `prefix`. An object contributes its leaves rather than itself, because + * Perspective's column model is flat; an empty object contributes nothing. + * + * `prefix` is grown and restored in place rather than copied per leaf. + */ +template +static void +for_each_leaf( + std::string& prefix, + const rapidjson::Value& value, + t_list_flatten mode, + F&& fn, + bool through_array = false +) { + if (mode != LIST_FLATTEN_STRINGIFY && value.IsArray()) { + // Inference only needs one element to learn the leaves' types; any + // path a later element introduces is grown into by `resolve_column`. + if (!value.Empty()) { + for_each_leaf(prefix, value[0], mode, fn, true); + } + + return; + } + + if (!value.IsObject()) { + fn(static_cast(prefix), value, through_array); + return; + } + + const auto len = prefix.size(); + for (const auto& child : value.GetObj()) { + prefix += FLATTEN_SEPARATOR; + prefix += child.name.GetString(); + for_each_leaf(prefix, child.value, mode, fn, through_array); + prefix.resize(len); + } +} + +/** + * The value written when an expansion has no element for a slot -- an empty or + * absent array yields one row rather than none, matching the Arrow path. + */ +static const rapidjson::Value NULL_LEAF; + +/** + * Row indices are 32 bit, and expansion is the only ingest path that can + * multiply an input's size. + */ +static void +check_row_count(std::uint64_t rows) { + if (rows > std::numeric_limits::max()) { + std::stringstream ss; + ss << "Array expansion produced " << rows + << " rows, which exceeds the maximum supported size\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } +} + +/** + * How many output rows `value` expands to. + * + * An object combines its children: `zip` requires every child wider than one + * to agree, while `cartesian` multiplies them. An array contributes the SUM of + * its elements' widths, so a nested array expands at each level -- the same + * fixpoint the Arrow path reaches by re-running its pass. + */ +static t_uindex +leaf_width(const rapidjson::Value& value, t_list_flatten mode) { + if (mode == LIST_FLATTEN_STRINGIFY || value.IsNull()) { + return 1; + } + + if (value.IsArray()) { + t_uindex total = 0; + for (const auto& element : value.GetArray()) { + total += leaf_width(element, mode); + } + + // An empty array still occupies a row, carrying a null. + return std::max(total, 1); + } + + if (!value.IsObject()) { + return 1; + } + + t_uindex width = 1; + const char* witness = nullptr; + for (const auto& child : value.GetObj()) { + const auto child_width = leaf_width(child.value, mode); + if (mode == LIST_FLATTEN_CARTESIAN) { + width *= child_width; + continue; + } + + if (child_width == 1) { + continue; + } + + if (witness != nullptr && child_width != width) { + std::stringstream ss; + ss << "Cannot zip `" << witness << "` (" << width << ") and `" + << child.name.GetString() << "` (" << child_width + << ") of differing length; use the `cartesian` list flatten " + "mode.\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + width = child_width; + witness = child.name.GetString(); + } + + return width; +} + +/** + * Whether `value` needs the descending, expanding path at all. A value that is + * neither an object nor an expandable array is written straight to the column + * named by its key, with no path string and no width arithmetic. + */ +static bool +is_nested(const rapidjson::Value& value, t_list_flatten mode) { + return value.IsObject() + || (mode != LIST_FLATTEN_STRINGIFY && value.IsArray()); +} + +/** + * A key's name as a view, using `rapidjson`'s length rather than `strlen`. + */ +static std::string_view +key_name(const rapidjson::Value& name) { + return {name.GetString(), name.GetStringLength()}; +} + +/** + * The slot a child of width `child_width` contributes to its record's slot `k`. + * + * `zip` passes `k` straight through, since every child wider than one shares + * the record's width. `cartesian` decomposes it mixed-radix: `stride` starts at + * the record's width and is consumed left to right, so the LAST child varies + * fastest, matching `itertools.product` and the Arrow implementation. + * + * INVARIANT: under `cartesian` this must be called for EVERY child in order, + * including ones the caller then skips, or the remaining children decompose + * against the wrong stride. + */ +static t_uindex +child_slot( + t_uindex k, + t_uindex child_width, + t_uindex& stride, + t_list_flatten mode +) { + if (mode != LIST_FLATTEN_CARTESIAN) { + return child_width > 1 ? k : 0; + } + + stride /= child_width; + return (k / stride) % child_width; +} + +/** + * Visit the leaves `value` contributes to output slot `k`, which must be less + * than `width`. + * + * `width` is `leaf_width(value, mode)`, passed in rather than recomputed: + * every caller already holds it, and recomputing walks the whole subtree once + * per slot. + */ +template +static void +emit_leaves( + std::string& prefix, + const rapidjson::Value& value, + t_uindex k, + t_uindex width, + t_list_flatten mode, + F&& fn +) { + if (mode == LIST_FLATTEN_STRINGIFY || value.IsNull()) { + fn(std::string_view{prefix}, value); + return; + } + + if (value.IsArray()) { + // The width is the sum of the elements' and each is at least one, so + // equality means every element is exactly one -- an array of scalars + // or of flat objects, which is the common case. Slot `k` is then + // element `k`, rather than a scan accumulating widths, which would + // make emitting a whole array quadratic in its length. + if (width == value.Size()) { + emit_leaves(prefix, value[k], 0, 1, mode, fn); + return; + } + + // Otherwise find the element covering slot `k`, and the slot within it. + for (const auto& element : value.GetArray()) { + const auto element_width = leaf_width(element, mode); + if (k < element_width) { + emit_leaves(prefix, element, k, element_width, mode, fn); + return; + } + + k -= element_width; + } + + // Empty array: the row survives carrying a null. + fn(std::string_view{prefix}, NULL_LEAF); + return; + } + + if (!value.IsObject()) { + fn(std::string_view{prefix}, value); + return; + } + + t_uindex stride = width; + const auto len = prefix.size(); + for (const auto& child : value.GetObj()) { + const auto child_width = leaf_width(child.value, mode); + const auto child_k = child_slot(k, child_width, stride, mode); + prefix += FLATTEN_SEPARATOR; + prefix += child.name.GetString(); + emit_leaves(prefix, child.value, child_k, child_width, mode, fn); + prefix.resize(len); + } +} + +/** + * A column name reached both as a literal key and by descending into an + * object cannot be filled coherently, as two different cells would write it. + */ +static void +check_path_collision( + const std::set& literal, const std::set& descended +) { + for (const auto& name : literal) { + if (descended.count(name) > 0) { + std::stringstream ss; + ss << "Column `" << name + << "` is both a key and the flattened path of an object\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + } +} + +JsonLoader::JsonLoader() = default; +JsonLoader::~JsonLoader() = default; + +const std::vector& +JsonLoader::names() const { + return m_names; +} + +const std::vector& +JsonLoader::types() const { + return m_types; +} + +bool +JsonLoader::is_implicit() const { + return m_is_implicit; +} + +bool +JsonLoader::empty() const { + return m_empty; +} + +void +JsonLoader::release() { + // Move-construct and let the temporary die: `rapidjson::Document` frees + // its allocator's chunks on destruction, where `SetNull` would not. + { auto _ = std::move(m_document); } + + m_stream = rapidjson::StringStream{nullptr}; +} + +void +JsonLoader::init( + std::string_view data, + t_json_format format, + const std::string& index, + const t_schema* existing, + t_list_flatten mode +) { + m_format = format; + m_mode = mode; + if (format == JSON_FORMAT_NDJSON) { + m_stream = rapidjson::StringStream(data.data()); + m_document.ParseStream(m_stream); + } else { + m_document.Parse(data.data()); + } + + switch (format) { + case JSON_FORMAT_ROWS: { + if (m_document.Size() == 0) { + m_empty = true; + } else if (!m_document[0].IsObject()) { + // TODO Legacy error message + PSP_COMPLAIN_AND_ABORT( + "Cannot determine data types without column names!\n" + ) + } + } break; + case JSON_FORMAT_COLUMNS: { + if (!m_document.IsObject()) { + // TODO Legacy error message + PSP_COMPLAIN_AND_ABORT( + "Cannot determine data types without column names!\n" + ) + } + } break; + case JSON_FORMAT_NDJSON: { + if (m_document.Size() == 0) { + m_empty = true; + } else if (!m_document.IsObject()) { + std::stringstream ss; + ss << "Received non-object " << m_document.GetType(); + PSP_COMPLAIN_AND_ABORT(ss.str()) + } + } break; + } + + if (existing != nullptr) { + // An update takes its columns from the Table, and its primary key from + // whether that Table was created with one. + m_names = existing->columns(); + m_types = existing->types(); + m_is_implicit = index.empty(); + return; + } + + if (m_empty) { + return; + } + + switch (format) { + case JSON_FORMAT_ROWS: + infer_rows(index); + break; + case JSON_FORMAT_COLUMNS: + infer_cols(index); + break; + case JSON_FORMAT_NDJSON: + infer_ndjson(index); + break; + } + + m_expands = !m_per_element.empty(); + if (mode == LIST_FLATTEN_CARTESIAN && m_per_element.size() > 1) { + // A product repeats every factor against the others' dimensions, so + // with more than one expansion point nothing varies per row. + m_per_element.clear(); + } +} + +std::optional +JsonLoader::repeated_index(const std::string& index) const { + if (!m_expands) { + return std::nullopt; + } + + // An index this payload does not carry cannot be established as + // per-element, so it is treated as repeated. + if (!index.empty() && m_per_element.count(index) == 0) { + return index; + } + + const std::string implicit{"__INDEX__"}; + const auto has_implicit = + std::find(m_names.begin(), m_names.end(), implicit) != m_names.end(); + + if (has_implicit && m_per_element.count(implicit) == 0) { + return implicit; + } + + return std::nullopt; +} + +std::uint32_t +JsonLoader::fill_table( + t_data_table& tbl, + const std::string& index, + std::uint32_t offset, + bool is_update +) { + if (m_empty) { + return 0; + } + + switch (m_format) { + case JSON_FORMAT_ROWS: + return fill_rows(tbl, index, offset, is_update); + case JSON_FORMAT_COLUMNS: + return fill_cols(tbl, index, offset, is_update); + case JSON_FORMAT_NDJSON: + return fill_ndjson(tbl, index, offset, is_update); + } + + return 0; +} + +/** + * Accumulate types from one record, used once by ndjson and per-record by the + * row format. `seen` grows with every key encountered; `known` with the keys + * that have produced a non-null value, so the caller can tell when it may stop. + */ +static void +infer_record( + const rapidjson::Value& record, + const std::string& index, + std::set& seen, + std::set& known, + std::vector& names, + std::vector& types, + bool& is_implicit, + std::set& literal, + std::set& descended, + std::set& per_element, + t_list_flatten mode +) { + std::string path; + for (const auto& col : record.GetObj()) { + path.assign(col.name.GetString()); + const auto top = path.size(); + for_each_leaf( + path, + col.value, + mode, + [&](const auto& name, const auto&, bool through_array) { + seen.insert(name); + (name.size() == top ? literal : descended).insert(name); + if (through_array) { + per_element.insert(name); + } + } + ); + } + + // https://github.com/Tencent/rapidjson/issues/1994 + for (const auto& col : record.GetObj()) { + path.assign(col.name.GetString()); + for_each_leaf(path, col.value, mode, [&](const auto& name, const auto& leaf, bool) { + if (name == index) { + is_implicit = false; + } + + if (known.count(name) > 0) { + return; + } + + auto dtype = rapidjson_type_to_dtype(leaf); + if (dtype != DTYPE_NONE) { + known.insert(name); + types.push_back(dtype); + names.emplace_back(name); + } + }); + } +} + +/** + * Columns which never produced a non-null value have no inferrable type. + */ +static void +default_untyped_to_string( + const std::set& seen, + const std::set& known, + std::vector& names, + std::vector& types +) { + for (const auto& col : seen) { + if (known.count(col) == 0) { + types.push_back(DTYPE_STR); + names.emplace_back(col); + } + } +} + +std::shared_ptr +JsonLoader::resolve_column( + t_data_table& tbl, + std::string_view name, + const rapidjson::Value& leaf, + bool is_update +) { + auto col = tbl.get_column_safe(name); + if (col) { + return col; + } + + if (is_update) { + LOG_DEBUG("Ignoring column " << name); + return nullptr; + } + + auto dtype = rapidjson_type_to_dtype(leaf); + if (dtype == DTYPE_NONE) { + // A `null` carries no type; wait for a record that does. + return nullptr; + } + + m_names.emplace_back(name); + m_types.push_back(dtype); + + // Sizes the new column to the table, leaving the preceding rows invalid. + return tbl.add_column_sptr(m_names.back(), dtype, true); +} + +void +JsonLoader::infer_rows(const std::string& index) { + std::set seen; + std::set known; + std::set literal; + std::set descended; + for (const auto& row : m_document.GetArray()) { + infer_record( + row, + index, + seen, + known, + m_names, + m_types, + m_is_implicit, + literal, + descended, + m_per_element, + m_mode + ); + + // Theoretically there can end too early if the first + // few rows are missing columns that are present in later rows. + if (known.size() == seen.size()) { + break; + } + } + + check_path_collision(literal, descended); + default_untyped_to_string(seen, known, m_names, m_types); +} + +void +JsonLoader::infer_ndjson(const std::string& index) { + // Only the first record is available without consuming the stream; later + // records grow the schema during the fill instead. + std::set seen; + std::set known; + std::set literal; + std::set descended; + infer_record( + m_document, + index, + seen, + known, + m_names, + m_types, + m_is_implicit, + literal, + descended, + m_per_element, + m_mode + ); + + check_path_collision(literal, descended); + default_untyped_to_string(seen, known, m_names, m_types); +} + +void +JsonLoader::infer_cols(const std::string& index) { + // https://github.com/Tencent/rapidjson/issues/1994 + for (const auto& it : m_document.GetObj()) { + if (!it.value.IsArray()) { + PSP_COMPLAIN_AND_ABORT("Malformed column") + } + + if (it.value.Empty()) { + PSP_COMPLAIN_AND_ABORT("Can't create table from empty columns") + } + + if (it.name.GetString() == index) { + m_is_implicit = false; + } + + std::set known; + std::string path; + bool saw_leaf = false; + for (const auto& cell : it.value.GetArray()) { + // Whether this cell contributes the column itself rather than + // paths beneath it -- the same test `for_each_leaf` applies. + const bool is_leaf = !cell.IsObject() + && (m_mode == LIST_FLATTEN_STRINGIFY || !cell.IsArray()); + + saw_leaf = saw_leaf || is_leaf; + path.assign(it.name.GetString()); + for_each_leaf( + path, + cell, + m_mode, + [&](const auto& name, const auto& v, bool through_array) { + if (through_array) { + m_per_element.insert(name); + } + + if (known.count(name) > 0) { + return; + } + + auto dtype = rapidjson_type_to_dtype(v); + if (dtype != DTYPE_NONE) { + known.insert(name); + m_types.push_back(dtype); + m_names.emplace_back(name); + } + } + ); + + // The first cell that yields any type ends the scan, for objects + // as well as scalars -- structure is inferred from the first row. + // A later cell carrying a path this one lacked is picked up by + // `resolve_column` during the fill, so nothing is dropped and no + // column costs more than one cell to infer. + if (!known.empty()) { + break; + } + } + + // Every cell was null, so there is no type to infer -- but only if the + // column has leaf cells at all. A column of objects contributing no + // paths contributes no column either. + if (known.empty() && saw_leaf) { + m_types.push_back(DTYPE_STR); + m_names.emplace_back(it.name.GetString()); + } + } +} + +/** + * Write one cell, resolving a type conflict the only way each mode can: table + * creation widens the column and rewrites, while an update cannot change the + * schema of a live Table and so must reject the value. + */ +static void +fill_cell( + t_data_table& tbl, + const std::shared_ptr& col, + std::string_view col_name, + t_uindex ii, + const rapidjson::Value& cell, + bool is_update +) { + auto promote = fill_column_json(col, ii, cell, is_update); + if (!promote) { + return; + } + + if (is_update) { + std::stringstream ss; + ss << "Cannot append value of type " << dtype_to_str(*promote) + << " to column \"" << col_name << "\" of type " + << dtype_to_str(col->get_dtype()) << " at index " << ii << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + LOG_DEBUG( + "Promoting column " << col_name << " from " + << dtype_to_str(col->get_dtype()) << " to " + << dtype_to_str(*promote) + ); + + const std::string name{col_name}; + tbl.promote_column(name, *promote, ii, true); + fill_column_json(tbl.get_column(name), ii, cell, is_update); +} + +std::uint32_t +JsonLoader::fill_rows( + t_data_table& tbl, + const std::string& index, + std::uint32_t offset, + bool is_update +) { + const auto nrows = static_cast(m_document.Size()); + tbl.extend(nrows); + t_uindex extended = nrows; + + const auto& psp_pkey_col = tbl.get_column("psp_pkey"); + const auto psp_okey_col = + is_update ? nullptr : tbl.get_column("psp_okey"); + + t_uindex ii = 0; + for (const auto& row : m_document.GetArray()) { + bool nested = false; + if (m_is_implicit && is_update) { + psp_pkey_col->set_nth(ii, (ii + offset)); + } + + for (const auto& it : row.GetObj()) { + if (is_nested(it.value, m_mode)) { + nested = true; + break; + } + + const auto name = key_name(it.name); + if (is_update && name == "__INDEX__") { + fill_cell( + tbl, psp_pkey_col, "psp_pkey", ii, it.value, is_update + ); + + continue; + } + + auto col = resolve_column(tbl, name, it.value, is_update); + if (!col) { + continue; + } + + fill_cell(tbl, col, name, ii, it.value, is_update); + if (!m_is_implicit && index == name) { + fill_column_json(psp_pkey_col, ii, it.value, is_update); + if (psp_okey_col) { + fill_column_json(psp_okey_col, ii, it.value, is_update); + } + } + } + + if (!nested) { + if (m_is_implicit && !is_update) { + psp_pkey_col->set_nth(ii, ii); + psp_okey_col->set_nth(ii, ii); + } + + ii++; + continue; + } + + const auto width = leaf_width(row, m_mode); + m_child_widths.clear(); + for (const auto& it : row.GetObj()) { + m_child_widths.push_back(leaf_width(it.value, m_mode)); + } + + check_row_count(static_cast(ii) + width); + if (ii + width > extended) { + extended = ii + width; + tbl.extend(extended); + } + + for (t_uindex k = 0; k < width; ++k) { + if (m_is_implicit && is_update) { + psp_pkey_col->set_nth(ii, (ii + offset)); + } + + std::string path; + t_uindex stride = width; + std::size_t ci = 0; + for (const auto& it : row.GetObj()) { + const auto child_width = m_child_widths[ci++]; + const auto sub = child_slot(k, child_width, stride, m_mode); + if (is_update + && std::string_view{it.name.GetString()} == "__INDEX__") { + fill_cell( + tbl, psp_pkey_col, "psp_pkey", ii, it.value, is_update + ); + + continue; + } + + path.assign(it.name.GetString()); + emit_leaves( + path, + it.value, + sub, + child_width, + m_mode, + [&](const auto& name, const auto& v) { + auto col = resolve_column(tbl, name, v, is_update); + if (!col) { + return; + } + + fill_cell(tbl, col, name, ii, v, is_update); + if (!m_is_implicit && index == name) { + fill_column_json(psp_pkey_col, ii, v, is_update); + if (psp_okey_col) { + fill_column_json(psp_okey_col, ii, v, is_update); + } + } + } + ); + } + + if (m_is_implicit && !is_update) { + psp_pkey_col->set_nth(ii, ii); + psp_okey_col->set_nth(ii, ii); + } + + ii++; + } + } + + return ii; +} + +std::uint32_t +JsonLoader::fill_cols( + t_data_table& tbl, + const std::string& index, + std::uint32_t offset, + bool is_update +) { + std::vector cells; + std::vector col_names; + std::vector is_pkey_column; + t_uindex nrows = 0; + for (const auto& it : m_document.GetObj()) { + if (is_update) { + // Creation validated these while inferring. + if (!it.value.IsArray()) { + PSP_COMPLAIN_AND_ABORT("Malformed column") + } + + if (it.value.Empty()) { + PSP_COMPLAIN_AND_ABORT("Can't create table from empty columns") + } + } + + cells.push_back(&it.value); + col_names.emplace_back(it.name.GetString()); + is_pkey_column.push_back( + is_update + && std::string_view{it.name.GetString()} == "__INDEX__" + ); + + nrows = std::max(nrows, static_cast(it.value.Size())); + } + + const auto& psp_pkey_col = tbl.get_column("psp_pkey"); + const auto psp_okey_col = is_update ? nullptr : tbl.get_column("psp_okey"); + const bool implicit_pkey_from_offset = m_is_implicit && is_update + && !m_document.GetObj().HasMember("__INDEX__"); + + tbl.extend(nrows); + bool expands = false; + if (implicit_pkey_from_offset) { + for (t_uindex ii = 0; ii < nrows; ii++) { + psp_pkey_col->set_nth(ii, (offset + ii)); + } + } + + for (std::size_t c = 0; c < cells.size() && !expands; ++c) { + const auto len = static_cast(cells[c]->Size()); + for (t_uindex r = 0; r < len; ++r) { + const auto& cell = (*cells[c])[r]; + if (is_nested(cell, m_mode)) { + expands = true; + break; + } + + if (is_pkey_column[c]) { + fill_cell(tbl, psp_pkey_col, "psp_pkey", r, cell, is_update); + continue; + } + + auto col = resolve_column(tbl, col_names[c], cell, is_update); + if (!col) { + continue; + } + + fill_cell(tbl, col, col_names[c], r, cell, is_update); + if (!m_is_implicit && index == col_names[c]) { + fill_column_json(psp_pkey_col, r, cell, is_update); + if (psp_okey_col) { + fill_column_json(psp_okey_col, r, cell, is_update); + } + } + } + } + + if (!expands) { + if (m_is_implicit && !is_update) { + for (t_uindex ii = 0; ii < nrows; ii++) { + psp_pkey_col->set_nth(ii, ii); + psp_okey_col->set_nth(ii, ii); + } + } + + return nrows; + } + + std::vector> widths(cells.size()); + for (std::size_t c = 0; c < cells.size(); ++c) { + const auto len = static_cast(cells[c]->Size()); + widths[c].assign(nrows, 1); + for (t_uindex r = 0; r < len; ++r) { + widths[c][r] = leaf_width((*cells[c])[r], m_mode); + } + } + + std::vector row_width(nrows, 1); + std::vector row_start(nrows, 0); + std::uint64_t total = 0; + for (t_uindex r = 0; r < nrows; ++r) { + t_uindex width = 1; + const std::string* witness = nullptr; + for (std::size_t c = 0; c < cells.size(); ++c) { + const auto w = widths[c][r]; + if (m_mode == LIST_FLATTEN_CARTESIAN) { + width *= w; + continue; + } + + if (w == 1) { + continue; + } + + if (witness != nullptr && w != width) { + std::stringstream ss; + ss << "Cannot zip `" << *witness << "` (" << width << ") and `" + << col_names[c] << "` (" << w << ") of differing length in " + << "row " << r + << "; use the `cartesian` list flatten mode.\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + width = w; + witness = &col_names[c]; + } + + row_width[r] = width; + row_start[r] = static_cast(total); + total += width; + } + + check_row_count(total); + const auto size = static_cast(total); + tbl.extend(size); + if (implicit_pkey_from_offset) { + for (t_uindex ii = 0; ii < size; ii++) { + psp_pkey_col->set_nth(ii, (offset + ii)); + } + } + + std::string path; + for (std::size_t c = 0; c < cells.size(); ++c) { + const auto len = static_cast(cells[c]->Size()); + for (t_uindex r = 0; r < len; ++r) { + const auto& cell = (*cells[c])[r]; + for (t_uindex k = 0; k < row_width[r]; ++k) { + const auto ii = row_start[r] + k; + if (is_pkey_column[c]) { + fill_cell( + tbl, psp_pkey_col, "psp_pkey", ii, cell, is_update + ); + + continue; + } + + t_uindex sub = 0; + if (m_mode == LIST_FLATTEN_CARTESIAN) { + t_uindex stride = 1; + for (std::size_t d = c + 1; d < cells.size(); ++d) { + stride *= widths[d][r]; + } + + sub = (k / stride) % widths[c][r]; + } else if (widths[c][r] > 1) { + sub = k; + } + + path.assign(col_names[c]); + emit_leaves( + path, + cell, + sub, + widths[c][r], + m_mode, + [&](const auto& name, const auto& v) { + auto col = resolve_column(tbl, name, v, is_update); + if (!col) { + return; + } + + fill_cell(tbl, col, name, ii, v, is_update); + if (!m_is_implicit && index == name) { + fill_column_json(psp_pkey_col, ii, v, is_update); + if (psp_okey_col) { + fill_column_json( + psp_okey_col, ii, v, is_update + ); + } + } + } + ); + } + } + } + + if (m_is_implicit && !is_update) { + for (t_uindex ii = 0; ii < size; ii++) { + psp_pkey_col->set_nth(ii, ii); + psp_okey_col->set_nth(ii, ii); + } + } + + return size; +} + +std::uint32_t +JsonLoader::fill_ndjson( + t_data_table& tbl, + const std::string& index, + std::uint32_t offset, + bool is_update +) { + const auto& psp_pkey_col = tbl.get_column("psp_pkey"); + const auto psp_okey_col = + is_update ? nullptr : tbl.get_column("psp_okey"); + + t_uindex ii = 0; + bool is_finished = false; + while (!is_finished) { + bool nested = false; + for (const auto& it : m_document.GetObj()) { + if (is_nested(it.value, m_mode)) { + nested = true; + break; + } + } + + if (!nested) { + tbl.extend(ii + 1); + if (m_is_implicit && is_update) { + psp_pkey_col->set_nth(ii, (ii + offset)); + } + + for (const auto& it : m_document.GetObj()) { + const auto name = key_name(it.name); + if (is_update && name == "__INDEX__") { + fill_cell( + tbl, psp_pkey_col, "psp_pkey", ii, it.value, is_update + ); + + continue; + } + + auto col = resolve_column(tbl, name, it.value, is_update); + if (!col) { + continue; + } + + fill_cell(tbl, col, name, ii, it.value, is_update); + if (!m_is_implicit && index == name) { + fill_column_json(psp_pkey_col, ii, it.value, is_update); + if (psp_okey_col) { + fill_column_json(psp_okey_col, ii, it.value, is_update); + } + } + } + + if (m_is_implicit && !is_update) { + psp_pkey_col->set_nth(ii, ii); + psp_okey_col->set_nth(ii, ii); + } + + ii++; + m_document.ParseStream(m_stream); + if (m_document.HasParseError()) { + is_finished = true; + } + + continue; + } + + const auto width = leaf_width(m_document, m_mode); + check_row_count(static_cast(ii) + width); + tbl.extend(ii + width); + + m_child_widths.clear(); + for (const auto& it : m_document.GetObj()) { + m_child_widths.push_back(leaf_width(it.value, m_mode)); + } + + for (t_uindex k = 0; k < width; ++k) { + if (m_is_implicit && is_update) { + psp_pkey_col->set_nth(ii, (ii + offset)); + } + + std::string path; + t_uindex stride = width; + std::size_t ci = 0; + for (const auto& it : m_document.GetObj()) { + const auto child_width = m_child_widths[ci++]; + const auto sub = child_slot(k, child_width, stride, m_mode); + if (is_update + && std::string_view{it.name.GetString()} == "__INDEX__") { + fill_cell( + tbl, psp_pkey_col, "psp_pkey", ii, it.value, is_update + ); + + continue; + } + + path.assign(it.name.GetString()); + emit_leaves( + path, + it.value, + sub, + child_width, + m_mode, + [&](const auto& name, const auto& v) { + auto col = resolve_column(tbl, name, v, is_update); + if (!col) { + return; + } + + fill_cell(tbl, col, name, ii, v, is_update); + if (!m_is_implicit && index == name) { + fill_column_json(psp_pkey_col, ii, v, is_update); + if (psp_okey_col) { + fill_column_json(psp_okey_col, ii, v, is_update); + } + } + } + ); + } + + if (m_is_implicit && !is_update) { + psp_pkey_col->set_nth(ii, ii); + psp_okey_col->set_nth(ii, ii); + } + + ii++; + } + + m_document.ParseStream(m_stream); + if (m_document.HasParseError()) { + is_finished = true; + } + } + + tbl.extend(ii); + return ii; +} + +} // namespace perspective::json diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index a103932587..ec251454a5 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -30,12 +30,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -1421,26 +1423,70 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { features->set_on_update(true); features->set_expressions(true); + const auto window_agg = [](const char* name, + std::initializer_list frames, + bool offset, + bool alpha, + std::optional result_type + ) { + proto::WindowAggregateArgs args; + args.set_name(name); + for (const auto* frame : frames) { + args.add_frames(frame); + } + args.set_offset(offset); + args.set_alpha(alpha); + if (result_type.has_value()) { + args.set_result_type(*result_type); + } + return args; + }; + + const std::initializer_list frames{ + "rows", "range", "cumulative" + }; + const std::initializer_list no_frames{}; + proto::GetFeaturesResp_WindowAggregateOptions numeric_aggs; - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_SUM); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_AVG); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_COUNT); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_MIN); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_MAX); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_STDDEV); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_VAR); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_LAG); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_LEAD); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_DIFF); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_RATE); - numeric_aggs.add_options(proto::WINDOW_AGGREGATE_EMA); + *numeric_aggs.add_options() = + window_agg("sum", frames, false, false, proto::ColumnType::FLOAT); + *numeric_aggs.add_options() = + window_agg("avg", frames, false, false, proto::ColumnType::FLOAT); + *numeric_aggs.add_options() = + window_agg("count", frames, false, false, proto::ColumnType::INTEGER); + *numeric_aggs.add_options() = + window_agg("min", frames, false, false, std::nullopt); + *numeric_aggs.add_options() = + window_agg("max", frames, false, false, std::nullopt); + *numeric_aggs.add_options() = + window_agg("stddev", frames, false, false, proto::ColumnType::FLOAT); + *numeric_aggs.add_options() = + window_agg("var", frames, false, false, proto::ColumnType::FLOAT); + *numeric_aggs.add_options() = + window_agg("lag", no_frames, true, false, std::nullopt); + *numeric_aggs.add_options() = + window_agg("lead", no_frames, true, false, std::nullopt); + *numeric_aggs.add_options() = + window_agg("diff", no_frames, true, false, proto::ColumnType::FLOAT); + // `rate` is defined on the order key's units, so only a `range` + // frame is meaningful for it. + *numeric_aggs.add_options() = + window_agg("rate", {"range"}, false, false, proto::ColumnType::FLOAT); + // `ema` is recursive - a smoothing factor, never a frame. + *numeric_aggs.add_options() = + window_agg("ema", no_frames, false, true, proto::ColumnType::FLOAT); proto::GetFeaturesResp_WindowAggregateOptions any_aggs; - any_aggs.add_options(proto::WINDOW_AGGREGATE_COUNT); - any_aggs.add_options(proto::WINDOW_AGGREGATE_MIN); - any_aggs.add_options(proto::WINDOW_AGGREGATE_MAX); - any_aggs.add_options(proto::WINDOW_AGGREGATE_LAG); - any_aggs.add_options(proto::WINDOW_AGGREGATE_LEAD); + *any_aggs.add_options() = + window_agg("count", frames, false, false, proto::ColumnType::INTEGER); + *any_aggs.add_options() = + window_agg("min", frames, false, false, std::nullopt); + *any_aggs.add_options() = + window_agg("max", frames, false, false, std::nullopt); + *any_aggs.add_options() = + window_agg("lag", no_frames, true, false, std::nullopt); + *any_aggs.add_options() = + window_agg("lead", no_frames, true, false, std::nullopt); auto& window_aggs = *features->mutable_window_aggregates(); window_aggs[proto::ColumnType::INTEGER] = numeric_aggs; @@ -1665,6 +1711,20 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { ? BACKING_STORE_DISK : BACKING_STORE_MEMORY; + apachearrow::t_list_flatten list_flatten; + switch (r.options().list_flatten()) { + case proto::LIST_FLATTEN_CARTESIAN: + list_flatten = apachearrow::LIST_FLATTEN_CARTESIAN; + break; + case proto::LIST_FLATTEN_STRINGIFY: + list_flatten = apachearrow::LIST_FLATTEN_STRINGIFY; + break; + case proto::LIST_FLATTEN_ZIP: + default: + list_flatten = apachearrow::LIST_FLATTEN_ZIP; + break; + } + switch (r.data().data_case()) { case proto::MakeTableData::kFromView: { auto view = m_resources.get_view(r.data().from_view()); @@ -1685,7 +1745,11 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { ); table = Table::from_arrow( - index, std::move(*arrow), limit, backing_store + index, + std::move(*arrow), + limit, + backing_store, + list_flatten ); break; } @@ -1694,7 +1758,11 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { { auto _ = std::move(req); } table = Table::from_arrow( - index, std::move(data), limit, backing_store + index, + std::move(data), + limit, + backing_store, + list_flatten ); break; } @@ -1703,7 +1771,11 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { { auto _ = std::move(req); } table = Table::from_csv( - index, std::move(data), limit, backing_store + index, + std::move(data), + limit, + backing_store, + list_flatten ); break; } @@ -1712,7 +1784,11 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { { auto _ = std::move(req); } table = Table::from_cols( - index, std::move(data), limit, backing_store + index, + std::move(data), + limit, + backing_store, + list_flatten ); break; } @@ -1721,7 +1797,11 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { { auto _ = std::move(req); } table = Table::from_rows( - index, std::move(data), limit, backing_store + index, + std::move(data), + limit, + backing_store, + list_flatten ); break; } @@ -1730,7 +1810,11 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { { auto _ = std::move(req); } table = Table::from_ndjson( - index, std::move(data), limit, backing_store + index, + std::move(data), + limit, + backing_store, + list_flatten ); break; } @@ -2246,50 +2330,33 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { } } - t_window_op op = t_window_op::WINDOW_OP_SUM; - switch (w.op()) { - case proto::WINDOW_AGGREGATE_SUM: - op = t_window_op::WINDOW_OP_SUM; - break; - case proto::WINDOW_AGGREGATE_AVG: - op = t_window_op::WINDOW_OP_AVG; - break; - case proto::WINDOW_AGGREGATE_COUNT: - op = t_window_op::WINDOW_OP_COUNT; - break; - case proto::WINDOW_AGGREGATE_MIN: - op = t_window_op::WINDOW_OP_MIN; - break; - case proto::WINDOW_AGGREGATE_MAX: - op = t_window_op::WINDOW_OP_MAX; - break; - case proto::WINDOW_AGGREGATE_STDDEV: - op = t_window_op::WINDOW_OP_STDDEV; - break; - case proto::WINDOW_AGGREGATE_VAR: - op = t_window_op::WINDOW_OP_VAR; - break; - case proto::WINDOW_AGGREGATE_LAG: - op = t_window_op::WINDOW_OP_LAG; - break; - case proto::WINDOW_AGGREGATE_LEAD: - op = t_window_op::WINDOW_OP_LEAD; - break; - case proto::WINDOW_AGGREGATE_DIFF: - op = t_window_op::WINDOW_OP_DIFF; - break; - case proto::WINDOW_AGGREGATE_RATE: - op = t_window_op::WINDOW_OP_RATE; - break; - case proto::WINDOW_AGGREGATE_EMA: - op = t_window_op::WINDOW_OP_EMA; - break; - default: - PSP_COMPLAIN_AND_ABORT( - "Window `op` not implemented in this build" - ); + static const std::unordered_map + WINDOW_OPS{ + {"sum", t_window_op::WINDOW_OP_SUM}, + {"avg", t_window_op::WINDOW_OP_AVG}, + {"count", t_window_op::WINDOW_OP_COUNT}, + {"min", t_window_op::WINDOW_OP_MIN}, + {"max", t_window_op::WINDOW_OP_MAX}, + {"stddev", t_window_op::WINDOW_OP_STDDEV}, + {"var", t_window_op::WINDOW_OP_VAR}, + {"first", t_window_op::WINDOW_OP_FIRST}, + {"last", t_window_op::WINDOW_OP_LAST}, + {"lag", t_window_op::WINDOW_OP_LAG}, + {"lead", t_window_op::WINDOW_OP_LEAD}, + {"diff", t_window_op::WINDOW_OP_DIFF}, + {"rate", t_window_op::WINDOW_OP_RATE}, + {"ema", t_window_op::WINDOW_OP_EMA}, + }; + + const auto op_entry = WINDOW_OPS.find(w.op()); + if (op_entry == WINDOW_OPS.end()) { + PSP_COMPLAIN_AND_ABORT( + "Window `op` not implemented in this build: " + w.op() + ); } + t_window_op op = op_entry->second; + // An OMITTED frame means cumulative for aggregating // ops - the initializer below IS that default. t_window_frame_type frame_type = diff --git a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp index d6684d07db..d4fde0b037 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp @@ -16,6 +16,7 @@ #include "perspective/computed_expression.h" #include "perspective/data_table.h" #include "perspective/raw_types.h" +#include "perspective/json_loader.h" #include "perspective/schema.h" #include "rapidjson/document.h" #include @@ -40,7 +41,8 @@ Table::Table( std::vector data_types, std::uint32_t limit, std::string index, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) : m_init(false), m_id(GLOBAL_TABLE_ID++), @@ -51,7 +53,8 @@ Table::Table( m_limit(limit), m_index(std::move(index)), m_gnode_set(false), - m_backing_store(backing_store) { + m_backing_store(backing_store), + m_list_flatten(list_flatten) { validate_columns(m_column_names); } @@ -393,7 +396,8 @@ Table::from_csv( const std::string& index, std::string&& data, std::uint32_t limit, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) { auto map = std::unordered_map>(); @@ -435,7 +439,7 @@ Table::from_csv( auto pool = std::make_shared(); pool->init(); auto tbl = std::make_shared
( - pool, column_names, data_types, limit, index, backing_store + pool, column_names, data_types, limit, index, backing_store, list_flatten ); // `psp_pkey` is guaranteed unique only when the index is implicit (a @@ -453,381 +457,12 @@ Table::from_csv( return tbl; } -static bool -ichar_equals(char a, char b) { - return std::tolower(static_cast(a)) - == std::tolower(static_cast(b)); -} - -static bool -istrequals(std::string_view a, std::string_view b) { - return a.size() == b.size() - && std::equal(a.begin(), a.end(), b.begin(), ichar_equals); -} - -t_dtype -rapidjson_type_to_dtype(const rapidjson::Value& value) { - switch (value.GetType()) { - case rapidjson::Type::kStringType: { - const auto& str = value.GetString(); - if (str[0] == '\0') { - return t_dtype::DTYPE_STR; - } - - if (istrequals(str, "true") || istrequals(str, "false")) { - return t_dtype::DTYPE_BOOL; - } - - // TODO JSON will no longer support date/datetime inference. The - // only way to load JSON data with these types will be with a - // Schema! - - char* endptr; - strtol(str, &endptr, 10); - if (*endptr == '\0') { - return t_dtype::DTYPE_INT32; - } - - strtof(str, &endptr); - if (*endptr == '\0') { - return t_dtype::DTYPE_FLOAT64; - } - - std::tm tm; - std::memset(&tm, 0, sizeof(tm)); - std::chrono::system_clock::time_point tp; - - if (parse_all_date_time(tm, tp, str)) { - if (tm.tm_hour == 0 && tm.tm_min == 0 && tm.tm_sec == 0) { - return t_dtype::DTYPE_DATE; - } - return t_dtype::DTYPE_TIME; - } - - auto datetime = apachearrow::parseAsArrowTimestamp(str); - if (datetime != std::nullopt) { - return t_dtype::DTYPE_TIME; - } - - return t_dtype::DTYPE_STR; - } - case rapidjson::Type::kNumberType: { - if (value.IsInt64()) { - if (value.GetInt64() - > std::numeric_limits::max()) { - return t_dtype::DTYPE_FLOAT64; - } - return t_dtype::DTYPE_INT32; - } - if (value.IsInt()) { - return t_dtype::DTYPE_INT32; - } - - return t_dtype::DTYPE_FLOAT64; - } - case rapidjson::Type::kTrueType: - case rapidjson::Type::kFalseType: - return t_dtype::DTYPE_BOOL; - case rapidjson::kNullType: - return t_dtype::DTYPE_NONE; - case rapidjson::kObjectType: - case rapidjson::kArrayType: - PSP_COMPLAIN_AND_ABORT("Unknown JSON type"); - return t_dtype::DTYPE_NONE; - default: - PSP_COMPLAIN_AND_ABORT("Unknown JSON type"); - return t_dtype::DTYPE_NONE; - } -} void Table::clear() { reset_gnode(m_gnode->get_id()); } -template -struct promote { - constexpr static t_dtype dtype = DTYPE_NONE; -}; - -#define PROMOTE_IMPL(A, B, C) \ - template <> \ - struct promote { \ - constexpr static t_dtype dtype = C; \ - }; - -PROMOTE_IMPL(DTYPE_INT32, DTYPE_INT64, DTYPE_INT64) -// PROMOTE_IMPL(std::int32_t, std::float_t, DTYPE_FLOAT32) -// PROMOTE_IMPL(std::int32_t, std::double_t, DTYPE_FLOAT64) - -template -static A -json_into(const rapidjson::Value& value) { - if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { - if (value.IsInt()) { - return value.GetInt(); - } - if (value.IsInt64()) { - return value.GetInt64(); - } - if (value.IsDouble()) { - return value.GetDouble(); - } - if (value.IsFloat()) { - return value.GetFloat(); - } - if (value.IsString()) { - if constexpr (std::is_same_v) { - return std::atoi(value.GetString()); - } else if constexpr (std::is_same_v) { - return std::atoll(value.GetString()); - } else if constexpr (std::is_same_v || std::is_same_v) { - return std::atof(value.GetString()); - } else { - static_assert(!std::is_same_v, "No coercion for type"); - } - } - if (value.IsNull()) { - return 0; - } - - std::stringstream ss; - ss << "Could not coerce " << value.GetType() << " to " - << "a number"; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } else if constexpr (std::is_same_v) { - switch (value.GetType()) { - case rapidjson::kNullType: - return ""; - case rapidjson::kFalseType: - return "false"; - case rapidjson::kTrueType: - return "true"; - case rapidjson::kObjectType: - PSP_COMPLAIN_AND_ABORT("Cannot coerce object to string"); - case rapidjson::kArrayType: - PSP_COMPLAIN_AND_ABORT("Cannot coerce array to string"); - case rapidjson::kStringType: - return value.GetString(); - case rapidjson::kNumberType: - if (value.IsInt()) { - return std::to_string(value.GetInt()); - } - if (value.IsInt64()) { - return std::to_string(value.GetInt64()); - } - if (value.IsDouble()) { - return std::to_string(value.GetDouble()); - } - if (value.IsFloat()) { - return std::to_string(value.GetFloat()); - } - } - - std::stringstream ss; - ss << "Could not coerce " << value.GetType() << " to " - << "a string"; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } else if constexpr (std::is_same_v) { - std::tm tm; - if (value.IsString()) { - if (!parse_all_date_time(tm, value.GetString())) { - PSP_COMPLAIN_AND_ABORT("Could not coerce to date"); - } - } else if (value.IsInt64()) { - return t_date::from_epoch_ms(value.GetInt64()); - } else { - PSP_COMPLAIN_AND_ABORT("Could not coerce to date"); - } - - return t_date(tm.tm_year + 1900, tm.tm_mon, tm.tm_mday); - } else if constexpr (std::is_same_v) { - if (value.IsString()) { - std::chrono::system_clock::time_point tp; - if (!parse_all_date_time(tp, value.GetString())) { - PSP_COMPLAIN_AND_ABORT("Could not coerce to time"); - } - - return t_time(std::chrono::duration_cast( - tp.time_since_epoch() - ) - .count()); - } - if (value.IsDouble()) { - return t_time(value.GetDouble()); - } - if (value.IsInt64()) { - return t_time(value.GetInt64()); - } - if (value.IsInt()) { - return t_time(value.GetInt()); - } - PSP_COMPLAIN_AND_ABORT( - "Could not coerce " + std::to_string(value.GetType()) - + " to a time." - ); - } else { - static_assert(!std::is_same_v, "No coercion for type"); - } -} - -std::optional -fill_column_json( - const std::shared_ptr& col, - const t_uindex i, - const rapidjson::Value& value, - const bool is_update -) { - if (value.IsNull()) { - if (is_update) { - col->unset(i); - } else { - col->clear(i); - } - return std::nullopt; - } - - switch (col->get_dtype()) { - case t_dtype::DTYPE_STR: { - if (!value.IsString()) { - auto v = json_into(value); - col->set_nth(i, v); - } else { - col->set_nth(i, value.GetString()); - } - return std::nullopt; - } - case t_dtype::DTYPE_INT32: { - if (value.IsInt()) { - col->set_nth(i, value.GetInt()); - return std::nullopt; - } - - if (value.IsInt64()) { - if (value.GetInt64() > std::numeric_limits::max()) - [[likely]] { - if (!is_update) { - LOG_DEBUG("Promoting due to int32 overflow"); - return {DTYPE_FLOAT64}; - } - } - - // Coerce in update mode - col->set_nth( - i, static_cast(value.GetInt64()) - ); - - return std::nullopt; - } - - if (value.IsDouble()) { - if (is_update) { - col->set_nth( - i, static_cast(value.GetDouble()) - ); - return std::nullopt; - } - - return {DTYPE_FLOAT64}; - } - - if (value.IsString()) { - const auto& str = value.GetString(); - if (str[0] == '\0') { - if (is_update) { - col->set_valid(i, false); - return std::nullopt; - } - - return {t_dtype::DTYPE_STR}; - } - - char* endptr; - std::int32_t result = strtol(str, &endptr, 10); - if (*endptr == '\0') { - col->set_nth(i, result); - return std::nullopt; - } - - float result2 = strtof(str, &endptr); - if (*endptr == '\0') { - if (is_update) { - col->set_nth( - i, static_cast(result2) - ); - return std::nullopt; - } - - return {t_dtype::DTYPE_FLOAT64}; - } - - return {t_dtype::DTYPE_STR}; - } - - std::stringstream ss; - ss << "Expected int, found " << value.GetType(); - PSP_COMPLAIN_AND_ABORT(ss.str()); - return std::nullopt; - } - case t_dtype::DTYPE_INT64: { - if (value.IsInt64()) [[likely]] { - col->set_nth(i, value.GetInt()); - } else if (value.IsDouble()) { - return {DTYPE_FLOAT64}; - } else if (value.IsString()) { - col->set_nth(i, std::atoll(value.GetString())); - } else { - std::stringstream ss; - ss << "Expected int64, found " << value.GetType(); - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - return std::nullopt; - } - case t_dtype::DTYPE_FLOAT64: { - if (value.IsDouble()) [[likely]] { - col->set_nth(i, value.GetDouble()); - } else if (value.IsInt64()) { - col->set_nth(i, static_cast(value.GetInt64())); - } else if (value.IsInt()) { - col->set_nth(i, value.GetInt()); - } else if (value.IsString()) { - col->set_nth(i, std::atof(value.GetString())); - } else { - std::stringstream ss; - ss << "Expected double, found " << value.GetType(); - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - return std::nullopt; - } - case t_dtype::DTYPE_BOOL: { - if (value.IsBool()) [[likely]] { - col->set_nth(i, value.GetBool()); - } else if (value.IsString() && istrequals(value.GetString(), "true")) { - col->set_nth(i, true); - } else if (value.IsString() && istrequals(value.GetString(), "false")) { - col->set_nth(i, false); - } else if (value.IsInt()) { - col->set_nth(i, value.GetInt() != 0); - } else { - std::stringstream ss; - ss << "Expected bool, found " << value.GetType(); - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - return std::nullopt; - } - case t_dtype::DTYPE_TIME: { - col->set_nth(i, json_into(value)); - return std::nullopt; - } - case t_dtype::DTYPE_DATE: { - col->set_nth(i, json_into(value)); - return std::nullopt; - } - default: - PSP_COMPLAIN_AND_ABORT("JSON field not yet implemented"); - return std::nullopt; - } -} void Table::remove_rows(const std::string_view& data) { @@ -862,7 +497,7 @@ Table::remove_rows(const std::string_view& data) { t_uindex ii = 0; auto col = data_table.get_column(m_index); for (const auto& cell : document.GetArray()) { - auto promote = fill_column_json(col, ii, cell, true); + auto promote = json::fill_column_json(col, ii, cell, true); if (promote) { std::stringstream ss; ss << "Cannot append value of type " << dtype_to_str(*promote) @@ -873,7 +508,7 @@ Table::remove_rows(const std::string_view& data) { } // if (!is_implicit && m_index == col_name) { - fill_column_json(psp_pkey_col, ii, cell, true); + json::fill_column_json(psp_pkey_col, ii, cell, true); // } ii++; @@ -920,7 +555,7 @@ Table::remove_cols(const std::string_view& data) { t_uindex ii = 0; auto col = data_table.get_column(m_index); for (const auto& cell : document.GetArray()) { - auto promote = fill_column_json(col, ii, cell, true); + auto promote = json::fill_column_json(col, ii, cell, true); if (promote) { std::stringstream ss; ss << "Cannot append value of type " << dtype_to_str(*promote) @@ -931,8 +566,8 @@ Table::remove_cols(const std::string_view& data) { } // if (!is_implicit && m_index == col_name) { - fill_column_json(psp_pkey_col, ii, cell, true); - fill_column_json(psp_okey_col, ii, cell, true); + json::fill_column_json(psp_pkey_col, ii, cell, true); + json::fill_column_json(psp_okey_col, ii, cell, true); // } ii++; @@ -944,212 +579,54 @@ Table::remove_cols(const std::string_view& data) { m_pool->send(get_gnode()->get_id(), 0, data_table); } -void -Table::update_cols(const std::string_view& data, std::uint32_t port_id) { - // 1.) Infer schema - rapidjson::Document document; - document.Parse(data.data()); - if (!document.IsObject()) { - // TODO Legacy error message - PSP_COMPLAIN_AND_ABORT( - "Cannot determine data types without column names!\n" - ) - } - - t_uindex nrows = 0; - for (const auto& it : document.GetObj()) { - if (!it.value.IsArray()) { - PSP_COMPLAIN_AND_ABORT("Malformed column") - } - - if (it.value.Empty()) { - PSP_COMPLAIN_AND_ABORT("Can't create table from empty columns") - } - - nrows = std::max(nrows, static_cast(it.value.Size())); - } - - bool is_implicit = m_index.empty(); - t_schema table_schema = get_schema(); - - // 2.) Create table - t_data_table data_table(table_schema); - data_table.init(); - data_table.extend(nrows); - - LOG_DEBUG("Updating table with schema " << table_schema); - LOG_DEBUG("Implicit index? " << is_implicit); - if (is_implicit) { - data_table.add_column("psp_pkey", DTYPE_INT32, true); - } else { - data_table.add_column( - "psp_pkey", table_schema.get_dtype(m_index), true - ); - } - - const auto& psp_pkey_col = data_table.get_column("psp_pkey"); - - auto schema = data_table.get_schema(); - - if (is_implicit && !document.GetObj().HasMember("__INDEX__")) { - for (std::uint32_t ii = 0; ii < nrows; ii++) { - psp_pkey_col->set_nth(ii, (m_offset + ii)); - } - } - - // 3.) Fill table - for (const auto& column : document.GetObj()) { - t_uindex ii = 0; - std::string_view col_name = column.name.GetString(); - if (std::string_view{column.name.GetString()} == "__INDEX__") { - col_name = "psp_pkey"; - } - - if (!schema.has_column(col_name)) { - LOG_DEBUG("Ignoring column " << col_name); - continue; - } - - for (const auto& cell : column.value.GetArray()) { - auto col = data_table.get_column(col_name); - auto promote = fill_column_json(col, ii, cell, true); - if (promote) { - std::stringstream ss; - ss << "Cannot append value of type " << dtype_to_str(*promote) - << " to column \"" << col_name << "\" of type " << dtype_to_str(col->get_dtype()) - << " at index " << ii - << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - if (!is_implicit && m_index == column.name.GetString()) { - fill_column_json(psp_pkey_col, ii, cell, true); - } - - ii++; - } - } - - data_table.clone_column("psp_pkey", "psp_okey"); - - process_op_column(data_table, t_op::OP_INSERT); - calculate_offset(nrows); - m_pool->send(get_gnode()->get_id(), port_id, data_table); -} - std::shared_ptr
-Table::from_cols( +Table::from_json_loader( + json::JsonLoader& loader, const std::string& index, std::string&& data, std::uint32_t limit, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) { - // 1.) Infer schema - rapidjson::Document document; - document.Parse(data.data()); - - std::vector column_names; - std::vector data_types; - bool is_implicit = true; - t_uindex nrows = 0; - - // https://github.com/Tencent/rapidjson/issues/1994 - for (const auto& it : document.GetObj()) { - if (!it.value.IsArray()) { - PSP_COMPLAIN_AND_ABORT("Malformed column") - } - - if (it.value.Empty()) { - PSP_COMPLAIN_AND_ABORT("Can't create table from empty columns") - } - - if (it.name.GetString() == index) { - is_implicit = false; - } - - nrows = std::max(nrows, static_cast(it.value.Size())); - bool found = false; - for (const auto& column_value : it.value.GetArray()) { - auto dtype = rapidjson_type_to_dtype(column_value); - if (dtype != DTYPE_NONE) { - data_types.push_back(dtype); - found = true; - break; - } - } - - if (!found) { - data_types.push_back(DTYPE_STR); - } - - column_names.emplace_back(it.name.GetString()); + if (const auto repeated = loader.repeated_index(index)) { + std::stringstream ss; + ss << "Cannot create a Table indexed on `" << *repeated + << "` from an expanded array.\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); } - t_schema schema(column_names, data_types); - - // 2.) Create table + t_schema schema(loader.names(), loader.types()); auto data_table = std::make_unique(schema); data_table->init(); - data_table->extend(nrows); - - if (is_implicit) { - // TODO should this be t_uindex? - data_table->add_column("psp_pkey", DTYPE_INT32, true); - data_table->add_column("psp_okey", DTYPE_INT32, true); - } else { - data_table->add_column("psp_pkey", schema.get_dtype(index), true); - data_table->add_column("psp_okey", schema.get_dtype(index), true); - } - - const auto& psp_pkey_col = data_table->get_column("psp_pkey"); - const auto& psp_okey_col = data_table->get_column("psp_okey"); - - // 3.) Fill table - for (const auto& col : document.GetObj()) { - t_uindex ii = 0; - const auto& col_name = col.name.GetString(); - LOG_DEBUG( - "Filling column " - << col_name << " dtype " - << dtype_to_str(data_table->get_column(col_name)->get_dtype()) - ); - for (const auto& cell : col.value.GetArray()) { - auto col = data_table->get_column(col_name); - auto promote = fill_column_json(col, ii, cell, false); - if (promote) { - LOG_DEBUG( - "Promoting column " << col_name << " from " - << dtype_to_str(col->get_dtype()) - << " to " << dtype_to_str(*promote) - ); - data_table->promote_column(col_name, *promote, ii, true); - col = data_table->get_column(col_name); - fill_column_json(col, ii, cell, false); - } + const auto pkey_dtype = + loader.is_implicit() ? DTYPE_INT32 : schema.get_dtype(index); - if (!is_implicit && index == col_name) { - fill_column_json(psp_pkey_col, ii, cell, false); - fill_column_json(psp_okey_col, ii, cell, false); - } + data_table->add_column("psp_pkey", pkey_dtype, true); + data_table->add_column("psp_okey", pkey_dtype, true); - ii++; - } - } + const auto nrows = loader.fill_table(*data_table, index, 0, false); - if (is_implicit) { - for (t_uindex ii = 0; ii < nrows; ii++) { - psp_pkey_col->set_nth(ii, ii); - psp_okey_col->set_nth(ii, ii); - } - } + // `names`/`types` may have grown during the fill -- an ndjson record can + // introduce a column -- so the Table's column list is read back from the + // loader rather than from `schema`. Copy them out before releasing. + auto column_names = loader.names(); + auto data_types = loader.types(); - { auto _ = std::move(document); } + // Drop the parsed document and the source text before the gnode allocates + // its master table, so the two peaks do not overlap. + loader.release(); { auto _ = std::move(data); } auto pool = std::make_shared(); pool->init(); auto tbl = std::make_shared
( - pool, schema.columns(), schema.types(), limit, index, backing_store + pool, + std::move(column_names), + std::move(data_types), + limit, + index, + backing_store, + list_flatten ); tbl->init(*data_table, nrows, t_op::OP_INSERT, 0); @@ -1158,350 +635,95 @@ Table::from_cols( return tbl; } -// rapidjson::StringBuffer buffer; -// buffer.Clear(); -// rapidjson::Writer writer(buffer); -// document.Accept(writer); -// std::cout << buffer.GetString() << std::endl; - void -Table::update_rows(const std::string_view& data, std::uint32_t port_id) { - // 1.) Infer schema - rapidjson::Document document; - document.Parse(data.data()); - if (document.Size() == 0) { +Table::update_json( + const std::string_view& data, + json::t_json_format format, + std::uint32_t port_id +) { + t_schema table_schema = get_schema(); + json::JsonLoader loader; + loader.init(data, format, m_index, &table_schema, m_list_flatten); + if (loader.empty()) { return; } - if (!document[0].IsObject()) { - // TODO Legacy error message - PSP_COMPLAIN_AND_ABORT( - "Cannot determine data types without column names!\n" - ) + if (const auto repeated = loader.repeated_index(m_index)) { + std::stringstream ss; + ss << "Cannot update a Table indexed on `" << *repeated + << "` from an expanded array.\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); } - bool is_implicit = m_index.empty(); - t_schema table_schema = get_schema(); - - // 2.) Create table - t_uindex size = document.Size(); t_data_table data_table(table_schema); data_table.init(); - data_table.extend(size); - if (is_implicit) { - data_table.add_column("psp_pkey", DTYPE_INT32, true); - } else { - data_table.add_column( - "psp_pkey", table_schema.get_dtype(m_index), true - ); - } - - t_uindex ii = 0; - const auto& psp_pkey_col = data_table.get_column("psp_pkey"); - auto schema = data_table.get_schema(); - // t_uindex col_count; - // bool supports_partial = - // m_limit == std::numeric_limits::max() && !m_index.empty(); - bool is_first_row = true; - std::vector missing_columns = m_column_names; - - // 3.) Fill table - for (const auto& row : document.GetArray()) { - if (is_implicit) { - psp_pkey_col->set_nth(ii, (ii + m_offset)); - } - - // col_count = m_column_names.size(); - for (const auto& it : row.GetObj()) { - std::shared_ptr col; - std::string_view col_name = it.name.GetString(); - if (std::string_view{it.name.GetString()} == "__INDEX__") { - col_name = "psp_pkey"; - } - - if (!schema.has_column(col_name)) { - LOG_DEBUG("Ignoring column " << col_name); - LOG_DEBUG("Schema:\n" << schema); - continue; - } - - if (is_first_row) { - missing_columns.erase( - std ::remove( - missing_columns.begin(), missing_columns.end(), col_name - ), - missing_columns.end() - ); - } - - col = data_table.get_column(col_name); - auto promote = fill_column_json(col, ii, it.value, true); - if (promote) { - std::stringstream ss; - ss << "Cannot append value of type " << dtype_to_str(*promote) - << " to column \"" << col_name << "\" of type " << dtype_to_str(col->get_dtype()) - << " at index " << ii - << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - if (!is_implicit && m_index == it.name.GetString()) { - fill_column_json(psp_pkey_col, ii, it.value, true); - } - } - - is_first_row = false; - ii++; - } + data_table.add_column( + "psp_pkey", + m_index.empty() ? DTYPE_INT32 : table_schema.get_dtype(m_index), + true + ); + const auto size = loader.fill_table(data_table, m_index, m_offset, true); data_table.clone_column("psp_pkey", "psp_okey"); process_op_column(data_table, t_op::OP_INSERT); calculate_offset(size); m_pool->send(get_gnode()->get_id(), port_id, data_table); } + +void +Table::update_cols(const std::string_view& data, std::uint32_t port_id) { + update_json(data, json::JSON_FORMAT_COLUMNS, port_id); +} + std::shared_ptr
-Table::from_rows( +Table::from_cols( const std::string& index, std::string&& data, std::uint32_t limit, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) { - // 1.) Infer schema - rapidjson::Document document; - document.Parse(data.data()); - // if (document.Size() == 0) { - // PSP_COMPLAIN_AND_ABORT("Can't create table from empty rows") - // } - - if (document.Size() > 0 && !document[0].IsObject()) { - LOG_DEBUG("Received non-object " << document[0].GetType()); - // TODO Legacy error message - PSP_COMPLAIN_AND_ABORT( - "Cannot determine data types without column names!\n" - ) - } - - std::vector column_names; - std::vector data_types; - bool is_implicit = true; - std::set columns_known_type; - std::set columns_seen; - - [&]() { - for (const auto& row : document.GetArray()) { - for (const auto& col : row.GetObj()) { - columns_seen.insert(col.name.GetString()); - } - - // https://github.com/Tencent/rapidjson/issues/1994 - for (const auto& col : row.GetObj()) { - if (col.name.GetString() == index) { - is_implicit = false; - } - - if (columns_known_type.count(col.name.GetString()) > 0) { - continue; - } - - auto dtype = rapidjson_type_to_dtype(col.value); - if (dtype != DTYPE_NONE) { - columns_known_type.insert(col.name.GetString()); - data_types.push_back(rapidjson_type_to_dtype(col.value)); - column_names.emplace_back(col.name.GetString()); - } - - // Theoretically there can end too early if the first - // few rows are missing columns that are present in later rows. - if (columns_known_type.size() == columns_seen.size()) { - return; - } - } - } - }(); - - auto untyped_columns = columns_seen; - for (const auto& col : columns_seen) { - if (columns_known_type.count(col) == 0) { - // Default all null columns to string - data_types.push_back(DTYPE_STR); - column_names.emplace_back(col); - } - } - - t_schema schema(column_names, data_types); - - // 2.) Create table - auto data_table = std::make_unique(schema); - data_table->init(); - data_table->extend(document.Size()); - - if (is_implicit) { - data_table->add_column("psp_pkey", DTYPE_INT32, true); - data_table->add_column("psp_okey", DTYPE_INT32, true); - } else { - data_table->add_column("psp_pkey", schema.get_dtype(index), true); - data_table->add_column("psp_okey", schema.get_dtype(index), true); - } - - std::int32_t ii = 0; - - const auto& psp_pkey_col = data_table->get_column("psp_pkey"); - const auto& psp_okey_col = data_table->get_column("psp_okey"); - - // 3.) Fill table - for (const auto& row : document.GetArray()) { - for (const auto& it : row.GetObj()) { - auto col = data_table->get_column(it.name.GetString()); - const auto* col_name = it.name.GetString(); - const auto& cell = it.value; - auto promote = fill_column_json(col, ii, cell, false); - if (promote) { - LOG_DEBUG( - "Promoting column " << col_name << " from " - << dtype_to_str(col->get_dtype()) - << " to " << dtype_to_str(*promote) - ); - data_table->promote_column(col_name, *promote, ii, true); - col = data_table->get_column(col_name); - fill_column_json(col, ii, cell, false); - } - - if (!is_implicit && index == it.name.GetString()) { - fill_column_json(psp_pkey_col, ii, it.value, false); - fill_column_json(psp_okey_col, ii, it.value, false); - } - } - - if (is_implicit) { - psp_pkey_col->set_nth(ii, ii); - psp_okey_col->set_nth(ii, ii); - } + json::JsonLoader loader; + loader.init( + data, json::JSON_FORMAT_COLUMNS, index, nullptr, list_flatten + ); + return from_json_loader( + loader, index, std::move(data), limit, backing_store, list_flatten + ); +} - ii++; - } +// rapidjson::StringBuffer buffer; +// buffer.Clear(); +// rapidjson::Writer writer(buffer); +// document.Accept(writer); +// std::cout << buffer.GetString() << std::endl; - { auto _ = std::move(document); } - { auto _ = std::move(data); } +void +Table::update_rows(const std::string_view& data, std::uint32_t port_id) { + update_json(data, json::JSON_FORMAT_ROWS, port_id); +} - auto pool = std::make_shared(); - pool->init(); - auto tbl = std::make_shared
( - pool, schema.columns(), schema.types(), limit, index, backing_store +std::shared_ptr
+Table::from_rows( + const std::string& index, + std::string&& data, + std::uint32_t limit, + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten +) { + json::JsonLoader loader; + loader.init( + data, json::JSON_FORMAT_ROWS, index, nullptr, list_flatten + ); + return from_json_loader( + loader, index, std::move(data), limit, backing_store, list_flatten ); - - tbl->init(*data_table, document.Size(), t_op::OP_INSERT, 0); - data_table.reset(); - pool->_process(); - return tbl; } void Table::update_ndjson(const std::string_view& data, std::uint32_t port_id) { - rapidjson::Document document; - rapidjson::StringStream s(data.data()); - document.ParseStream(s); - if (document.Size() == 0) { - return; - } - - if (!document.IsObject()) { - // TODO Legacy error message - PSP_COMPLAIN_AND_ABORT( - "Cannot determine data types without column names!\n" - ) - } - - bool is_implicit = m_index.empty(); - t_schema table_schema = get_schema(); - - // 2.) Create table - t_data_table data_table(table_schema); - data_table.init(); - - // 2a.) Estimate row size to reduce malloc pressure. - auto newlines = 0; - for (char i : data) { - if (i == '\n') { - newlines++; - } - } - - data_table.reserve(newlines + 1); - if (is_implicit) { - data_table.add_column("psp_pkey", DTYPE_INT32, true); - } else { - data_table.add_column( - "psp_pkey", table_schema.get_dtype(m_index), true - ); - } - - t_uindex ii = 0; - const auto& psp_pkey_col = data_table.get_column("psp_pkey"); - auto schema = data_table.get_schema(); - bool is_first_row = true; - std::vector missing_columns = m_column_names; - - // 3.) Fill table - bool is_finished = false; - while (!is_finished) { - if (is_implicit) { - psp_pkey_col->set_nth(ii, (ii + m_offset)); - } - - for (const auto& it : document.GetObj()) { - std::shared_ptr col; - std::string_view col_name = it.name.GetString(); - if (std::string_view{it.name.GetString()} == "__INDEX__") { - col_name = "psp_pkey"; - } - - if (!schema.has_column(col_name)) { - LOG_DEBUG("Ignoring column " << col_name); - LOG_DEBUG("Schema:\n" << schema); - continue; - } - - if (is_first_row) { - missing_columns.erase( - std ::remove( - missing_columns.begin(), missing_columns.end(), col_name - ), - missing_columns.end() - ); - } - - col = data_table.get_column(col_name); - auto promote = fill_column_json(col, ii, it.value, true); - if (promote) { - std::stringstream ss; - ss << "Cannot append value of type " << dtype_to_str(*promote) - << " to column \"" << col_name << "\" of type " << dtype_to_str(col->get_dtype()) - << " at index " << ii - << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - if (!is_implicit && m_index == it.name.GetString()) { - fill_column_json(psp_pkey_col, ii, it.value, true); - } - } - - is_first_row = false; - - ii++; - - document.ParseStream(s); - if (document.HasParseError()) { - is_finished = true; - } - } - - data_table.extend(ii); - data_table.clone_column("psp_pkey", "psp_okey"); - process_op_column(data_table, t_op::OP_INSERT); - calculate_offset(ii); - m_pool->send(get_gnode()->get_id(), port_id, data_table); + update_json(data, json::JSON_FORMAT_NDJSON, port_id); } std::shared_ptr
@@ -1509,149 +731,16 @@ Table::from_ndjson( const std::string& index, std::string&& data, std::uint32_t limit, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) { - // 1.) Infer schema - rapidjson::Document document; - rapidjson::StringStream s(data.data()); - document.ParseStream(s); - - if (document.Size() > 0 && !document.IsObject()) { - std::stringstream ss; - ss << "Received non-object " << document[0].GetType(); - PSP_COMPLAIN_AND_ABORT(ss.str()) - } - - std::vector column_names; - std::vector data_types; - bool is_implicit = true; - std::set columns_known_type; - std::set columns_seen; - - // TODO I don't think it makes sense to do the same incremental-schema - // enhancement we do for regular JSON. For now this only checks the first - // row. - [&]() { - for (const auto& col : document.GetObj()) { - columns_seen.insert(col.name.GetString()); - } - - // https://github.com/Tencent/rapidjson/issues/1994 - for (const auto& col : document.GetObj()) { - if (col.name.GetString() == index) { - is_implicit = false; - } - - if (columns_known_type.count(col.name.GetString()) > 0) { - continue; - } - - auto dtype = rapidjson_type_to_dtype(col.value); - if (dtype != DTYPE_NONE) { - columns_known_type.insert(col.name.GetString()); - data_types.push_back(rapidjson_type_to_dtype(col.value)); - column_names.emplace_back(col.name.GetString()); - } - - // Theoretically there can end too early if the first - // few rows are missing columns that are present in later rows. - if (columns_known_type.size() == columns_seen.size()) { - return; - } - } - }(); - - auto untyped_columns = columns_seen; - for (const auto& col : columns_seen) { - if (columns_known_type.count(col) == 0) { - // Default all null columns to string - data_types.push_back(DTYPE_STR); - column_names.emplace_back(col); - } - } - - t_schema schema(column_names, data_types); - - // 2.) Create table - auto data_table = std::make_unique(schema); - data_table->init(); - - if (is_implicit) { - data_table->add_column("psp_pkey", DTYPE_INT32, true); - data_table->add_column("psp_okey", DTYPE_INT32, true); - } else { - data_table->add_column("psp_pkey", schema.get_dtype(index), true); - data_table->add_column("psp_okey", schema.get_dtype(index), true); - } - - std::int32_t ii = 0; - const auto& psp_pkey_col = data_table->get_column("psp_pkey"); - const auto& psp_okey_col = data_table->get_column("psp_okey"); - - // 2a.) Estimate row size to reduce malloc pressure. - auto newlines = 0; - for (char i : data) { - if (i == '\n') { - newlines++; - } - } - - data_table->reserve(newlines + 1); - - // 3.) Fill table - bool is_finished = false; - while (!is_finished) { - data_table->extend(ii + 1); - for (const auto& it : document.GetObj()) { - auto col = data_table->get_column(it.name.GetString()); - const auto* col_name = it.name.GetString(); - const auto& cell = it.value; - auto promote = fill_column_json(col, ii, cell, false); - if (promote) { - LOG_DEBUG( - "Promoting column " << col_name << " from " - << dtype_to_str(col->get_dtype()) - << " to " << dtype_to_str(*promote) - ); - - data_table->promote_column(col_name, *promote, ii, true); - col = data_table->get_column(col_name); - fill_column_json(col, ii, cell, false); - } - - if (!is_implicit && index == it.name.GetString()) { - fill_column_json(psp_pkey_col, ii, it.value, false); - fill_column_json(psp_okey_col, ii, it.value, false); - } - } - - if (is_implicit) { - psp_pkey_col->set_nth(ii, ii); - psp_okey_col->set_nth(ii, ii); - } - - ii++; - document.ParseStream(s); - if (document.HasParseError()) { - is_finished = true; - } - } - - data_table->extend(ii); - - { auto _ = std::move(document); } - { auto _ = std::move(data); } - - auto pool = std::make_shared(); - pool->init(); - auto tbl = std::make_shared
( - pool, schema.columns(), schema.types(), limit, index, backing_store + json::JsonLoader loader; + loader.init( + data, json::JSON_FORMAT_NDJSON, index, nullptr, list_flatten + ); + return from_json_loader( + loader, index, std::move(data), limit, backing_store, list_flatten ); - - tbl->init(*data_table, ii, t_op::OP_INSERT, 0); - data_table.reset(); - pool->_process(); - return tbl; } std::shared_ptr
@@ -1659,7 +748,8 @@ Table::from_schema( const std::string& index, const t_schema& schema, std::uint32_t limit, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) { auto pool = std::make_shared(); pool->init(); @@ -1684,7 +774,13 @@ Table::from_schema( } auto tbl = std::make_shared
( - pool, schema.columns(), schema.types(), limit, index, backing_store + pool, + schema.columns(), + schema.types(), + limit, + index, + backing_store, + list_flatten ); tbl->init(data_table, 0, t_op::OP_INSERT, 0); @@ -1696,9 +792,18 @@ void Table::update_arrow(const std::string_view& data, std::uint32_t port_id) { apachearrow::ArrowLoader arrow_loader; arrow_loader.initialize( - reinterpret_cast(data.data()), data.size() + reinterpret_cast(data.data()), + data.size(), + m_list_flatten ); + if (const auto repeated = arrow_loader.repeated_index(m_index)) { + std::stringstream ss; + ss << "Cannot update a Table indexed on `" << *repeated + << "` from an expanded list column.\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + t_data_table data_table{this->get_schema()}; data_table.init(); auto row_count = arrow_loader.row_count(); @@ -1729,15 +834,28 @@ Table::from_arrow( const std::string& index, std::string&& data, std::uint32_t limit, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) { apachearrow::ArrowLoader arrow_loader; // Parse the arrow and get its metadata arrow_loader.initialize( - reinterpret_cast(data.data()), data.size() + reinterpret_cast(data.data()), + data.size(), + list_flatten ); + if (const auto repeated = arrow_loader.repeated_index(index)) { + std::stringstream ss; + ss << "Cannot create a Table indexed on `" << *repeated + << "` from an expanded list column, as that index repeats across " + "the rows of an expansion and would silently collide. Index on a " + "column drawn from the list itself, or use the `stringify` list " + "flatten mode.\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + // Infer schema auto columns = arrow_loader.names(); auto types = arrow_loader.types(); @@ -1770,8 +888,10 @@ Table::from_arrow( // Make Table auto pool = std::make_shared(); pool->init(); - auto table = - std::make_shared
(pool, columns, types, limit, index, backing_store); + auto table = std::make_shared
( + pool, columns, types, limit, index, backing_store, list_flatten + ); + table->init(*data_table, data_table->num_rows(), t_op::OP_INSERT, 0); data_table.reset(); pool->_process(); @@ -1785,7 +905,8 @@ Table::make_table( std::uint32_t limit, const std::string& index, const std::string_view& data, - t_backing_store backing_store + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten ) { auto pool = std::make_shared(); pool->init(); @@ -1810,7 +931,7 @@ Table::make_table( auto columns = data_table.get_schema().columns(); auto dtypes = data_table.get_schema().types(); auto table = std::make_shared
( - pool, columns, dtypes, limit, index, backing_store + pool, columns, dtypes, limit, index, backing_store, list_flatten ); table->init(data_table, data_table.num_rows(), t_op::OP_INSERT, 0); pool->_process(); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h index 87b3c4db5d..0a0e2c6187 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h @@ -18,12 +18,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include namespace perspective { namespace apachearrow { @@ -37,9 +39,32 @@ namespace apachearrow { /** * @brief Initialize the arrow loader with a pointer to a binary. * + * Nested `STRUCT` and `LIST` columns are normalized into Perspective's + * flat column model here, so `names`, `types` and `row_count` all + * describe the post-flattening shape. + * * @param ptr + * @param mode - how `LIST` columns are ingested. + */ + void initialize( + const std::uint8_t* ptr, + std::uint32_t, + t_list_flatten mode = LIST_FLATTEN_ZIP + ); + + /** + * @brief The name of an index column whose value repeats across the + * rows of an expansion, or `nullopt` if indexing is safe. + * + * A column derived from an exploded list takes a different element per + * output row, so it is a legitimate key; a sibling of that list carries + * the identical value on every row of the group and would silently + * collide. + * + * @param index - the explicit index column, or empty for none. */ - void initialize(const std::uint8_t* ptr, std::uint32_t); + std::optional repeated_index(const std::string& index + ) const; /** * @brief Initialize the arrow loader with a CSV. @@ -77,6 +102,12 @@ namespace apachearrow { std::uint32_t row_count() const; private: + /** + * @brief The post-normalization fields, which for a nested input differ + * from `m_table`'s. + */ + const std::vector>& fields() const; + void fill_column( t_data_table& tbl, const std::shared_ptr& col, @@ -88,16 +119,27 @@ namespace apachearrow { ); std::shared_ptr m_table; + + /** + * @brief Set only when the input had a struct or list column. + * + * INVARIANT: while this is null, `m_table` is the exact object parsed + * out of the IPC bytes and every read path below behaves as it did + * before flattening existed. + */ + std::unique_ptr m_normalized; std::vector m_names; std::vector m_types; + bool m_expanded{false}; }; - template + template void iter_col_copy( const std::shared_ptr& dest, std::shared_ptr src, const int64_t offset, - const int64_t len + const int64_t len, + const GATHER& gather ); void copy_array( diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h new file mode 100644 index 0000000000..4532c90505 --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h @@ -0,0 +1,66 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace perspective { +namespace apachearrow { + + using perspective::t_list_flatten; + using perspective::LIST_FLATTEN_CARTESIAN; + using perspective::LIST_FLATTEN_STRINGIFY; + using perspective::LIST_FLATTEN_ZIP; + extern const char* const FLATTEN_SEPARATOR; + + /** + * @brief A table rewritten into Perspective's flat column model, plus the + * row expansion left deferred as a gather plan. + */ + struct t_normalized_table { + std::vector> fields; + std::vector> columns; + + /** + * Per column, output row -> index into that column, or -1 for a null + * slot. An empty entry means the column is already row-aligned and is + * copied with no indirection. + */ + std::vector> gathers; + + /** + * Per column, whether it takes a distinct element per output row + * rather than repeating one input value across an expansion group. + */ + std::vector per_element; + std::int64_t num_rows; + }; + + PERSPECTIVE_EXPORT t_normalized_table normalize_table( + std::shared_ptr input, t_list_flatten mode + ); + + PERSPECTIVE_EXPORT bool + normalize_table_is_noop(const arrow::Table& input, t_list_flatten mode); + + PERSPECTIVE_EXPORT bool + normalize_table_expands(const arrow::Table& input, t_list_flatten mode); + +} // namespace apachearrow +} // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h new file mode 100644 index 0000000000..d3b2452e5f --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h @@ -0,0 +1,23 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#pragma once + +namespace perspective { + +enum t_list_flatten { + LIST_FLATTEN_ZIP = 0, + LIST_FLATTEN_CARTESIAN = 1, + LIST_FLATTEN_STRINGIFY = 2 +}; + +} // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h new file mode 100644 index 0000000000..434baff81f --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h @@ -0,0 +1,121 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace perspective { +namespace json { + t_dtype rapidjson_type_to_dtype(const rapidjson::Value& value); + std::optional fill_column_json( + const std::shared_ptr& col, + t_uindex i, + const rapidjson::Value& value, + bool is_update + ); + + /** + * @brief The wire shape of a JSON payload. + */ + enum t_json_format { + /** `[{"a": 1}, {"a": 2}]` */ + JSON_FORMAT_ROWS, + + /** `{"a": [1, 2]}` */ + JSON_FORMAT_COLUMNS, + + /** `{"a": 1}\n{"a": 2}` — parsed one record at a time. */ + JSON_FORMAT_NDJSON, + }; + + class PERSPECTIVE_EXPORT JsonLoader { + public: + JsonLoader(); + ~JsonLoader(); + + void init( + std::string_view data, + t_json_format format, + const std::string& index, + const t_schema* existing, + t_list_flatten mode + ); + + const std::vector& names() const; + const std::vector& types() const; + + bool is_implicit() const; + + bool empty() const; + + std::optional repeated_index(const std::string& index + ) const; + + void release(); + + std::uint32_t fill_table( + t_data_table& tbl, + const std::string& index, + std::uint32_t offset, + bool is_update + ); + + private: + std::shared_ptr resolve_column( + t_data_table& tbl, + std::string_view name, + const rapidjson::Value& leaf, + bool is_update + ); + + void infer_rows(const std::string& index); + void infer_cols(const std::string& index); + void infer_ndjson(const std::string& index); + + std::uint32_t + fill_rows(t_data_table&, const std::string&, std::uint32_t, bool); + std::uint32_t + fill_cols(t_data_table&, const std::string&, std::uint32_t, bool); + std::uint32_t + fill_ndjson(t_data_table&, const std::string&, std::uint32_t, bool); + + rapidjson::Document m_document; + rapidjson::StringStream m_stream{nullptr}; + t_json_format m_format{JSON_FORMAT_ROWS}; + std::vector m_names; + std::vector m_types; + t_list_flatten m_mode{LIST_FLATTEN_ZIP}; + + std::set m_per_element; + std::vector m_child_widths; + bool m_expands{false}; + bool m_is_implicit{true}; + bool m_empty{false}; + }; + +} // namespace json +} // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h index 3fd2d3ff74..5e162fcfcf 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h @@ -19,6 +19,8 @@ #include #include #include +#include +#include namespace perspective { @@ -54,7 +56,9 @@ class PERSPECTIVE_EXPORT Table { std::vector data_types, std::uint32_t limit, std::string index, - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); /** @@ -228,42 +232,54 @@ class PERSPECTIVE_EXPORT Table { const std::string& index, std::string&& data, std::uint32_t limit = std::numeric_limits::max(), - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); static std::shared_ptr
from_cols( const std::string& index, std::string&& data, std::uint32_t limit = std::numeric_limits::max(), - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); static std::shared_ptr
from_rows( const std::string& index, std::string&& data, std::uint32_t limit = std::numeric_limits::max(), - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); static std::shared_ptr
from_ndjson( const std::string& index, std::string&& data, std::uint32_t limit = std::numeric_limits::max(), - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); static std::shared_ptr
from_schema( const std::string& index, const t_schema& schema, std::uint32_t limit = std::numeric_limits::max(), - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); static std::shared_ptr
from_arrow( const std::string& index, std::string&& data, std::uint32_t limit = std::numeric_limits::max(), - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); static std::shared_ptr
make_table( @@ -272,10 +288,35 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit, const std::string& index, const std::string_view& data, - t_backing_store backing_store = BACKING_STORE_MEMORY + t_backing_store backing_store = BACKING_STORE_MEMORY, + apachearrow::t_list_flatten list_flatten = + apachearrow::LIST_FLATTEN_ZIP ); private: + /** + * @brief Build a `Table` from an already-parsed JSON payload, shared by the + * three JSON creation formats. + */ + static std::shared_ptr
from_json_loader( + json::JsonLoader& loader, + const std::string& index, + std::string&& data, + std::uint32_t limit, + t_backing_store backing_store, + apachearrow::t_list_flatten list_flatten + ); + + /** + * @brief Apply a JSON payload to this `Table`, shared by the three JSON + * update formats. + */ + void update_json( + const std::string_view& data, + json::t_json_format format, + std::uint32_t port_id + ); + /** * @brief Make sure that the table does not have an explicit index AND an * implicit index (with the `__INDEX__` column in data). @@ -324,6 +365,15 @@ class PERSPECTIVE_EXPORT Table { const std::string m_index; bool m_gnode_set; const t_backing_store m_backing_store; + + /** + * @brief How `arrow::Type::LIST` columns are ingested. + * + * INVARIANT: this must be applied to `update_arrow` as well as to the + * `from_arrow` which created the Table, or an update would produce a + * different column shape than the schema it is written against. + */ + const apachearrow::t_list_flatten m_list_flatten; }; } // namespace perspective \ No newline at end of file diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/traversal.h b/rust/perspective-server/cpp/perspective/src/include/perspective/traversal.h index 2a9bdb0810..8d3e5c1466 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/traversal.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/traversal.h @@ -173,6 +173,10 @@ t_traversal::sort_by( const SRC_T& src, t_ctx2* ctx2 ) { + if (m_leaves_only || m_total_only || m_nodes->empty()) { + return; + } + std::vector new_nodes(m_nodes->size()); // Pair is -> (old tvidx, new tvidx) diff --git a/rust/perspective-viewer/src/rust/components/window_editor.rs b/rust/perspective-viewer/src/rust/components/window_editor.rs index a6efb5d06d..28c34b0d8c 100644 --- a/rust/perspective-viewer/src/rust/components/window_editor.rs +++ b/rust/perspective-viewer/src/rust/components/window_editor.rs @@ -13,9 +13,7 @@ use std::collections::HashSet; use std::rc::Rc; -use perspective_client::config::{ - ColumnType, WindowAggregate, WindowFrame, WindowSort, WindowSortDir, WindowSpec, -}; +use perspective_client::config::{ColumnType, WindowFrame, WindowSort, WindowSortDir, WindowSpec}; use wasm_bindgen::JsCast; use web_sys::{DragEvent, HtmlInputElement, MouseEvent}; use yew::prelude::*; @@ -30,63 +28,33 @@ use crate::presentation::Presentation; use crate::session::{Session, SessionMetadataRc}; use crate::utils::{AddListener, DragEffect, DragTarget, Subscription}; -fn op_label(op: WindowAggregate) -> &'static str { - match op { - WindowAggregate::Sum => "sum", - WindowAggregate::Avg => "avg", - WindowAggregate::Count => "count", - WindowAggregate::Min => "min", - WindowAggregate::Max => "max", - WindowAggregate::Stddev => "stddev", - WindowAggregate::Var => "var", - WindowAggregate::First => "first", - WindowAggregate::Last => "last", - WindowAggregate::Lag => "lag", - WindowAggregate::Lead => "lead", - WindowAggregate::Diff => "diff", - WindowAggregate::Rate => "rate", - WindowAggregate::Ema => "ema", - } -} - -fn op_from_label(label: &str) -> Option { - Some(match label { - "sum" => WindowAggregate::Sum, - "avg" => WindowAggregate::Avg, - "count" => WindowAggregate::Count, - "min" => WindowAggregate::Min, - "max" => WindowAggregate::Max, - "stddev" => WindowAggregate::Stddev, - "var" => WindowAggregate::Var, - "first" => WindowAggregate::First, - "last" => WindowAggregate::Last, - "lag" => WindowAggregate::Lag, - "lead" => WindowAggregate::Lead, - "diff" => WindowAggregate::Diff, - "rate" => WindowAggregate::Rate, - "ema" => WindowAggregate::Ema, - _ => return None, - }) +/// The declared capabilities of one window aggregate, for a `source` column +/// type. Which controls an aggregate needs is the data model's to state - the +/// editor cannot infer it from a name it has never seen. +fn op_spec( + metadata: &SessionMetadataRc, + source: &str, + op: &str, +) -> Option { + let ty = metadata.get_column_table_type(source)?; + metadata.get_window_aggregate(ty, op) } -fn is_aggregating(op: WindowAggregate) -> bool { - matches!( - op, - WindowAggregate::Sum - | WindowAggregate::Avg - | WindowAggregate::Count - | WindowAggregate::Min - | WindowAggregate::Max - | WindowAggregate::Stddev - | WindowAggregate::Var - ) +/// The editor's frame-type labels, as the `frames` a declaration lists. +fn frame_label(frame: &str) -> &'static str { + match frame { + "rows" => "Rows", + "range" => "Range", + _ => "Cumulative", + } } -fn is_positional(op: WindowAggregate) -> bool { - matches!( - op, - WindowAggregate::Lag | WindowAggregate::Lead | WindowAggregate::Diff - ) +fn frame_name(label: &str) -> &'static str { + match label { + "Rows" => "rows", + "Range" => "range", + _ => "cumulative", + } } fn is_orderable_for_range(ty: ColumnType) -> bool { @@ -182,7 +150,7 @@ impl WindowDraft { }; Self { - op: op_label(spec.aggregate).to_string(), + op: spec.aggregate.clone(), source: spec.column.clone(), order_by: spec .order_by @@ -212,7 +180,7 @@ impl WindowDraft { } fn validate(&self, metadata: &SessionMetadataRc) -> Result { - let op = op_from_label(&self.op).ok_or("Unknown op")?; + let op = self.op.clone(); // Every slot takes true `Table` columns ONLY - expression aliases // and other window columns would create dependency cycles (and @@ -245,18 +213,13 @@ impl WindowDraft { // Backstop for API-authored specs opened in the editor - the op // menu only offers the feature-declared set, so this is - // unreachable from the UI. - let available = metadata - .get_features() - .map(|x| x.get_window_aggregates(source_ty)) - .unwrap_or_default(); - - if !available.contains(&op) { - return Err(format!( - "\"{}\" is not a supported window aggregate for this column", - op_label(op) - )); - } + // unreachable from the UI. The declaration also supplies the + // controls this op takes, below. + let declared = metadata + .get_window_aggregate(source_ty, &op) + .ok_or_else(|| { + format!("\"{op}\" is not a supported window aggregate for this column") + })?; // An EMPTY order slot is valid when the backend has a natural row // order to fall back on (primary key order in the engine, `rowid` @@ -300,11 +263,15 @@ impl WindowDraft { // The numeric fields are typed and input-clamped to their domains, // so no parse or range errors are reachable here. - let frame = if is_aggregating(op) || op == WindowAggregate::Rate { + let frame = if declared.frames.is_empty() { + None + } else { + let chosen = frame_name(&self.frame_type); + if !declared.frames.iter().any(|x| x == chosen) { + return Err(format!("\"{op}\" does not support a {chosen} frame")); + } + match self.frame_type.as_str() { - "Rows" | "Cumulative" if op == WindowAggregate::Rate => { - return Err("\"rate\" requires a range frame".to_string()); - }, "Rows" => Some(WindowFrame::Rows(self.frame_rows)), "Range" => { // The natural-order fallback has no units, so `range` @@ -323,15 +290,13 @@ impl WindowDraft { }, _ => None, } - } else { - None }; // Emit `None` at the engine default so a spec saved without an // explicit `offset` round-trips unchanged (the name-stripped // change-detection baseline compares specs structurally). - let offset = (is_positional(op) && self.offset != 1).then_some(self.offset); - let alpha = (op == WindowAggregate::Ema).then_some(self.alpha); + let offset = (declared.offset && self.offset != 1).then_some(self.offset); + let alpha = declared.alpha.then_some(self.alpha); let mut partition_by = self.partition_by.clone(); partition_by.retain(|col| !col.is_empty()); @@ -676,12 +641,10 @@ impl Component for WindowEditor { }) .unwrap_or_default(); - if !op_from_label(&self.draft.op) - .map(|op| available.contains(&op)) - .unwrap_or_default() + if !available.iter().any(|x| x.name == self.draft.op) && let Some(first) = available.first() { - self.draft.op = op_label(*first).to_string(); + self.draft.op = first.name.clone(); } }, DragTarget::WindowOrderBy => self.draft.order_by = column, @@ -698,11 +661,19 @@ impl Component for WindowEditor { WindowEditorMsg::SetOp(op) => { self.draft.op = op; - // Keep the frame coherent as the op class changes (`rate` - // requires a Range frame; the frame-type dropdown omits the - // rest). - if op_from_label(&self.draft.op) == Some(WindowAggregate::Rate) { - self.draft.frame_type = "Range".to_string(); + // Keep the frame coherent as the op changes - an op that + // does not accept the current frame kind coerces to its + // first declared one (the dropdown omits the rest). + if let Some(declared) = + op_spec(&ctx.props().metadata, &self.draft.source, &self.draft.op) + && !declared.frames.is_empty() + && !declared + .frames + .iter() + .any(|x| x == frame_name(&self.draft.frame_type)) + && let Some(first) = declared.frames.first() + { + self.draft.frame_type = frame_label(first).to_string(); } }, WindowEditorMsg::ClearSource => self.draft.source = String::default(), @@ -745,7 +716,7 @@ impl Component for WindowEditor { } fn view(&self, ctx: &Context) -> Html { - let op = op_from_label(&self.draft.op); + let declared = op_spec(&ctx.props().metadata, &self.draft.source, &self.draft.op); // The op selector renders the FEATURE-DECLARED window aggregates // for the source column's type, in the server's declared order - @@ -763,7 +734,7 @@ impl Component for WindowEditor { }) .unwrap_or_default() .into_iter() - .map(|x| SelectItem::Option(op_label(x).to_string())) + .map(|x| SelectItem::Option(x.name.clone())) .collect(), ); @@ -782,17 +753,16 @@ impl Component for WindowEditor { }; // Frame type as a dropdown; like the op selector, invalid choices - // are omitted rather than disabled (`rate` requires a Range frame, - // and `SetOp` already coerces the draft there). + // are omitted rather than disabled - the declared `frames` are the + // menu, and `SetOp` already coerces the draft into them. let frame_types: Rc>> = Rc::new( - if op == Some(WindowAggregate::Rate) { - vec!["Range"] - } else { - vec!["Rows", "Range", "Cumulative"] - } - .into_iter() - .map(|x| SelectItem::Option(x.to_string())) - .collect(), + declared + .as_ref() + .map(|x| x.frames.clone()) + .unwrap_or_default() + .iter() + .map(|x| SelectItem::Option(frame_label(x).to_string())) + .collect(), ); // Slots take true `Table` columns ONLY - expression aliases and @@ -927,10 +897,12 @@ impl Component for WindowEditor { > }; - let show_frame = - op.map(is_aggregating).unwrap_or_default() || op == Some(WindowAggregate::Rate); - let show_offset = op.map(is_positional).unwrap_or_default(); - let show_alpha = op == Some(WindowAggregate::Ema); + let show_frame = declared + .as_ref() + .map(|x| !x.frames.is_empty()) + .unwrap_or_default(); + let show_offset = declared.as_ref().map(|x| x.offset).unwrap_or_default(); + let show_alpha = declared.as_ref().map(|x| x.alpha).unwrap_or_default(); html! { <> diff --git a/rust/perspective-viewer/src/rust/custom_events.rs b/rust/perspective-viewer/src/rust/custom_events.rs index ef4aa68698..1774e9b6f2 100644 --- a/rust/perspective-viewer/src/rust/custom_events.rs +++ b/rust/perspective-viewer/src/rust/custom_events.rs @@ -261,7 +261,36 @@ pub fn wire_element_events( } }); + let layout_sub = workspace.layout_changed().add_listener({ + clone!(elem); + move |panels: Vec| { + let ids = panels + .iter() + .map(|id| JsValue::from_str(id.as_str())) + .collect::(); + + let detail = js_sys::Object::new(); + let _ = js_sys::Reflect::set(&detail, &JsValue::from_str("panels"), &ids); + dispatch_event(&elem, "layout-update", JsValue::from(detail)).unwrap(); + } + }); + + let active_panel_sub = workspace.active_changed().add_listener({ + clone!(elem); + move |active: Option| { + let panel = active + .map(|id| JsValue::from_str(id.as_str())) + .unwrap_or(JsValue::NULL); + + let detail = js_sys::Object::new(); + let _ = js_sys::Reflect::set(&detail, &JsValue::from_str("panel"), &panel); + dispatch_event(&elem, "active-panel-update", JsValue::from(detail)).unwrap(); + } + }); + vec![ + layout_sub, + active_panel_sub, theme_sub, before_settings_sub, settings_sub, diff --git a/rust/perspective-viewer/src/rust/session/metadata.rs b/rust/perspective-viewer/src/rust/session/metadata.rs index 388f6e375c..1f332c537c 100644 --- a/rust/perspective-viewer/src/rust/session/metadata.rs +++ b/rust/perspective-viewer/src/rust/session/metadata.rs @@ -98,34 +98,20 @@ impl SessionMetadata { /// Records the pre-aggregation types of the config's window columns, so /// `get_column_table_type` resolves them like any other derived column. - /// Types mirror the engine's `t_window_engine::resolve_dtype` rules from - /// the window's op and its source's table type - independent of any - /// aggregation the view may also apply. pub(super) fn update_windows( &mut self, windows: &perspective_client::config::Windows, ) -> ApiResult<()> { - use perspective_client::config::WindowAggregate; let window_schema = windows .iter() .filter_map(|(name, w)| { let source = self.get_column_table_type(&w.column)?; - let dtype = match w.aggregate { - WindowAggregate::Sum - | WindowAggregate::Avg - | WindowAggregate::Stddev - | WindowAggregate::Var - | WindowAggregate::Rate - | WindowAggregate::Ema - | WindowAggregate::Diff => ColumnType::Float, - WindowAggregate::Count => ColumnType::Integer, - WindowAggregate::Min - | WindowAggregate::Max - | WindowAggregate::First - | WindowAggregate::Last - | WindowAggregate::Lag - | WindowAggregate::Lead => source, - }; + let dtype = self + .get_window_aggregate(source, &w.aggregate) + .and_then(|spec| spec.result_type) + .and_then(|ty| ColumnType::try_from(ty).ok()) + .unwrap_or(source); + Some((name.clone(), dtype)) }) .collect(); @@ -133,6 +119,18 @@ impl SessionMetadata { Ok(()) } + /// The declaration for one window aggregate over a `source` column type. + pub fn get_window_aggregate( + &self, + source: ColumnType, + name: &str, + ) -> Option { + self.get_features()? + .get_window_aggregates(source) + .into_iter() + .find(|x| x.name == name) + } + /// Whether `view_schema` has been populated by a prior successful /// `create_view`. Used by `Session::create_view` to force a full /// build on first run even when `is_clean` happens to be true, so diff --git a/rust/perspective-viewer/src/rust/workspace.rs b/rust/perspective-viewer/src/rust/workspace.rs index 60c36cde2d..33f1a06202 100644 --- a/rust/perspective-viewer/src/rust/workspace.rs +++ b/rust/perspective-viewer/src/rust/workspace.rs @@ -21,7 +21,7 @@ use perspective_client::config::Filter; use crate::renderer::Renderer; use crate::session::Session; -use crate::utils::{EffectLedger, PubSub, Subscription}; +use crate::utils::{EffectLedger, PubSub, Subscription, spawn_owned}; /// A unique identifier for a [`Panel`] within a [`Workspace`]. #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -243,6 +243,22 @@ struct WorkspaceData { /// In-flight effects (public mutators + scheduled internal flows), /// drained by `flush()` — see [`EffectLedger`]. effects: EffectLedger, + + /// The PLACED panel set changed since the last emit. Mutation sites set + /// this and emit NOTHING; the coalescing flush task owns delivery (see + /// [`Workspace::schedule_layout_flush`]). + layout_dirty: bool, + layout_changed: Rc>>, + + /// The active panel changed since the last emit. A separate channel from + /// `layout_dirty`: "which panel is selected" and "which panels exist" are + /// distinct facts, and one event may not mean both. + active_dirty: bool, + active_changed: Rc>>, + + /// A flush task is already queued — the flag that makes N mutations + /// within one operation schedule ONE task rather than N. + flush_scheduled: bool, } impl Default for Workspace { @@ -269,6 +285,11 @@ impl Workspace { staged_changed: Rc::new(PubSub::default()), reserved: None, effects: EffectLedger::default(), + layout_dirty: false, + layout_changed: Rc::new(PubSub::default()), + active_dirty: false, + active_changed: Rc::new(PubSub::default()), + flush_scheduled: false, }))) } @@ -277,6 +298,62 @@ impl Workspace { self.0.borrow().effects.clone() } + pub fn layout_changed(&self) -> Rc>> { + self.0.borrow().layout_changed.clone() + } + + pub fn active_changed(&self) -> Rc>> { + self.0.borrow().active_changed.clone() + } + + /// Queue the coalescing layout-event flush, if anything is dirty and no + /// flush is already pending. + fn schedule_layout_flush(&self) { + let schedule = { + let mut data = self.0.borrow_mut(); + let dirty = data.layout_dirty || data.active_dirty; + let queued = data.flush_scheduled; + data.flush_scheduled |= dirty; + dirty && !queued + }; + + if !schedule { + return; + } + + let effects = self.effects(); + let this = self.clone(); + spawn_owned("workspace_layout_flush", async move { + effects.settle().await; + this.flush_layout_events(); + Ok(()) + }); + } + + /// Emit whatever is dirty, outside any borrow. + fn flush_layout_events(&self) { + let (layout, active, panels, active_id, layout_pubsub, active_pubsub) = { + let mut data = self.0.borrow_mut(); + data.flush_scheduled = false; + ( + std::mem::take(&mut data.layout_dirty), + std::mem::take(&mut data.active_dirty), + data.panels.iter().map(|p| p.id.clone()).collect::>(), + data.active.clone(), + data.layout_changed.clone(), + data.active_changed.clone(), + ) + }; + + if layout { + layout_pubsub.emit(panels); + } + + if active { + active_pubsub.emit(active_id); + } + } + /// Stage a layout tree for `MainPanel` to apply at its next `rendered` /// pass (see `WorkspaceData::pending_layout`). pub fn set_pending_layout(&self, layout: crate::js::Layout) { @@ -512,16 +589,22 @@ impl Workspace { /// Append a [`Panel`]. When the element had zero panels, the inserted panel /// becomes the active one (there is no other candidate). pub fn insert_panel(&self, panel: Panel) { - let mut data = self.0.borrow_mut(); - if data.active.is_none() { - data.active = Some(panel.id.clone()); + { + let mut data = self.0.borrow_mut(); + if data.active.is_none() { + data.active = Some(panel.id.clone()); + data.active_dirty = true; + } + + panel + .renderer + .set_active_flag(data.active.as_ref() == Some(&panel.id)); + data.panels.push(panel); + data.layout_dirty = true; + Self::sync_solo_flags(&data); } - panel - .renderer - .set_active_flag(data.active.as_ref() == Some(&panel.id)); - data.panels.push(panel); - Self::sync_solo_flags(&data); + self.schedule_layout_flush(); } /// Hold `panel` in the reservation slot (see [`WorkspaceData::reserved`]): @@ -585,8 +668,10 @@ impl Workspace { if data.active.as_ref() == Some(id) { data.active = None; + data.active_dirty = true; } + data.layout_dirty |= removed.is_some(); Self::sync_solo_flags(&data); ( removed, @@ -605,25 +690,32 @@ impl Workspace { staged_pubsub.emit(()); } + self.schedule_layout_flush(); removed } /// Set the active panel. Returns `false` (no-op) if `id` is not a known /// panel. pub fn set_active(&self, id: PanelId) -> bool { - let mut data = self.0.borrow_mut(); - if data.panels.iter().any(|p| p.id == id) { - data.active = Some(id); - for panel in data.panels.iter() { - panel - .renderer - .set_active_flag(data.active.as_ref() == Some(&panel.id)); + let known = { + let mut data = self.0.borrow_mut(); + if data.panels.iter().any(|p| p.id == id) { + data.active_dirty |= data.active.as_ref() != Some(&id); + data.active = Some(id); + for panel in data.panels.iter() { + panel + .renderer + .set_active_flag(data.active.as_ref() == Some(&panel.id)); + } + + true + } else { + false } + }; - true - } else { - false - } + self.schedule_layout_flush(); + known } /// The default [`Client`], if one has been loaded. diff --git a/rust/perspective-viewer/test/js/multi_panel/layout_events.spec.ts b/rust/perspective-viewer/test/js/multi_panel/layout_events.spec.ts new file mode 100644 index 0000000000..4ecc71785b --- /dev/null +++ b/rust/perspective-viewer/test/js/multi_panel/layout_events.spec.ts @@ -0,0 +1,240 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "../helpers.ts"; +import { armInvariants } from "./harness.ts"; + +const TABLE = "load-viewer-csv"; + +test.beforeEach(async ({ page }) => { + await page.goto("/rust/perspective-viewer/test/html/superstore.html"); + await page.evaluate(async () => { + while (!window["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); +}); + +armInvariants(test); + +async function record(page) { + await page.evaluate(() => { + const viewer = document.querySelector("perspective-viewer")!; + const log: { type: string; detail: any }[] = []; + window["__LAYOUT_LOG__"] = log; + for (const type of [ + "perspective-layout-update", + "perspective-active-panel-update", + ]) { + viewer.addEventListener(type, (e: Event) => + log.push({ type, detail: (e as CustomEvent).detail }), + ); + } + }); +} + +async function drain(page) { + await page.evaluate(async () => { + // @ts-ignore + await document.querySelector("perspective-viewer")!.flush(); + await new Promise((x) => setTimeout(x, 50)); + }); +} + +async function layoutEvents(page) { + return page.evaluate(() => + (window["__LAYOUT_LOG__"] ?? []).filter( + (e) => e.type === "perspective-layout-update", + ), + ); +} + +async function activeEvents(page) { + return page.evaluate(() => + (window["__LAYOUT_LOG__"] ?? []).filter( + (e) => e.type === "perspective-active-panel-update", + ), + ); +} + +async function panelNames(page): Promise { + return page.evaluate(() => + // @ts-ignore + document.querySelector("perspective-viewer")!.getPanelNames(), + ); +} + +async function activePanel(page): Promise { + return page.evaluate(() => + // @ts-ignore + document.querySelector("perspective-viewer")!.getActivePanel(), + ); +} + +test.describe("layout events", () => { + test("addPanel fires one layout-update matching getPanelNames", async ({ + page, + }) => { + await record(page); + await page.evaluate(async (table) => { + // @ts-ignore + await document.querySelector("perspective-viewer")!.addPanel({ + table, + title: "Added", + }); + }, TABLE); + + await drain(page); + const events = await layoutEvents(page); + expect(events).toHaveLength(1); + expect(events[0].detail.panels).toEqual(await panelNames(page)); + }); + + test("removePanel fires one layout-update, empty at zero panels", async ({ + page, + }) => { + const ids = await panelNames(page); + await record(page); + await page.evaluate(async (id) => { + // @ts-ignore + await document.querySelector("perspective-viewer")!.removePanel(id); + }, ids[0]); + + await drain(page); + const events = await layoutEvents(page); + expect(events).toHaveLength(1); + expect(events[0].detail.panels).toEqual([]); + expect(await panelNames(page)).toEqual([]); + }); + + test("restoreWorkspace coalesces to exactly one event", async ({ + page, + }) => { + await page.evaluate(async (table) => { + // @ts-ignore + await document + .querySelector("perspective-viewer")! + .restoreWorkspace({ + layout: { + type: "tab-layout", + tabs: ["a", "b", "c"], + selected: 0, + }, + panels: { + a: { table }, + b: { table }, + c: { table }, + }, + }); + }, TABLE); + + await drain(page); + await record(page); + + await page.evaluate(async (table) => { + // @ts-ignore + await document + .querySelector("perspective-viewer")! + .restoreWorkspace({ + layout: { + type: "tab-layout", + tabs: ["x", "y"], + selected: 0, + }, + panels: { x: { table }, y: { table } }, + }); + }, TABLE); + + await drain(page); + const events = await layoutEvents(page); + expect(events).toHaveLength(1); + expect(events[0].detail.panels).toHaveLength(2); + }); + + test("an inert load(client) places no panel and fires no event", async ({ + page, + }) => { + await page.evaluate(async () => { + // @ts-ignore + const viewer = document.querySelector("perspective-viewer")!; + // @ts-ignore + for (const id of viewer.getPanelNames()) { + // @ts-ignore + await viewer.removePanel(id); + } + }); + + await drain(page); + await record(page); + await page.evaluate(async () => { + const worker = (window as any).__TEST_WORKER__; + // @ts-ignore + await document.querySelector("perspective-viewer")!.load(worker); + }); + + await drain(page); + expect(await layoutEvents(page)).toHaveLength(0); + expect(await panelNames(page)).toEqual([]); + }); + + test("setActivePanel fires active-panel-update and NOT layout-update", async ({ + page, + }) => { + const first = (await panelNames(page))[0]; + const second = await page.evaluate(async (table) => { + // @ts-ignore + return document.querySelector("perspective-viewer")!.addPanel({ + table, + title: "Second", + }); + }, TABLE); + + await drain(page); + expect(await activePanel(page)).toEqual(first); + await record(page); + await page.evaluate(async (id) => { + // @ts-ignore + await document + .querySelector("perspective-viewer")! + .setActivePanel(id); + }, second); + + await drain(page); + expect(await layoutEvents(page)).toHaveLength(0); + const active = await activeEvents(page); + expect(active).toHaveLength(1); + expect(active[0].detail.panel).toEqual(second); + }); + + test("re-entrant addPanel from a handler does not panic", async ({ + page, + }) => { + await page.evaluate(async (table) => { + const viewer = document.querySelector("perspective-viewer")!; + let reentered = false; + viewer.addEventListener("perspective-layout-update", () => { + if (!reentered) { + reentered = true; + // @ts-ignore + viewer.addPanel({ table, title: "Reentrant" }); + } + }); + + // @ts-ignore + await viewer.addPanel({ table, title: "Outer" }); + }, TABLE); + + await drain(page); + const names = await panelNames(page); + expect(names.length).toBeGreaterThanOrEqual(3); + }); +});