diff --git a/.github/actions/config/action.yaml b/.github/actions/config/action.yaml index 6b2b4025cc..a4d81c7ab5 100644 --- a/.github/actions/config/action.yaml +++ b/.github/actions/config/action.yaml @@ -18,6 +18,8 @@ outputs: value: ${{ steps.setuppush.outputs.SKIP_CI || steps.setuppr.outputs.SKIP_CI || steps.setupmanual.outputs.SKIP_CI }} SKIP_CACHE: value: ${{ steps.setuppush.outputs.SKIP_CACHE || steps.setuppr.outputs.SKIP_CACHE || steps.setupmanual.outputs.SKIP_CACHE }} + SAVE_CACHE: + value: ${{ steps.setuppush.outputs.SAVE_CACHE || steps.setuppr.outputs.SAVE_CACHE || steps.setupmanual.outputs.SAVE_CACHE }} FULL_RUN: value: ${{ steps.setuppush.outputs.FULL_RUN || steps.setuppr.outputs.FULL_RUN || steps.setupmanual.outputs.FULL_RUN }} PUBLISH_RELEASE: @@ -54,6 +56,7 @@ runs: echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT echo "SKIP_CACHE=$SKIP_CACHE" >> $GITHUB_OUTPUT + echo "SAVE_CACHE=$SAVE_CACHE" >> $GITHUB_OUTPUT echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT echo "PUBLISH_RELEASE=$PUBLISH_RELEASE" >> $GITHUB_OUTPUT echo "SKIP_PYTHON=$SKIP_PYTHON" >> $GITHUB_OUTPUT @@ -61,6 +64,7 @@ runs: env: SKIP_CI: ${{ contains(github.event.head_commit.message, '[ci-skip]') }} SKIP_CACHE: ${{ contains(github.event.head_commit.message, '[ci-skip-cache]') }} + SAVE_CACHE: ${{ github.ref_name == 'master' || startsWith(github.ref_name, 'v') }} FULL_RUN: ${{ startsWith(github.ref_name, 'v') || contains(github.event.head_commit.message, '[ci-full]') || github.ref_name == 'master' }} PUBLISH_RELEASE: ${{ startsWith(github.ref_name, 'v') }} SKIP_PYTHON: ${{ contains(github.event.head_commit.message, '[ci-skip-python]') }} @@ -81,6 +85,7 @@ runs: echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT echo "SKIP_CACHE=$SKIP_CACHE" >> $GITHUB_OUTPUT + echo "SAVE_CACHE=$SAVE_CACHE" >> $GITHUB_OUTPUT echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT echo "PUBLISH_RELEASE=$PUBLISH_RELEASE" >> $GITHUB_OUTPUT echo "SKIP_PYTHON=$SKIP_PYTHON" >> $GITHUB_OUTPUT @@ -88,6 +93,7 @@ runs: env: SKIP_CI: ${{ contains(github.event.pull_request.title, '[ci-skip]') || contains(github.event.head_commit.message, '[ci-skip]') }} SKIP_CACHE: ${{ contains(github.event.pull_request.title, '[ci-skip-cache]') || contains(github.event.head_commit.message, '[ci-skip-cache]') }} + SAVE_CACHE: "false" FULL_RUN: ${{ contains(github.event.pull_request.title, '[ci-full]') || contains(github.event.head_commit.message, '[ci-full]') }} PUBLISH_RELEASE: ${{ startsWith(github.ref_name, 'v') }} SKIP_PYTHON: ${{ contains(github.event.pull_request.title, '[ci-skip-python]') || contains(github.event.head_commit.message, '[ci-skip-python]') }} @@ -108,6 +114,7 @@ runs: echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT echo "SKIP_CACHE=$SKIP_CACHE" >> $GITHUB_OUTPUT + echo "SAVE_CACHE=$SAVE_CACHE" >> $GITHUB_OUTPUT echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT echo "PUBLISH_RELEASE=$PUBLISH_RELEASE" >> $GITHUB_OUTPUT echo "SKIP_PYTHON=$SKIP_PYTHON" >> $GITHUB_OUTPUT @@ -115,6 +122,7 @@ runs: env: SKIP_CI: false SKIP_CACHE: ${{ github.event.inputs.ci-skip-cache }} + SAVE_CACHE: "true" FULL_RUN: ${{ github.event.inputs.ci-full }} PUBLISH_RELEASE: ${{ startsWith(github.ref_name, 'v') }} SKIP_PYTHON: ${{ github.event.inputs.ci-skip-python }} diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index 99a9b2dbbb..1807dfb44d 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -44,6 +44,18 @@ inputs: ccache: default: "" description: "Install ccache with this cache key (empty = disabled); build steps must also set PSP_USE_CCACHE" + ccache_variant: + default: "ccache" + description: "'ccache' or 'sccache'; sccache also caches rustc when the build step sets RUSTC_WRAPPER, and is the stable choice for MSVC" + emsdk_cache: + default: "" + description: "Cache the .emsdk toolchain + generated sysroot under this key suffix (empty = disabled)" + cargo_cache: + default: "true" + description: "Cache the cargo registry. Disable for jobs which use rust-cache, which caches ~/.cargo itself" + cache_save: + default: "true" + description: "Save the ccache/sccache dir at job end (restore is unconditional). Pass config's SAVE_CACHE so PR runs don't duplicate master's entries against the 10G repo cap" runs: using: "composite" @@ -83,31 +95,35 @@ runs: with: version: 9 + # emsdk installs to `/.emsdk` (`install_emsdk.mjs`), and + # the emscripten sysroot libraries generated during the first build + # land inside it, so a post-build save skips both the clone and the + # ~2.5min sysroot generation. Keyed per wasm32/wasm64 job because the + # sysroots differ and only the first save under a key wins. - name: Setup emsdk cache uses: actions/cache@v4 id: emsdk-cache - if: ${{ inputs.skip_cache == 'false' && inputs.javascript == 'true' }} + if: ${{ inputs.skip_cache == 'false' && inputs.emsdk_cache != '' }} with: - path: | - ~/boost_1_82_0/ - ~/.emsdk/ - ~/.llvm/ - key: ${{ runner.os }}-emsdk-${{ hashFiles('package.json') }} + path: .emsdk/ + key: ${{ runner.os }}-emsdk-${{ inputs.emsdk_cache }}-${{ hashFiles('package.json') }} restore-keys: | - ${{ runner.os }}-emsdk- + ${{ runner.os }}-emsdk-${{ inputs.emsdk_cache }}- + # Only the manylinux container jobs need this; everywhere else + # `setup-python` provides its own pip cache. - name: Setup pip cache uses: actions/cache@v4 - if: ${{ inputs.skip_cache == 'false' && inputs.python == 'true' }} + if: ${{ inputs.skip_cache == 'false' && inputs.python == 'true' && inputs.manylinux == 'true' }} with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + key: ${{ runner.os }}-pip-${{ hashFiles('rust/perspective-python/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - name: Setup cargo cache uses: actions/cache@v4 - if: ${{ inputs.skip_cache == 'false' && inputs.rust == 'true' }} + if: ${{ inputs.skip_cache == 'false' && inputs.rust == 'true' && inputs.cargo_cache == 'true' }} with: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} path: | @@ -123,6 +139,8 @@ runs: if: ${{ inputs.ccache != '' && inputs.skip_cache == 'false' }} with: key: ${{ inputs.ccache }} + variant: ${{ inputs.ccache_variant }} + save: ${{ inputs.cache_save }} max-size: "1G" # https://github.com/apache/arrow/issues/38391 diff --git a/.github/actions/install-wheel/action.yaml b/.github/actions/install-wheel/action.yaml index f1a74c9c06..64510d93a5 100644 --- a/.github/actions/install-wheel/action.yaml +++ b/.github/actions/install-wheel/action.yaml @@ -17,26 +17,29 @@ inputs: inplace: default: "true" description: "Install in-place?" + glob: + default: "*.whl" + description: "Wheel filename glob; narrow this when wheels for multiple platforms share the working directory" runs: using: "composite" steps: - name: Install wheel (Linux) shell: sh - run: python -m pip install -U --no-dependencies *.whl --target rust/perspective-python + run: python -m pip install -U --no-dependencies ${{ inputs.glob }} --target rust/perspective-python if: ${{ inputs.inplace == 'true' && runner.os == 'Linux' }} - name: Install wheel (Linux) shell: sh - run: python -m pip install -U --no-dependencies *.whl + run: python -m pip install -U --no-dependencies ${{ inputs.glob }} if: ${{ inputs.inplace != 'true' && runner.os == 'Linux' }} - name: Install wheel (OSX) shell: sh - run: python -m pip install -U --no-dependencies *.whl --target rust/perspective-python + run: python -m pip install -U --no-dependencies ${{ inputs.glob }} --target rust/perspective-python if: ${{ runner.os == 'macOS' }} - name: Install wheel (Windows) shell: pwsh - run: python -m pip install -U --no-dependencies (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target rust/perspective-python + run: python -m pip install -U --no-dependencies (Get-ChildItem .\${{ inputs.glob }} | Select-Object -Expand FullName) --target rust/perspective-python if: ${{ runner.os == 'Windows' }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5b5745f7dd..c40756cf96 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -87,8 +87,21 @@ jobs: id: init-step uses: ./.github/actions/install-deps with: + cargo_cache: "false" skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + # Caches dependency-crate artifacts (workspace crates are pruned + # on save); this job heads the critical path so its ~4min of + # cold dep compilation is pure wall-clock. + - name: Setup rust cache + if: ${{ steps.config-step.outputs.SKIP_CACHE != 'true' }} + uses: Swatinem/rust-cache@v2 + with: + # rust-cache doesn't read `.cargo/config.toml`'s + # `target-dir = "rust/target"` redirect + workspaces: ". -> rust/target" + save-if: ${{ steps.config-step.outputs.SAVE_CACHE == 'true' }} + - name: Metadata Build run: pnpm run build --ci env: @@ -140,14 +153,18 @@ jobs: name: perspective-metadata path: rust/ + # `clean: "true"` (~2-4min) dropped 2026-08: runners now start + # with ~88G free and no job in the workflow peaks near that; the + # `df -h` probe below is the canary if disk pressure returns. - name: Initialize Build id: init-step uses: ./.github/actions/install-deps with: - clean: "true" python: "false" playwright: "true" ccache: "emscripten-wasm32" + emsdk_cache: "wasm32" + cache_save: ${{ steps.config-step.outputs.SAVE_CACHE }} skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} - name: WebAssembly Build @@ -156,6 +173,9 @@ jobs: PACKAGE: "server,client,viewer,viewer-datagrid,viewer-charts,react,anywidget" PSP_USE_CCACHE: "1" + - name: Run df -h + run: df -h + - uses: actions/upload-artifact@v4 with: name: perspective-js-dist @@ -208,14 +228,16 @@ jobs: name: perspective-metadata path: rust/ + # `clean: "true"` dropped 2026-08 — see `build_js`. - name: Initialize Build id: init-step uses: ./.github/actions/install-deps with: - clean: "true" python: "false" playwright: "true" ccache: "emscripten-wasm64" + emsdk_cache: "wasm64" + cache_save: ${{ steps.config-step.outputs.SAVE_CACHE }} skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} - name: WebAssembly Build (Memory64) @@ -225,6 +247,9 @@ jobs: PSP_WASM64: "only" PSP_USE_CCACHE: "1" + - name: Run df -h + run: df -h + - uses: actions/upload-artifact@v4 with: name: perspective-server-wasm64-dist @@ -292,6 +317,9 @@ jobs: name: perspective-metadata path: rust/ + # The C++ engine (the `perspective-server` build-script cmake + # step) dominates this job and caches at ~100MB/platform; + # `CMakeLists.txt` picks up the launcher via `PSP_USE_CCACHE`. - name: Initialize Build id: init-step uses: ./.github/actions/install-deps @@ -299,6 +327,9 @@ jobs: javascript: "false" arch: ${{ matrix.arch }} manylinux: ${{ matrix.container && 'true' || 'false' }} + ccache: native-${{ matrix.os }}-${{ matrix.arch }} + ccache_variant: ${{ contains(matrix.os, 'windows') && 'sccache' || 'ccache' }} + cache_save: ${{ steps.config-step.outputs.SAVE_CACHE }} skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} - name: Run df -h @@ -313,7 +344,11 @@ jobs: PSP_ARCH: ${{ matrix.arch }} PSP_ROOT_DIR: ${{ github.workspace }} PSP_BUILD_WHEEL: 1 + PSP_USE_CCACHE: "1" + # sccache instead of ccache on Windows: stable with MSVC, and + # `RUSTC_WRAPPER` additionally caches the rust dependency graph + # (rust-cache can't follow the `D:\psp-rust` redirect here). - name: Python Build (Windows) run: | New-Item -ItemType Directory -Path $env:CARGO_TARGET_DIR -Force @@ -326,6 +361,9 @@ jobs: PACKAGE: "python" PSP_ARCH: ${{ matrix.arch }} PSP_BUILD_WHEEL: 1 + PSP_USE_CCACHE: "1" + PSP_CCACHE_BINARY: sccache + RUSTC_WRAPPER: sccache - name: Run df -h if: ${{ runner.os == 'Linux' }} @@ -391,15 +429,34 @@ jobs: javascript: "false" arch: ${{ matrix.arch }} manylinux: "false" + ccache: native-rust-${{ matrix.os }}-${{ matrix.arch }} + ccache_variant: ${{ contains(matrix.os, 'windows') && 'sccache' || 'ccache' }} + cargo_cache: ${{ contains(matrix.os, 'windows') && 'true' || 'false' }} + cache_save: ${{ steps.config-step.outputs.SAVE_CACHE }} skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + # Linux only: rust-cache cannot follow the Windows job's + # `CARGO_TARGET_DIR: D:\psp-rust` redirect (its `workspaces` + # input path-joins, mangling absolute targets) — Windows relies + # on ccache above, which covers its dominant C++ cost. + - name: Setup rust cache + if: ${{ !contains(matrix.os, 'windows') && steps.config-step.outputs.SKIP_CACHE != 'true' }} + uses: Swatinem/rust-cache@v2 + with: + # rust-cache doesn't read `.cargo/config.toml`'s + # `target-dir = "rust/target"` redirect + workspaces: ". -> rust/target" + save-if: ${{ steps.config-step.outputs.SAVE_CACHE == 'true' }} + - name: Rust Build run: pnpm run build if: ${{ !contains(matrix.os, 'windows') }} env: PACKAGE: "rust" PSP_ROOT_DIR: ${{ github.workspace }} + PSP_USE_CCACHE: "1" + # sccache on Windows — see `build_python`'s Windows build step. - name: Rust Build (Windows) run: | New-Item -ItemType Directory -Path $env:CARGO_TARGET_DIR -Force @@ -409,6 +466,9 @@ jobs: CARGO_TARGET_DIR: D:\psp-rust PSP_ROOT_DIR: ${{ github.workspace }} PACKAGE: "rust" + PSP_USE_CCACHE: "1" + PSP_CCACHE_BINARY: sccache + RUSTC_WRAPPER: sccache - name: Rust Test if: ${{ !contains(matrix.os, 'windows') }} @@ -416,6 +476,7 @@ jobs: env: PACKAGE: "rust" PSP_ROOT_DIR: ${{ github.workspace }} + PSP_USE_CCACHE: "1" - name: Rust Test (Windows) run: | @@ -426,16 +487,9 @@ jobs: CARGO_TARGET_DIR: D:\psp-rust PSP_ROOT_DIR: ${{ github.workspace }} PACKAGE: "rust" - - - name: Package - if: ${{ !contains(matrix.os, 'windows') && steps.config-step.outputs.FULL_RUN }} - run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-python - - - uses: actions/upload-artifact@v4 - if: ${{ !contains(matrix.os, 'windows') && steps.config-step.outputs.FULL_RUN }} - with: - name: perspective-rust - path: rust/target/package/*.crate + PSP_USE_CCACHE: "1" + PSP_CCACHE_BINARY: sccache + RUSTC_WRAPPER: sccache # ,-,---. . . .-,--. . . # '|___/ . . . | ,-| '|__/ . . ,-. ,-| . ,-| ,-. @@ -472,6 +526,8 @@ jobs: name: perspective-metadata path: rust/ + # Shares the `emscripten-wasm32` ccache prefix and `wasm32` + # emsdk sysroot with `build_js` — same toolchain, same target. - name: Initialize Build id: init-step uses: ./.github/actions/install-deps @@ -479,8 +535,21 @@ jobs: javascript: "false" arch: ${{ matrix.arch }} manylinux: "false" + ccache: "emscripten-wasm32" + emsdk_cache: "wasm32" + cargo_cache: "false" + cache_save: ${{ steps.config-step.outputs.SAVE_CACHE }} skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + - name: Setup rust cache + if: ${{ steps.config-step.outputs.SKIP_CACHE != 'true' }} + uses: Swatinem/rust-cache@v2 + with: + # rust-cache doesn't read `.cargo/config.toml`'s + # `target-dir = "rust/target"` redirect + workspaces: ". -> rust/target" + save-if: ${{ steps.config-step.outputs.SAVE_CACHE == 'true' }} + # Place the anywidget bundle in-tree _before_ maturin runs, so the # pyproject.toml `include` rules package it into the wheel — # without it `import perspective.widget` fails under jupyterlite. @@ -495,6 +564,7 @@ jobs: PSP_PYODIDE: 1 PACKAGE: "python" CI: 1 + PSP_USE_CCACHE: "1" - name: Verify wheel is PyPI-compatible (PEP 783) shell: bash @@ -588,6 +658,83 @@ jobs: # /` | | /| # `--' ' `-' # + build_python_sdist: + needs: [build_js] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + python-version: + - 3.11 + node-version: [22.x] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Config + id: config-step + uses: ./.github/actions/config + + - name: Initialize Build + id: init-step + uses: ./.github/actions/install-deps + with: + playwright: "true" + cpp: "false" + emsdk_cache: "wasm32" + cache_save: ${{ steps.config-step.outputs.SAVE_CACHE }} + skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + + - uses: actions/download-artifact@v4 + with: + name: perspective-js-dist + path: . + + - uses: actions/download-artifact@v4 + with: + name: perspective-metadata + path: rust/ + + - name: Build extension + run: pnpm run build + env: + # `anywidget` builds the widget bundle into the python static + # dir; `jupyterlab` builds the labextension into the maturin + # `data` dir the sdist packages. Order matters. + PACKAGE: "anywidget,jupyterlab" + + - name: Python Build sdist + run: pnpm run build + env: + PACKAGE: "python" + PSP_BUILD_SDIST: 1 + + - uses: actions/upload-artifact@v4 + with: + name: perspective-python-sdist + path: rust/target/wheels/*.tar.gz + + - name: Package + if: ${{ steps.config-step.outputs.FULL_RUN }} + run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-python + + - uses: actions/upload-artifact@v4 + if: ${{ steps.config-step.outputs.FULL_RUN }} + with: + name: perspective-rust + path: rust/target/package/*.crate + + # A single Linux job: the injected labextension + anywidget assets are + # platform-independent and `repack_wheel.mjs` is a zip rewrite, so this + # job repacks EVERY platform's wheel (the old per-platform matrix burned + # 2 extra runners on it — 8.8min of pure setup on Windows — and never + # repacked the linux-aarch64 wheel at all, publishing it without the + # labextension). Repacked wheels ship as one `-repacked` artifact + # consumed by `publish`; the per-platform artifacts keep the original + # un-repacked wheels for `test_python`/`benchmark_python`. build_and_test_jupyterlab: needs: [build_js, build_python] runs-on: ${{ matrix.os }} @@ -596,27 +743,9 @@ jobs: matrix: os: - ubuntu-22.04 - - macos-14 - - windows-2025 - arch: - - aarch64 - - x86_64 python-version: - 3.11 node-version: [22.x] - is-release: - - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full }} - exclude: - - os: windows-2025 - arch: aarch64 - - os: ubuntu-22.04 - arch: aarch64 - - os: macos-14 - is-release: false - - os: macos-14 - arch: x86_64 - - os: windows-2025 - is-release: false steps: - name: Checkout @@ -647,7 +776,26 @@ jobs: - uses: actions/download-artifact@v4 with: - name: perspective-python-dist-${{ matrix.arch }}-${{ matrix.os }}-${{ matrix.python-version }} + name: perspective-python-dist-x86_64-ubuntu-22.04-${{ matrix.python-version }} + path: . + + - uses: actions/download-artifact@v4 + with: + name: perspective-python-dist-aarch64-ubuntu-22.04-arm-${{ matrix.python-version }} + path: . + + # The win/mac wheels only exist on release-shaped runs — this + # condition must mirror `build_python`'s `is-release` matrix. + - uses: actions/download-artifact@v4 + if: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full == 'true' }} + with: + name: perspective-python-dist-aarch64-macos-14-${{ matrix.python-version }} + path: . + + - uses: actions/download-artifact@v4 + if: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full == 'true' }} + with: + name: perspective-python-dist-x86_64-windows-2025-${{ matrix.python-version }} path: . - name: Build extension @@ -660,44 +808,29 @@ jobs: - run: node tools/scripts/repack_wheel.mjs - - name: Python Build sdist - run: pnpm run build - env: - PACKAGE: "python" - if: ${{ runner.os == 'Linux' }} - # PSP_USE_CCACHE: 1 - # PSP_ARCH: ${{ matrix.arch }} - PSP_BUILD_SDIST: 1 - - uses: actions/upload-artifact@v4 with: - name: perspective-python-dist-${{ matrix.arch}}-${{ matrix.os }}-${{ matrix.python-version }} + name: perspective-python-dist-repacked path: "*.whl" - overwrite: true - - - uses: actions/upload-artifact@v4 - if: ${{ runner.os == 'Linux' }} - with: - name: perspective-python-sdist - path: rust/target/wheels/*.tar.gz + # Only the Linux x86_64 wheel is installable on this runner — + # the other platforms' wheels share the working directory. - uses: ./.github/actions/install-wheel - if: ${{ runner.os == 'Linux' }} + with: + glob: "*manylinux*x86_64.whl" - uses: ./.github/actions/install-wheel - if: ${{ runner.os == 'Linux' }} with: inplace: "false" + glob: "*manylinux*x86_64.whl" # jlab_start.ts passes an explicit `--config`, so no # `jupyter lab --generate-config` is needed. - name: Run Jupyter Tests - if: ${{ runner.os == 'Linux' }} run: pnpm run test --jupyter env: PACKAGE: "jupyterlab" - # ,--,--' . ,-_/ .---. . # `- | ,-. ,-. |- ' | ,-. . , ,-. \___ ,-. ,-. . ,-. |- # , | |-' `-. | | ,-| | / ,-| \ | | | | | | @@ -845,7 +978,7 @@ jobs: test_python_sdist: if: startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full - needs: [build_and_test_jupyterlab, build_and_test_rust] + needs: [build_python_sdist] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -864,6 +997,9 @@ jobs: id: config-step uses: ./.github/actions/config + # sccache rather than ccache: pip unpacks the sdist into a + # randomized temp dir each run, and `RUSTC_WRAPPER` also covers + # the rust dependency compile inside the pip build. - name: Initialize Build uses: ./.github/actions/install-deps with: @@ -871,6 +1007,9 @@ jobs: cpp: "true" python: "true" javascript: "false" + ccache: "sdist-python" + ccache_variant: "sccache" + cache_save: ${{ steps.config-step.outputs.SAVE_CACHE }} skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} - uses: actions/download-artifact@v4 @@ -902,6 +1041,10 @@ jobs: - name: Install perspective-python sdist shell: bash run: python -m pip install -vv ./perspective*.tar.gz + env: + PSP_USE_CCACHE: "1" + PSP_CCACHE_BINARY: sccache + RUSTC_WRAPPER: sccache - name: Verify licenses are installed shell: bash @@ -938,7 +1081,7 @@ jobs: # `-' benchmark_python: needs: [build_python, build_js] - if: startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full + if: startsWith(github.ref, 'refs/tags/v') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -998,7 +1141,7 @@ jobs: # `--' ' benchmark_js: needs: [build_js] - if: startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full + if: startsWith(github.ref, 'refs/tags/v') strategy: matrix: os: [ubuntu-22.04] @@ -1082,25 +1225,13 @@ jobs: name: perspective-js-dist path: . + # All native-platform wheels, with the labextension + anywidget + # assets injected by `build_and_test_jupyterlab`. The per-platform + # `perspective-python-dist-*` artifacts hold the un-repacked + # originals and must not be published. - uses: actions/download-artifact@v4 with: - name: perspective-python-dist-aarch64-macos-14-3.11 - - - uses: actions/download-artifact@v4 - with: - name: perspective-python-dist-aarch64-ubuntu-22.04-arm-3.11 - - # - uses: actions/download-artifact@v4 - # with: - # name: perspective-python-dist-x86_64-macos-14-3.9 - - - uses: actions/download-artifact@v4 - with: - name: perspective-python-dist-x86_64-windows-2025-3.11 - - - uses: actions/download-artifact@v4 - with: - name: perspective-python-dist-x86_64-ubuntu-22.04-3.11 + name: perspective-python-dist-repacked - uses: actions/download-artifact@v4 with: diff --git a/docs/md/explanation/view/config/grouping_and_pivots.md b/docs/md/explanation/view/config/grouping_and_pivots.md index 5e1689b81e..bfc877131a 100644 --- a/docs/md/explanation/view/config/grouping_and_pivots.md +++ b/docs/md/explanation/view/config/grouping_and_pivots.md @@ -38,6 +38,50 @@ let view = table.view(Some(ViewConfigUpdate { +### `group_rollup_mode` + +The `group_rollup_mode` option controls how the grouped rows themselves render: + +- `"rollup"` (the default) - the full hierarchy, with a subtotal row for + every group at every level and a grand total row, each addressable by its + `__ROW_PATH__`. +- `"flat"` - leaf rows only, one row per deepest-level group, with no + subtotal or grand total rows. Useful for chart plugins and exports where + subtotal rows would double-count. +- `"total"` - the grand total row _only_. `"total"` is mutually exclusive + with `group_by` (which is cleared when it is set) - it is the one shape + an empty `group_by` cannot express, since no `group_by` at all yields the + unaggregated dataset. + +
+ +```javascript +const view = await table.view({ + group_by: ["a"], + group_rollup_mode: "flat", +}); +``` + +
+
+ +```python +view = table.view(group_by=["a"], group_rollup_mode="flat") +``` + +
+
+ +```rust +let view = table.view(Some(ViewConfigUpdate { + group_by: Some(vec!["a".into()]), + group_rollup_mode: Some(GroupRollupMode::Flat), + ..ViewConfigUpdate::default() +})).await?; +``` + +
+ ## Split By A split by _splits_ the dataset by the unique values of each column used as a @@ -73,6 +117,56 @@ let view = table.view(Some(ViewConfigUpdate { +### `split_rollup_mode` + +The `split_rollup_mode` option is the `split_by` counterpart to +[`group_rollup_mode`](#group_rollup_mode), controlling whether subtotal +_column groups_ are emitted: + +- `"flat"` (the default) - only full-depth split combinations appear as + columns, e.g. `"CA|Sales"`. This is Perspective's historical behavior. +- `"rollup"` - additionally emits a grand-total column per aggregate (named + by the bare column name, e.g. `"Sales"`, aggregating across every split + group) and, when more than one `split_by` column is applied, a subtotal + column per intermediate split group (e.g. `"CA|Sales"` alongside + `"CA|First Class|Sales"`). Total and subtotal columns precede their + groups, in pre-order. + +
+ +```javascript +const view = await table.view({ + group_by: ["State"], + split_by: ["Ship Mode"], + split_rollup_mode: "rollup", +}); +``` + +
+
+ +```python +view = table.view( + group_by=["State"], + split_by=["Ship Mode"], + split_rollup_mode="rollup", +) +``` + +
+
+ +```rust +let view = table.view(Some(ViewConfigUpdate { + group_by: Some(vec!["State".into()]), + split_by: Some(vec!["Ship Mode".into()]), + split_rollup_mode: Some(SplitRollupMode::Rollup), + ..ViewConfigUpdate::default() +})).await?; +``` + +
+ ## Aggregates Aggregates perform a calculation over an entire column, and are displayed when @@ -126,22 +220,186 @@ let view = table.view(Some(ViewConfigUpdate { -The available aggregate functions depend on the column type: +Every aggregate is described below, grouped by what it computes. Which of them +a given column accepts depends on its type — see +[Availability by column type](#availability-by-column-type). + +### Sums and products + +| Aggregate | Description | Result type | +| --- | --- | --- | +| `sum` | Total of the group's values | `integer` or `float` | +| `sum not null` | As `sum`, but non-finite (`NaN`) values are skipped rather than poisoning the total | `integer` or `float` | +| `sum abs` | Sum of the absolute values — `Σ abs(v)` | `integer` or `float` | +| `abs sum` | Absolute value of the sum — `abs(Σ v)` | `integer` or `float` | +| `mul` | Product of the group's values | `integer` or `float` | +| `gmv` | Gross market value — leaf rows are a plain `sum`, parent rows sum the _absolute_ subtotal of each immediate child group | `integer` or `float` | +| `pct sum parent` | The group's `sum` as a percentage of its parent row's, `0`–`100`; `100` at the root, and `null` when the parent's sum is `0` | `float` | +| `pct sum total` | The group's `sum` as a percentage of the grand total, `0`–`100` | `float` | + +A numeric aggregate widens to the input's numeric class — `integer` columns +accumulate as `integer`, `float` columns as `float`. + +### Averages and dispersion + +| Aggregate | Description | Result type | +| --- | --- | --- | +| `avg` | Arithmetic mean of the non-null values | `float` | +| `weighted mean` | `Σ(value × weight) / Σ(weight)`, over rows where both the value and the weight are non-null and finite; `null` when the weights sum to `0`. Takes a **weight column** as an argument | `float` | +| `stddev` | Population standard deviation | `float` | +| `var` | Population variance — divides by `N`, not `N - 1` | `float` | + +`stddev` and `var` are `null` for a group of fewer than two non-null values. + +### Extrema and order statistics + +| Aggregate | Description | Result type | +| --- | --- | --- | +| `min`, `max` | Smallest and largest of the group's current values | input type | +| `min by`, `max by` | The value from the row at which a **second column**, supplied as an argument, is smallest or largest | input type | +| `high`, `low` | High and low _water mark_ — the largest and smallest value this `View` has ever observed for the group, which never moves back when rows are updated or removed | input type | +| `high minus low` | `max - min` of the group's current values, i.e. its range. Despite the name this uses `min`/`max`, not the water marks | input type | +| `median`, `q1`, `q3` | The value at the 50%, 25% and 75% position of the group's values; on `float` columns an exact split averages the two adjacent values | input type | + +### Positional + +| Aggregate | Description | Result type | +| --- | --- | --- | +| `first` | Value from the group's earliest row | input type | +| `last by index` | Value from the group's latest row | input type | +| `last minus first` | `last by index` minus `first` | input type | +| `last` | Value from the group's most recently _updated_ row | input type | + +"Earliest" and "latest" are by the `Table`'s `index` column, or by row order +when the `Table` is unindexed. This is not the same as `last`, which tracks +update recency rather than position. + +### Cardinality and identity + +| Aggregate | Description | Result type | +| --- | --- | --- | +| `count` | Number of rows in the group | `integer` | +| `distinct count` | Number of distinct values in the group | `integer` | +| `unique` | The group's value when every row shares one, otherwise `null` | input type | +| `distinct leaf` | As `unique`, but only on leaf rows — parent rows are blank | input type | +| `dominant` | The most frequent non-null value, i.e. the mode; a tie resolves to whichever value reached the winning count first | input type | +| `any` | The group's first _truthy_ value — any non-null value for `string` columns, the first non-zero for numbers and dates, the first `true` for `boolean` — or `null` if it has none | input type | +| `or` | Identical to `any` | input type | +| `and` | `true` when every value in the group is truthy, else `false` | `boolean` | +| `join` | The group's distinct values, sorted and rendered as a `", "`-delimited string, truncated at 280 characters. Nulls render as `null` | `string` | + +### Nulls + +Null handling is not uniform, and is usually what makes two similar-looking +aggregates differ: + +- `count` counts **rows**, not values — a group of 3 rows whose value is + `null` counts `3`. This is not the same as the `count` [window + aggregate](./windows.md#aggregates), which counts non-null values. +- `distinct count` counts `null` as **one distinct value**, so a group of + `[1, null, null]` counts `2`. +- `sum`, `avg`, `stddev`, `var`, `dominant` and `weighted mean` skip nulls + entirely. `avg` divides by the count of non-null values, so a group of all + nulls is `null` rather than `0`. +- `any` and `or` return the first _truthy_ value, not the first non-null one — + a numeric group of all `0`, or a `boolean` group of all `false`, aggregates + to `null`. +- `join` renders nulls into its output as the literal text `null`. + +### Availability by column type + +The aggregates a column accepts depend on its type: **Numeric columns** (`integer`, `float`): `sum`, `abs sum`, `sum abs`, -`sum not null`, `any`, `avg`, `mean`, `count`, `distinct count`, `dominant`, -`first`, `last`, `last by index`, `high`, `low`, `max`, `min`, -`high minus low`, `last minus first`, `median`, `q1`, `q3`, -`pct sum parent`, `pct sum total`, `stddev`, `var`, `unique`, -`weighted mean`, `min by`, `max by`. - -**String columns**: `count`, `any`, `distinct count`, `dominant`, `first`, -`last`, `last by index`, `join`, `median`, `q1`, `q3`, `unique`, `min by`, -`max by`. - -**Date/Datetime columns**: `count`, `any`, `avg`, `distinct count`, `dominant`, -`first`, `last`, `last by index`, `high`, `low`, `max`, `min`, `median`, -`q1`, `q3`, `unique`. - -**Boolean columns**: `count`, `any`, `distinct count`, `dominant`, `first`, -`last`, `last by index`, `unique`. +`sum not null`, `mul`, `gmv`, `any`, `avg`, `mean`, `count`, `distinct count`, +`distinct leaf`, `dominant`, `first`, `last`, `last by index`, `high`, `low`, +`max`, `min`, `min by`, `max by`, `high minus low`, `last minus first`, +`median`, `q1`, `q3`, `pct sum parent`, `pct sum total`, `stddev`, `var`, +`unique`, `weighted mean`. + +**String columns**: `count`, `any`, `distinct count`, `distinct leaf`, +`dominant`, `first`, `last`, `last by index`, `join`, `median`, `q1`, `q3`, +`unique`, `min by`, `max by`. + +**Date/Datetime columns**: `count`, `any`, `avg`, `distinct count`, +`distinct leaf`, `dominant`, `first`, `last`, `last by index`, `high`, `low`, +`max`, `min`, `median`, `q1`, `q3`, `unique`. + +**Boolean columns**: `count`, `any`, `and`, `or`, `distinct count`, +`distinct leaf`, `dominant`, `first`, `last`, `last by index`, `unique`. + +
avg on a date or +datetime column returns a float — the mean of the +column's underlying numeric representation — not a date.
+ +### Argument-taking aggregates + +`weighted mean`, `min by` and `max by` each read a second column, and are +written as a `[name, [argument]]` pair rather than a bare string: + +
+ +```javascript +const view = await table.view({ + aggregates: { a: ["weighted mean", ["b"]] }, +}); +``` + +
+
+ +```python +view = table.view(aggregates={"a": ("weighted mean", ["b"])}) +``` + +
+
+ +```rust +let view = table.view(Some(ViewConfigUpdate { + aggregates: Some(HashMap::from([( + "a".into(), + Aggregate::MultiAggregate("weighted mean".into(), vec!["b".into()]), + )])), + ..ViewConfigUpdate::default() +})).await?; +``` + +
+ +
In Rust, Aggregate::from(&str) splits on +" by " to build a MultiAggregate. Single aggregates +whose names contain that substring — "last by index" — must +therefore be constructed as +Aggregate::SingleAggregate("last by index".into()) rather than +"last by index".into(), which silently resolves to +last.
+ +### Aliases + +Several aggregates answer to more than one name. Every name below is accepted +anywhere an aggregate is, and each group refers to one function: + +| Canonical | Also accepted | +| --- | --- | +| `avg` | `mean` | +| `distinct count` | `distinct`, `distinctcount`, `distinct_count` | +| `first` | `first by index` | +| `last` | `last_value` | +| `high` | `high_water_mark` | +| `low` | `low_water_mark` | +| `pct sum total` | `pct sum grand total`, `pct_sum_grand_total` | +| `var` | `variance` | +| `stddev` | `standard deviation` | + +Most multi-word aggregates also answer to a snake_case spelling — +`sum_not_null`, `sum_abs`, `abs_sum`, `weighted_mean`, `distinct_leaf`, +`pct_sum_parent`, `pct_sum_total`, `min_by`, `max_by`. Three do not, and are +only accepted spelled with spaces: `high minus low`, `last minus first` and +`last by index`. + +A few names the engine parses are _not implemented_ — `identity`, +`mean by count`, and `div`/`add`, which have no way to receive their operands +from a `ViewConfig`. Naming one is rejected exactly as a misspelled aggregate +is: the `View` fails to construct with an error naming the aggregate and the +column it was given for, and the `Table` is left untouched. diff --git a/packages/anywidget/src/js/index.js b/packages/anywidget/src/js/index.js index 3bc179d9fb..40fb3593ff 100644 --- a/packages/anywidget/src/js/index.js +++ b/packages/anywidget/src/js/index.js @@ -38,6 +38,7 @@ const PERSISTENT_ATTRIBUTES = [ "group_by", "split_by", "group_rollup_mode", + "split_rollup_mode", "aggregates", "sort", "filter", @@ -57,6 +58,7 @@ const STRING_ATTRIBUTES = new Set([ "title", "version", "group_rollup_mode", + "split_rollup_mode", ]); function isEqual(a, b) { diff --git a/packages/jupyterlab/test/jupyter/widget.spec.mjs b/packages/jupyterlab/test/jupyter/widget.spec.mjs index 4e4bd43856..b9f1a2eb9c 100644 --- a/packages/jupyterlab/test/jupyter/widget.spec.mjs +++ b/packages/jupyterlab/test/jupyter/widget.spec.mjs @@ -379,6 +379,7 @@ async_table = await async_client.open_table(sync_table.get_name())`, filter: [], group_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", plugin: "Datagrid", // Default plugin_config serializes empty — only // non-default values appear in `save()`. @@ -418,6 +419,7 @@ w.theme = "Pro Dark"`, group_by: ["date"], // Chart plugins only support "flat" rollup mode group_rollup_mode: "flat", + split_rollup_mode: "flat", plugin: "X Bar", plugin_config: {}, settings: true, @@ -472,6 +474,7 @@ w.theme = "Pro Dark"`, filter: [], group_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", plugin: "Datagrid", // Default plugin_config serializes empty — only // non-default values appear in `save()`. diff --git a/packages/viewer-charts/src/ts/plugin/plugin.ts b/packages/viewer-charts/src/ts/plugin/plugin.ts index 8b95d39af4..b448ab6863 100644 --- a/packages/viewer-charts/src/ts/plugin/plugin.ts +++ b/packages/viewer-charts/src/ts/plugin/plugin.ts @@ -442,6 +442,7 @@ export class HTMLPerspectiveViewerWebGLPluginElement max_cells: this._chartType.max_cells, max_columns: this._chartType.max_columns, group_rollup_modes: ["flat"], + split_rollup_modes: ["flat"], group_by_role: this._chartType.group_by_role, split_by_role: this._chartType.split_by_role, connects_row_order: !!this._chartType.connects_row_order, diff --git a/packages/viewer-datagrid/src/css/mitered-headers.css b/packages/viewer-datagrid/src/css/mitered-headers.css index 0c8a1f6f08..c46bfc7c82 100644 --- a/packages/viewer-datagrid/src/css/mitered-headers.css +++ b/packages/viewer-datagrid/src/css/mitered-headers.css @@ -11,54 +11,126 @@ * ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ */ -.psp-header-border:not(.psp-is-top):not(.psp-header-leaf) { - /* right */ - box-shadow: 1px 0px var(--psp-inactive--border-color, #8b868045); +/* Mitered header borders - borders gapped at one or both corners, showing + * that a cell binds more tightly to its neighbor in that direction. + * + * Each edge of a cell is one independent `--psp-miter-*` slot, set by + * exactly one single-purpose class per edge (emitted by + * `style_handlers/border_model.ts`) and composed by the single base rule + * below. Custom properties on the same element merge across classes where + * `box-shadow` itself cannot - so edge combinations need NO per-combination + * rules here, and a new cell kind costs zero new CSS. + * + * Each border segment is a `box-shadow` piece: with offset `(dx, dy)` and + * spread `-s`, the painted region is the cell rect shrunk by `s` then + * translated - a 1px edge line whose ends stop `2s` short of the gapped + * corner(s). `--psp-mh-gap` is the miter gap; `--psp-mh-color` the line + * color. Both are theme-overridable on `regular-table`. + * + * Class naming: `psp-b-{t|r|b|l}` = full edge; suffix `-m{t|b|l|r|tb|lr}` = + * which corner(s) are mitered (gapped). */ + +regular-table { + --psp-mh-color: var(--psp-inactive--border-color, #8b868045); + --psp-mh-gap: 8px; } -.psp-header-group { - /* bottom */ - box-shadow: 0px 10px 0 -9px var(--psp-inactive--border-color, #8b868045); +regular-table thead th { + box-shadow: + var(--psp-miter-top, 0 0 0 0 transparent), + var(--psp-miter-right, 0 0 0 0 transparent), + var(--psp-miter-bottom, 0 0 0 0 transparent), + var(--psp-miter-left, 0 0 0 0 transparent); } -.psp-is-top { - /* top-miter-right */ - box-shadow: 5px 4px 0px -4px var(--psp-inactive--border-color, #8b868045); +/* Right edge */ + +.psp-b-r { + --psp-miter-right: 1px 0px 0px 0px var(--psp-mh-color); } -.psp-is-top.psp-header-group:not(.psp-header-group-corner) { - /* top-miter-right and bottom */ - box-shadow: - 5px 4px 0px -4px var(--psp-inactive--border-color, #8b868045), - 0px 10px 0 -9px var(--psp-inactive--border-color, #8b868045); +.psp-b-r-mt { + --psp-miter-right: calc(var(--psp-mh-gap) / 2 + 1px) + calc(var(--psp-mh-gap) / 2) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); +} + +.psp-b-r-mb { + --psp-miter-right: calc(var(--psp-mh-gap) / 2 + 1px) + calc(var(--psp-mh-gap) / -2) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); +} + +.psp-b-r-mtb { + --psp-miter-right: calc(var(--psp-mh-gap) + 1px) 0px 0px + calc(-1 * var(--psp-mh-gap)) var(--psp-mh-color); +} + +/* Left edge */ + +.psp-b-l { + --psp-miter-left: -1px 0px 0px 0px var(--psp-mh-color); +} + +.psp-b-l-mt { + --psp-miter-left: calc(var(--psp-mh-gap) / -2 - 1px) + calc(var(--psp-mh-gap) / 2) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); } -.psp-header-border.psp-header-group { - &:not(.psp-is-top):not(.psp-header-group-corner) { - /* right and bottom */ - box-shadow: - 1px 0px var(--psp-inactive--border-color, #8b868045), - 0px 10px 0 -9px var(--psp-inactive--border-color, #8b868045); - } +.psp-b-l-mb { + --psp-miter-left: calc(var(--psp-mh-gap) / -2 - 1px) + calc(var(--psp-mh-gap) / -2) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); } -perspective-viewer[settings] - tr.rt-autosize - .psp-header-leaf.psp-header-border:not(.psp-menu-enabled) { - box-shadow: 1px 0px var(--psp-inactive--border-color, #8b868045); +.psp-b-l-mtb { + --psp-miter-left: calc(-1 * var(--psp-mh-gap) - 1px) 0px 0px + calc(-1 * var(--psp-mh-gap)) var(--psp-mh-color); +} + +/* Bottom edge */ + +.psp-b-b { + --psp-miter-bottom: 0px 1px 0px 0px var(--psp-mh-color); +} + +.psp-b-b-ml { + --psp-miter-bottom: calc(var(--psp-mh-gap) / 2) + calc(var(--psp-mh-gap) / 2 + 1px) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); +} + +.psp-b-b-mr { + --psp-miter-bottom: calc(var(--psp-mh-gap) / -2) + calc(var(--psp-mh-gap) / 2 + 1px) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); +} + +.psp-b-b-mlr { + --psp-miter-bottom: 0px calc(var(--psp-mh-gap) + 1px) 0px + calc(-1 * var(--psp-mh-gap)) var(--psp-mh-color); +} + +/* Top edge */ + +.psp-b-t { + --psp-miter-top: 0px -1px 0px 0px var(--psp-mh-color); } -:host-context(perspective-viewer[settings]) - tr.rt-autosize - .psp-header-leaf.psp-header-border:not(.psp-menu-enabled) { - box-shadow: 1px 0px var(--psp-inactive--border-color, #8b868045); +.psp-b-t-ml { + --psp-miter-top: calc(var(--psp-mh-gap) / 2) + calc(var(--psp-mh-gap) / -2 - 1px) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); } -.psp-header-leaf.psp-header-border { - /* bottom-miter-right */ - box-shadow: 5px -4px 0px -4px var(--psp-inactive--border-color, #8b868045); +.psp-b-t-mr { + --psp-miter-top: calc(var(--psp-mh-gap) / -2) + calc(var(--psp-mh-gap) / -2 - 1px) 0px calc(var(--psp-mh-gap) / -2) + var(--psp-mh-color); } -tr:only-child th { - box-shadow: none !important; +.psp-b-t-mlr { + --psp-miter-top: 0px calc(-1 * var(--psp-mh-gap) - 1px) 0px + calc(-1 * var(--psp-mh-gap)) var(--psp-mh-color); } diff --git a/packages/viewer-datagrid/src/css/regular_table.css b/packages/viewer-datagrid/src/css/regular_table.css index dc364819b2..f8e1e32d96 100644 --- a/packages/viewer-datagrid/src/css/regular_table.css +++ b/packages/viewer-datagrid/src/css/regular_table.css @@ -132,6 +132,11 @@ perspective-viewer.dragging, border-right-width: 0px; } +/* Subtotal / grand-total column groups (`split_rollup_mode: "rollup"`) + carry `psp-split-total` / `psp-split-subtotal` marker classes for themes; + no default styling - group boundaries come from the mitered border + system. */ + .psp-header-sort-desc:after { background-repeat: no-repeat; background-color: var(--psp--color); diff --git a/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts b/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts index abd381327e..8e1597b430 100644 --- a/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts +++ b/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts @@ -133,6 +133,7 @@ export class HTMLPerspectiveViewerDatagridPluginElement // group_by_role: "Row Groups", // split_by_role: "Column Groups", group_rollup_modes: ["rollup", "flat", "total"], + split_rollup_modes: ["flat", "rollup"], // Higher priority than the chart plugins so the Datagrid is // loaded by default. priority: 1, diff --git a/packages/viewer-datagrid/src/ts/data_listener/format_cell.ts b/packages/viewer-datagrid/src/ts/data_listener/format_cell.ts index 5828a4dcc2..54bc0e4490 100644 --- a/packages/viewer-datagrid/src/ts/data_listener/format_cell.ts +++ b/packages/viewer-datagrid/src/ts/data_listener/format_cell.ts @@ -40,6 +40,7 @@ export function format_cell( const type: ColumnType = ((use_table_schema && this._table_schema[title]) || this._schema[title] || + this._window_schema?.[title] || "string") as ColumnType; const plugin: ColumnConfig = plugins[title] || {}; const is_numeric = type === "integer" || type === "float"; @@ -97,7 +98,11 @@ export function format_cell( italic.textContent = val as string; return italic; } else { + // `String(val)`, not a cast: with no formatter (e.g. an unknown + // column type falling back to `"string"`), a raw non-string value + // must not leak into consumers that expect text - see the + // `{ toString }` wrapper in `format_tree_header.ts`. const formatter = FORMAT_CACHE.get(type, plugin); - return formatter ? formatter.format(val) : (val as string); + return formatter ? formatter.format(val) : String(val); } } diff --git a/packages/viewer-datagrid/src/ts/data_listener/format_tree_header.ts b/packages/viewer-datagrid/src/ts/data_listener/format_tree_header.ts index 78e166f474..4f258f4a18 100644 --- a/packages/viewer-datagrid/src/ts/data_listener/format_tree_header.ts +++ b/packages/viewer-datagrid/src/ts/data_listener/format_tree_header.ts @@ -44,7 +44,14 @@ export function* format_tree_header_row_path( if (formatted instanceof HTMLElement) { newPath = newPath.concat(formatted); } else { - newPath = newPath.concat({ toString: () => formatted as string }); + // `format_cell` contractually returns `string | HTMLElement | + // null`, but this wrapper is the row-header styling's only + // source of text - coerce defensively so a non-string can + // never surface from `toString()`. `null` (a null group key) + // coerces to `""`, keeping the is-empty styling predicate. + newPath = newPath.concat({ + toString: () => String(formatted ?? ""), + }); } newPath.length = row_headers.length + 1; diff --git a/packages/viewer-datagrid/src/ts/data_listener/index.ts b/packages/viewer-datagrid/src/ts/data_listener/index.ts index f756254acc..d09562ab1e 100644 --- a/packages/viewer-datagrid/src/ts/data_listener/index.ts +++ b/packages/viewer-datagrid/src/ts/data_listener/index.ts @@ -115,8 +115,17 @@ export function createDataListener( this._schema = { ...(a as Schema), ...(b as Schema) }; for (let i = 0; i < new_col_paths.length; i++) { const column_path_parts = new_col_paths[i].split("|"); + + // Subtotal/total columns (`split_rollup_mode: "rollup"`) + // have fewer than `split_by.length` levels - the column + // name is always the last part. const column = - column_path_parts[this._config.split_by.length]; + column_path_parts[ + Math.min( + this._config.split_by.length, + column_path_parts.length - 1, + ) + ]; this._is_editable[i + new_window.start_col!] = !!this._table_schema[column]; @@ -165,6 +174,34 @@ export function createDataListener( ) { const path = this._column_paths[ipath]; const path_parts = path.split("|"); + + // Under `split_rollup_mode: "rollup"`, grand-total and subtotal + // columns have fewer than `split_by.length` levels. Pad between + // the split levels and the trailing column name so the name + // always lands at index `split_by.length` - every downstream + // `column_header` consumer indexes it there. + // + // Pads are zero-width spaces repeated by group depth, NOT `""`: + // regular-table merges header cells on bare value equality + // across the whole row, so a plain `""` pad would merge with an + // adjacent group's equal pad - or, for the grand-total group's + // row-0 cell, with the blank CORNER `` beside it, swallowing + // the column and its group-boundary border. Depth-keying makes + // pads equal exactly within one group - same-depth groups are + // never adjacent in the pre-order column space - so pads merge + // inside a group and never across one. The grand-total group has + // no real levels, so its entire header stack is pads (rendering + // blank). + if (path_parts.length < this._config.split_by.length + 1) { + const column_name = path_parts.pop() as string; + const pad = "\u200b".repeat(path_parts.length + 1); + while (path_parts.length < this._config.split_by.length) { + path_parts.push(pad); + } + + path_parts.push(column_name); + } + const column = columns[path] || new Array(y1 - y0).fill(null); const agg_depth = Math.min( (regularTable as any)[PRIVATE_PLUGIN_SYMBOL]?.[ diff --git a/packages/viewer-datagrid/src/ts/model/create.ts b/packages/viewer-datagrid/src/ts/model/create.ts index e5fc2b3212..dc98dceec2 100644 --- a/packages/viewer-datagrid/src/ts/model/create.ts +++ b/packages/viewer-datagrid/src/ts/model/create.ts @@ -28,6 +28,40 @@ import { } from "../types.js"; import type { HTMLPerspectiveViewerElement } from "@perspective-dev/viewer"; +// Mirror of the engine's window-aggregate result types (the +// `GetFeaturesResp.window_aggregates` table in `server.cpp`): these +// aggregates yield a fixed type, everything else - `min`, `max`, `lag`, +// `lead` - preserves the source column's type. Window columns appear in no +// schema the client can query when they are not visible (the `View`'s +// `schema()` covers visible columns only, and `table.schema()` / +// `expression_schema()` are pre-window), so a `group_by` on a window column +// needs this to format and style its row headers. +const WINDOW_FLOAT_AGGREGATES = new Set([ + "sum", + "avg", + "stddev", + "var", + "diff", + "rate", + "ema", +]); + +function window_output_type( + aggregate: string, + source_column: string, + table_schema: Schema, +): ColumnType { + if (aggregate === "count") { + return "integer"; + } + + if (WINDOW_FLOAT_AGGREGATES.has(aggregate)) { + return "float"; + } + + return table_schema[source_column] ?? "string"; +} + function arraysChanged(a: T[], b: T[]): boolean { if (a.length !== b.length) { return true; @@ -180,6 +214,9 @@ export async function createModel( const group_rollup_mode_changed = old.group_rollup_mode !== config.group_rollup_mode; + const split_rollup_mode_changed = + old.split_rollup_mode !== config.split_rollup_mode; + const theme_changed = this.model._theme !== style._theme; this._reset_scroll_top = group_by_changed; this._reset_scroll_left = split_by_changed; @@ -192,6 +229,7 @@ export async function createModel( this._reset_column_size = group_rollup_mode_changed || + split_rollup_mode_changed || split_by_changed || group_by_changed || columns_changed || @@ -220,6 +258,13 @@ export async function createModel( ...(expression_schema as Schema), }; + const _window_schema: Schema = Object.fromEntries( + Object.entries(config.windows ?? {}).map(([name, spec]) => [ + name, + window_output_type(spec!.aggregate, spec!.column, _table_schema), + ]), + ); + const _column_paths: string[] = []; const _is_editable: boolean[] = []; const _column_types: ColumnType[] = []; @@ -249,6 +294,7 @@ export async function createModel( _config: config, _num_rows: num_rows, _schema, + _window_schema, _ids: [], ...style, _column_paths, @@ -260,7 +306,7 @@ export async function createModel( dirty: false, }, _row_header_types: config.group_by.map((column_path) => { - return _table_schema[column_path]; + return _table_schema[column_path] ?? _window_schema[column_path]; }), _series_color_map: new Map(), _series_color_seed: new Map(), diff --git a/packages/viewer-datagrid/src/ts/style_handlers/body.ts b/packages/viewer-datagrid/src/ts/style_handlers/body.ts index cf37eef17c..18d42ec213 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/body.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/body.ts @@ -47,10 +47,36 @@ export function applyBodyCellStyles( for (const { element: td, metadata, isHeader } of cells) { const column_name = metadata.column_header?.[model._config.split_by.length]; + + // Mark subtotal/grand-total column cells (`split_rollup_mode: + // "rollup"`) so themes can emphasize them like row-tree totals. + // `x` is absent on row-header cell metadata. + const meta_x = (metadata as { x?: number }).x; + const n_split_levels = + meta_x === undefined + ? undefined + : model._column_paths[meta_x]?.split("|").length - 1; + + const is_rollup_col = + model._config.split_by.length > 0 && + n_split_levels !== undefined && + n_split_levels < model._config.split_by.length; + + td.classList.toggle( + "psp-split-total", + is_rollup_col && n_split_levels === 0, + ); + + td.classList.toggle( + "psp-split-subtotal", + is_rollup_col && n_split_levels! > 0, + ); + const type = get_psp_type(model, metadata); const plugin = column_name ? plugins[column_name.toString()] : undefined; + const is_numeric = type === "integer" || type === "float"; // Calculate aggregate depth visibility diff --git a/packages/viewer-datagrid/src/ts/style_handlers/border_model.ts b/packages/viewer-datagrid/src/ts/style_handlers/border_model.ts new file mode 100644 index 0000000000..1837edd49e --- /dev/null +++ b/packages/viewer-datagrid/src/ts/style_handlers/border_model.ts @@ -0,0 +1,262 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// The single source of truth for the "mitered" header borders: borders gapped +// at one or both corners, indicating the cell binds more tightly to its +// neighbor in that direction. Pure - no DOM, no regular-table - so the +// classification is unit-testable in plain node. +// +// The central rule: for adjacent data columns `x`, `x + 1`, the depth of the +// boundary between them is the length of the longest common prefix of their +// raw column paths' split levels. A vertical border exists in header row `y` +// iff `boundary_depth <= y` - borders "grow downward" from the row where the +// paths diverge - mitered at the top in the row where they begin, and mitered +// at the bottom in the bottommost header row (where the columns bind to the +// body below). This needs no special-casing for `split_rollup_mode: +// "rollup"`: total and subtotal groups produce shorter paths, so their +// boundaries are shallower and their borders taller, automatically. + +/// One edge of a cell. `miter-start` gaps the corner nearest the axis origin +/// (top for vertical edges, left for horizontal edges). +export type EdgeState = + | "none" + | "full" + | "miter-start" + | "miter-end" + | "miter-both"; + +export interface CellBorders { + top: EdgeState; + right: EdgeState; + bottom: EdgeState; + left: EdgeState; +} + +export interface HeaderCellInput { + /// Raw (unpadded) column paths, `model._column_paths`. May be sparse + /// where the virtual viewport has not loaded a column. + paths: (string | undefined)[]; + + /// `config.split_by.length`. + split_by_len: number; + + /// Leftmost data column covered by this cell, `metadata.x`. `undefined` + /// for corner cells. + x?: number; + + /// `th.colSpan` - regular-table merges adjacent equal header values + /// horizontally, and a merged cell's right edge belongs to its LAST + /// covered column. + colspan: number; + + /// Header row index, `0..split_by_len - 1` for group rows. + y: number; + + row_kind: "group" | "name" | "menu"; + + is_corner: boolean; + + /// For corner cells: whether this cell sits at the boundary between the + /// row-header region and the data columns (`row_header_x` equals the + /// effective row-header depth). + corner_needs_border?: boolean; + + /// Whether this is the bottommost header row (the name row with settings + /// closed, the menu row with settings open). + is_last_header_row: boolean; + + /// A single header row draws no borders at all (previously + /// `tr:only-child th { box-shadow: none !important }`). + single_header_row: boolean; +} + +const NONE: CellBorders = { + top: "none", + right: "none", + bottom: "none", + left: "none", +}; + +/// The split levels of a raw column path - the path minus its trailing +/// column name. Total/subtotal columns (`split_rollup_mode: "rollup"`) have +/// fewer than `split_by_len` levels. NOTE a column NAME containing `"|"` +/// mis-splits here; this is a pre-existing ambiguity shared with the data +/// listener, not worsened. +export function split_levels(path: string, split_by_len: number): string[] { + const parts = path.split("|"); + return parts.slice(0, Math.min(split_by_len, parts.length - 1)); +} + +/// Depth of the boundary between data columns `x` and `x + 1`: +/// - `null` when either side is unloaded (draw nothing, self-corrects when +/// the viewport fills), except past the end of the loaded set, which is +/// the table's trailing edge and reads as a depth-0 boundary. +/// - Otherwise the longest-common-prefix length of the two columns' split +/// levels: `0..split_by_len`, where `split_by_len` means "same group" +/// (an aggregate-internal boundary). +export function boundary_depth( + paths: (string | undefined)[], + split_by_len: number, + x: number, +): number | null { + const left = paths[x]; + if (left === undefined) { + return null; + } + + if (x + 1 >= paths.length) { + return 0; + } + + const right = paths[x + 1]; + if (right === undefined) { + return null; + } + + const a = split_levels(left, split_by_len); + const b = split_levels(right, split_by_len); + let depth = 0; + while (depth < a.length && depth < b.length && a[depth] === b[depth]) { + depth++; + } + + // IDENTICAL level lists mean the same traversal node - an + // aggregate-internal boundary, never a group boundary. This matters for + // total/subtotal groups (`split_rollup_mode: "rollup"`), whose level + // lists are SHORTER than `split_by_len`: two aggregate columns of the + // grand-total group have LCP 0, which without this clause would read as + // a top-level group boundary between them. + if (depth === a.length && depth === b.length) { + return split_by_len; + } + + return depth; +} + +/// Whether the group-row cell at `(x, y)` displays a real split level +/// rather than a vertical-continuation pad. Pads draw no bottom border - +/// they are interior to their group's header block - and the synthesized +/// `"Total"` label of a zero-level grand-total column counts as a pad too: +/// the whole Total column header stack reads as one open block. +function has_real_text( + paths: (string | undefined)[], + split_by_len: number, + x: number, + y: number, +): boolean { + const path = paths[x]; + if (path === undefined) { + return false; + } + + return y < split_levels(path, split_by_len).length; +} + +export function classify_header_cell(input: HeaderCellInput): CellBorders { + if (input.single_header_row || input.split_by_len === 0) { + return NONE; + } + + // The NW corner region renders as one open block - no dividers between + // corner cells - with only the boundary column's right edge separating + // it from the data columns. + if (input.is_corner) { + if (!input.corner_needs_border) { + return NONE; + } + + return { + ...NONE, + right: + input.y === 0 && input.row_kind === "group" + ? "miter-start" + : input.is_last_header_row + ? "miter-end" + : "full", + }; + } + + if (input.x === undefined) { + return NONE; + } + + const x_end = input.x + input.colspan - 1; + const depth = boundary_depth(input.paths, input.split_by_len, x_end); + const is_trailing = x_end >= input.paths.length - 1; + + let right: EdgeState = "none"; + if (depth !== null) { + if (input.row_kind === "group") { + if (depth <= input.y) { + right = depth === input.y ? "miter-start" : "full"; + } + } else if (depth < input.split_by_len || is_trailing) { + right = input.is_last_header_row ? "miter-end" : "full"; + } + } + + const bottom: EdgeState = + input.row_kind === "group" && + has_real_text(input.paths, input.split_by_len, input.x, input.y) + ? "miter-both" + : "none"; + + return { top: "none", right, bottom, left: "none" }; +} + +/// The utility classes, one per (edge, state). Each sets a single per-edge +/// custom property (`--psp-miter-*`) composed by one base rule in +/// `mitered-headers.css` - custom properties merge across classes where +/// `box-shadow` itself cannot. +const EDGE_CLASSES: Record< + keyof CellBorders, + Record, string> +> = { + top: { + full: "psp-b-t", + "miter-start": "psp-b-t-ml", + "miter-end": "psp-b-t-mr", + "miter-both": "psp-b-t-mlr", + }, + right: { + full: "psp-b-r", + "miter-start": "psp-b-r-mt", + "miter-end": "psp-b-r-mb", + "miter-both": "psp-b-r-mtb", + }, + bottom: { + full: "psp-b-b", + "miter-start": "psp-b-b-ml", + "miter-end": "psp-b-b-mr", + "miter-both": "psp-b-b-mlr", + }, + left: { + full: "psp-b-l", + "miter-start": "psp-b-l-mt", + "miter-end": "psp-b-l-mb", + "miter-both": "psp-b-l-mtb", + }, +}; + +export const ALL_BORDER_CLASSES: string[] = Object.values(EDGE_CLASSES) + .map((states) => Object.values(states)) + .flat(); + +/// Toggle exactly the classes for `borders` on `element`, clearing the rest. +export function apply_borders(element: Element, borders: CellBorders): void { + for (const edge of ["top", "right", "bottom", "left"] as const) { + const state = borders[edge]; + for (const [s, class_name] of Object.entries(EDGE_CLASSES[edge])) { + element.classList.toggle(class_name, s === state); + } + } +} diff --git a/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts b/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts index 345b79ac5f..2d6f4b5cb5 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts @@ -14,6 +14,8 @@ import { RegularTableElement } from "regular-table"; import { get_psp_type, type DatagridModel } from "../types.js"; import { CollectedHeaderRow } from "./types.js"; import type { HTMLPerspectiveViewerElement } from "@perspective-dev/viewer"; +import { apply_borders, classify_header_cell } from "./border_model.js"; +import { corner_boundary_x } from "./group_header.js"; /** * Apply selected column styling in response to column settings toggle events. @@ -96,11 +98,11 @@ export function styleColumnHeaderRow( headerRow: CollectedHeaderRow, regularTable: RegularTableElement, is_menu_row: boolean, + is_last_header_row: boolean, + single_header_row: boolean, ): void { - const header_depth = - model._config.group_by.length - - (model._config.group_rollup_mode === "flat" ? 1 : 0); - + const header_depth = corner_boundary_x(model); + const split_by_len = model._config.split_by.length; const selectedColumn = model._column_settings_selected_column; for (const { element: td, metadata } of headerRow.cells) { if ( @@ -116,15 +118,45 @@ export function styleColumnHeaderRow( const sort = model._config.sort.find((x) => x[0] === column_name); const is_corner = typeof metadata.x === "undefined"; - const needs_border = - (metadata.type === "corner" && - metadata.row_header_x === header_depth) || - (!is_corner && - (metadata.x + 1) % model._config.columns.length === 0); + const borders = classify_header_cell({ + paths: model._column_paths, + split_by_len, + x: is_corner ? undefined : metadata.x, + colspan: td.colSpan || 1, + y: split_by_len + (is_menu_row ? 1 : 0), + row_kind: is_menu_row ? "menu" : "name", + is_corner: metadata.type === "corner", + corner_needs_border: + metadata.type === "corner" && + model._config.group_by.length > 0 && + metadata.row_header_x === header_depth, + is_last_header_row, + single_header_row, + }); - td.classList.toggle("psp-header-border", needs_border); + apply_borders(td, borders); + td.classList.toggle("psp-header-border", borders.right !== "none"); td.classList.toggle("psp-header-group", false); td.classList.toggle("psp-header-leaf", true); + + // Under `split_rollup_mode: "rollup"`, columns whose raw path has + // fewer levels than `split_by` are subtotal (or, with zero levels, + // grand-total) column groups. + const n_split_levels = is_corner + ? undefined + : model._column_paths[metadata.x!]?.split("|").length - 1; + const is_rollup_col = + model._config.split_by.length > 0 && + n_split_levels !== undefined && + n_split_levels < model._config.split_by.length; + td.classList.toggle( + "psp-split-total", + is_rollup_col && n_split_levels === 0, + ); + td.classList.toggle( + "psp-split-subtotal", + is_rollup_col && n_split_levels! > 0, + ); td.classList.toggle("psp-is-top", false); td.classList.toggle("psp-header-corner", is_corner); td.classList.toggle( diff --git a/packages/viewer-datagrid/src/ts/style_handlers/editable.ts b/packages/viewer-datagrid/src/ts/style_handlers/editable.ts index aaabedf138..1b499d5f64 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/editable.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/editable.ts @@ -85,19 +85,34 @@ export function applyColumnHeaderStyles( } // Style the actual column header rows + const single_header_row = len <= 1; const colHeadersIndex = model._config.split_by.length; + const menuHeadersIndex = model._config.split_by.length + 1; if (colHeadersIndex < headerRows.length) { const colHeaders = headerRows[colHeadersIndex]; if (colHeaders) { - styleColumnHeaderRow(model, colHeaders, regularTable, false); + styleColumnHeaderRow( + model, + colHeaders, + regularTable, + false, + menuHeadersIndex >= headerRows.length, + single_header_row, + ); } } - const menuHeadersIndex = model._config.split_by.length + 1; if (menuHeadersIndex < headerRows.length) { const menuHeaders = headerRows[menuHeadersIndex]; if (menuHeaders) { - styleColumnHeaderRow(model, menuHeaders, regularTable, true); + styleColumnHeaderRow( + model, + menuHeaders, + regularTable, + true, + true, + single_header_row, + ); } } } diff --git a/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts b/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts index 53e42e238d..f31b607405 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts @@ -14,22 +14,36 @@ import { RegularTableElement } from "regular-table"; import type { DatagridModel } from "../types.js"; import { CollectedHeaderRow } from "./types.js"; +import { apply_borders, classify_header_cell } from "./border_model.js"; + +/// The row-header column index whose right edge separates the row-header +/// region from the data columns. Flat rollup mode has no tree-expander +/// gutter, so its last row-header column is one earlier. +export function corner_boundary_x(model: DatagridModel): number { + return ( + model._config.group_by.length - + (model._config.group_rollup_mode === "flat" ? 1 : 0) + ); +} /** - * Apply styles to group header rows. + * Apply styles to group header rows. Border segments come from the pure + * classification in `border_model.ts`; this handler only reads regular-table + * metadata and toggles classes. Rows at and below the column-name row are + * border-classified by `styleColumnHeaderRow` instead. */ export function applyGroupHeaderStyles( model: DatagridModel, headerRows: CollectedHeaderRow[], regularTable: RegularTableElement, ): void { - const header_depth = model._config.group_by.length; - const m: boolean[][] = []; - let marked = new Set(); + const split_by_len = model._config.split_by.length; + const single_header_row = headerRows.length <= 1; + const boundary_x = corner_boundary_x(model); for (let y = 0; y < headerRows.length; y++) { const { cells } = headerRows[y]; - const tops = new Set(); + const is_group_row = y < split_by_len; for (let x = 0; x < cells.length; x++) { const { element: td, metadata } = cells[x]; @@ -38,18 +52,10 @@ export function applyGroupHeaderStyles( } td.style.backgroundColor = ""; - - const needs_border = - (metadata.type === "corner" && - header_depth > 0 && - metadata.row_header_x === header_depth) || - (metadata.type === "column_header" && metadata.x >= 0); - td.classList.toggle("psp-align-right", false); td.classList.toggle("psp-align-left", false); td.classList.toggle("psp-header-group", true); td.classList.toggle("psp-header-leaf", false); - td.classList.toggle("psp-header-border", needs_border); td.classList.toggle( "psp-header-group-corner", metadata.type === "corner", @@ -61,28 +67,51 @@ export function applyGroupHeaderStyles( td.classList.toggle("psp-header-sort-desc", false); td.classList.toggle("psp-header-sort-col-asc", false); td.classList.toggle("psp-header-sort-col-desc", false); + td.classList.toggle("psp-header-sort-abs-asc", false); + td.classList.toggle("psp-header-sort-abs-desc", false); + td.classList.toggle("psp-header-sort-abs-col-asc", false); + td.classList.toggle("psp-header-sort-abs-col-desc", false); td.classList.toggle("psp-sort-enabled", false); - // Calculate spanning for psp-is-top - let xx = x; - for (; m[y] && m[y][xx]; ++xx) {} + // regular-table recycles ``s across roles - when `split_by` + // grows by 2+ in one step, the old menu row's cells land in a + // group row still carrying their name/menu-row classes, and the + // `.psp-menu-enabled` styling bleeds onto the group cell's + // content span. + td.classList.toggle("psp-menu-enabled", false); + td.classList.toggle("psp-menu-open", false); - tops.add(xx); + if (!is_group_row) { + // The name and menu rows are re-styled (including borders) + // by `styleColumnHeaderRow` in the same pass. + continue; + } - const cell = td; - for (let tx = xx; tx < xx + cell.colSpan; ++tx) { - for (let ty = y; ty < y + cell.rowSpan; ++ty) { - if (!m[ty]) { - m[ty] = []; - } + const is_corner = metadata.type === "corner"; + const is_data = + metadata.type === "column_header" && metadata.x! >= 0; + const borders = classify_header_cell({ + paths: model._column_paths, + split_by_len, + x: is_data ? metadata.x : undefined, + colspan: td.colSpan || 1, + y, + row_kind: "group", + is_corner, + corner_needs_border: + is_corner && + model._config.group_by.length > 0 && + metadata.row_header_x === boundary_x, + is_last_header_row: false, + single_header_row, + }); - m[ty][tx] = true; - } - } + apply_borders(td, borders); - cell.classList.toggle("psp-is-top", y === 0 || !marked.has(xx)); + // Legacy semantic classes, kept for themes and event handlers; + // no CSS in this package draws from them anymore. + td.classList.toggle("psp-header-border", borders.right !== "none"); + td.classList.toggle("psp-is-top", borders.right === "miter-start"); } - - marked = tops; } } diff --git a/packages/viewer-datagrid/src/ts/types.ts b/packages/viewer-datagrid/src/ts/types.ts index f7556853ed..b4b66ab312 100644 --- a/packages/viewer-datagrid/src/ts/types.ts +++ b/packages/viewer-datagrid/src/ts/types.ts @@ -242,6 +242,11 @@ export interface DatagridModel { _num_rows: number; _num_columns?: number; _schema: Schema; + + /// Output types of the view's `windows` columns, synthesized in + /// `create.ts` - a window column used only in `group_by` appears in no + /// queryable schema (`view.schema()` covers visible columns only). + _window_schema: Schema; _theme: string; _ids: unknown[][]; _plugin_background: number[]; diff --git a/packages/viewer-datagrid/test/js/border_model.spec.ts b/packages/viewer-datagrid/test/js/border_model.spec.ts new file mode 100644 index 0000000000..3fe932e3a9 --- /dev/null +++ b/packages/viewer-datagrid/test/js/border_model.spec.ts @@ -0,0 +1,461 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// Pure node tests for the mitered-header border model - no browser, no +// perspective engine. These pin the classification for the cases the old +// bespoke-CSS approach got wrong (N>=2 header group rows, rollup +// total/subtotal groups). + +import { test, expect } from "@perspective-dev/test"; +import { + boundary_depth, + classify_header_cell, + split_levels, + HeaderCellInput, +} from "../../src/ts/style_handlers/border_model.js"; + +function cell(overrides: Partial): HeaderCellInput { + return { + paths: [], + split_by_len: 0, + colspan: 1, + y: 0, + row_kind: "group", + is_corner: false, + is_last_header_row: false, + single_header_row: false, + ...overrides, + }; +} + +test.describe("split_levels", () => { + test("strips the trailing column name", () => { + expect(split_levels("false|b|w", 2)).toStrictEqual(["false", "b"]); + expect(split_levels("false|w", 2)).toStrictEqual(["false"]); + expect(split_levels("w", 2)).toStrictEqual([]); + expect(split_levels("w", 0)).toStrictEqual([]); + }); +}); + +test.describe("boundary_depth", () => { + const paths_2_level = [ + "w", // grand total + "false|w", // subtotal + "false|b|w", + "false|d|w", + "true|w", // subtotal + "true|a|w", + "true|c|w", + ]; + + test("total-to-group boundary is depth 0", () => { + expect(boundary_depth(paths_2_level, 2, 0)).toEqual(0); + }); + + test("subtotal-to-first-child boundary is depth 1", () => { + expect(boundary_depth(paths_2_level, 2, 1)).toEqual(1); + }); + + test("sibling leaf boundary is depth 1", () => { + expect(boundary_depth(paths_2_level, 2, 2)).toEqual(1); + }); + + test("last-child-to-next-group boundary is depth 0", () => { + expect(boundary_depth(paths_2_level, 2, 3)).toEqual(0); + }); + + test("trailing edge is depth 0", () => { + expect(boundary_depth(paths_2_level, 2, 6)).toEqual(0); + }); + + test("aggregate-internal boundary is depth == split_by_len", () => { + const paths = ["false|w", "false|x", "true|w", "true|x"]; + expect(boundary_depth(paths, 1, 0)).toEqual(1); + expect(boundary_depth(paths, 1, 1)).toEqual(0); + }); + + test("total/subtotal aggregate-internal boundaries are never group boundaries", () => { + // With N>=2 aggregates, total/subtotal columns have IDENTICAL + // (short) level lists - the boundary between them is internal to + // one traversal node, not a depth-0/depth-1 group edge. + const paths = [ + "w", + "x", // grand-total group, 2 aggregates + "false|w", + "false|x", // subtotal group, 2 aggregates + "false|b|w", + "false|b|x", + ]; + expect(boundary_depth(paths, 2, 0)).toEqual(2); + expect(boundary_depth(paths, 2, 1)).toEqual(0); + expect(boundary_depth(paths, 2, 2)).toEqual(2); + expect(boundary_depth(paths, 2, 3)).toEqual(1); + expect(boundary_depth(paths, 2, 4)).toEqual(2); + }); + + test("unloaded neighbor is null", () => { + const paths: (string | undefined)[] = ["false|w", undefined, "true|w"]; + expect(boundary_depth(paths, 1, 0)).toBeNull(); + expect(boundary_depth(paths, 1, 1)).toBeNull(); + }); +}); + +test.describe("classify_header_cell", () => { + test.describe("1-level split, flat", () => { + const paths = ["false|w", "true|w"]; + + test("group row: boundary starts mitered at its own row", () => { + const borders = classify_header_cell( + cell({ paths, split_by_len: 1, x: 0, y: 0 }), + ); + expect(borders.right).toEqual("miter-start"); + expect(borders.bottom).toEqual("miter-both"); + }); + + test("name row (last): miter-end at group boundary", () => { + const borders = classify_header_cell( + cell({ + paths, + split_by_len: 1, + x: 0, + y: 1, + row_kind: "name", + is_last_header_row: true, + }), + ); + expect(borders.right).toEqual("miter-end"); + expect(borders.bottom).toEqual("none"); + }); + + test("name row (settings open, not last): full", () => { + const borders = classify_header_cell( + cell({ + paths, + split_by_len: 1, + x: 0, + y: 1, + row_kind: "name", + is_last_header_row: false, + }), + ); + expect(borders.right).toEqual("full"); + }); + + test("menu row is miter-end", () => { + const borders = classify_header_cell( + cell({ + paths, + split_by_len: 1, + x: 0, + y: 2, + row_kind: "menu", + is_last_header_row: true, + }), + ); + expect(borders.right).toEqual("miter-end"); + }); + }); + + test.describe("aggregate-internal boundaries", () => { + const paths = ["false|w", "false|x", "true|w", "true|x"]; + + test("no border inside a group, in any row", () => { + for (const [y, row_kind, last] of [ + [0, "group", false], + [1, "name", true], + ] as const) { + const borders = classify_header_cell( + cell({ + paths, + split_by_len: 1, + x: 0, + y, + row_kind, + is_last_header_row: last, + }), + ); + expect(borders.right).toEqual("none"); + } + }); + + test("group-end column draws in both rows", () => { + expect( + classify_header_cell( + cell({ paths, split_by_len: 1, x: 1, y: 0 }), + ).right, + ).toEqual("miter-start"); + expect( + classify_header_cell( + cell({ + paths, + split_by_len: 1, + x: 1, + y: 1, + row_kind: "name", + is_last_header_row: true, + }), + ).right, + ).toEqual("miter-end"); + }); + }); + + test.describe("2-level split, flat (the N>=2 regression)", () => { + // group A spans x 0..1, group B x 2..3, one aggregate. + const paths = ["A|p|w", "A|q|w", "B|p|w", "B|q|w"]; + + test("sibling boundary: absent in row 0, mitered start in row 1", () => { + expect( + classify_header_cell( + cell({ paths, split_by_len: 2, x: 0, y: 0 }), + ).right, + ).toEqual("none"); + expect( + classify_header_cell( + cell({ paths, split_by_len: 2, x: 0, y: 1 }), + ).right, + ).toEqual("miter-start"); + }); + + test("top-level boundary: mitered start in row 0, FULL in row 1", () => { + // The dropped-edge bug case: a cell needing top-gap right border + // in one row and a continuing full border in the next. + const merged_a = classify_header_cell( + cell({ paths, split_by_len: 2, x: 0, colspan: 2, y: 0 }), + ); + expect(merged_a.right).toEqual("miter-start"); + expect(merged_a.bottom).toEqual("miter-both"); + + expect( + classify_header_cell( + cell({ paths, split_by_len: 2, x: 1, y: 1 }), + ).right, + ).toEqual("full"); + }); + }); + + test.describe("rollup total/subtotal groups", () => { + const paths = [ + "w", + "false|w", + "false|b|w", + "false|d|w", + "true|w", + "true|a|w", + "true|c|w", + ]; + + test("Total label cell: depth-0 border from row 0, NO bottom", () => { + // The Total column's header stack - label and pads alike - + // reads as one open block over its column name. + const borders = classify_header_cell( + cell({ paths, split_by_len: 2, x: 0, y: 0 }), + ); + expect(borders.right).toEqual("miter-start"); + expect(borders.bottom).toEqual("none"); + }); + + test("Total pad cell: continuing full border, no bottom", () => { + const borders = classify_header_cell( + cell({ paths, split_by_len: 2, x: 0, y: 1 }), + ); + expect(borders.right).toEqual("full"); + expect(borders.bottom).toEqual("none"); + }); + + test("subtotal pad: border starts mitered at its own row, no bottom", () => { + const borders = classify_header_cell( + cell({ paths, split_by_len: 2, x: 1, y: 1 }), + ); + expect(borders.right).toEqual("miter-start"); + expect(borders.bottom).toEqual("none"); + }); + + test("no borders between aggregate columns inside total/subtotal groups", () => { + const multi_agg_paths = [ + "w", + "x", + "false|w", + "false|x", + "false|b|w", + "false|b|x", + ]; + // Between "w" and "x" (grand-total interior) and between + // "false|w" and "false|x" (subtotal interior): nothing, in any + // header row. + for (const x of [0, 2]) { + for (const [y, row_kind, last] of [ + [0, "group", false], + [1, "group", false], + [2, "name", true], + ] as const) { + const borders = classify_header_cell( + cell({ + paths: multi_agg_paths, + split_by_len: 2, + x, + y, + row_kind, + is_last_header_row: last, + }), + ); + expect(borders.right).toEqual("none"); + } + } + }); + + test("merged group cell over subtotal + children keeps its bottom", () => { + // regular-table merges the "false" level cell across the + // subtotal column and both children; its right edge is the last + // covered column's boundary (depth 0 vs "true"). + const borders = classify_header_cell( + cell({ paths, split_by_len: 2, x: 1, colspan: 3, y: 0 }), + ); + expect(borders.right).toEqual("miter-start"); + expect(borders.bottom).toEqual("miter-both"); + }); + }); + + test.describe("degenerate cases", () => { + test("single header row draws nothing", () => { + const borders = classify_header_cell( + cell({ + paths: ["a", "b"], + split_by_len: 0, + x: 0, + y: 0, + row_kind: "name", + is_last_header_row: true, + single_header_row: true, + }), + ); + expect(borders).toStrictEqual({ + top: "none", + right: "none", + bottom: "none", + left: "none", + }); + }); + + test("no split_by levels draws nothing at all", () => { + const paths = ["a", "b", "c"]; + for (const x of [0, 1, 2]) { + expect( + classify_header_cell( + cell({ + paths, + split_by_len: 0, + x, + y: 0, + row_kind: "name", + is_last_header_row: false, + }), + ), + ).toStrictEqual({ + top: "none", + right: "none", + bottom: "none", + left: "none", + }); + } + + expect( + classify_header_cell( + cell({ + paths, + split_by_len: 0, + y: 0, + row_kind: "name", + is_corner: true, + corner_needs_border: true, + is_last_header_row: false, + }), + ).right, + ).toEqual("none"); + }); + + test("unloaded neighbor draws nothing", () => { + const paths: (string | undefined)[] = [ + "false|w", + undefined, + "true|w", + ]; + expect( + classify_header_cell( + cell({ paths, split_by_len: 1, x: 0, y: 0 }), + ).right, + ).toEqual("none"); + }); + }); + + test.describe("corner cells", () => { + test("row-header boundary corner: mitered start at top, full below, mitered end at bottom", () => { + expect( + classify_header_cell( + cell({ + paths: [], + split_by_len: 2, + y: 0, + is_corner: true, + corner_needs_border: true, + }), + ).right, + ).toEqual("miter-start"); + expect( + classify_header_cell( + cell({ + paths: [], + split_by_len: 2, + y: 1, + is_corner: true, + corner_needs_border: true, + }), + ).right, + ).toEqual("full"); + expect( + classify_header_cell( + cell({ + paths: [], + split_by_len: 2, + y: 2, + row_kind: "name", + is_corner: true, + corner_needs_border: true, + is_last_header_row: true, + }), + ).right, + ).toEqual("miter-end"); + }); + + test("corner cells have no dividers between them", () => { + // The NW corner region reads as one open block: no bottom + // borders anywhere, and no right borders except the row-header + // boundary column's. + for (const corner_needs_border of [true, false]) { + for (const y of [0, 1]) { + const borders = classify_header_cell( + cell({ + paths: [], + split_by_len: 2, + y, + is_corner: true, + corner_needs_border, + }), + ); + expect(borders.bottom).toEqual("none"); + if (!corner_needs_border) { + expect(borders.right).toEqual("none"); + } + } + } + }); + }); +}); diff --git a/packages/viewer-datagrid/test/js/header_recycling.spec.ts b/packages/viewer-datagrid/test/js/header_recycling.spec.ts new file mode 100644 index 0000000000..40b704d84b --- /dev/null +++ b/packages/viewer-datagrid/test/js/header_recycling.spec.ts @@ -0,0 +1,79 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// regular-table recycles header `` elements across roles when the +// header row count changes. Stale role classes must be cleared by whichever +// style pass claims the recycled cell - regression coverage for the +// menu-row classes bleeding into group header rows (visible as menu-button +// styling on group cells' content spans). + +import { test, expect } from "@perspective-dev/test"; +import type { Page } from "@playwright/test"; + +async function await_ready(page: Page): Promise { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); +} + +test.describe("header th recycling", () => { + test("menu-row classes do not leak into group header rows", async ({ + page, + }) => { + await await_ready(page); + const restore = async (config: Record) => + await page.evaluate(async (config) => { + const viewer = document.querySelector( + "perspective-viewer", + )! as any; + await viewer.restore(config); + await viewer.flush(); + }, config); + + // With settings open, grow `split_by` by 2 in one step: the menu + // row's ``s (which carry `psp-menu-enabled`) are recycled into + // a group header row. + await restore({ + plugin: "Datagrid", + columns: ["Sales", "Profit"], + group_by: ["Region"], + split_by: ["Category"], + settings: true, + }); + await restore({ + settings: true, + split_by: ["Category", "Ship Mode", "Segment"], + }); + + const leaked = await page.evaluate(() => { + const datagrid = document + .querySelector("perspective-viewer")! + .querySelector("perspective-viewer-datagrid") as any; + return Array.from( + datagrid.shadowRoot.querySelectorAll( + "regular-table thead .psp-header-group", + ), + ) + .filter( + (el: any) => + el.classList.contains("psp-menu-enabled") || + el.classList.contains("psp-menu-open"), + ) + .map((el: any) => el.className); + }); + + expect(leaked).toStrictEqual([]); + }); +}); diff --git a/packages/viewer-datagrid/test/js/split_rollup_mode.spec.ts b/packages/viewer-datagrid/test/js/split_rollup_mode.spec.ts new file mode 100644 index 0000000000..e25b390891 --- /dev/null +++ b/packages/viewer-datagrid/test/js/split_rollup_mode.spec.ts @@ -0,0 +1,185 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import type { Page } from "@playwright/test"; + +async function await_ready(page: Page): Promise { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); +} + +async function restore_and_read_headers( + page: Page, + config: Record, +): Promise<{ + header_rows: string[][]; + total_cells: string[]; + border_classes: string[][][]; + corner_cells: boolean[][]; +}> { + return await page.evaluate(async (config) => { + const viewer = document.querySelector("perspective-viewer")! as any; + await viewer.restore(config); + await viewer.flush(); + const datagrid = viewer.querySelector( + "perspective-viewer-datagrid", + ) as any; + + const thead = datagrid.shadowRoot.querySelector("regular-table thead"); + const header_rows = Array.from(thead.querySelectorAll("tr")).map( + (tr: any) => + Array.from(tr.children).map((th: any) => + th.textContent.replace(/\u200b/g, "").trim(), + ), + ) as string[][]; + const total_cells = Array.from( + datagrid.shadowRoot.querySelectorAll( + "regular-table .psp-split-total, regular-table .psp-split-subtotal", + ), + ).map((x: any) => + x.textContent.replace(/\u200b/g, "").trim(), + ) as string[]; + const border_classes = Array.from(thead.querySelectorAll("tr")).map( + (tr: any) => + Array.from(tr.children).map((th: any) => + Array.from(th.classList as DOMTokenList) + .filter((x: any) => (x as string).startsWith("psp-b-")) + .sort(), + ), + ) as string[][][]; + const corner_cells = Array.from(thead.querySelectorAll("tr")).map( + (tr: any) => + Array.from(tr.children).map((th: any) => + th.classList.contains("psp-header-group-corner"), + ), + ) as boolean[][]; + return { header_rows, total_cells, border_classes, corner_cells }; + }, config); +} + +test.describe("split_rollup_mode datagrid rendering", () => { + test("rollup mode renders a leading Total column group", async ({ + page, + }) => { + await await_ready(page); + const { header_rows, total_cells } = await restore_and_read_headers( + page, + { + plugin: "Datagrid", + columns: ["Sales"], + group_by: ["Region"], + split_by: ["Category"], + split_rollup_mode: "rollup", + settings: false, + }, + ); + + expect(header_rows.length).toBeGreaterThanOrEqual(2); + const group_row = header_rows[0]; + expect(group_row.filter((x) => x !== "")[0]).toEqual("Furniture"); + expect(total_cells.length).toBeGreaterThan(0); + }); + + test("flat mode (default) renders no Total group and no rollup classes", async ({ + page, + }) => { + await await_ready(page); + const { total_cells } = await restore_and_read_headers(page, { + plugin: "Datagrid", + columns: ["Sales"], + group_by: ["Region"], + split_by: ["Category"], + settings: false, + }); + + expect(total_cells).toStrictEqual([]); + }); + + test("2-level split pads subtotal headers to the column-name row", async ({ + page, + }) => { + await await_ready(page); + const { header_rows } = await restore_and_read_headers(page, { + plugin: "Datagrid", + columns: ["Sales"], + group_by: ["Region"], + split_by: ["Category", "Ship Mode"], + split_rollup_mode: "rollup", + settings: false, + }); + + // 2 split-level rows + 1 column-name row; the column-name row + // contains "Sales" for every group, subtotals included. + expect(header_rows.length).toBeGreaterThanOrEqual(3); + const name_row = header_rows[header_rows.length - 1]; + expect( + name_row.filter((x) => x !== "").every((x) => x === "Sales"), + ).toBe(true); + }); + + test("mitered border classes follow boundary depth", async ({ page }) => { + await await_ready(page); + const { header_rows, border_classes, corner_cells } = + await restore_and_read_headers(page, { + plugin: "Datagrid", + columns: ["Sales"], + group_by: ["Region"], + split_by: ["Category", "Ship Mode"], + split_rollup_mode: "rollup", + settings: false, + }); + + const total_x = header_rows[0].findIndex( + (text, i) => text === "" && !corner_cells[0][i], + ); + expect(total_x).toBeGreaterThanOrEqual(0); + expect(border_classes[0][total_x]).toContain("psp-b-r-mt"); + expect(border_classes[0][total_x]).not.toContain("psp-b-b-mlr"); + + const pad_x = header_rows[1].findIndex( + (text, i) => + text === "" && + !corner_cells[1][i] && + border_classes[1][i].includes("psp-b-r"), + ); + + expect(pad_x).toBeGreaterThanOrEqual(0); + expect(border_classes[1][pad_x]).not.toContain("psp-b-b-mlr"); + expect(border_classes[1].some((x) => x.includes("psp-b-r-mt"))).toBe( + true, + ); + + const name_y = border_classes.length - 1; + expect( + border_classes[name_y].some((x) => x.includes("psp-b-r-mb")), + ).toBe(true); + }); + + test("single header row draws no mitered borders", async ({ page }) => { + await await_ready(page); + const { border_classes } = await restore_and_read_headers(page, { + plugin: "Datagrid", + columns: ["Sales", "Profit"], + group_by: [], + split_by: [], + settings: false, + }); + + expect(border_classes.length).toEqual(1); + expect(border_classes[0].every((x) => x.length === 0)).toBe(true); + }); +}); diff --git a/packages/viewer-datagrid/test/js/window_group_by.spec.ts b/packages/viewer-datagrid/test/js/window_group_by.spec.ts new file mode 100644 index 0000000000..0ee7568d2e --- /dev/null +++ b/packages/viewer-datagrid/test/js/window_group_by.spec.ts @@ -0,0 +1,83 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// A window column used in the `group_by` position appears in no schema the +// client can query (the pivoted `View`'s `schema()` covers visible columns +// only), which historically made `format_cell` fall back to the raw value +// and crash `cell_style_row_header` mid-draw with `.trim is not a +// function`. The model now synthesizes `_window_schema` from the window +// specs, and `format_cell` coerces its fallback to a string. + +import { test, expect } from "@perspective-dev/test"; +import type { Page } from "@playwright/test"; + +async function await_ready(page: Page): Promise { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); +} + +test.describe("window column in group_by", () => { + test("draws without error and formats row headers", async ({ page }) => { + await await_ready(page); + const errors: string[] = []; + page.on("console", (msg) => { + if (msg.type() === "error" || msg.type() === "warning") { + errors.push(msg.text()); + } + }); + page.on("pageerror", (err) => errors.push(err.message)); + + const out = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + await viewer.restore({ + plugin: "Datagrid", + plugin_config: { edit_mode: "EDIT" }, + group_by: ["New Column 1"], + group_rollup_mode: "rollup", + windows: { + "New Column 1": { column: "Sales", aggregate: "sum" }, + }, + columns: ["Sales"], + settings: true, + }); + await viewer.flush(); + + const datagrid = viewer.querySelector( + "perspective-viewer-datagrid", + ) as any; + + const tree_labels = Array.from( + datagrid.shadowRoot.querySelectorAll( + "regular-table tbody th.psp-tree-label, regular-table tbody th.psp-tree-leaf", + ), + ).map((x: any) => x.textContent.trim()); + return { tree_labels }; + }); + + expect( + errors.filter((x) => x.includes("is not a function")), + ).toStrictEqual([]); + + // Row headers rendered, with float-formatted (not raw) values - + // the harness superstore Sales sums always format with decimals. + expect(out.tree_labels.length).toBeGreaterThan(0); + expect( + out.tree_labels + .filter((x: string) => x !== "" && x !== "TOTAL") + .every((x: string) => /\d/.test(x)), + ).toBe(true); + }); +}); diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index 7cd08cdd4c..fac18573e2 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -217,6 +217,17 @@ enum GroupRollupMode { TOTAL = 2; } +// The `split_by` corollary to `GroupRollupMode`. `FLAT` (the wire default) +// is the historical behavior - only full-depth split combinations are +// emitted as columns. `ROLLUP` additionally emits grand-total and subtotal +// column groups in pre-order ("totals before"). Enum value names are +// prefixed because proto enum values share their parent scope with +// `GroupRollupMode`'s. +enum SplitRollupMode { + SPLIT_ROLLUP_MODE_FLAT = 0; + SPLIT_ROLLUP_MODE_ROLLUP = 1; +} + // Informs the client of the feature set, e.g. what to expect in the // `ViewConfig` message. message GetFeaturesReq {} @@ -232,6 +243,7 @@ message GetFeaturesResp { map window_aggregates = 9; bool unordered = 10; + repeated SplitRollupMode split_rollup_mode = 11; message WindowAggregateOptions { repeated WindowAggregateArgs options = 1; @@ -598,6 +610,7 @@ message ViewConfig { optional uint32 group_by_depth = 9; optional GroupRollupMode group_rollup_mode = 10; map windows = 11; + optional SplitRollupMode split_rollup_mode = 12; message AggList { repeated string aggregations = 1; diff --git a/rust/perspective-client/src/rust/client.rs b/rust/perspective-client/src/rust/client.rs index 3268dc090a..a9dfed0376 100644 --- a/rust/perspective-client/src/rust/client.rs +++ b/rust/perspective-client/src/rust/client.rs @@ -103,6 +103,25 @@ impl Features { }) .collect::>() } + + /// Unlike [`Features::get_group_rollup_modes`], an empty feature list + /// resolves to `[Flat]` rather than "no constraint" - servers predating + /// (or not implementing) split rollup can only produce leaf columns, so + /// absence must not offer the `Rollup` option. + pub fn get_split_rollup_modes(&self) -> Vec { + if self.split_rollup_mode.is_empty() { + return vec![crate::config::SplitRollupMode::Flat]; + } + + self.split_rollup_mode + .iter() + .map(|x| { + crate::config::SplitRollupMode::from( + crate::proto::SplitRollupMode::try_from(*x).unwrap(), + ) + }) + .collect::>() + } } impl Deref for Features { diff --git a/rust/perspective-client/src/rust/config/view_config.rs b/rust/perspective-client/src/rust/config/view_config.rs index c2bad83513..5ca3034df3 100644 --- a/rust/perspective-client/src/rust/config/view_config.rs +++ b/rust/perspective-client/src/rust/config/view_config.rs @@ -67,6 +67,49 @@ impl From for proto::GroupRollupMode { } } +/// The `split_by` corollary to [`GroupRollupMode`]. `Flat` (the default, +/// matching this crate's historical behavior) emits only full-depth split +/// combinations as columns; `Rollup` additionally emits grand-total and +/// subtotal column groups in "totals before" order. There is no `Total` +/// variant - an empty `split_by` already expresses a single grand-total +/// column group. +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq, TS)] +pub enum SplitRollupMode { + #[default] + #[serde(rename = "flat")] + Flat, + + #[serde(rename = "rollup")] + Rollup, +} + +impl Display for SplitRollupMode { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + write!(fmt, "{}", match self { + Self::Flat => "Flat", + Self::Rollup => "Rollup", + }) + } +} + +impl From for SplitRollupMode { + fn from(value: proto::SplitRollupMode) -> Self { + match value { + proto::SplitRollupMode::Flat => Self::Flat, + proto::SplitRollupMode::Rollup => Self::Rollup, + } + } +} + +impl From for proto::SplitRollupMode { + fn from(value: SplitRollupMode) -> Self { + match value { + SplitRollupMode::Flat => proto::SplitRollupMode::Flat, + SplitRollupMode::Rollup => proto::SplitRollupMode::Rollup, + } + } +} + #[derive(Clone, Debug, Deserialize, Default, PartialEq, Serialize, TS)] #[serde(deny_unknown_fields)] pub struct ViewConfig { @@ -86,6 +129,9 @@ pub struct ViewConfig { #[serde(default)] pub group_rollup_mode: GroupRollupMode, + #[serde(default)] + pub split_rollup_mode: SplitRollupMode, + #[serde(skip_serializing_if = "is_default_value")] #[serde(default)] pub filter_op: FilterReducer, @@ -245,6 +291,11 @@ pub struct ViewConfigUpdate { #[serde(default)] #[ts(optional)] pub group_rollup_mode: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + #[ts(optional)] + pub split_rollup_mode: Option, } impl From for proto::ViewConfig { @@ -293,6 +344,9 @@ impl From for proto::ViewConfig { group_rollup_mode: value .group_rollup_mode .map(|x| proto::GroupRollupMode::from(x).into()), + split_rollup_mode: value + .split_rollup_mode + .map(|x| proto::SplitRollupMode::from(x).into()), } } } @@ -329,6 +383,7 @@ impl From for ViewConfigUpdate { aggregates: Some(value.aggregates), group_by_depth: value.group_by_depth, group_rollup_mode: Some(value.group_rollup_mode), + split_rollup_mode: Some(value.split_rollup_mode), } } } @@ -371,6 +426,12 @@ impl From for ViewConfig { .and_then(|x| x.ok()) .map(|x| x.into()) .unwrap_or_default(), + split_rollup_mode: value + .split_rollup_mode + .map(proto::SplitRollupMode::try_from) + .and_then(|x| x.ok()) + .map(|x| x.into()) + .unwrap_or_default(), } } } @@ -389,6 +450,7 @@ impl From for ViewConfig { aggregates: value.aggregates.unwrap_or_default(), group_by_depth: value.group_by_depth, group_rollup_mode: value.group_rollup_mode.unwrap_or_default(), + split_rollup_mode: value.split_rollup_mode.unwrap_or_default(), } } } @@ -431,6 +493,10 @@ impl From for ViewConfigUpdate { .group_rollup_mode .and_then(|x| proto::GroupRollupMode::try_from(x).ok()) .map(|x| x.into()), + split_rollup_mode: value + .split_rollup_mode + .and_then(|x| proto::SplitRollupMode::try_from(x).ok()) + .map(|x| x.into()), } } } @@ -487,6 +553,7 @@ impl ViewConfig { changed = Self::_apply(&mut self.expressions, update.expressions) || changed; changed = Self::_apply(&mut self.windows, update.windows) || changed; changed = Self::_apply(&mut self.group_rollup_mode, update.group_rollup_mode) || changed; + changed = Self::_apply(&mut self.split_rollup_mode, update.split_rollup_mode) || changed; if self.group_rollup_mode == GroupRollupMode::Total && !self.group_by.is_empty() { tracing::info!("`total` incompatible with `group_by`"); changed = true; diff --git a/rust/perspective-client/src/rust/virtual_server/features.rs b/rust/perspective-client/src/rust/virtual_server/features.rs index b94e96e0b7..dfdd833651 100644 --- a/rust/perspective-client/src/rust/virtual_server/features.rs +++ b/rust/perspective-client/src/rust/virtual_server/features.rs @@ -16,7 +16,7 @@ use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::config::GroupRollupMode; +use crate::config::{GroupRollupMode, SplitRollupMode}; use crate::proto::get_features_resp::{AggregateArgs, AggregateOptions, ColumnTypeOptions}; use crate::proto::{ColumnType, GetFeaturesResp, WindowAggregateArgs}; @@ -42,6 +42,13 @@ pub struct Features<'a> { #[ts(optional, as = "Option<_>")] pub split_by: bool, + /// Which `split_rollup_mode` options are supported. Empty (the default) + /// means `["flat"]` - a server must opt in to `"rollup"` explicitly, as + /// it requires emitting subtotal and grand-total column groups. + #[serde(default)] + #[ts(optional, as = "Option<_>")] + pub split_rollup_mode: Vec, + /// Available filter operators per column type. #[serde(default)] #[ts(optional, as = "Option<_>")] @@ -142,6 +149,11 @@ impl<'a> From> for GetFeaturesResp { .iter() .map(|x| crate::proto::GroupRollupMode::from(*x) as i32) .collect(), + split_rollup_mode: value + .split_rollup_mode + .iter() + .map(|x| crate::proto::SplitRollupMode::from(*x) as i32) + .collect(), split_by: value.split_by, expressions: value.expressions, on_update: value.on_update, diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs index e3949feade..2f1c95f1f0 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs @@ -41,7 +41,9 @@ use std::fmt; use indexmap::IndexMap; use serde::Deserialize; -use crate::config::{FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, ViewConfig}; +use crate::config::{ + FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, SplitRollupMode, ViewConfig, +}; use crate::proto::{ColumnType, ViewPort}; use crate::virtual_server::generic_sql_model::table_make_view::ViewQueryContext; @@ -114,9 +116,15 @@ pub(crate) fn column_path_source<'a>( name: &str, config: &'a ViewConfig, ) -> Option<(usize, &'a str)> { + if config.split_by.is_empty() { + return None; + } + + let rollup = config.split_rollup_mode == SplitRollupMode::Rollup; + let mut best: Option<(usize, &'a str)> = None; for (idx, col) in config.columns.iter().flatten().enumerate() { - if name.len() > col.len() + if (name.len() > col.len() || rollup) && name.ends_with(col.as_str()) && best.is_none_or(|(_, b)| col.len() > b.len()) { diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs index 7b1c90d80b..87f0327349 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs @@ -12,8 +12,8 @@ use super::GenericSQLError; use crate::config::{ - Aggregate, Filter, FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, ViewConfig, WindowFrame, - WindowSortDir, WindowSpec, + Aggregate, Filter, FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, SplitRollupMode, + ViewConfig, WindowFrame, WindowSortDir, WindowSpec, }; fn aggregate_to_string(agg: &Aggregate) -> String { @@ -397,47 +397,69 @@ impl<'a> ViewQueryContext<'a> { ); let cols: Vec<&String> = self.config.columns.iter().flatten().collect(); - let from = if cols.is_empty() { - "__PSP_PIVOT_SRC__".to_string() + if self.is_split_rollup() && !cols.is_empty() { + let n = self.config.split_by.len(); + let union = std::iter::once(0u64) + .chain((1..=n).map(|k| (1u64 << k) - 1)) + .map(|mask| { + format!( + "SELECT *, {} AS __CGROUPING_ID__ FROM __PSP_PIVOT_BASE__", + mask + ) + }) + .collect::>() + .join(" UNION ALL "); + + format!( + "WITH __PSP_PIVOT_BASE__ AS ({}), __PSP_PIVOT_SRC__ AS ({}) SELECT * \ + EXCLUDE (__ROW_NUM__) FROM {}", + src, + union, + self.pivot_join(&cols, &["__ROW_NUM__".to_string()]) + ) } else { - self.pivot_join(&cols, &["__ROW_NUM__".to_string()]) - }; + let from = if cols.is_empty() { + "__PSP_PIVOT_SRC__".to_string() + } else { + self.pivot_join(&cols, &["__ROW_NUM__".to_string()]) + }; - format!( - "WITH __PSP_PIVOT_SRC__ AS ({}) SELECT * EXCLUDE (__ROW_NUM__) FROM {}", - src, from - ) + format!( + "WITH __PSP_PIVOT_SRC__ AS ({}) SELECT * EXCLUDE (__ROW_NUM__) FROM {}", + src, from + ) + } }, QueryOrientation::GroupedAndPivoted => { let groups_joined = self.group_col_names.join(", "); - let split_cols_joined = self.pivot_on_expr(); + let split_cols_joined = if self.is_split_rollup() { + format!("ROLLUP({})", self.pivot_on_expr()) + } else { + self.pivot_on_expr() + }; + let mut inner_clauses = self.select_clauses(); inner_clauses.extend(self.row_path_select_clauses()); if !self.is_flat_mode() { inner_clauses.push(self.grouping_id_clause()); } inner_clauses.extend(self.split_select_clauses()); + if self.is_split_rollup() { + inner_clauses.push(self.cgrouping_id_clause()); + } for (sidx, Sort(sort_col, sort_dir)) in self.config.sort.iter().enumerate() { if *sort_dir != SortDir::None && !is_col_sort(sort_dir) { - let agg = self.get_aggregate(sort_col); + let sort_source = self.sort_source_expr(sort_col); if self.is_flat_mode() { inner_clauses.push(format!( - "sum({}({})) OVER (PARTITION BY {}) AS __SORT_{}__", - agg, - self.col_name(sort_col), - groups_joined, - sidx, + "sum({}) OVER (PARTITION BY {}) AS __SORT_{}__", + sort_source, groups_joined, sidx, )); } else { inner_clauses.push(format!( - "sum({}({})) OVER (PARTITION BY {}({}), {}) AS __SORT_{}__", - agg, - self.col_name(sort_col), - self.grouping_fn, - groups_joined, - groups_joined, - sidx, + "sum({}) OVER (PARTITION BY {}({}), {}) AS __SORT_{}__", + sort_source, self.grouping_fn, groups_joined, groups_joined, sidx, )); } } @@ -489,6 +511,31 @@ impl<'a> ViewQueryContext<'a> { let select = self.select_clauses().join(", "); format!("SELECT {} FROM {}{}", select, self.from_expr, where_sql) }, + QueryOrientation::TotalPivoted if self.is_split_rollup() => { + let cols: Vec<&String> = self.config.columns.iter().flatten().collect(); + let mut src_clauses = self.select_clauses(); + src_clauses.push("1 AS __TOTAL_KEY__".to_string()); + src_clauses.extend(self.split_select_clauses()); + src_clauses.push(self.cgrouping_id_clause()); + let src = format!( + "SELECT {} FROM {}{} GROUP BY ROLLUP({})", + src_clauses.join(", "), + self.from_expr, + where_sql, + self.pivot_on_expr(), + ); + + let from = if cols.is_empty() { + "__PSP_PIVOT_SRC__".to_string() + } else { + self.pivot_join(&cols, &["__TOTAL_KEY__".to_string()]) + }; + + format!( + "WITH __PSP_PIVOT_SRC__ AS ({}) SELECT * EXCLUDE (__TOTAL_KEY__) FROM {}", + src, from + ) + }, QueryOrientation::TotalPivoted => { let mut src_clauses: Vec = self .config @@ -611,6 +658,25 @@ impl<'a> ViewQueryContext<'a> { .unwrap_or_else(|| "any_value".to_string()) } + fn sort_source_expr(&self, sort_col: &str) -> String { + let base = format!( + "{}({})", + self.get_aggregate(sort_col), + self.col_name(sort_col) + ); + + if self.is_split_rollup() { + format!( + "CASE WHEN {}({}) = 0 THEN {} END", + self.grouping_fn, + self.pivot_on_expr(), + base + ) + } else { + base + } + } + fn select_clauses(&self) -> Vec { let mut clauses = Vec::new(); if self.needs_aggregation() { @@ -655,15 +721,57 @@ impl<'a> ViewQueryContext<'a> { /// DuckDB's `PIVOT` then drops (matching its native `ON` behavior). fn pivot_on_expr_for(&self, col: &str) -> String { let sep = quote_literal(self.column_separator); - let splits = self - .config - .split_by + let leaf = format!( + "{} || '{}{}'", + self.split_prefix_expr(self.config.split_by.len(), &sep), + sep, + quote_literal(col) + ); + + if !self.is_split_rollup() { + return leaf; + } + + let n = self.config.split_by.len(); + let mut arms = Vec::with_capacity(n); + for rolled in (1..=n).rev() { + let mask = (1u64 << rolled) - 1; + let kept = n - rolled; + let name = if kept == 0 { + format!("'{}'", quote_literal(col)) + } else { + format!( + "{} || '{}{}'", + self.split_prefix_expr(kept, &sep), + sep, + quote_literal(col) + ) + }; + + arms.push(format!("WHEN {} THEN {}", mask, name)); + } + + format!("CASE __CGROUPING_ID__ {} ELSE {} END", arms.join(" "), leaf) + } + + fn split_prefix_expr(&self, kept: usize, sep: &str) -> String { + self.config.split_by[..kept] .iter() .map(|c| format!("\"{}\"", quote_ident(c))) .collect::>() - .join(&format!(" || '{}' || ", sep)); + .join(&format!(" || '{}' || ", sep)) + } + + fn is_split_rollup(&self) -> bool { + self.config.split_rollup_mode == SplitRollupMode::Rollup && !self.config.split_by.is_empty() + } - format!("{} || '{}{}'", splits, sep, quote_literal(col)) + fn cgrouping_id_clause(&self) -> String { + format!( + "{}({}) AS __CGROUPING_ID__", + self.grouping_fn, + self.pivot_on_expr() + ) } /// Builds a `FROM` expression pivoting `__PSP_PIVOT_SRC__` once per data diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs index 7a4bdef98f..f17447868a 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs @@ -738,6 +738,7 @@ fn test_table_make_view_total_pivoted_aggregate() { #[test] fn test_column_path_source() { let mut config = ViewConfig::default(); + config.split_by = vec!["state".to_string()]; config.columns = vec![ Some("price".to_string()), Some("total_price".to_string()), @@ -760,6 +761,30 @@ fn test_column_path_source() { assert_eq!(column_path_source("__ROW_PATH_0__", &config), None); } +#[test] +fn test_column_path_source_no_split_by_never_matches() { + let mut config = ViewConfig::default(); + config.group_by = vec!["State".to_string()]; + config.columns = vec![ + Some("Category".to_string()), + Some("Sub-Category".to_string()), + ]; + + assert_eq!(column_path_source("Sub-Category", &config), None); + assert_eq!(column_path_source("Category", &config), None); + + // The same shadowing pair resolves correctly once a pivot exists. + config.split_by = vec!["Region".to_string()]; + assert_eq!( + column_path_source("West|Sub-Category", &config), + Some((1, "Sub-Category")) + ); + assert_eq!( + column_path_source("West|Category", &config), + Some((0, "Category")) + ); +} + #[test] fn test_sort_column_paths_value_major() { let mut config = ViewConfig::default(); diff --git a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts index b41a0855a5..e0c0e83ceb 100644 --- a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts +++ b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts @@ -305,6 +305,7 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { boolean: WINDOW_AGGREGATES_ANY, }, group_rollup_mode: ["rollup", "flat", "total"], + split_rollup_mode: ["flat", "rollup"], filter_ops: { integer: FILTER_OPS, float: FILTER_OPS, diff --git a/rust/perspective-js/test/js/aggregates.spec.js b/rust/perspective-js/test/js/aggregates.spec.js index 67390c63b1..4c565a73b2 100644 --- a/rust/perspective-js/test/js/aggregates.spec.js +++ b/rust/perspective-js/test/js/aggregates.spec.js @@ -2431,6 +2431,95 @@ const std = (nums) => { }); }); + test.describe("Invalid aggregates", function () { + const rejects = async (aggregates, message) => { + const table = await perspective.table(data); + await expect( + table.view({ group_by: ["y"], columns: ["x"], aggregates }), + ).rejects.toThrow(message); + + // The `Table` survives a rejected `View` - the aggregate is + // refused before any context is built. + const view = await table.view({ + group_by: ["y"], + columns: ["x"], + aggregates: { x: "sum" }, + }); + + expect(await view.to_columns()).toEqual({ + __ROW_PATH__: [[], ["a"], ["b"], ["c"], ["d"]], + x: [10, 1, 2, 3, 4], + }); + + await view.delete(); + await table.delete(); + }; + + test("unrecognized aggregate name", async function () { + await rejects( + { x: "sumz" }, + "Abort(): Invalid aggregate 'sumz' for column 'x' found in View aggregates.", + ); + }); + + test("`identity` is named but unimplemented", async function () { + await rejects( + { x: "identity" }, + "Abort(): Unimplemented aggregate 'identity' for column 'x' found in View aggregates.", + ); + }); + + test("`mean by count` is named but unimplemented", async function () { + await rejects( + { x: "mean by count" }, + "Abort(): Unimplemented aggregate 'mean by count' for column 'x' found in View aggregates.", + ); + }); + + test("`div` cannot be configured from a ViewConfig", async function () { + await rejects( + { x: "div" }, + "Abort(): Unimplemented aggregate 'div' for column 'x' found in View aggregates.", + ); + }); + + test("argument-taking aggregate with no argument", async function () { + await rejects( + { x: "weighted mean" }, + "Abort(): Aggregate 'weighted mean' for column 'x' requires a column argument.", + ); + }); + + test("argument-taking aggregate with an unknown argument", async function () { + await rejects( + { x: ["weighted mean", ["nosuchcolumn"]] }, + "Abort(): Invalid column 'nosuchcolumn' found in the 'weighted mean' aggregate for column 'x'.", + ); + }); + + test("snake_case argument-taking aggregates read their argument", async function () { + const table = await perspective.table([ + { x: 1, w: 1, y: "a" }, + { x: 3, w: 3, y: "a" }, + ]); + + const view = await table.view({ + group_by: ["y"], + columns: ["x"], + aggregates: { x: ["weighted_mean", ["w"]] }, + }); + + // (1*1 + 3*3) / (1 + 3), not null. + expect(await view.to_columns()).toEqual({ + __ROW_PATH__: [[], ["a"]], + x: [2.5, 2.5], + }); + + await view.delete(); + await table.delete(); + }); + }); + test.describe("Aggregates with negatives", function () { test("sum abs", async function () { var table = await perspective.table([ diff --git a/rust/perspective-js/test/js/duckdb/shadowed_columns.spec.js b/rust/perspective-js/test/js/duckdb/shadowed_columns.spec.js new file mode 100644 index 0000000000..325b9880e0 --- /dev/null +++ b/rust/perspective-js/test/js/duckdb/shadowed_columns.spec.js @@ -0,0 +1,47 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import { describeDuckDB } from "./setup.js"; + +describeDuckDB("shadowed columns", (getClient) => { + test("group_by view schema() keeps a column which suffixes another", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + group_by: ["State"], + columns: ["Category", "Sub-Category", "Sales"], + }); + + expect(await view.schema()).toEqual({ + Category: "string", + "Sub-Category": "string", + Sales: "float", + }); + + await view.delete(); + }); + + test("split_by view schema() resolves both shadower and shadowed", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + split_by: ["Region"], + columns: ["Category", "Sub-Category"], + }); + + expect(await view.schema()).toEqual({ + Category: "string", + "Sub-Category": "string", + }); + + await view.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/duckdb/split_rollup_mode.spec.js b/rust/perspective-js/test/js/duckdb/split_rollup_mode.spec.js new file mode 100644 index 0000000000..6d4fca9bbb --- /dev/null +++ b/rust/perspective-js/test/js/duckdb/split_rollup_mode.spec.js @@ -0,0 +1,206 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// The DuckDB virtual-server mirror of the engine's `split_rollup_mode` +// suite. The `underscore_test` table (4 rows, known values) pins exact +// values AND regression-covers PIVOT's `_`-separator quirk (#3187) against +// the CASE-truncated subtotal names. + +import { test, expect } from "@perspective-dev/test"; +import { describeDuckDB } from "./setup.js"; + +describeDuckDB("split_rollup_mode", (getClient) => { + test("rollup emits grand-total column group in totals-before order", async function () { + const table = await getClient().open_table("memory.underscore_test"); + const view = await table.view({ + columns: ["total_sales"], + group_by: ["region_name"], + split_by: ["sub_region"], + aggregates: { total_sales: "sum" }, + split_rollup_mode: "rollup", + }); + const paths = await view.column_paths(); + expect(paths).toEqual([ + "total_sales", + "bay_area|total_sales", + "la_metro|total_sales", + "new_jersey|total_sales", + "new_york|total_sales", + ]); + + const cols = await view.to_columns(); + expect(cols).toEqual({ + __ROW_PATH__: [[], ["east_coast"], ["west_coast"]], + total_sales: [1001.5, 300.75, 700.75], + "bay_area|total_sales": [300.75, null, 300.75], + "la_metro|total_sales": [400, null, 400], + "new_jersey|total_sales": [200.25, 200.25, null], + "new_york|total_sales": [100.5, 100.5, null], + }); + await view.delete(); + }); + + test("2-level split emits interleaved subtotal column groups", async function () { + const table = await getClient().open_table("memory.underscore_test"); + const view = await table.view({ + columns: ["total_sales"], + group_by: ["account_number"], + split_by: ["region_name", "sub_region"], + aggregates: { total_sales: "sum" }, + split_rollup_mode: "rollup", + }); + const paths = await view.column_paths(); + expect(paths).toEqual([ + "total_sales", + "east_coast|total_sales", + "east_coast|new_jersey|total_sales", + "east_coast|new_york|total_sales", + "west_coast|total_sales", + "west_coast|bay_area|total_sales", + "west_coast|la_metro|total_sales", + ]); + + const cols = await view.to_columns(); + expect(cols).toEqual({ + __ROW_PATH__: [[], [1], [2], [3], [4]], + total_sales: [1001.5, 100.5, 200.25, 300.75, 400], + "east_coast|total_sales": [300.75, 100.5, 200.25, null, null], + "east_coast|new_jersey|total_sales": [ + 200.25, + null, + 200.25, + null, + null, + ], + "east_coast|new_york|total_sales": [100.5, 100.5, null, null, null], + "west_coast|total_sales": [700.75, null, null, 300.75, 400], + "west_coast|bay_area|total_sales": [ + 300.75, + null, + null, + 300.75, + null, + ], + "west_coast|la_metro|total_sales": [400, null, null, null, 400], + }); + await view.delete(); + }); + + test("row sort orders by the un-inflated group total", async function () { + const table = await getClient().open_table("memory.underscore_test"); + const view = await table.view({ + columns: ["total_sales"], + group_by: ["region_name"], + split_by: ["sub_region"], + aggregates: { total_sales: "sum" }, + split_rollup_mode: "rollup", + sort: [["total_sales", "desc"]], + }); + + // west_coast (700.75) before east_coast (300.75) - the sort key + // must sum split-LEAF source rows only, or the rollup duplicates + // inflate it non-uniformly. + const cols = await view.to_columns(); + expect(cols["__ROW_PATH__"]).toEqual([ + [], + ["west_coast"], + ["east_coast"], + ]); + expect(cols["total_sales"]).toEqual([1001.5, 700.75, 300.75]); + await view.delete(); + }); + + test("group_rollup_mode total interop", async function () { + const table = await getClient().open_table("memory.underscore_test"); + const view = await table.view({ + columns: ["total_sales"], + split_by: ["sub_region"], + aggregates: { total_sales: "sum" }, + group_rollup_mode: "total", + split_rollup_mode: "rollup", + }); + const json = await view.to_json(); + expect(json).toEqual([ + { + total_sales: 1001.5, + "bay_area|total_sales": 300.75, + "la_metro|total_sales": 400, + "new_jersey|total_sales": 200.25, + "new_york|total_sales": 100.5, + }, + ]); + await view.delete(); + }); + + test("column_only emits a grand-total coalesce column", async function () { + const table = await getClient().open_table("memory.underscore_test"); + const view = await table.view({ + columns: ["total_sales"], + split_by: ["region_name"], + split_rollup_mode: "rollup", + }); + const cols = await view.to_columns(); + expect(cols).toEqual({ + total_sales: [100.5, 200.25, 300.75, 400], + "east_coast|total_sales": [100.5, 200.25, null, null], + "west_coast|total_sales": [null, null, 300.75, 400], + }); + await view.delete(); + }); + + test("flat mode is unchanged", async function () { + const table = await getClient().open_table("memory.underscore_test"); + const rollup_view = await table.view({ + columns: ["total_sales"], + group_by: ["region_name"], + split_by: ["sub_region"], + aggregates: { total_sales: "sum" }, + split_rollup_mode: "flat", + }); + const default_view = await table.view({ + columns: ["total_sales"], + group_by: ["region_name"], + split_by: ["sub_region"], + aggregates: { total_sales: "sum" }, + }); + expect(await rollup_view.to_columns()).toEqual( + await default_view.to_columns(), + ); + await rollup_view.delete(); + await default_view.delete(); + }); + + test("superstore parity: grand totals equal the unsplit view", async function () { + const table = await getClient().open_table("memory.superstore"); + const rollup_view = await table.view({ + columns: ["Sales"], + group_by: ["Region"], + split_by: ["Category"], + aggregates: { Sales: "sum" }, + split_rollup_mode: "rollup", + }); + const unsplit_view = await table.view({ + columns: ["Sales"], + group_by: ["Region"], + aggregates: { Sales: "sum" }, + }); + + const rollup_cols = await rollup_view.to_columns(); + const unsplit_cols = await unsplit_view.to_columns(); + expect(rollup_cols["__ROW_PATH__"]).toEqual( + unsplit_cols["__ROW_PATH__"], + ); + expect(rollup_cols["Sales"]).toEqual(unsplit_cols["Sales"]); + await rollup_view.delete(); + await unsplit_view.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/filters.spec.js b/rust/perspective-js/test/js/filters.spec.js index dedda9308b..2e546a3c2e 100644 --- a/rust/perspective-js/test/js/filters.spec.js +++ b/rust/perspective-js/test/js/filters.spec.js @@ -619,6 +619,7 @@ const datetime_data_local = [ sort: [], split_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", }); view.delete(); diff --git a/rust/perspective-js/test/js/split_rollup_mode.spec.js b/rust/perspective-js/test/js/split_rollup_mode.spec.js new file mode 100644 index 0000000000..b98ffd82fe --- /dev/null +++ b/rust/perspective-js/test/js/split_rollup_mode.spec.js @@ -0,0 +1,405 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "./perspective_client"; + +const data = { + w: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5], + x: [1, 2, 3, 4, 4, 3, 2, 1], + y: ["a", "b", "c", "d", "a", "b", "c", "d"], + z: [true, false, true, false, true, false, true, false], +}; + +((perspective) => { + test.describe("split_rollup_mode", function () { + test.describe("rollup", function () { + test("emits grand-total column group in totals-before order", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + const paths = await view.column_paths(); + expect(paths).toStrictEqual(["w", "false|w", "true|w"]); + expect(await view.num_columns()).toEqual(3); + view.delete(); + table.delete(); + }); + + test("grand-total column values aggregate across all splits", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + const cols = await view.to_columns(); + expect(cols).toStrictEqual({ + __ROW_PATH__: [[], ["a"], ["b"], ["c"], ["d"]], + w: [40, 7, 9, 11, 13], + "false|w": [22, null, 9, null, 13], + "true|w": [18, 7, null, 11, null], + }); + view.delete(); + table.delete(); + }); + + test("2-level split emits interleaved subtotal column groups", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["x"], + split_by: ["z", "y"], + split_rollup_mode: "rollup", + }); + const paths = await view.column_paths(); + expect(paths).toStrictEqual([ + "w", + "false|w", + "false|b|w", + "false|d|w", + "true|w", + "true|a|w", + "true|c|w", + ]); + + const cols = await view.to_columns(); + expect(cols).toStrictEqual({ + __ROW_PATH__: [[], [1], [2], [3], [4]], + w: [40, 10, 10, 10, 10], + "false|w": [22, 8.5, 2.5, 6.5, 4.5], + "false|b|w": [9, null, 2.5, 6.5, null], + "false|d|w": [13, 8.5, null, null, 4.5], + "true|w": [18, 1.5, 7.5, 3.5, 5.5], + "true|a|w": [7, 1.5, null, null, 5.5], + "true|c|w": [11, null, 7.5, 3.5, null], + }); + view.delete(); + table.delete(); + }); + + test("row sort works and total row remains first", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + sort: [["w", "desc"]], + }); + const cols = await view.to_columns(); + expect(cols).toStrictEqual({ + __ROW_PATH__: [[], ["d"], ["c"], ["b"], ["a"]], + w: [40, 13, 11, 9, 7], + "false|w": [22, 13, null, 9, null], + "true|w": [18, null, 11, null, 7], + }); + view.delete(); + table.delete(); + }); + + test("hidden sort columns stay hidden", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + sort: [["x", "asc"]], + }); + const paths = await view.column_paths(); + expect(paths).toStrictEqual(["w", "false|w", "true|w"]); + expect(await view.num_columns()).toEqual(3); + view.delete(); + table.delete(); + }); + + test("group_rollup_mode flat interop", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + group_rollup_mode: "flat", + split_rollup_mode: "rollup", + }); + const cols = await view.to_columns(); + expect(cols).toStrictEqual({ + __ROW_PATH__: [["a"], ["b"], ["c"], ["d"]], + w: [7, 9, 11, 13], + "false|w": [null, 9, null, 13], + "true|w": [7, null, 11, null], + }); + view.delete(); + table.delete(); + }); + + test("column_only emits a grand-total coalesce column", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + const paths = await view.column_paths(); + expect(paths).toStrictEqual(["w", "false|w", "true|w"]); + + // Each column-only row is a single-row group, so the + // grand-total column coalesces the row's value across all + // split groups. + const cols = await view.to_columns(); + expect(cols).toStrictEqual({ + w: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5], + "false|w": [null, 2.5, null, 4.5, null, 6.5, null, 8.5], + "true|w": [1.5, null, 3.5, null, 5.5, null, 7.5, null], + }); + view.delete(); + table.delete(); + }); + + test("group_rollup_mode total interop", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_rollup_mode: "total", + split_by: ["z"], + split_rollup_mode: "rollup", + }); + const paths = await view.column_paths(); + expect(paths).toStrictEqual(["w", "false|w", "true|w"]); + + const cols = await view.to_columns(); + expect(cols).toStrictEqual({ + w: [40], + "false|w": [22], + "true|w": [18], + }); + view.delete(); + table.delete(); + }); + + test("multiple aggregate columns group per traversal node", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w", "x"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + const paths = await view.column_paths(); + expect(paths).toStrictEqual([ + "w", + "x", + "false|w", + "false|x", + "true|w", + "true|x", + ]); + view.delete(); + table.delete(); + }); + + test("column viewport windows index the widened space", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + const first = await view.to_columns({ + start_col: 0, + end_col: 1, + }); + expect(Object.keys(first)).toStrictEqual(["__ROW_PATH__", "w"]); + expect(first["w"]).toStrictEqual([40, 7, 9, 11, 13]); + + const second = await view.to_columns({ + start_col: 1, + end_col: 2, + }); + expect(Object.keys(second)).toStrictEqual([ + "__ROW_PATH__", + "false|w", + ]); + view.delete(); + table.delete(); + }); + + test("empty column viewport window with sort", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + sort: [["w", "desc"]], + }); + const cols = await view.to_columns({ + start_col: 1, + end_col: 1, + }); + expect(cols["false|w"]).toBeUndefined(); + expect(cols["true|w"]).toBeUndefined(); + expect(cols["w"]).toBeUndefined(); + view.delete(); + table.delete(); + }); + + test("empty result with sort does not crash", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + sort: [["w", "desc"]], + filter: [["x", "<", 0]], + }); + expect(await view.num_rows()).toEqual(1); + await view.to_columns(); + view.delete(); + table.delete(); + }); + + test("updates after table.update()", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + table.update([{ w: 10, y: "a", z: false }]); + const cols = await view.to_columns(); + expect(cols).toStrictEqual({ + __ROW_PATH__: [[], ["a"], ["b"], ["c"], ["d"]], + w: [50, 17, 9, 11, 13], + "false|w": [32, 10, 9, null, 13], + "true|w": [18, 7, null, 11, null], + }); + view.delete(); + table.delete(); + }); + + test("on_update row delta stride matches the widened column set", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + + const result = new Promise((resolve) => { + view.on_update( + async (updated) => { + const t2 = await perspective.table(updated.delta); + const v2 = await t2.view(); + const out = await v2.to_columns(); + v2.delete(); + t2.delete(); + resolve(out); + }, + { mode: "row" }, + ); + }); + + table.update([{ w: 10, y: "a", z: false }]); + const delta = await result; + + // The delta batch carries one column per emitted column + // path, total and subtotal groups included. + expect(Object.keys(delta).sort()).toStrictEqual( + ["w", "false|w", "true|w"].sort(), + ); + view.delete(); + table.delete(); + }); + + test("view.get_config() round-trips the mode", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "rollup", + }); + const config = await view.get_config(); + expect(config.split_rollup_mode).toEqual("rollup"); + view.delete(); + table.delete(); + }); + }); + + test.describe("flat (default)", function () { + test("default emits leaves only, unchanged", async function () { + const table = await perspective.table(data); + const view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + }); + const paths = await view.column_paths(); + expect(paths).toStrictEqual(["false|w", "true|w"]); + const config = await view.get_config(); + expect(config.split_rollup_mode).toEqual("flat"); + view.delete(); + table.delete(); + }); + + test("explicit flat equals default", async function () { + const table = await perspective.table(data); + const default_view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + }); + const flat_view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_by: ["z"], + split_rollup_mode: "flat", + }); + expect(await flat_view.to_columns()).toStrictEqual( + await default_view.to_columns(), + ); + flat_view.delete(); + default_view.delete(); + table.delete(); + }); + + test("no split_by ignores the mode", async function () { + const table = await perspective.table(data); + const rollup_view = await table.view({ + columns: ["w"], + group_by: ["y"], + split_rollup_mode: "rollup", + }); + const default_view = await table.view({ + columns: ["w"], + group_by: ["y"], + }); + expect(await rollup_view.to_columns()).toStrictEqual( + await default_view.to_columns(), + ); + rollup_view.delete(); + default_view.delete(); + table.delete(); + }); + }); + }); +})(perspective); diff --git a/rust/perspective-js/test/js/to_format/to_format_regressions.spec.js b/rust/perspective-js/test/js/to_format/to_format_regressions.spec.js index d326e8fe04..aaa27570f7 100644 --- a/rust/perspective-js/test/js/to_format/to_format_regressions.spec.js +++ b/rust/perspective-js/test/js/to_format/to_format_regressions.spec.js @@ -35,6 +35,7 @@ test.describe("to_format regressions", function () { split_by: ["Sub-Category"], // sort: [["Customer Name", "desc"]], group_rollup_mode: "rollup", + split_rollup_mode: "flat", columns: ["Sales", "Quantity", "Discount", "Profit"], }); @@ -51,6 +52,7 @@ test.describe("to_format regressions", function () { split_by: ["Sub-Category"], sort: [["Customer Name", "desc"]], group_rollup_mode: "rollup", + split_rollup_mode: "flat", columns: ["Sales", "Quantity", "Discount", "Profit"], }); diff --git a/rust/perspective-js/test/js/view_config.spec.js b/rust/perspective-js/test/js/view_config.spec.js index 56caa8ae6f..daffa676b9 100644 --- a/rust/perspective-js/test/js/view_config.spec.js +++ b/rust/perspective-js/test/js/view_config.spec.js @@ -59,6 +59,7 @@ const data = [ sort: [], split_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", }); view.delete(); @@ -83,6 +84,7 @@ const data = [ sort: [], split_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", }); view.delete(); @@ -112,6 +114,7 @@ const data = [ sort: [], split_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", }); view.delete(); @@ -146,6 +149,7 @@ const data = [ sort: [], split_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", }); view.delete(); @@ -170,6 +174,7 @@ const data = [ sort: [], split_by: [], group_rollup_mode: "rollup", + split_rollup_mode: "flat", }); view.delete(); diff --git a/rust/perspective-python/perspective/virtual_servers/duckdb.py b/rust/perspective-python/perspective/virtual_servers/duckdb.py index 87b9e1bfad..31c8c58f09 100644 --- a/rust/perspective-python/perspective/virtual_servers/duckdb.py +++ b/rust/perspective-python/perspective/virtual_servers/duckdb.py @@ -179,6 +179,7 @@ def get_features(self): "sort": True, "expressions": True, "group_rollup_mode": ["rollup", "flat", "total"], + "split_rollup_mode": ["flat", "rollup"], "filter_ops": { "integer": FILTER_OPS, "float": FILTER_OPS, diff --git a/rust/perspective-python/perspective/widget/viewer/viewer.py b/rust/perspective-python/perspective/widget/viewer/viewer.py index 62f9ad73a0..50ef157a7a 100644 --- a/rust/perspective-python/perspective/widget/viewer/viewer.py +++ b/rust/perspective-python/perspective/widget/viewer/viewer.py @@ -57,6 +57,7 @@ def __init__( group_by=None, split_by=None, group_rollup_mode=None, + split_rollup_mode=None, aggregates=None, sort=None, filter=None, @@ -83,6 +84,8 @@ def __init__( to use as split by. group_rollup_mode (:obj:`str`): How group-by rollups render; one of ``rollup``, ``flat``, or ``total``. + split_rollup_mode (:obj:`str`): Whether split-by emits subtotal + and grand-total column groups; one of ``flat`` or ``rollup``. aggregates (:obj:`dict` of :obj:`str` to :obj:`str`): A dictionary of column names to aggregate types, which specify aggregates for individual columns. @@ -128,6 +131,7 @@ def __init__( self.group_by = group_by or [] # validate_group_by(group_by) or [] self.split_by = split_by or [] # validate_split_by(split_by) or [] self.group_rollup_mode = group_rollup_mode or "rollup" + self.split_rollup_mode = split_rollup_mode or "flat" self.aggregates = aggregates or {} # validate_aggregates(aggregates) or {} self.sort = sort or [] # validate_sort(sort) or [] self.filter = filter or [] # validate_filter(filter) or [] diff --git a/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py b/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py index 6864077b4f..d00cea786f 100644 --- a/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py +++ b/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py @@ -40,6 +40,7 @@ class PerspectiveTraitlets(HasTraits): group_rollup_mode = Enum(("rollup", "flat", "total"), default_value="rollup").tag( sync=True ) + split_rollup_mode = Enum(("flat", "rollup"), default_value="flat").tag(sync=True) aggregates = Dict(default_value={}).tag(sync=True) sort = List(default_value=[]).tag(sync=True) filter = List(default_value=[]).tag(sync=True) diff --git a/rust/perspective-server/cpp/perspective/CMakeLists.txt b/rust/perspective-server/cpp/perspective/CMakeLists.txt index cabb6e9218..9df7c5611c 100644 --- a/rust/perspective-server/cpp/perspective/CMakeLists.txt +++ b/rust/perspective-server/cpp/perspective/CMakeLists.txt @@ -124,8 +124,15 @@ else() endif() if(DEFINED ENV{PSP_USE_CCACHE}) - set(CMAKE_C_COMPILER_LAUNCHER ccache) - set(CMAKE_CXX_COMPILER_LAUNCHER ccache) + # `PSP_CCACHE_BINARY` selects an alternate launcher (CI uses `sccache` + # on Windows, where MSVC support in ccache is unreliable) + if(DEFINED ENV{PSP_CCACHE_BINARY}) + set(CMAKE_C_COMPILER_LAUNCHER $ENV{PSP_CCACHE_BINARY}) + set(CMAKE_CXX_COMPILER_LAUNCHER $ENV{PSP_CCACHE_BINARY}) + else() + set(CMAKE_C_COMPILER_LAUNCHER ccache) + set(CMAKE_CXX_COMPILER_LAUNCHER ccache) + endif() endif() if(NOT DEFINED PSP_CPP_BUILD) diff --git a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp index 0f7b145596..8452ef55d4 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp @@ -492,8 +492,8 @@ sorttype_to_str(t_sorttype type) { } } -t_aggtype -str_to_aggtype(const std::string& str) { +std::optional +maybe_str_to_aggtype(const std::string& str) { if (str == "distinct count" || str == "distinctcount" || str == "distinct" || str == "distinct_count") { return t_aggtype::AGGTYPE_DISTINCT_COUNT; @@ -623,12 +623,54 @@ str_to_aggtype(const std::string& str) { return t_aggtype::AGGTYPE_GMV; } - std::stringstream ss; - ss << "Encountered unknown aggregate operation: '" << str << "'" - << "\n"; - PSP_COMPLAIN_AND_ABORT(ss.str()); - // use any as default - return t_aggtype::AGGTYPE_ANY; + return std::nullopt; +} + +t_aggtype +str_to_aggtype(const std::string& str) { + const auto agg = maybe_str_to_aggtype(str); + if (!agg) { + std::stringstream ss; + ss << "Invalid aggregate '" << str << "'." << '\n'; + PSP_COMPLAIN_AND_ABORT(ss.str()); + // use any as default + return t_aggtype::AGGTYPE_ANY; + } + + return *agg; +} + +bool +is_implemented_aggtype(t_aggtype agg) { + switch (agg) { + case t_aggtype::AGGTYPE_IDENTITY: + case t_aggtype::AGGTYPE_MEAN_BY_COUNT: + case t_aggtype::AGGTYPE_PY_AGG: + case t_aggtype::AGGTYPE_SCALED_DIV: + case t_aggtype::AGGTYPE_SCALED_ADD: + case t_aggtype::AGGTYPE_SCALED_MUL: + case t_aggtype::AGGTYPE_UDF_COMBINER: + case t_aggtype::AGGTYPE_UDF_REDUCER: { + return false; + } + default: { + return true; + } + } +} + +bool +aggtype_takes_argument(t_aggtype agg) { + switch (agg) { + case t_aggtype::AGGTYPE_WEIGHTED_MEAN: + case t_aggtype::AGGTYPE_MAX_BY: + case t_aggtype::AGGTYPE_MIN_BY: { + return true; + } + default: { + return false; + } + } } t_aggtype diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp index 38c04ce9e6..cab874e870 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp @@ -423,7 +423,8 @@ t_ctx2::get_data(const std::vector& rows) const { // Perspective generates extra headers for columns in the sort, which // needs to be skipped when generating row deltas. - bool should_skip_column_headers = !m_sortby.empty() && start_col < end_col; + bool should_skip_column_headers = !m_sortby.empty() + && !m_config.is_split_rollup() && start_col < end_col; if (should_skip_column_headers) { auto depth = m_config.get_num_cpivots(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index ec251454a5..92530efd17 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -200,7 +200,9 @@ make_context( auto column_pivot_depth = view_config->get_column_pivot_depth(); auto expressions = view_config->get_used_expressions(); - t_totals total = !sortspec.empty() ? TOTALS_BEFORE : TOTALS_HIDDEN; + bool split_rollup = view_config->is_split_rollup(); + t_totals total = + (split_rollup || !sortspec.empty()) ? TOTALS_BEFORE : TOTALS_HIDDEN; auto cfg = t_config( row_pivots, @@ -214,6 +216,7 @@ make_context( view_config->get_windows() ); cfg.set_backing_store(table->get_backing_store()); + cfg.set_split_rollup(split_rollup); auto ctx2 = std::make_shared(*schema, cfg); ctx2->init(); @@ -1499,6 +1502,12 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { features->add_group_rollup_mode(proto::GroupRollupMode::ROLLUP); features->add_group_rollup_mode(proto::GroupRollupMode::FLAT); features->add_group_rollup_mode(proto::GroupRollupMode::TOTAL); + features->add_split_rollup_mode( + proto::SplitRollupMode::SPLIT_ROLLUP_MODE_FLAT + ); + features->add_split_rollup_mode( + proto::SplitRollupMode::SPLIT_ROLLUP_MODE_ROLLUP + ); proto::GetFeaturesResp_ColumnTypeOptions opts; opts.add_options("=="); opts.add_options("!="); @@ -1551,6 +1560,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { string_opts.add_aggregates()->set_name("count"); string_opts.add_aggregates()->set_name("any"); string_opts.add_aggregates()->set_name("distinct count"); + string_opts.add_aggregates()->set_name("distinct leaf"); string_opts.add_aggregates()->set_name("dominant"); string_opts.add_aggregates()->set_name("first"); string_opts.add_aggregates()->set_name("join"); @@ -1569,8 +1579,20 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { (*features->mutable_aggregates())[proto::ColumnType::STRING] = string_opts; + proto::GetFeaturesResp_AggregateOptions bool_opts; + bool_opts.add_aggregates()->set_name("count"); + bool_opts.add_aggregates()->set_name("and"); + bool_opts.add_aggregates()->set_name("any"); + bool_opts.add_aggregates()->set_name("distinct count"); + bool_opts.add_aggregates()->set_name("distinct leaf"); + bool_opts.add_aggregates()->set_name("dominant"); + bool_opts.add_aggregates()->set_name("first"); + bool_opts.add_aggregates()->set_name("last"); + bool_opts.add_aggregates()->set_name("last by index"); + bool_opts.add_aggregates()->set_name("or"); + bool_opts.add_aggregates()->set_name("unique"); (*features->mutable_aggregates())[proto::ColumnType::BOOLEAN] = - string_opts; + bool_opts; proto::GetFeaturesResp_AggregateOptions number_opts; number_opts.add_aggregates()->set_name("sum"); @@ -1579,6 +1601,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { number_opts.add_aggregates()->set_name("avg"); number_opts.add_aggregates()->set_name("count"); number_opts.add_aggregates()->set_name("distinct count"); + number_opts.add_aggregates()->set_name("distinct leaf"); number_opts.add_aggregates()->set_name("dominant"); number_opts.add_aggregates()->set_name("first"); number_opts.add_aggregates()->set_name("gmv"); @@ -1592,6 +1615,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { number_opts.add_aggregates()->set_name("last"); number_opts.add_aggregates()->set_name("mean"); number_opts.add_aggregates()->set_name("median"); + number_opts.add_aggregates()->set_name("mul"); number_opts.add_aggregates()->set_name("q1"); number_opts.add_aggregates()->set_name("q3"); number_opts.add_aggregates()->set_name("pct sum parent"); @@ -1604,6 +1628,12 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { auto args3 = number_opts.add_aggregates(); args3->set_name("weighted mean"); args3->add_args(proto::ColumnType::FLOAT); + auto args4 = number_opts.add_aggregates(); + args4->set_name("min by"); + args4->add_args(proto::ColumnType::FLOAT); + auto args5 = number_opts.add_aggregates(); + args5->set_name("max by"); + args5->add_args(proto::ColumnType::FLOAT); (*features->mutable_aggregates())[proto::ColumnType::INTEGER] = number_opts; @@ -1615,6 +1645,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { datetime_opts.add_aggregates()->set_name("any"); datetime_opts.add_aggregates()->set_name("avg"); datetime_opts.add_aggregates()->set_name("distinct count"); + datetime_opts.add_aggregates()->set_name("distinct leaf"); datetime_opts.add_aggregates()->set_name("dominant"); datetime_opts.add_aggregates()->set_name("first"); datetime_opts.add_aggregates()->set_name("high"); @@ -2649,6 +2680,9 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 1 : false; bool total_only = cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 2 : false; + bool split_rollup = cfg.has_split_rollup_mode() + && cfg.split_rollup_mode() + == proto::SplitRollupMode::SPLIT_ROLLUP_MODE_ROLLUP; auto config = std::make_shared( vocab, @@ -2663,7 +2697,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { column_only, leaves_only, total_only, - windows + windows, + split_rollup ); config->init(schema); @@ -2986,6 +3021,12 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { view_config_proto->set_group_rollup_mode(mode); } + view_config_proto->set_split_rollup_mode( + view_config->is_split_rollup() + ? proto::SplitRollupMode::SPLIT_ROLLUP_MODE_ROLLUP + : proto::SplitRollupMode::SPLIT_ROLLUP_MODE_FLAT + ); + for (const auto& expr : view_config->get_expressions()) { auto* proto_exprs = view_config_proto->mutable_expressions(); (*proto_exprs)[expr->get_expression_alias()] = diff --git a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp index 0dc71c207e..dadecc3208 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp @@ -73,6 +73,8 @@ View::View( _find_hidden_sort(column_sort); } + m_split_rollup = m_view_config->is_split_rollup() && sides() == 2; + // configure data window for `get_data` and `row_delta` // Column-only views skip the grand total row (offset=1), but // total_only mode needs to return exactly that row. @@ -169,7 +171,8 @@ View::num_columns() const { auto count = 0; for (t_uindex i = 0; i < col_length; ++i) { - if (m_ctx->unity_get_column_path(i + 1).size() != depth) { + if (!m_split_rollup + && m_ctx->unity_get_column_path(i + 1).size() != depth) { continue; } @@ -211,7 +214,8 @@ View::column_names(bool skip, std::int32_t depth) const { } std::vector col_path = m_ctx->unity_get_column_path(key + 1); - if (skip && col_path.size() < static_cast(depth)) { + if (skip && !m_split_rollup + && col_path.size() < static_cast(depth)) { continue; } @@ -294,7 +298,8 @@ View::column_names_range( } std::vector col_path = m_ctx->unity_get_column_path(key + 1); - if (skip && col_path.size() < static_cast(depth)) { + if (skip && !m_split_rollup + && col_path.size() < static_cast(depth)) { start_col += 1; end_col += 1; continue; @@ -320,7 +325,8 @@ View::column_names_range( } std::vector col_path = m_ctx->unity_get_column_path(key + 1); - if (skip && col_path.size() < static_cast(depth)) { + if (skip && !m_split_rollup + && col_path.size() < static_cast(depth)) { end_col += 1; max = std::min(end_col, col_count); continue; @@ -724,7 +730,7 @@ View::get_data( column_indices.push_back(0); for (t_uindex i = 0; i < col_length; ++i) { auto col_path = m_ctx->unity_get_column_path(i + 1); - if (col_path.size() != depth) { + if (!m_split_rollup && col_path.size() != depth) { continue; } @@ -773,6 +779,10 @@ View::get_data( std::vector slice_with_headers = m_ctx->get_data(start_row, end_row, start_col_index, end_col_index); + if (column_indices.empty()) { + slice_with_headers.clear(); + } + auto iter = slice_with_headers.begin(); while (iter != slice_with_headers.end()) { t_uindex prev = column_indices.front(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp b/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp index 69297bece3..a5508ade70 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp @@ -31,7 +31,8 @@ t_view_config::t_view_config( bool column_only, bool leaves_only, bool total_only, - const std::vector& windows + const std::vector& windows, + bool split_rollup ) : m_init(false), m_vocab(std::move(vocab)), @@ -48,7 +49,8 @@ t_view_config::t_view_config( m_filter_op(std::move(filter_op)), m_column_only(column_only), m_leaves_only(leaves_only), - m_total_only(total_only) {} + m_total_only(total_only), + m_split_rollup(split_rollup) {} void t_view_config::init(const std::shared_ptr& schema) { @@ -86,6 +88,52 @@ t_view_config::validate(const std::shared_ptr& schema) { << '\n'; PSP_COMPLAIN_AND_ABORT(ss.str()); } + + const std::vector& aggregate = agg.second; + if (aggregate.empty()) { + std::stringstream ss; + ss << "Missing aggregate for column '" << col + << "' found in View aggregates." << '\n'; + PSP_COMPLAIN_AND_ABORT(ss.str()); + continue; + } + + const std::string& agg_name = aggregate[0]; + const auto agg_type = maybe_str_to_aggtype(agg_name); + if (!agg_type) { + std::stringstream ss; + ss << "Invalid aggregate '" << agg_name << "' for column '" << col + << "' found in View aggregates." << '\n'; + PSP_COMPLAIN_AND_ABORT(ss.str()); + continue; + } + + if (!is_implemented_aggtype(*agg_type)) { + std::stringstream ss; + ss << "Unimplemented aggregate '" << agg_name << "' for column '" + << col << "' found in View aggregates." << '\n'; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + if (aggtype_takes_argument(*agg_type)) { + if (aggregate.size() < 2) { + std::stringstream ss; + ss << "Aggregate '" << agg_name << "' for column '" << col + << "' requires a column argument." << '\n'; + PSP_COMPLAIN_AND_ABORT(ss.str()); + continue; + } + + const std::string& arg = aggregate[1]; + if (!schema->has_column(arg) + && expression_aliases.count(arg) == 0) { + std::stringstream ss; + ss << "Invalid column '" << arg << "' found in the '" + << agg_name << "' aggregate for column '" << col << "'." + << '\n'; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + } } for (const std::string& col : m_row_pivots) { @@ -293,6 +341,11 @@ t_view_config::is_total_only() const { return m_total_only; } +bool +t_view_config::is_split_rollup() const { + return m_split_rollup; +} + std::int32_t t_view_config::get_row_pivot_depth() const { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); @@ -371,17 +424,9 @@ t_view_config::fill_aggspecs(const std::shared_ptr& schema) { agg_type = t_aggtype::AGGTYPE_UNIQUE; } else if (m_aggregates.count(column) > 0) { auto col = m_aggregates.at(column); - if (col.at(0) == "weighted mean") { - dependencies.emplace_back(col.at(1), DEPTYPE_COLUMN); - agg_type = AGGTYPE_WEIGHTED_MEAN; - } else if (col.at(0) == "max by") { - dependencies.emplace_back(col.at(1), DEPTYPE_COLUMN); - agg_type = AGGTYPE_MAX_BY; - } else if (col.at(0) == "min by") { + agg_type = str_to_aggtype(col.at(0)); + if (aggtype_takes_argument(agg_type)) { dependencies.emplace_back(col.at(1), DEPTYPE_COLUMN); - agg_type = AGGTYPE_MIN_BY; - } else { - agg_type = str_to_aggtype(col.at(0)); } } else { t_dtype dtype = schema->get_dtype(column); @@ -461,17 +506,9 @@ t_view_config::make_aggspec( if (m_column_only && !m_total_only) { agg_type = t_aggtype::AGGTYPE_ANY; } else { - if (aggregate.at(0) == "weighted mean") { + agg_type = str_to_aggtype(aggregate.at(0)); + if (aggtype_takes_argument(agg_type)) { dependencies.emplace_back(aggregate.at(1), DEPTYPE_COLUMN); - agg_type = AGGTYPE_WEIGHTED_MEAN; - } else if (aggregate.at(0) == "max by") { - dependencies.emplace_back(aggregate.at(1), DEPTYPE_COLUMN); - agg_type = AGGTYPE_MAX_BY; - } else if (aggregate.at(0) == "min by") { - dependencies.emplace_back(aggregate.at(1), DEPTYPE_COLUMN); - agg_type = AGGTYPE_MIN_BY; - } else { - agg_type = str_to_aggtype(aggregate.at(0)); } } diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/base.h b/rust/perspective-server/cpp/perspective/src/include/perspective/base.h index 127bc5c3c2..9b0d15f586 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/base.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/base.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -302,7 +303,13 @@ enum t_aggtype { AGGTYPE_GMV }; +PERSPECTIVE_EXPORT std::optional +maybe_str_to_aggtype(const std::string& str); + PERSPECTIVE_EXPORT t_aggtype str_to_aggtype(const std::string& str); +PERSPECTIVE_EXPORT bool is_implemented_aggtype(t_aggtype agg); +PERSPECTIVE_EXPORT bool aggtype_takes_argument(t_aggtype agg); + PERSPECTIVE_EXPORT t_aggtype _get_default_aggregate(t_dtype dtype); PERSPECTIVE_EXPORT std::string _get_default_aggregate_string(t_dtype dtype); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/config.h b/rust/perspective-server/cpp/perspective/src/include/perspective/config.h index 04fca7ec01..4ede3e2309 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/config.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/config.h @@ -211,6 +211,16 @@ class PERSPECTIVE_EXPORT t_config { return m_grand_agg_str; } + inline void + set_split_rollup(bool split_rollup) { + m_split_rollup = split_rollup; + } + + inline bool + is_split_rollup() const { + return m_split_rollup; + } + // The backing store for any persistent storage created by the context built // from this config (currently the expression `m_master` table). Inherited // from the parent `Table`'s backing store. Defaults to memory. @@ -255,6 +265,7 @@ class PERSPECTIVE_EXPORT t_config { std::string m_grand_agg_str; t_fmode m_fmode; bool m_has_pkey_agg; + bool m_split_rollup = false; t_backing_store m_backing_store = BACKING_STORE_MEMORY; }; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h index 47c9a6021b..c88a371b4b 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h @@ -452,6 +452,8 @@ class PERSPECTIVE_EXPORT View { t_uindex m_row_offset; t_uindex m_col_offset; + bool m_split_rollup; + std::shared_ptr m_view_config; }; } // end namespace perspective \ No newline at end of file diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h b/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h index 17d894b76e..67e2e06895 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h @@ -61,7 +61,8 @@ class PERSPECTIVE_EXPORT t_view_config { bool column_only, bool leaves_only = false, bool total_only = false, - const std::vector& windows = {} + const std::vector& windows = {}, + bool split_rollup = false ); /** @@ -127,6 +128,7 @@ class PERSPECTIVE_EXPORT t_view_config { bool is_leaves_only() const; bool is_total_only() const; + bool is_split_rollup() const; std::int32_t get_row_pivot_depth() const; std::int32_t get_column_pivot_depth() const; @@ -250,5 +252,6 @@ class PERSPECTIVE_EXPORT t_view_config { bool m_column_only; bool m_leaves_only; bool m_total_only; + bool m_split_rollup; }; } // end namespace perspective \ No newline at end of file diff --git a/rust/perspective-viewer/src/css/config-selector.css b/rust/perspective-viewer/src/css/config-selector.css index 31d29b966b..6a102436f0 100644 --- a/rust/perspective-viewer/src/css/config-selector.css +++ b/rust/perspective-viewer/src/css/config-selector.css @@ -354,7 +354,8 @@ } } - .group_rollup_wrapper { + .group_rollup_wrapper, + .split_rollup_wrapper { margin-bottom: -30px; flex: 0 1 auto; color: var(--psp-inactive--color); diff --git a/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs b/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs index 76d818d435..abdd59ada8 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs @@ -86,6 +86,7 @@ pub enum ConfigSelectorMsg { TransposePivots, New(DragTarget, InPlaceColumn), UpdateGroupRollupMode(GroupRollupMode), + UpdateSplitRollupMode(SplitRollupMode), } #[derive(Clone)] @@ -166,6 +167,22 @@ impl Component for ConfigSelector { false }, + ConfigSelectorMsg::UpdateSplitRollupMode(mode) => { + let config = ViewConfigUpdate { + split_rollup_mode: Some(mode), + ..ViewConfigUpdate::default() + }; + + { + let session = ctx.props().session.clone(); + let renderer = ctx.props().renderer.clone(); + if let Ok(task) = apply_and_render(&session, &renderer, config) { + spawn_owned("config-selector", task); + } + } + + false + }, ConfigSelectorMsg::Close(index, DragTarget::GroupBy) => { if ctx.props().view_config.group_rollup_mode == GroupRollupMode::Total { let requirements = ctx.props().renderer.metadata(); @@ -584,6 +601,17 @@ impl Component for ConfigSelector { let group_rollups = requirements.get_group_rollups(&rollup_features); + let on_split_rollup_mode = ctx + .link() + .callback(ConfigSelectorMsg::UpdateSplitRollupMode); + + let split_rollup_features = metadata + .get_features() + .map(|x| x.get_split_rollup_modes()) + .unwrap(); + + let split_rollups = requirements.get_split_rollups(&split_rollup_features); + html! { <>
@@ -638,14 +666,31 @@ impl Component for ConfigSelector { } if features.split_by { - if !config.split_by.is_empty() { + if !config.split_by.is_empty() || split_rollups.len() > 1 {
- + if split_rollups.len() > 1 { + + id="split_rollup_mode_selector" + wrapper_class="split_rollup_wrapper" + is_autosize=true + values={Rc::new( + split_rollups + .iter() + .map(|x| SelectItem::Option(*x)) + .collect(), + )} + selected={config.split_rollup_mode} + on_select={on_split_rollup_mode} + /> + } + if !config.split_by.is_empty() { + + }
}