Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion e2e/questdb
Submodule questdb updated 90 files
+ .github/agents-demo.jpg
+ .github/readme-benchmark.webp
+204 −105 README.md
+397 −0 benchmarks/src/main/java/org/questdb/SymbolPatternIndexBenchmark.java
+64 −0 ci/git-auth-env.sh
+68 −0 ci/submodule-update.sh
+38 −4 ci/templates/aux-job.yml
+17 −3 ci/templates/build-client-native.yml
+9 −1 ci/templates/detect-local-client.yml
+7 −1 ci/templates/rust-test-and-lint.yml
+34 −12 ci/templates/steps.yml
+15 −2 ci/test-fuzz.yml
+64 −15 ci/test-hosted-pipeline.yml
+65 −18 ci/test-pipeline.yml
+4 −1 core/CMakeLists.txt
+14 −0 core/src/main/java/io/questdb/PropServerConfiguration.java
+2 −0 core/src/main/java/io/questdb/PropertyKey.java
+48 −0 core/src/main/java/io/questdb/cairo/AbstractIntervalPartitionFrameCursor.java
+21 −15 core/src/main/java/io/questdb/cairo/AbstractPartitionFrameCursorFactory.java
+4 −0 core/src/main/java/io/questdb/cairo/CairoConfiguration.java
+10 −0 core/src/main/java/io/questdb/cairo/CairoConfigurationWrapper.java
+10 −0 core/src/main/java/io/questdb/cairo/DefaultCairoConfiguration.java
+23 −0 core/src/main/java/io/questdb/cairo/FullPartitionFrameCursorFactory.java
+12 −0 core/src/main/java/io/questdb/cairo/SymbolMapReaderImpl.java
+11 −6 core/src/main/java/io/questdb/cairo/TableReader.java
+628 −148 core/src/main/java/io/questdb/cairo/idx/AbstractPostingIndexReader.java
+183 −0 core/src/main/java/io/questdb/cairo/idx/BitmapIndexFwdReader.java
+86 −2 core/src/main/java/io/questdb/cairo/idx/BitmapIndexUtils.java
+4 −0 core/src/main/java/io/questdb/cairo/idx/IndexFwdNullReader.java
+86 −30 core/src/main/java/io/questdb/cairo/idx/PostingIndexBwdReader.java
+10 −10 core/src/main/java/io/questdb/cairo/idx/PostingIndexFwdReader.java
+380 −5 core/src/main/java/io/questdb/cairo/idx/PostingIndexUtils.java
+27 −15 core/src/main/java/io/questdb/cairo/lv/LiveViewRefreshJob.java
+5 −0 core/src/main/java/io/questdb/cairo/lv/LiveViewRefreshSqlExecutionContext.java
+12 −0 core/src/main/java/io/questdb/cairo/sql/PartitionFrameCursor.java
+4 −13 core/src/main/java/io/questdb/cairo/sql/PartitionFrameCursorFactory.java
+11 −0 core/src/main/java/io/questdb/cairo/sql/StaticSymbolTable.java
+7 −0 core/src/main/java/io/questdb/cairo/sql/TableReferenceOutOfDateException.java
+10 −1 core/src/main/java/io/questdb/cutlass/parquet/BaseParquetExporter.java
+1 −1 core/src/main/java/io/questdb/cutlass/parquet/SQLSerialParquetExporter.java
+580 −11 core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java
+9 −0 core/src/main/java/io/questdb/griffin/SqlExecutionContext.java
+5 −0 core/src/main/java/io/questdb/griffin/SqlHints.java
+466 −100 core/src/main/java/io/questdb/griffin/SqlOptimiser.java
+127 −37 core/src/main/java/io/questdb/griffin/engine/functions/regex/AbstractLikeSymbolFunctionFactory.java
+129 −5 core/src/main/java/io/questdb/griffin/engine/functions/regex/MatchSymbolFunctionFactory.java
+39 −0 core/src/main/java/io/questdb/griffin/engine/functions/regex/SymbolKeySetProvider.java
+2 −3 core/src/main/java/io/questdb/griffin/engine/groupby/GroupByLongTopKJob.java
+2 −3 core/src/main/java/io/questdb/griffin/engine/groupby/GroupByMergeShardJob.java
+118 −0 core/src/main/java/io/questdb/griffin/engine/groupby/PostAggregationCircuitBreaker.java
+11 −2 core/src/main/java/io/questdb/griffin/engine/table/AbstractPageFrameRecordCursorFactory.java
+1,134 −0 core/src/main/java/io/questdb/griffin/engine/table/AdaptiveSymbolPatternRecordCursorFactory.java
+11 −8 core/src/main/java/io/questdb/griffin/engine/table/AsyncGroupByRecordCursor.java
+10 −7 core/src/main/java/io/questdb/griffin/engine/table/AsyncHorizonJoinRecordCursor.java
+10 −7 core/src/main/java/io/questdb/griffin/engine/table/AsyncMultiHorizonJoinRecordCursor.java
+352 −94 core/src/main/java/io/questdb/griffin/engine/table/CoveringIndexRecordCursorFactory.java
+2 −2 core/src/main/java/io/questdb/griffin/engine/table/GroupByShardingContext.java
+27 −2 core/src/main/java/io/questdb/griffin/engine/table/HeapRowCursorFactory.java
+4 −0 core/src/main/java/io/questdb/griffin/engine/table/SequentialRowCursorFactory.java
+187 −0 core/src/main/java/io/questdb/griffin/engine/table/SymbolPatternIndexRecordCursorFactory.java
+54 −69 core/src/main/java/io/questdb/std/IntLongSortedList.java
+4 −4 core/src/main/java/io/questdb/tasks/GroupByLongTopKTask.java
+4 −4 core/src/main/java/io/questdb/tasks/GroupByMergeShardTask.java
+2 −0 core/src/test/java/io/questdb/test/ServerMainTest.java
+49 −0 core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexFilterCompilationLeakTest.java
+84 −0 core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexMemoryLimitTest.java
+297 −5 core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexMultiKeyOrderingTest.java
+80 −9 core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexParallelDecodeTest.java
+154 −2 core/src/test/java/io/questdb/test/cairo/covering/PostingReaderConcurrentReadTest.java
+506 −28 core/src/test/java/io/questdb/test/cairo/covering/PostingReaderSelectKthMatchTest.java
+9 −0 core/src/test/java/io/questdb/test/cairo/lv/LiveViewConcurrencyTest.java
+96 −0 core/src/test/java/io/questdb/test/cairo/lv/LiveViewParquetBaseTest.java
+48 −0 core/src/test/java/io/questdb/test/cairo/lv/LiveViewRefreshSqlExecutionContextTest.java
+115 −0 core/src/test/java/io/questdb/test/cairo/lv/LiveViewSmokeTest.java
+127 −0 core/src/test/java/io/questdb/test/cairo/lv/LiveViewTest.java
+154 −0 core/src/test/java/io/questdb/test/cutlass/http/line/LineHttpSenderTokenProviderServerTest.java
+46 −0 core/src/test/java/io/questdb/test/cutlass/pgwire/PGParquetTest.java
+95 −11 core/src/test/java/io/questdb/test/griffin/CopyExportTest.java
+247 −0 core/src/test/java/io/questdb/test/griffin/ExplainPlanTest.java
+55 −12 core/src/test/java/io/questdb/test/griffin/ParallelGroupByMemoryTrackerTest.java
+225 −15 core/src/test/java/io/questdb/test/griffin/ParquetRowGroupPruningTest.java
+4,291 −0 core/src/test/java/io/questdb/test/griffin/SymbolPatternIndexTest.java
+111 −0 core/src/test/java/io/questdb/test/griffin/engine/functions/regex/ILikeSymbolFunctionFactoryTest.java
+249 −0 core/src/test/java/io/questdb/test/griffin/engine/functions/regex/LikeSymbolFunctionFactoryTest.java
+524 −0 core/src/test/java/io/questdb/test/griffin/engine/functions/regex/MatchSymbolFunctionFactoryTest.java
+212 −0 core/src/test/java/io/questdb/test/griffin/engine/groupby/PostAggregationCircuitBreakerTest.java
+1,192 −203 core/src/test/java/io/questdb/test/griffin/engine/join/JoinTest.java
+94 −0 core/src/test/java/io/questdb/test/std/IntLongSortedListTest.java
+14 −16 i18n/README.zh-cn.md
+1 −1 java-questdb-client
152 changes: 103 additions & 49 deletions src/consts/shared-definitions.json

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions src/modules/ConsoleEventTracker/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ export enum ConsoleEvent {
NOTEBOOK_CELL_EXPAND_WIDTH = "notebook.cell_expand_width",
NOTEBOOK_CELL_SIZE_RESET = "notebook.cell_size_reset",
NOTEBOOK_CELL_VIEW_CHANGE = "notebook.cell_view_change",
NOTEBOOK_CELL_VIEW_MAXIMIZE = "notebook.cell_view_maximize",
NOTEBOOK_CELL_EDITOR_TOGGLE = "notebook.cell_editor_toggle",
NOTEBOOK_CELL_MAXIMIZE = "notebook.cell_maximize",
NOTEBOOK_DRAW_TOGGLE = "notebook.draw_toggle",
NOTEBOOK_DRAW_REFUSED = "notebook.draw_refused",
Expand Down Expand Up @@ -163,12 +163,12 @@ export enum ConsoleEvent {
MCP_DUPLICATE_CELL = "mcp.duplicate_cell",
MCP_SET_LAYOUT_MODE = "mcp.set_layout_mode",
MCP_SET_CELL_LAYOUT = "mcp.set_cell_layout",
MCP_SET_CELL_DIMENSIONS = "mcp.set_cell_dimensions",
MCP_SET_CELL_MODE = "mcp.set_cell_mode",
MCP_SET_CELL_CHART_CONFIG = "mcp.set_cell_chart_config",
MCP_SET_CELL_AUTOREFRESH = "mcp.set_cell_autorefresh",
MCP_SET_NOTEBOOK_AUTOREFRESH = "mcp.set_notebook_autorefresh",
MCP_SET_CELL_NAME = "mcp.set_cell_name",
MCP_SET_CELL_VIEW_MAXIMIZED = "mcp.set_cell_view_maximized",
MCP_SET_CELL_MAXIMIZED = "mcp.set_cell_maximized",
MCP_APPLY_NOTEBOOK_STATE = "mcp.apply_notebook_state",
MCP_GET_TABLES = "mcp.get_tables",
Expand Down
30 changes: 30 additions & 0 deletions src/scenes/Editor/Monaco/importTabs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -775,6 +775,36 @@ describe("sanitizeBuffer", () => {
})

describe("notebookViewState sanitization", () => {
it("imports one preferred view and maps main's legacy boolean", () => {
const input = {
label: "Notebook",
value: "",
position: 0,
notebookViewState: {
cells: [
{ id: "preferred", value: "SELECT 1", paneView: "editor" },
{ id: "legacy-on", value: "SELECT 2", isViewMaximized: true },
{ id: "legacy-off", value: "SELECT 3", isViewMaximized: false },
{
id: "markdown",
value: "# Title",
type: "markdown",
paneView: "result",
},
],
},
}

const cells = sanitizeBuffer(input).notebookViewState?.cells
expect(cells?.map((cell) => cell.paneView)).toEqual([
"editor",
"result",
"editor_result",
undefined,
])
expect(cells?.every((cell) => !("isViewMaximized" in cell))).toBe(true)
})

it("whitelists cell fields, reindexes positions, drops session state", () => {
const input = {
label: "Notebook",
Expand Down
24 changes: 18 additions & 6 deletions src/scenes/Editor/Monaco/importTabs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ import type {
NotebookViewState,
} from "../../../store/notebook"
import type { ChartConfig, QueryChart } from "../Notebook/CellChart/chartTypes"
import { isAutoRefresh } from "../Notebook/notebookUtils"
import { isAutoRefresh, MAX_PANE_HEIGHT_PX } from "../Notebook/notebookUtils"
import { LINE_NUMBER_HARD_LIMIT } from "./index"
import {
isAgentCellView,
MAX_NOTEBOOK_CELLS,
MAX_CELL_LINES,
MAX_CELL_NAME_LENGTH,
Expand Down Expand Up @@ -271,11 +272,22 @@ const sanitizeNotebookCell = (
const chartConfig = sanitizeChartConfig(item.chartConfig)
if (chartConfig) cell.chartConfig = chartConfig
if (isAutoRefresh(item.autoRefresh)) cell.autoRefresh = item.autoRefresh
if (typeof item.isViewMaximized === "boolean")
cell.isViewMaximized = item.isViewMaximized
if (typeof item.topHeight === "number") cell.topHeight = item.topHeight
if (typeof item.bottomHeight === "number")
cell.bottomHeight = item.bottomHeight
if (item.type !== "markdown") {
if (isAgentCellView(item.paneView)) {
cell.paneView = item.paneView
} else if (item.isViewMaximized === true) {
cell.paneView = "result"
} else {
cell.paneView = "editor_result"
}
}
if (typeof item.topHeight === "number" && Number.isFinite(item.topHeight))
cell.topHeight = Math.min(MAX_PANE_HEIGHT_PX, item.topHeight)
if (
typeof item.bottomHeight === "number" &&
Number.isFinite(item.bottomHeight)
)
cell.bottomHeight = Math.min(MAX_PANE_HEIGHT_PX, item.bottomHeight)
if (typeof item.topResized === "boolean") cell.topResized = item.topResized
if (typeof item.bottomResized === "boolean")
cell.bottomResized = item.bottomResized
Expand Down
58 changes: 40 additions & 18 deletions src/scenes/Editor/Notebook/CellChart/ChartRenderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@ import React, {
useCallback,
useEffect,
useImperativeHandle,
useLayoutEffect,
useMemo,
useRef,
useState,
} from "react"
import ReactECharts from "echarts-for-react/lib/core"
import type { EChartsOption } from "echarts"
import { useTheme } from "styled-components"
import { echarts } from "./echartsSetup"
import { withZoomSlider } from "./buildEchartsOption"
import { snapChartWidth } from "./chartDensity"
import { createQuestdbTheme } from "./questdbTheme"
import { useNotebookBufferId } from "../NotebookProvider"
import {
Expand Down Expand Up @@ -93,6 +97,7 @@ export const ChartRenderer = React.forwardRef<ChartRendererHandle, Props>(
const reactEchartsRef = useRef<ReactECharts | null>(null)
const wrapperRef = useRef<HTMLDivElement | null>(null)
const zoomWindowRef = useRef(zoomWindow)
const [measuredWidth, setMeasuredWidth] = useState(0)
// Decided once per mount: a chart mounting into an already-settled
// notebook (scroll remount) skips the entry animation.
const animateEntryRef = useRef(
Expand Down Expand Up @@ -128,7 +133,9 @@ export const ChartRenderer = React.forwardRef<ChartRendererHandle, Props>(

const observer = new ResizeObserver((entries) => {
const box = entries[0]?.contentRect
if (box) resizeTo(box.width, box.height)
if (!box) return
resizeTo(box.width, box.height)
if (box.width > 0) setMeasuredWidth(snapChartWidth(box.width))
})
observer.observe(wrapper)

Expand Down Expand Up @@ -186,14 +193,27 @@ export const ChartRenderer = React.forwardRef<ChartRendererHandle, Props>(
[bufferId],
)

const key = useMemo(() => structuralKey(option), [option])
// The chart mounts once the wrapper is measured, so the first instance
// already carries the right slider decision instead of remounting for it.
useLayoutEffect(() => {
const width = wrapperRef.current?.getBoundingClientRect().width
if (width) setMeasuredWidth(snapChartWidth(width))
}, [])

const optionToDraw = useMemo(
() => withZoomSlider(option, measuredWidth),
[option, measuredWidth],
)
const key = useMemo(() => structuralKey(optionToDraw), [optionToDraw])

const suppressEntryAnimation =
!animateEntryRef.current && !firstInstanceDoneRef.current
const renderOption = useMemo(
() =>
suppressEntryAnimation ? { ...option, animationDuration: 0 } : option,
[option, suppressEntryAnimation],
suppressEntryAnimation
? { ...optionToDraw, animationDuration: 0 }
: optionToDraw,
[optionToDraw, suppressEntryAnimation],
)

const events = useMemo(() => {
Expand All @@ -220,20 +240,22 @@ export const ChartRenderer = React.forwardRef<ChartRendererHandle, Props>(
height: typeof height === "number" ? `${height}px` : height,
}}
>
<ReactECharts
key={`${theme.mode}:${key}`}
ref={reactEchartsRef}
echarts={echarts}
option={renderOption}
theme={chartTheme}
notMerge={false}
lazyUpdate
autoResize={false}
onEvents={events}
onChartReady={handleChartReady}
style={{ height: "100%", width: "100%" }}
opts={{ renderer: "canvas" }}
/>
{measuredWidth > 0 && (
<ReactECharts
key={`${theme.mode}:${key}`}
ref={reactEchartsRef}
echarts={echarts}
option={renderOption}
theme={chartTheme}
notMerge={false}
lazyUpdate
autoResize={false}
onEvents={events}
onChartReady={handleChartReady}
style={{ height: "100%", width: "100%" }}
opts={{ renderer: "canvas" }}
/>
)}
</div>
)
},
Expand Down
20 changes: 20 additions & 0 deletions src/scenes/Editor/Notebook/CellChart/buildEchartsOption.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -370,3 +370,23 @@ describe("buildEchartsOption — timestamps render in UTC, like the result grid"
expect(opt.useUTC).toBe(true)
})
})

describe("buildEchartsOption — axis labels", () => {
it("hides overlapping x-axis labels so a narrow chart thins them instead of colliding", () => {
// Given a temporal x column with more ticks than a narrow chart can fit
const columns = [col("ts", "TIMESTAMP"), col("v", "DOUBLE")]
const dataset = Array.from({ length: 15 }, (_, i) => [i * 120_000, i])
// When the option builds
const opt = buildEchartsOption({ xColumn: "ts" }, [
resolved({
columns,
dataset,
xColumn: "ts",
type: "bar",
yColumns: ["v"],
}),
])
// Then the x-axis label config asks ECharts to hide overlaps
expect(firstAxis(opt.xAxis).axisLabel).toMatchObject({ hideOverlap: true })
})
})
40 changes: 26 additions & 14 deletions src/scenes/Editor/Notebook/CellChart/buildEchartsOption.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { EChartsOption } from "echarts"
import { needsZoomSlider } from "./chartDensity"
import type { ColumnDefinition } from "../../../../utils/questdb/types"
import type { ChartConfig, ChartType, SeriesAxis } from "./chartTypes"
import { MAX_PARTITION_SERIES, classifyColumn } from "./inferChartConfig"
Expand Down Expand Up @@ -32,8 +33,6 @@ type SeriesContext = {
categoryUnion: string[] | null // set only when overlaying series onto a shared category axis; null for a single positional query
}

const DATAZOOM_THRESHOLD = 200

const LEGEND_BOTTOM = 3
const SLIDER_HEIGHT = 18
const SLIDER_BOTTOM = 40
Expand Down Expand Up @@ -357,7 +356,7 @@ export const buildEchartsOption = (
queries: ResolvedQuery[],
): EChartsOption => {
const chartText = { fontSize: CHART_FONT_SIZE }
const axisLabel = { fontSize: CHART_FONT_SIZE }
const axisLabel = { fontSize: CHART_FONT_SIZE, hideOverlap: true }
const axisName = { fontSize: CHART_FONT_SIZE }
const baseLegend = {
type: "scroll" as const,
Expand Down Expand Up @@ -459,22 +458,13 @@ export const buildEchartsOption = (
? (rightQueries[0].name ?? rightQueries[0].yColumns[0] ?? "")
: "")

const maxRows = queries.reduce((m, q) => Math.max(m, q.dataset.length), 0)
const hasZoom = maxRows > DATAZOOM_THRESHOLD
const sliderZoom = {
type: "slider" as const,
height: SLIDER_HEIGHT,
bottom: SLIDER_BOTTOM,
textStyle: chartText,
}

const rightPadding = 36

const grid: EChartsOption["grid"] = {
left: 24,
right: rightPadding,
top: 40,
bottom: hasZoom ? GRID_BOTTOM_WITH_ZOOM : GRID_BOTTOM_NO_ZOOM,
bottom: GRID_BOTTOM_NO_ZOOM,
containLabel: true,
}

Expand Down Expand Up @@ -548,7 +538,29 @@ export const buildEchartsOption = (
grid,
xAxis,
yAxis,
dataZoom: hasZoom ? [{ type: "inside" }, sliderZoom] : undefined,
series: series as EChartsOption["series"],
}
}

// Zoom exists only for a chart too dense for its measured width: the slider
// costs plot height, and zooming a sparse chart is meaningless. The renderer
// applies this from its own size.
export const withZoomSlider = (
option: EChartsOption,
containerWidthPx: number,
): EChartsOption => {
if (!needsZoomSlider(option, containerWidthPx)) return option
return {
...option,
grid: { ...(option.grid as object), bottom: GRID_BOTTOM_WITH_ZOOM },
dataZoom: [
{ type: "inside" },
{
type: "slider",
height: SLIDER_HEIGHT,
bottom: SLIDER_BOTTOM,
textStyle: { fontSize: CHART_FONT_SIZE },
},
],
}
}
Loading
Loading