From 7c2b0869d8a64e19283599448f6c4977eb21a12f Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:54:59 +0000 Subject: [PATCH 1/3] Fix flaky test 02421_simple_queries_for_opentelemetry Test 7 reused the same query_id for the TCP 'set opentelemetry_start_trace_probability=1' query and the subsequent HTTP SELECT. Under parallel CI load the two queries can overlap, producing Code 216 QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING. Additionally, check_http_attributes queried the 'query' span for the query_id, but the http.referer / http.user.agent / http.method attributes live on the HTTPHandler (SERVER) span, which carries no clickhouse.query_id. The old substring check matched the JSONEachRow column-name keys, so it always reported the attributes as present regardless of value, i.e. it never actually verified them. Give the HTTP query its own query_id, send a traceparent header (so the request is traced) plus Referer / User-Agent headers, and verify the http.* attribute VALUES on the HTTPHandler span (matched via clickhouse.uri). The reference output is unchanged. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 6944c75587cad4497c1bde904e2e4e471749d985) --- .../02421_simple_queries_for_opentelemetry.sh | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh index e6d7b4e10ddc..bf60e8aa9cfa 100755 --- a/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh +++ b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh @@ -108,7 +108,15 @@ function check_tcp_attributes() function execute_query_HTTP() { - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&database=${CLICKHOUSE_DATABASE}&query_id=$1" -d "$2" + # A traceparent header (with the sampled flag) forces the request to be traced, and the + # Referer / User-Agent headers populate the http.* attributes on the HTTPHandler span. + local trace_id + trace_id=$(${CLICKHOUSE_CLIENT} -q "SELECT lower(hex(generateUUIDv4()))") + ${CLICKHOUSE_CURL} -sS \ + -H "traceparent: 00-${trace_id}-0000000000000010-01" \ + -H "referer: some-referer" \ + -H "user-agent: some-user-agent" \ + "${CLICKHOUSE_URL}&database=${CLICKHOUSE_DATABASE}&query_id=$1" -d "$2" } function check_http_attributes() @@ -119,15 +127,17 @@ function check_http_attributes() local agent="not found" local method="not found" + # The http.* attributes live on the HTTPHandler (SERVER) span, not on the child query span. + # Match it by the query_id embedded in the request URI (clickhouse.uri). result=$(${CLICKHOUSE_CLIENT} -q " SYSTEM FLUSH LOGS opentelemetry_span_log; - SELECT attribute['http.referer'], - attribute['http.user.agent'], - attribute['http.method'] + SELECT attribute['http.referer'] AS referer, + attribute['http.user.agent'] AS user_agent, + attribute['http.method'] AS method FROM system.opentelemetry_span_log WHERE finish_date >= yesterday() - AND operation_name = 'query' - AND attribute['clickhouse.query_id'] = '${query_id}' + AND operation_name = 'HTTPHandler' + AND attribute['clickhouse.uri'] LIKE '%${query_id}%' FORMAT JSONEachRow; ") @@ -135,16 +145,16 @@ function check_http_attributes() echo "Error: No result returned from ClickHouse server" return 1 fi - - if [[ $result == *"http.referer"* ]]; then + + if [[ $result == *'"referer":"some-referer"'* ]]; then referer="present" fi - - if [[ $result == *"http.user.agent"* ]]; then + + if [[ $result == *'"user_agent":"some-user-agent"'* ]]; then agent="present" fi - if [[ $result == *"http.method"* ]]; then + if [[ $result == *'"method":"POST"'* ]]; then method="present" fi @@ -191,9 +201,8 @@ query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()") execute_query $query_id 'select * from opentelemetry_test format Null' check_tcp_attributes $query_id -# Test 7: Executes a TCP SELECT query and checks for http attributes in OpenTelemetry spans. +# Test 7: Executes an HTTP SELECT query and checks for http attributes in OpenTelemetry spans. query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()") -execute_query "$query_id" 'set opentelemetry_start_trace_probability=1' execute_query_HTTP "$query_id" 'select * from opentelemetry_test FORMAT Null' check_http_attributes "$query_id" # From af130b2d4fd14b3bfdf2050e16b74bcfbd5c97b8 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:02:35 +0000 Subject: [PATCH 2/3] Fix flaky test 01017_uniqCombined_memory_usage The two K=16 positive-case budgets (5230000 for UInt32, 5900000 for UInt64) left effectively zero headroom: CI failures showed the tracked peak landing exactly at the budget ("would use 4.99 MiB, maximum: 4.99 MiB" and "5.63 MiB, maximum: 5.63 MiB") on arm builds. With memory_profiler_step=1 every allocation is flushed to the query tracker, so allocator/arena rounding variance intermittently tips these two cases over. Give the two flaky K=16 positive budgets ~20% headroom over the observed peak (6300000 and 7100000). The negative-case budgets (2000000) are left unchanged, so the assertions still catch any regression that inflates the HLL state. The K=12 and K=18 budgets already have sufficient margin and are untouched. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 5e662b6d271004d2bd81f12092b330d37f974eec) --- tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql b/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql index eca370d94af8..6acadd840c64 100644 --- a/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql +++ b/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql @@ -32,14 +32,14 @@ SELECT 'K=16'; SELECT 'UInt32'; SET max_memory_usage = 2000000; SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(16)(number % 4096) u FROM numbers(4096 * 100) GROUP BY k); -- { serverError MEMORY_LIMIT_EXCEEDED } -SET max_memory_usage = 5230000; +SET max_memory_usage = 6300000; SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(16)(number % 4096) u FROM numbers(4096 * 100) GROUP BY k); -- HashTable for UInt64 (used until (1<<11) elements), hence 2048 elements SELECT 'UInt64'; SET max_memory_usage = 2000000; SELECT sum(u) FROM (SELECT intDiv(number, 2048) AS k, uniqCombined(16)(reinterpretAsString(number % 2048)) u FROM numbers(2048 * 100) GROUP BY k); -- { serverError MEMORY_LIMIT_EXCEEDED } -SET max_memory_usage = 5900000; +SET max_memory_usage = 7100000; SELECT sum(u) FROM (SELECT intDiv(number, 2048) AS k, uniqCombined(16)(reinterpretAsString(number % 2048)) u FROM numbers(2048 * 100) GROUP BY k); SELECT 'K=18'; From 99c4d972620c3f7706fc0542ff3486c5b4c565c8 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:49:36 +0000 Subject: [PATCH 3/3] Fix flaky test_rename_distributed_parallel_insert_and_select The cleanup renames back to num2 run ON CLUSTER (async distributed DDL plus async ReplicatedMergeTree metadata replication) with ignore_exception=True, so a failed or late rename is swallowed. The strict final selects then run with no wait for schema convergence. A distributed SELECT for num2 forwarded to a shard replica whose local _replicated metadata has not yet converged (still foo2/foo3) fails with UNKNOWN_IDENTIFIER at the analyzer. This is correct engine behaviour for per-shard schema during async metadata replication; the test asserted before its precondition held. Replace the fixed cleanup renames with wait_for_rename_to_num2, which re-drives the idempotent ON CLUSTER rename-back and polls system.columns until num2 is present on every node before the strict selects, repairing both a swallowed rename and replication lag. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 51eea395dd9091ce12f453f5407c861d2a6c1368) --- tests/integration/test_rename_column/test.py | 40 ++++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 870a2f0f2107..3a8f5d18b941 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -269,6 +269,37 @@ def rename_column_on_cluster( break +def _num2_converged(nodes, table_name): + return all( + node.query( + "SELECT count() FROM system.columns " + "WHERE database = 'default' AND table IN ('{t}', '{t}_replicated') " + "AND name = 'num2'".format(t=table_name) + ).strip() + == "2" + for node in nodes + ) + + +def wait_for_rename_to_num2(nodes, table_name, attempts=12, poll_seconds=5): + # Best-effort cleanup renames + async ON CLUSTER replication can leave the + # column as foo2/foo3 on some replicas. Re-drive the idempotent rename-back + # until num2 is present on every node's Distributed and _replicated tables, + # so the strict queries below do not race schema convergence (UNKNOWN_IDENTIFIER). + tables = [table_name, "%s_replicated" % table_name] + for _ in range(attempts): + if _num2_converged(nodes, table_name): + return + for old_name in ("foo2", "foo3"): + for table in tables: + rename_column_on_cluster(nodes[0], table, old_name, "num2", 1, True) + time.sleep(poll_seconds) + if not _num2_converged(nodes, table_name): + raise Exception( + "columns did not converge to num2 for {} on all nodes".format(table_name) + ) + + def alter_move(node, table_name, iterations=1, ignore_exception=False): i = 0 while True: @@ -809,14 +840,7 @@ def test_rename_distributed_parallel_insert_and_select(started_cluster): for task in tasks: task.get(timeout=240) - rename_column_on_cluster(node1, table_name, "foo2", "num2", 1, True) - rename_column_on_cluster( - node1, "%s_replicated" % table_name, "foo2", "num2", 1, True - ) - rename_column_on_cluster(node1, table_name, "foo3", "num2", 1, True) - rename_column_on_cluster( - node1, "%s_replicated" % table_name, "foo3", "num2", 1, True - ) + wait_for_rename_to_num2(nodes, table_name) insert(node1, table_name, 1000, col_names=["num", "num2"]) select(node1, table_name, "num2")