From 7d3bfee6d51efe00c44553530178387f90d9cd43 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 26 Jun 2026 11:22:45 -0400 Subject: [PATCH 1/2] DRILL-8239: Convert convert_fromJSON UDF to EVF Rewrite the convert_fromJSON UDFs to use the EVF JsonLoader (ResultSetLoader) instead of the legacy JsonReader, mirroring the HTTP storage plugin UDFs. JsonConverterUtils builds the loader from either the system JSON options or the explicit allTextMode/readNumbersAsDouble arguments, and centralises the per-row conversion. To preserve the full convert_fromJSON contract, the EVF complex-writer support in ProjectRecordBatch is extended: * Multiple complex-writer functions per project list. addLoader now keeps a list of (loader, output-column) pairs -- captured at codegen in DrillComplexWriterFuncHolder -- so each loader's output lands in the column reserved for it (fixes SELECT convert_from(a) m1, convert_from(b) m2 and cases that project columns before/after the function). * Top-level scalars and arrays. The UDF wraps each input value in a single marker field so the record-oriented loader reads {scalar, array, object} uniformly; ProjectRecordBatch unwraps that marker column by transferring it directly (preserving the value's own type), and otherwise wraps the loader's columns in a map (the HTTP-UDF behaviour). * Per-output-batch lifecycle. Loaders are re-started before each batch, fixing the "Unexpected state: HARVESTED" failure on multi-row/multi-batch input. * Null/empty input writes an aligned (null) row so the loader row count matches the surrounding batch. --- .../expr/fn/DrillComplexWriterFuncHolder.java | 2 +- .../expr/fn/impl/conv/JsonConvertFrom.java | 298 +++++++----------- .../expr/fn/impl/conv/JsonConverterUtils.java | 180 +++++++++++ .../impl/project/ProjectBatchBuilder.java | 7 +- .../impl/project/ProjectRecordBatch.java | 111 +++++-- 5 files changed, 387 insertions(+), 211 deletions(-) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java index 4f6491c0920..8b468019bae 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java @@ -87,7 +87,7 @@ protected HoldingContainer generateEvalBody(ClassGenerator classGenerator, Ho classGenerator.getEvalBlock().add(complexWriter.invoke("setPosition").arg(classGenerator.getMappingSet().getValueWriteIndex())); sub.decl(classGenerator.getModel()._ref(ComplexWriter.class), getReturnValue().getName(), complexWriter); } else { - classGenerator.getSetupBlock().add(projBatch.invoke("addLoader").arg(rsLoader)); + classGenerator.getSetupBlock().add(projBatch.invoke("addLoader").arg(rsLoader).arg(JExpr.lit(refName))); sub.decl(classGenerator.getModel()._ref(ResultSetLoader.class), getReturnValue().getName(), rsLoader); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java index 61e64a55dcc..a64ce5b2f18 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.expr.fn.impl.conv; -import io.netty.buffer.DrillBuf; import org.apache.drill.exec.expr.DrillSimpleFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; @@ -30,11 +29,13 @@ import org.apache.drill.exec.expr.holders.NullableVarCharHolder; import org.apache.drill.exec.expr.holders.VarBinaryHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; +import org.apache.drill.exec.physical.resultSet.ResultSetLoader; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; import javax.inject.Inject; +@SuppressWarnings("unused") public class JsonConvertFrom { private JsonConvertFrom() { @@ -46,39 +47,31 @@ public static class ConvertFromJson implements DrillSimpleFunc { @Param VarBinaryHolder in; - @Inject - DrillBuf buffer; + @Output + ComplexWriter writer; @Inject OptionManager options; + @Inject + ResultSetLoader rsLoader; + @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output - ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE); - boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE); - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, 1, in.start, in.end, in.buffer, false, false, false); } } @@ -94,79 +87,66 @@ public static class ConvertFromJsonWithArgs implements DrillSimpleFunc { @Param BitHolder readNumbersAsDoubleHolder; + @Output + ComplexWriter writer; + + @Inject + OptionManager options; + @Inject - DrillBuf buffer; + ResultSetLoader rsLoader; @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output - ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = allTextModeHolder.value == 1; - boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1; - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, 1, in.start, in.end, in.buffer, true, + allTextModeHolder.value == 1, readNumbersAsDoubleHolder.value == 1); } } - @FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true) public static class ConvertFromJsonVarchar implements DrillSimpleFunc { @Param VarCharHolder in; - @Inject - DrillBuf buffer; + @Output + ComplexWriter writer; @Inject OptionManager options; + @Inject + ResultSetLoader rsLoader; + @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output - ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE); - boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE); - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, 1, in.start, in.end, in.buffer, false, false, false); } } @@ -182,36 +162,32 @@ public static class ConvertFromJsonVarcharWithConfig implements DrillSimpleFunc @Param BitHolder readNumbersAsDoubleHolder; + @Output + ComplexWriter writer; + @Inject - DrillBuf buffer; + OptionManager options; + + @Inject + ResultSetLoader rsLoader; @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output - ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = allTextModeHolder.value == 1; - boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1; - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, 1, in.start, in.end, in.buffer, true, + allTextModeHolder.value == 1, readNumbersAsDoubleHolder.value == 1); } } @@ -221,47 +197,31 @@ public static class ConvertFromJsonNullableInput implements DrillSimpleFunc { @Param NullableVarBinaryHolder in; - @Inject - DrillBuf buffer; + @Output + ComplexWriter writer; @Inject OptionManager options; + @Inject + ResultSetLoader rsLoader; + @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output - ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE); - boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE); - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - if (in.isSet == 0) { - // Return empty map - org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap(); - mapWriter.start(); - mapWriter.end(); - return; - } - - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, in.isSet, in.start, in.end, in.buffer, false, false, false); } } @@ -277,44 +237,32 @@ public static class ConvertFromJsonNullableInputWithArgs implements DrillSimpleF @Param BitHolder readNumbersAsDoubleHolder; + @Output + ComplexWriter writer; + + @Inject + OptionManager options; + @Inject - DrillBuf buffer; + ResultSetLoader rsLoader; @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output - ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = allTextModeHolder.value == 1; - boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1; - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - if (in.isSet == 0) { - // Return empty map - org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap(); - mapWriter.start(); - mapWriter.end(); - return; - } - - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, in.isSet, in.start, in.end, in.buffer, true, + allTextModeHolder.value == 1, readNumbersAsDoubleHolder.value == 1); } } @@ -324,46 +272,31 @@ public static class ConvertFromJsonVarcharNullableInput implements DrillSimpleFu @Param NullableVarCharHolder in; - @Inject - DrillBuf buffer; + @Output + ComplexWriter writer; @Inject OptionManager options; + @Inject + ResultSetLoader rsLoader; + @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE); - boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE); - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - if (in.isSet == 0) { - // Return empty map - org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap(); - mapWriter.start(); - mapWriter.end(); - return; - } - - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, in.isSet, in.start, in.end, in.buffer, false, false, false); } } @@ -379,43 +312,32 @@ public static class ConvertFromJsonVarcharNullableInputWithConfigs implements Dr @Param BitHolder readNumbersAsDoubleHolder; + @Output + ComplexWriter writer; + @Inject - DrillBuf buffer; + OptionManager options; + + @Inject + ResultSetLoader rsLoader; @Workspace - org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader; + org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator streamIter; - @Output ComplexWriter writer; + @Workspace + org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl jsonLoader; @Override public void setup() { - boolean allTextMode = allTextModeHolder.value == 1; - boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1; - - jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer) - .defaultSchemaPathColumns() - .allTextMode(allTextMode) - .readNumbersAsDouble(readNumbersAsDouble) - .build(); + streamIter = new org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator(); + rsLoader.startBatch(); } @Override public void eval() { - if (in.isSet == 0) { - // Return empty map - org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap(); - mapWriter.start(); - mapWriter.end(); - return; - } - - try { - jsonReader.setSource(in.start, in.end, in.buffer); - jsonReader.write(writer); - buffer = jsonReader.getWorkBuf(); - } catch (Exception e) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e); - } + jsonLoader = org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils.convertJson( + rsLoader, jsonLoader, options, streamIter, in.isSet, in.start, in.end, in.buffer, true, + allTextModeHolder.value == 1, readNumbersAsDoubleHolder.value == 1); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java new file mode 100644 index 00000000000..d6ccaaf4d02 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.expr.fn.impl.conv; + + +import io.netty.buffer.DrillBuf; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.physical.resultSet.ResultSetLoader; +import org.apache.drill.exec.physical.resultSet.RowSetLoader; +import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.store.easy.json.loader.ClosingStreamIterator; +import org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl; +import org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder; +import org.apache.drill.exec.store.easy.json.loader.JsonLoaderOptions; +import org.apache.drill.exec.vector.complex.fn.DrillBufInputStream; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JsonConverterUtils { + + private static final Logger logger = LoggerFactory.getLogger(JsonConverterUtils.class); + + /** + * Field name used to wrap the input value so that the record-oriented JSON + * loader can read top-level scalars and arrays (not just objects) uniformly. + * The single resulting column carries the converted value and is recognised + * and unwrapped by {@code ProjectRecordBatch} (which transfers it directly to + * the output column instead of wrapping the loader's columns in a map). + */ + public static final String WRAP_FIELD = "drill_json_value_wrapper"; + + private static final byte[] WRAP_PREFIX = + ("{\"" + WRAP_FIELD + "\":").getBytes(StandardCharsets.UTF_8); + private static final byte[] WRAP_SUFFIX = "}".getBytes(StandardCharsets.UTF_8); + + private JsonConverterUtils() { + } + + /** + * Creates a {@link JsonLoaderImpl} for use in JSON conversion UDFs, using the + * system JSON options from the {@link OptionManager}. + * + * @param rsLoader The {@link ResultSetLoader} used in the UDF + * @param options The {@link OptionManager} used in the UDF. This is used to extract the global JSON options + * @param stream An input stream containing the input JSON data + * @return A {@link JsonLoaderImpl} for use in the UDF. + */ + public static JsonLoaderImpl createJsonLoader(ResultSetLoader rsLoader, + OptionManager options, + ClosingStreamIterator stream) { + JsonLoaderBuilder jsonLoaderBuilder = new JsonLoaderBuilder() + .resultSetLoader(rsLoader) + .standardOptions(options) + .fromStream(() -> stream); + + return (JsonLoaderImpl) jsonLoaderBuilder.build(); + } + + /** + * Creates a {@link JsonLoaderImpl} for use in JSON conversion UDFs, overriding the + * {@code allTextMode} and {@code readNumbersAsDouble} options with the values supplied + * as function arguments. Remaining options are taken from the system JSON options. + * + * @param rsLoader The {@link ResultSetLoader} used in the UDF + * @param options The {@link OptionManager} used in the UDF. This is used to extract the global JSON options + * @param stream An input stream containing the input JSON data + * @param allTextMode Whether to read all scalars as text + * @param readNumbersAsDouble Whether to read all numbers as doubles + * @return A {@link JsonLoaderImpl} for use in the UDF. + */ + public static JsonLoaderImpl createJsonLoader(ResultSetLoader rsLoader, + OptionManager options, + ClosingStreamIterator stream, + boolean allTextMode, + boolean readNumbersAsDouble) { + JsonLoaderOptions jsonOptions = new JsonLoaderOptions(options); + jsonOptions.allTextMode = allTextMode; + jsonOptions.readNumbersAsDouble = readNumbersAsDouble; + + JsonLoaderBuilder jsonLoaderBuilder = new JsonLoaderBuilder() + .resultSetLoader(rsLoader) + .options(jsonOptions) + .fromStream(() -> stream); + + return (JsonLoaderImpl) jsonLoaderBuilder.build(); + } + + /** + * Converts a single JSON value (one row of UDF input) into the result set loader. + * + *

Exactly one row is always written to the loader -- even for null/empty + * input -- so that the loader's row count stays aligned with the surrounding + * project batch. The value is wrapped in a single-field object so that + * top-level scalars and arrays parse the same way as objects.

+ * + * @param rsLoader the result set loader injected into the UDF + * @param jsonLoader the lazily created JSON loader, or {@code null} on the first call + * @param options the system JSON options + * @param stream the streaming iterator bound to the JSON loader + * @param isSet 0 if the input value is null, 1 otherwise + * @param start start offset of the value in {@code buffer} + * @param end end offset of the value in {@code buffer} + * @param buffer buffer holding the input value + * @param useArgs whether to honour the explicit allTextMode/readNumbersAsDouble arguments + * @param allTextMode explicit all-text-mode argument (used only when {@code useArgs}) + * @param readNumbersAsDouble explicit read-numbers-as-double argument (used only when {@code useArgs}) + * @return the JSON loader (created on first use), to be cached by the caller + */ + public static JsonLoaderImpl convertJson(ResultSetLoader rsLoader, + JsonLoaderImpl jsonLoader, + OptionManager options, + ClosingStreamIterator stream, + int isSet, int start, int end, DrillBuf buffer, + boolean useArgs, + boolean allTextMode, + boolean readNumbersAsDouble) { + RowSetLoader rowWriter = rsLoader.writer(); + rowWriter.start(); + // For null or empty input emit an (unset) row to keep the row count aligned. + if (isSet == 0 || start == end) { + rowWriter.save(); + return jsonLoader; + } + + try { + stream.setValue(getWrappedJsonStream(start, end, buffer)); + if (jsonLoader == null) { + jsonLoader = useArgs + ? createJsonLoader(rsLoader, options, stream, allTextMode, readNumbersAsDouble) + : createJsonLoader(rsLoader, options, stream); + } + // next() reads the single wrapped record; always save the row so the + // count stays aligned even for an empty document. + jsonLoader.parser().next(); + rowWriter.save(); + } catch (Exception e) { + throw UserException.dataReadError(e) + .message("Error while reading JSON. ") + .addContext(e.getMessage()) + .build(logger); + } + return jsonLoader; + } + + /** + * Wraps the raw input value in a single-field JSON object ({@code {"json": }}) + * so that the record-oriented JSON loader accepts top-level scalars and arrays. + */ + private static InputStream getWrappedJsonStream(int start, int end, DrillBuf buffer) { + InputStream value = DrillBufInputStream.getStream(start, end, buffer); + return new SequenceInputStream(Collections.enumeration(Arrays.asList( + new ByteArrayInputStream(WRAP_PREFIX), + value, + new ByteArrayInputStream(WRAP_SUFFIX)))); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchBuilder.java index 056f4c09a75..96bfb378ab4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchBuilder.java @@ -95,9 +95,10 @@ public ValueVectorWriteExpression addOutputVector(String name, LogicalExpression @Override public void addComplexField(FieldReference ref) { - if (projectBatch.rsLoader == null) { - initComplexWriters(); - } + // The result set loaders are not registered until the generated projector + // setup runs (after this point), so the complex-writer list is always + // (re)initialised here; it is simply left unused when loaders are present. + initComplexWriters(); if (projectBatch.complexFieldReferencesList == null) { projectBatch.complexFieldReferencesList = Lists.newArrayList(); } else { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index a23227a275e..bce6f191ba6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -28,12 +28,14 @@ import org.apache.drill.exec.physical.config.Project; import org.apache.drill.exec.physical.resultSet.ResultSetLoader; import org.apache.drill.exec.record.AbstractSingleRecordBatch; +import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.SimpleRecordBatch; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils; import org.apache.drill.exec.util.record.RecordBatchStats; import org.apache.drill.exec.util.record.RecordBatchStats.RecordBatchIOType; import org.apache.drill.exec.vector.AllocationHelper; @@ -54,7 +56,14 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch { protected List allocationVectors; @Deprecated // use new writer rsLoader protected List complexWriters; - protected ResultSetLoader rsLoader; + // One result set loader per complex-writer (EVF) function in the project list. + // rsLoaderRefs holds each loader's output column name, captured at codegen, so + // the harvested column lands in the slot reserved for that function. + protected List rsLoaders; + private List rsLoaderRefs; + // True once the loaders have been harvested and must be re-started before the + // next output batch is written. + private boolean loadersNeedStart; protected List complexFieldReferencesList; protected ProjectMemoryManager memoryManager; private Projector projector; @@ -111,7 +120,7 @@ protected IterOutcome doWork() { memoryManager.update(); if (first && incomingRecordCount == 0) { - if (!CollectionUtils.isEmpty(complexWriters) || rsLoader != null ) { + if (!CollectionUtils.isEmpty(complexWriters) || !CollectionUtils.isEmpty(rsLoaders)) { IterOutcome next = null; while (incomingRecordCount == 0) { if (getLastKnownOutcome() == EMIT) { @@ -146,7 +155,7 @@ protected IterOutcome doWork() { } } - if ((!CollectionUtils.isEmpty(complexWriters) || rsLoader != null) && getLastKnownOutcome() == EMIT) { + if ((!CollectionUtils.isEmpty(complexWriters) || !CollectionUtils.isEmpty(rsLoaders)) && getLastKnownOutcome() == EMIT) { throw UserException.unsupportedError() .message("Currently functions producing complex types as output are not " + "supported in project list for subquery between LATERAL and UNNEST. Please re-write the query using this " + @@ -162,6 +171,7 @@ protected IterOutcome doWork() { memoryManager.getOutputRowCount(), incomingRecordCount, incoming, this); doAlloc(maxOuputRecordCount); + startLoaderBatchIfNeeded(); long projectStartTime = System.currentTimeMillis(); int outputRecords = projector.projectRecords(incoming, 0, maxOuputRecordCount, 0); long projectEndTime = System.currentTimeMillis(); @@ -178,14 +188,8 @@ protected IterOutcome doWork() { } // In case of complex writer expression, vectors would be added to batch run-time. // We have to re-build the schema. - if (rsLoader != null) { - MapVector map = container.addOrGet(container.getLast().getField().getName(), Types.required(TypeProtos.MinorType.MAP), MapVector.class); - map.setMapValueCount(recordCount); - for (VectorWrapper vectorWrapper : rsLoader.harvest()) { - ValueVector valueVector = vectorWrapper.getValueVector(); - map.putChild(valueVector.getField().getName(), valueVector); - } - container.buildSchema(SelectionVectorMode.NONE); + if (!CollectionUtils.isEmpty(rsLoaders)) { + harvestLoaders(); } else if (!CollectionUtils.isEmpty(complexWriters)) { container.buildSchema(SelectionVectorMode.NONE); } @@ -202,6 +206,7 @@ private void handleRemainder() { assert memoryManager.incomingBatch() == incoming; int recordsToProcess = Math.min(remainingRecordCount, memoryManager.getOutputRowCount()); doAlloc(recordsToProcess); + startLoaderBatchIfNeeded(); logger.trace("handleRemainder: remaining RC {}, toProcess {}, remainder index {}, incoming {}, Project {}", remainingRecordCount, recordsToProcess, remainderIndex, incoming, this); @@ -225,7 +230,9 @@ private void handleRemainder() { } // In case of complex writer expression, vectors would be added to batch run-time. // We have to re-build the schema. - if (!CollectionUtils.isEmpty(complexWriters) || rsLoader != null) { + if (!CollectionUtils.isEmpty(rsLoaders)) { + harvestLoaders(); + } else if (!CollectionUtils.isEmpty(complexWriters)) { container.buildSchema(SelectionVectorMode.NONE); } @@ -239,8 +246,65 @@ public void addComplexWriter(ComplexWriter writer) { complexWriters.add(writer); } - public void addLoader(ResultSetLoader loader) { - rsLoader = loader; + public void addLoader(ResultSetLoader loader, String refName) { + if (rsLoaders == null) { + rsLoaders = new ArrayList<>(); + rsLoaderRefs = new ArrayList<>(); + } + rsLoaders.add(loader); + rsLoaderRefs.add(refName); + } + + /** + * Re-starts each result set loader batch when the previous batch has already + * been harvested. The loaders start their first batch from the generated UDF + * setup; this drives every subsequent output batch. + */ + private void startLoaderBatchIfNeeded() { + if (loadersNeedStart && !CollectionUtils.isEmpty(rsLoaders)) { + for (ResultSetLoader loader : rsLoaders) { + loader.startBatch(); + } + loadersNeedStart = false; + } + } + + /** + * Harvests each complex-writer function's result set loader and moves its + * single (wrapped) output column into the container slot reserved for that + * function (see {@link ProjectBatchBuilder#addComplexField}). + */ + private void harvestLoaders() { + for (int i = 0; i < rsLoaders.size(); i++) { + String refName = rsLoaderRefs.get(i); + VectorContainer harvested = rsLoaders.get(i).harvest(); + List columns = new ArrayList<>(); + for (VectorWrapper w : harvested) { + columns.add(w.getValueVector()); + } + + if (columns.size() == 1 + && JsonConverterUtils.WRAP_FIELD.equals(columns.get(0).getField().getName())) { + // convert_fromJSON wraps its value in a single marker column. Unwrap it + // so the output preserves the value's own type (scalar, array or map). + ValueVector src = columns.get(0); + ValueVector dst = container.addOrGet( + MaterializedField.create(refName, src.getField().getType()), callBack); + src.makeTransferPair(dst).transfer(); + } else { + // The loader produced the JSON object's fields directly (e.g. the HTTP + // UDFs, or an empty/all-null document). Wrap them in a map named for the + // output column. + MapVector map = container.addOrGet(refName, + Types.required(TypeProtos.MinorType.MAP), MapVector.class); + map.setMapValueCount(recordCount); + for (ValueVector src : columns) { + map.putChild(src.getField().getName(), src); + } + } + } + loadersNeedStart = true; + container.buildSchema(SelectionVectorMode.NONE); } private void doAlloc(int recordCount) { @@ -274,8 +338,10 @@ private void setValueCount(int count) { for (ComplexWriter writer : complexWriters) { writer.setValueCount(count); } - } else if (rsLoader != null) { - rsLoader.setTargetRowCount(count); + } else if (!CollectionUtils.isEmpty(rsLoaders)) { + for (ResultSetLoader loader : rsLoaders) { + loader.setTargetRowCount(count); + } } } @@ -359,7 +425,7 @@ protected IterOutcome handleNullInput() { protected IterOutcome getFinalOutcome(boolean hasMoreRecordInBoundary) { // In a case of complex writers vectors are added at runtime, so the schema // may change (e.g. when a batch contains new column(s) not present in previous batches) - if (!CollectionUtils.isEmpty(complexWriters) || rsLoader != null) { + if (!CollectionUtils.isEmpty(complexWriters) || !CollectionUtils.isEmpty(rsLoaders)) { return IterOutcome.OK_NEW_SCHEMA; } return super.getFinalOutcome(hasMoreRecordInBoundary); @@ -375,9 +441,16 @@ private void setupNewSchema(RecordBatch incomingBatch, int configuredBatchSize) } allocationVectors = new ArrayList<>(); - if (rsLoader != null) { + if (!CollectionUtils.isEmpty(rsLoaders)) { container.clear(); - rsLoader.close(); + for (ResultSetLoader loader : rsLoaders) { + loader.close(); + } + // The generated projector setup repopulates these (one loader per + // complex-writer function) on the rebuild that follows. + rsLoaders = null; + rsLoaderRefs = null; + loadersNeedStart = false; } else if (!CollectionUtils.isEmpty(complexWriters)) { container.clear(); } else { From c5b742401c4d34cfa12d34b5680f9b835d127845 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Wed, 19 Aug 2026 11:45:15 -0400 Subject: [PATCH 2/2] DRILL-8239: Address review feedback - Remove the unreachable result set loader branch in setValueCount(). Every complex-writer function goes through ProjectBatchBuilder.addComplexField(), which always initialises complexWriters, so the else-if could never run. Loaders need no row count here: the count follows the rows the generated function wrote and the batch is harvested immediately afterwards. - Use CollectionUtils.isEmpty() for complexWriters consistently. - Restore alphabetical import order in ProjectRecordBatch. - Correct the getWrappedJsonStream Javadoc to name the real wrapper field. - Add a test covering a JSON conversion over more rows than the loader's default row count limit. --- .../expr/fn/impl/conv/JsonConverterUtils.java | 5 ++-- .../impl/project/ProjectRecordBatch.java | 14 +++++----- .../store/json/TestJsonConversionUDF.java | 27 +++++++++++++++++++ .../data.json | 1 + .../meta.json | 15 +++++++++++ .../data.json | 1 + .../meta.json | 15 +++++++++++ 7 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/data.json create mode 100644 result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/meta.json create mode 100644 result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/data.json create mode 100644 result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/meta.json diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java index d6ccaaf4d02..fbcaaf6fd13 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConverterUtils.java @@ -167,8 +167,9 @@ public static JsonLoaderImpl convertJson(ResultSetLoader rsLoader, } /** - * Wraps the raw input value in a single-field JSON object ({@code {"json": }}) - * so that the record-oriented JSON loader accepts top-level scalars and arrays. + * Wraps the raw input value in a single-field JSON object + * ({@code {"drill_json_value_wrapper": }}, see {@link #WRAP_FIELD}) so that + * the record-oriented JSON loader accepts top-level scalars and arrays. */ private static InputStream getWrappedJsonStream(int start, int end, DrillBuf buffer) { InputStream value = DrillBufInputStream.getStream(start, end, buffer); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index bce6f191ba6..0b373cac770 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -24,18 +24,18 @@ import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.config.Project; import org.apache.drill.exec.physical.resultSet.ResultSetLoader; import org.apache.drill.exec.record.AbstractSingleRecordBatch; -import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; +import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.SimpleRecordBatch; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.VectorWrapper; -import org.apache.drill.exec.expr.fn.impl.conv.JsonConverterUtils; import org.apache.drill.exec.util.record.RecordBatchStats; import org.apache.drill.exec.util.record.RecordBatchStats.RecordBatchIOType; import org.apache.drill.exec.vector.AllocationHelper; @@ -314,7 +314,7 @@ private void doAlloc(int recordCount) { } // Allocate vv for complexWriters. - if (complexWriters != null) { + if (!CollectionUtils.isEmpty(complexWriters)) { for (ComplexWriter writer : complexWriters) { writer.allocate(); } @@ -334,14 +334,12 @@ private void setValueCount(int count) { // the transfer pairs or vector copies. container.setRecordCount(count); - if (complexWriters != null) { + // Result set loaders need nothing here: their row count follows the rows the + // generated function wrote, and the batch is harvested right after this call. + if (!CollectionUtils.isEmpty(complexWriters)) { for (ComplexWriter writer : complexWriters) { writer.setValueCount(count); } - } else if (!CollectionUtils.isEmpty(rsLoaders)) { - for (ResultSetLoader loader : rsLoaders) { - loader.setTargetRowCount(count); - } } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonConversionUDF.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonConversionUDF.java index d6fa857a390..4501e5f7fc8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonConversionUDF.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonConversionUDF.java @@ -21,6 +21,7 @@ import org.apache.commons.io.FileUtils; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.physical.resultSet.ResultSetLoader; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.rpc.user.QueryDataBatch; @@ -220,4 +221,30 @@ public void testReadNumbersAsDoubleFromArgs() throws Exception { resetSessionOption(ExecConstants.JSON_ALL_TEXT_MODE); resetSessionOption(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE); } + + @Test + public void testConvertFromJsonBatchLargerThanLoaderRowLimit() throws Exception { + // More rows than the result set loader's default row count limit, so that a + // single project batch outgrows the loader's initial target row count. + int rowCount = ResultSetLoader.DEFAULT_ROW_COUNT * 2 + 17; + String table = "large_json_input.csv"; + File file = new File(dirTestWatcher.getRootDir(), table); + StringBuilder csv = new StringBuilder(); + for (int i = 0; i < rowCount; i++) { + csv.append("col_").append(i).append(", {\"id\":").append(i).append("}\n"); + } + try { + FileUtils.writeStringToFile(file, csv.toString(), Charset.defaultCharset()); + String sql = String.format( + "SELECT COUNT(*) AS cnt FROM (SELECT convert_fromJSON(columns[1]) AS col FROM dfs.`%s`)", table); + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("cnt") + .baselineValues((long) rowCount) + .go(); + } finally { + FileUtils.deleteQuietly(file); + } + } } diff --git a/result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/data.json b/result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/data.json new file mode 100644 index 00000000000..387662532d7 --- /dev/null +++ b/result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/data.json @@ -0,0 +1 @@ +[{"state":"NY","customer_count":"8542","avg_orders_per_customer":"24.983259189885274"},{"state":"CA","customer_count":"387","avg_orders_per_customer":"0.29457364341085274"},{"state":"TX","customer_count":"246","avg_orders_per_customer":"0.26422764227642276"},{"state":"FL","customer_count":"203","avg_orders_per_customer":"0.22167487684729065"},{"state":"IL","customer_count":"114","avg_orders_per_customer":"0.2543859649122807"},{"state":"PA","customer_count":"108","avg_orders_per_customer":"0.25925925925925924"},{"state":"OH","customer_count":"103","avg_orders_per_customer":"0.2912621359223301"},{"state":"AZ","customer_count":"81","avg_orders_per_customer":"0.30864197530864196"},{"state":"VA","customer_count":"78","avg_orders_per_customer":"0.2692307692307692"},{"state":"GA","customer_count":"74","avg_orders_per_customer":"0.3918918918918919"},{"state":"WA","customer_count":"69","avg_orders_per_customer":"0.36231884057971014"},{"state":"MI","customer_count":"68","avg_orders_per_customer":"0.23529411764705882"},{"state":"NC","customer_count":"58","avg_orders_per_customer":"0.25862068965517243"},{"state":"CO","customer_count":"58","avg_orders_per_customer":"0.2413793103448276"},{"state":"MO","customer_count":"54","avg_orders_per_customer":"0.16666666666666666"},{"state":"MA","customer_count":"54","avg_orders_per_customer":"0.5"},{"state":"MD","customer_count":"52","avg_orders_per_customer":"0.4807692307692308"},{"state":"TN","customer_count":"47","avg_orders_per_customer":"0.14893617021276595"},{"state":"UT","customer_count":"45","avg_orders_per_customer":"0.2"},{"state":"IN","customer_count":"43","avg_orders_per_customer":"0.27906976744186046"},{"state":"NJ","customer_count":"42","avg_orders_per_customer":"1.5714285714285714"},{"state":"MN","customer_count":"41","avg_orders_per_customer":"0.4146341463414634"},{"state":"LA","customer_count":"39","avg_orders_per_customer":"0.38461538461538464"},{"state":"WI","customer_count":"38","avg_orders_per_customer":"0.39473684210526316"},{"state":"DC","customer_count":"34","avg_orders_per_customer":"0.20588235294117646"},{"state":"OR","customer_count":"33","avg_orders_per_customer":"0.30303030303030304"},{"state":"CT","customer_count":"29","avg_orders_per_customer":"0.3448275862068966"},{"state":"SC","customer_count":"29","avg_orders_per_customer":"0.3793103448275862"},{"state":"KS","customer_count":"26","avg_orders_per_customer":"0.15384615384615385"},{"state":"NV","customer_count":"25","avg_orders_per_customer":"0.44"},{"state":"IA","customer_count":"25","avg_orders_per_customer":"0.24"},{"state":"PR","customer_count":"24","avg_orders_per_customer":"0.25"},{"state":"AL","customer_count":"21","avg_orders_per_customer":"0.38095238095238093"},{"state":"AR","customer_count":"20","avg_orders_per_customer":"0.25"},{"state":"OK","customer_count":"19","avg_orders_per_customer":"0.47368421052631576"},{"state":"NM","customer_count":"18","avg_orders_per_customer":"0.4444444444444444"},{"state":"KY","customer_count":"18","avg_orders_per_customer":"0.3888888888888889"},{"state":"MS","customer_count":"17","avg_orders_per_customer":"0.35294117647058826"},{"state":"NE","customer_count":"15","avg_orders_per_customer":"0.2"},{"state":"RI","customer_count":"13","avg_orders_per_customer":"0.07692307692307693"},{"state":"ID","customer_count":"12","avg_orders_per_customer":"0.4166666666666667"},{"state":"WV","customer_count":"9","avg_orders_per_customer":"0.3333333333333333"},{"state":"NH","customer_count":"9","avg_orders_per_customer":"0.6666666666666666"},{"state":"SD","customer_count":"8","avg_orders_per_customer":"0.125"},{"state":"HI","customer_count":"7","avg_orders_per_customer":"0.5714285714285714"},{"state":"ME","customer_count":"7","avg_orders_per_customer":"0.2857142857142857"},{"state":"ND","customer_count":"6","avg_orders_per_customer":"0.5"},{"state":"AK","customer_count":"3","avg_orders_per_customer":"0.3333333333333333"},{"state":"WY","customer_count":"3","avg_orders_per_customer":"0.6666666666666666"},{"state":"DE","customer_count":"2","avg_orders_per_customer":"0"},{"state":"VT","customer_count":"2","avg_orders_per_customer":"1.5"},{"state":"MT","customer_count":"1","avg_orders_per_customer":"0"}] \ No newline at end of file diff --git a/result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/meta.json b/result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/meta.json new file mode 100644 index 00000000000..630a4e3b1f9 --- /dev/null +++ b/result-cache/40658b21-8d70-4eef-ba2f-52881392a6a8/meta.json @@ -0,0 +1,15 @@ +{ + "cacheId" : "40658b21-8d70-4eef-ba2f-52881392a6a8", + "queryId" : "157b9289-9c18-477b-bd16-b758d2ad9787", + "sqlHash" : "1ffac56b73e4af6792d8018e2a36135137665a4508624aa7141849e45f22d5b8", + "sql" : "SELECT\n c.`state`,\n COUNT(DISTINCT c.`customerid`) AS `customer_count`,\n CAST(COUNT(o.`orderid`) AS DOUBLE) / COUNT(DISTINCT c.`customerid`) AS `avg_orders_per_customer`\nFROM `mysql.store`.`customers` AS c\nLEFT JOIN `mysql.store`.`orders` AS o\n ON c.`customerid` = o.`customerid`\nGROUP BY c.`state`\nORDER BY `customer_count` DESC", + "defaultSchema" : "", + "userName" : "anonymous", + "queryState" : "COMPLETED", + "columns" : [ "state", "customer_count", "avg_orders_per_customer" ], + "metadata" : [ "VARCHAR(2, 0)", "BIGINT(21, 0)", "FLOAT8(23, 31)" ], + "totalRows" : 52, + "sizeBytes" : 4208, + "cachedAt" : 1787063670634, + "lastAccessedAt" : 1787063670634 +} \ No newline at end of file diff --git a/result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/data.json b/result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/data.json new file mode 100644 index 00000000000..e5525f7c60f --- /dev/null +++ b/result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/data.json @@ -0,0 +1 @@ +[{"state":"NY","customer_count":"8542"},{"state":"CA","customer_count":"387"},{"state":"TX","customer_count":"246"},{"state":"FL","customer_count":"203"},{"state":"IL","customer_count":"114"},{"state":"PA","customer_count":"108"},{"state":"OH","customer_count":"103"},{"state":"AZ","customer_count":"81"},{"state":"VA","customer_count":"78"},{"state":"GA","customer_count":"74"},{"state":"WA","customer_count":"69"},{"state":"MI","customer_count":"68"},{"state":"NC","customer_count":"58"},{"state":"CO","customer_count":"58"},{"state":"MA","customer_count":"54"},{"state":"MO","customer_count":"54"},{"state":"MD","customer_count":"52"},{"state":"TN","customer_count":"47"},{"state":"UT","customer_count":"45"},{"state":"IN","customer_count":"43"},{"state":"NJ","customer_count":"42"},{"state":"MN","customer_count":"41"},{"state":"LA","customer_count":"39"},{"state":"WI","customer_count":"38"},{"state":"DC","customer_count":"34"},{"state":"OR","customer_count":"33"},{"state":"CT","customer_count":"29"},{"state":"SC","customer_count":"29"},{"state":"KS","customer_count":"26"},{"state":"IA","customer_count":"25"},{"state":"NV","customer_count":"25"},{"state":"PR","customer_count":"24"},{"state":"AL","customer_count":"21"},{"state":"AR","customer_count":"20"},{"state":"OK","customer_count":"19"},{"state":"KY","customer_count":"18"},{"state":"NM","customer_count":"18"},{"state":"MS","customer_count":"17"},{"state":"NE","customer_count":"15"},{"state":"RI","customer_count":"13"},{"state":"ID","customer_count":"12"},{"state":"WV","customer_count":"9"},{"state":"NH","customer_count":"9"},{"state":"SD","customer_count":"8"},{"state":"ME","customer_count":"7"},{"state":"HI","customer_count":"7"},{"state":"ND","customer_count":"6"},{"state":"WY","customer_count":"3"},{"state":"AK","customer_count":"3"},{"state":"VT","customer_count":"2"},{"state":"DE","customer_count":"2"},{"state":"MT","customer_count":"1"}] \ No newline at end of file diff --git a/result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/meta.json b/result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/meta.json new file mode 100644 index 00000000000..e98d6cff74a --- /dev/null +++ b/result-cache/4bdbeab0-2721-4be8-b92e-2c61f2f8d30e/meta.json @@ -0,0 +1,15 @@ +{ + "cacheId" : "4bdbeab0-2721-4be8-b92e-2c61f2f8d30e", + "queryId" : "157b92ce-9cd8-9ccc-57cb-c910ba8bc627", + "sqlHash" : "ceb1b41179fe1c884c794b4cf71797098eaa8461c1ac82c8684402d1e160a403", + "sql" : "SELECT `state`, COUNT(*) AS `customer_count`\nFROM `mysql.store`.`customers`\nGROUP BY `state`\nORDER BY `customer_count` DESC", + "defaultSchema" : "", + "userName" : "anonymous", + "queryState" : "COMPLETED", + "columns" : [ "state", "customer_count" ], + "metadata" : [ "VARCHAR(2, 0)", "BIGINT(21, 0)" ], + "totalRows" : 52, + "sizeBytes" : 1922, + "cachedAt" : 1787063600807, + "lastAccessedAt" : 1787063600807 +} \ No newline at end of file