diff --git a/.licenserc.yaml b/.licenserc.yaml index afb8d66123913e..38cc9e90663dd3 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -64,6 +64,7 @@ header: # the file an administrator edits in plugins/connector//. Matched by name # rather than by a **/*.conf.template glob, so a new one is a deliberate entry # here rather than something a wildcard silently absorbs. + - "fe/fe-connector/fe-connector-adbc/src/main/resources/adbc.conf.template" - "fe/fe-connector/fe-connector-hive/src/main/resources/hms.conf.template" - "fe/fe-connector/fe-connector-iceberg/src/main/resources/iceberg.conf.template" - "fe/fe-connector/fe-connector-jdbc/src/main/resources/jdbc.conf.template" diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 92dffd98da875b..cdab814c78fe4c 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -109,6 +109,7 @@ add_thirdparty(arrow_flight LIB64) add_thirdparty(arrow_flight_sql LIB64) add_thirdparty(arrow_dataset LIB64) add_thirdparty(arrow_acero LIB64) +add_thirdparty(adbc_driver_manager LIB64) add_thirdparty(parquet LIB64) # liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it # so the final linker resolves C math symbols from the system library first. diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index 736c9647a2feec..370f77bed8b1c8 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -113,6 +113,17 @@ bool FileScanLocalState::TEST_should_use_file_scanner_v2(const TQueryOptions& qu bool FileScanLocalState::_should_use_file_scanner_v2(const TQueryOptions& query_options, bool is_load, const TFileScanRangeParams& scan_params) { + // ADBC only has a FileScannerV2 reader, and enable_file_scanner_v2 is a session variable marked + // fuzzy=true, so the regression harness flips it to false at random. Without letting adbc + // through unconditionally, those queries land in v1, which has no "adbc" branch, and come back + // with an undiagnosable NotSupported -- showing up in CI as a random failure. This is the same + // mechanism as is_transactional_hive below, pointed the other way. Loads are left alone: there + // is no ADBC load path, so widening the rule to cover them would only route them somewhere they + // still cannot run. + if (!is_load && scan_params.__isset.table_format_params && + scan_params.table_format_params.table_format_type == "adbc") { + return true; + } const bool is_transactional_hive = scan_params.__isset.table_format_params && scan_params.table_format_params.table_format_type == "transactional_hive"; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 6859fe8d5434a4..875e017901c1d6 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -54,6 +54,7 @@ #include "format_v2/jni/jdbc_reader.h" #include "format_v2/jni/max_compute_jni_reader.h" #include "format_v2/jni/trino_connector_jni_reader.h" +#include "format_v2/table/adbc_reader.h" #include "format_v2/table/hive_reader.h" #include "format_v2/table/hudi_reader.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" @@ -103,7 +104,8 @@ bool is_supported_table_format(const TFileRangeDesc& range) { } bool is_supported_arrow_table_format(const TFileRangeDesc& range) { - return table_format_name(range) == "remote_doris"; + const auto table_format = table_format_name(range); + return table_format == "remote_doris" || table_format == "adbc"; } bool is_supported_jni_table_format(const TFileRangeDesc& range) { @@ -619,6 +621,8 @@ Status FileScannerV2::_create_table_reader_for_format( *reader = std::make_unique(); } else if (table_format == "remote_doris") { *reader = std::make_unique(); + } else if (table_format == "adbc") { + *reader = std::make_unique(); } else { return Status::NotSupported("FileScannerV2 does not support table format {}", table_format); } diff --git a/be/src/format/arrow/arrow_array_normalizer.cpp b/be/src/format/arrow/arrow_array_normalizer.cpp new file mode 100644 index 00000000000000..4b4483cb922a2b --- /dev/null +++ b/be/src/format/arrow/arrow_array_normalizer.cpp @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/arrow/arrow_array_normalizer.h" + +#include +#include +#include + +#include + +#include "common/check.h" +#include "common/status.h" + +namespace doris { + +namespace { + +// The accepted counterpart of an encoding-only variant. Null when there is none. +std::shared_ptr target_type_for(const arrow::DataType& type) { + switch (type.id()) { + case arrow::Type::LARGE_STRING: + case arrow::Type::STRING_VIEW: + return arrow::utf8(); + case arrow::Type::LARGE_BINARY: + case arrow::Type::BINARY_VIEW: + return arrow::binary(); + case arrow::Type::DICTIONARY: + return static_cast(type).value_type(); + case arrow::Type::RUN_END_ENCODED: + return static_cast(type).value_type(); + default: + return nullptr; + } +} + +} // namespace + +bool is_serde_acceptable_arrow_type(const arrow::DataType& type) { + switch (type.id()) { + // Encoding-only variants: convertible to an accepted type. + case arrow::Type::LARGE_STRING: + case arrow::Type::LARGE_BINARY: + case arrow::Type::STRING_VIEW: + case arrow::Type::BINARY_VIEW: + case arrow::Type::DICTIONARY: + case arrow::Type::RUN_END_ENCODED: + return false; + // No Doris column can hold these, so they must not reach a serde either. + case arrow::Type::INTERVAL_MONTHS: + case arrow::Type::INTERVAL_DAY_TIME: + case arrow::Type::INTERVAL_MONTH_DAY_NANO: + case arrow::Type::DURATION: + case arrow::Type::SPARSE_UNION: + case arrow::Type::DENSE_UNION: + return false; + default: + return true; + } +} + +Status normalize_arrow_array(const std::shared_ptr& arr, + std::shared_ptr* out) { + DORIS_CHECK(arr != nullptr); + DORIS_CHECK(out != nullptr); + + std::shared_ptr current = arr; + // dictionary decodes to large_utf8, which still needs converting. The bound + // stops a driver from turning a malformed type into an endless loop. + constexpr int kMaxPasses = 4; + for (int pass = 0; pass < kMaxPasses; ++pass) { + const auto& type = *current->type(); + if (is_serde_acceptable_arrow_type(type)) { + *out = std::move(current); + return Status::OK(); + } + + auto target = target_type_for(type); + if (target == nullptr) { + return Status::NotSupported( + "ADBC: arrow type '{}' cannot be materialized into a Doris column", + type.ToString()); + } + + auto casted = arrow::compute::Cast(*current, target); + if (!casted.ok()) { + return Status::InternalError("ADBC: failed to normalize arrow type '{}' to '{}': {}", + type.ToString(), target->ToString(), + casted.status().ToString()); + } + current = casted.MoveValueUnsafe(); + } + return Status::InternalError( + "ADBC: arrow type '{}' is still not materializable after {} " + "normalization passes", + arr->type()->ToString(), kMaxPasses); +} + +} // namespace doris diff --git a/be/src/format/arrow/arrow_array_normalizer.h b/be/src/format/arrow/arrow_array_normalizer.h new file mode 100644 index 00000000000000..eed9ab02293fbb --- /dev/null +++ b/be/src/format/arrow/arrow_array_normalizer.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" + +namespace doris { + +/// DataTypeSerDe::read_column_from_arrow was written for the Arrow that Doris itself emits, so it +/// accepts only a subset of the Arrow type variants. Third-party ADBC drivers emit others: DuckDB +/// emits string_view, Go-based drivers may emit large_* and dictionary. This normalizes them into +/// a shape the serdes accept. +/// +/// Only top-level types are normalized. A nested type whose child is an unaccepted variant (say +/// list) passes through and fails inside the serde, loudly rather than silently. + +/// Whether the serdes take this Arrow type as-is. +bool is_serde_acceptable_arrow_type(const arrow::DataType& type); + +/// Normalizes `arr` into a serde-acceptable shape. Returns it unchanged (no copy) when it already +/// is one, and fails with the offending type named when no accepted shape exists. +Status normalize_arrow_array(const std::shared_ptr& arr, + std::shared_ptr* out); + +} // namespace doris diff --git a/be/src/format_v2/table/adbc_reader.cpp b/be/src/format_v2/table/adbc_reader.cpp new file mode 100644 index 00000000000000..98847c40603f2b --- /dev/null +++ b/be/src/format_v2/table/adbc_reader.cpp @@ -0,0 +1,724 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/adbc_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/check.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/data_type_serde.h" +#include "format/arrow/arrow_array_normalizer.h" +#include "format_v2/materialized_reader_util.h" +#include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" +#include "runtime/runtime_state.h" +#include "util/adbc_driver_registry.h" +#include "util/timezone_utils.h" +#include "util/url_coding.h" + +namespace doris::format::adbc { +namespace { + +// Keys of TTableFormatFileDesc.adbc_params. Kept next to the code that reads them so the FE-side +// producer and this consumer stay diffable. +constexpr const char* kParamDriverPath = "driver_path"; +constexpr const char* kParamDriverEntrypoint = "driver_entrypoint"; +constexpr const char* kParamUri = "uri"; +constexpr const char* kParamUsername = "username"; +constexpr const char* kParamPassword = "password"; +constexpr const char* kParamQuerySql = "query_sql"; +// Base64 of one opaque partition descriptor the driver produced on FE. Mutually exclusive with +// kParamQuerySql: a range either runs a statement here or reads one partition of a statement the +// source has already run. +constexpr const char* kParamPartitionDescriptor = "partition_descriptor"; +// Anything under this prefix is an ADBC option name in full (the prefix is part of the option name, +// e.g. "adbc.connection.autocommit") and is handed to the driver untouched. +constexpr std::string_view kAdbcOptionPrefix = "adbc."; + +const std::string* find_param(const std::map& params, + const std::string& key) { + const auto it = params.find(key); + return it == params.end() ? nullptr : &it->second; +} + +Status validate_adbc_range(const TFileRangeDesc& range) { + if (!range.__isset.table_format_params || + range.table_format_params.table_format_type != "adbc") { + return Status::InvalidArgument("ADBC reader requires the adbc table format"); + } + if (!range.table_format_params.__isset.adbc_params) { + return Status::InvalidArgument("ADBC reader requires adbc_params"); + } + const auto& params = range.table_format_params.adbc_params; + for (const auto* key : {kParamDriverPath, kParamUri}) { + const auto* value = find_param(params, key); + if (value == nullptr || value->empty()) { + return Status::InvalidArgument("ADBC reader requires a non-empty '{}' parameter", key); + } + } + const auto* query_sql = find_param(params, kParamQuerySql); + const auto* partition = find_param(params, kParamPartitionDescriptor); + const bool has_query = query_sql != nullptr && !query_sql->empty(); + const bool has_partition = partition != nullptr && !partition->empty(); + // Not a defensive nicety: reading a partition means the source has ALREADY run the statement, so + // a range carrying both would let this reader run it a second time depending on which branch it + // happened to take. FE refuses to build such a range; this refuses to act on one. + if (has_query == has_partition) { + return Status::InvalidArgument( + "ADBC reader requires exactly one of '{}' and '{}', but the range carries {}", + kParamQuerySql, kParamPartitionDescriptor, has_query ? "both" : "neither"); + } + return Status::OK(); +} + +// Drivers allocate the strings inside AdbcError, so every populated error has to be released. +class AdbcErrorGuard { +public: + AdbcErrorGuard() = default; + ~AdbcErrorGuard() { reset(); } + AdbcErrorGuard(const AdbcErrorGuard&) = delete; + AdbcErrorGuard& operator=(const AdbcErrorGuard&) = delete; + + AdbcError* get() { return &_error; } + + std::string take_message() { + std::string message = _error.message != nullptr ? _error.message : ""; + reset(); + return message; + } + + void reset() { + if (_error.release != nullptr) { + _error.release(&_error); + } + _error = ADBC_ERROR_INIT; + } + +private: + AdbcError _error = ADBC_ERROR_INIT; +}; + +Status adbc_call_status(const char* what, AdbcStatusCode code, AdbcErrorGuard& error) { + const std::string message = error.take_message(); + // AdbcStatusCode is a uint8_t, so spell out the name as well as the number. + return Status::InternalError("ADBC: {} failed ({}, code {}): {}", what, + AdbcStatusCodeMessage(code), static_cast(code), + message.empty() ? "driver reported no message" : message); +} + +#define RETURN_IF_ADBC_ERROR(expr, what, error) \ + do { \ + const AdbcStatusCode adbc_call_code = (expr); \ + if (adbc_call_code != ADBC_STATUS_OK) { \ + return adbc_call_status((what), adbc_call_code, (error)); \ + } \ + (error).reset(); \ + } while (0) + +// The production stream: one ADBC database/connection/statement per scan range. +// +// P0 keeps them un-pooled on purpose. Reusing databases across ranges is a throughput optimization +// that only pays off once multiple partitions run concurrently, and an unverifiable caching layer +// added now would only obscure the functional path. +class RealAdbcStream final : public AdbcStream { +public: + explicit RealAdbcStream(const TFileRangeDesc& range) : _range(range) {} + ~RealAdbcStream() override { static_cast(close()); } + + Status open() { + RETURN_IF_ERROR(validate_adbc_range(_range)); + const auto& params = _range.table_format_params.adbc_params; + const std::string& driver_path = *find_param(params, kParamDriverPath); + const std::string& uri = *find_param(params, kParamUri); + const auto* entrypoint = find_param(params, kParamDriverEntrypoint); + // validate_adbc_range has established that exactly one of these is present. + const auto* partition = find_param(params, kParamPartitionDescriptor); + + RETURN_IF_ERROR(AdbcDriverRegistry::instance().get_or_load( + driver_path, entrypoint != nullptr ? *entrypoint : std::string(), &_driver)); + DORIS_CHECK(_driver != nullptr); + + AdbcErrorGuard error; + RETURN_IF_ADBC_ERROR(_driver->DatabaseNew(&_database, error.get()), "DatabaseNew", error); + _database_created = true; + RETURN_IF_ERROR(_set_database_options(params, uri, error)); + RETURN_IF_ADBC_ERROR(_driver->DatabaseInit(&_database, error.get()), "DatabaseInit", error); + + RETURN_IF_ADBC_ERROR(_driver->ConnectionNew(&_connection, error.get()), "ConnectionNew", + error); + _connection_created = true; + RETURN_IF_ADBC_ERROR(_driver->ConnectionInit(&_connection, &_database, error.get()), + "ConnectionInit", error); + + if (partition != nullptr && !partition->empty()) { + RETURN_IF_ERROR(_read_partition(*partition, error)); + } else { + RETURN_IF_ERROR(_execute_query(*find_param(params, kParamQuerySql), error)); + } + + // Before Arrow ever sees it: the driver's stream may not clear its release callback, and + // Arrow aborts the process when that happens. Both branches need it -- the Flight SQL + // driver's ReadPartition stream breaks the contract exactly like its ExecuteQuery one. + enforce_stream_release_contract(&_c_stream); + + auto reader = arrow::ImportRecordBatchReader(&_c_stream); + if (!reader.ok()) { + return Status::InternalError("ADBC: failed to import the result stream: {}", + reader.status().ToString()); + } + // ImportRecordBatchReader moves the stream's contents; the reader owns it from here. + _reader = reader.MoveValueUnsafe(); + return Status::OK(); + } + + Status next(std::shared_ptr* batch) override { + DORIS_CHECK(batch != nullptr); + if (_reader == nullptr) { + return Status::InternalError("ADBC: result stream is not open"); + } + std::shared_ptr next_batch; + const auto status = _reader->ReadNext(&next_batch); + if (!status.ok()) { + return Status::InternalError("ADBC: failed to read the next batch: {}", + status.ToString()); + } + *batch = std::move(next_batch); + return Status::OK(); + } + + Status close() override { + Status result = Status::OK(); + // Release in reverse order of creation. The reader owns the imported stream, so it has to + // go before the statement that produced it. + _reader.reset(); + if (_c_stream.release != nullptr) { + // Only reachable when the import itself failed; nothing else owns the stream then. + _c_stream.release(&_c_stream); + _c_stream = {}; + } + AdbcErrorGuard error; + if (_statement_created) { + const auto code = _driver->StatementRelease(&_statement, error.get()); + if (code != ADBC_STATUS_OK && result.ok()) { + result = adbc_call_status("StatementRelease", code, error); + } + error.reset(); + _statement_created = false; + } + if (_connection_created) { + const auto code = _driver->ConnectionRelease(&_connection, error.get()); + if (code != ADBC_STATUS_OK && result.ok()) { + result = adbc_call_status("ConnectionRelease", code, error); + } + error.reset(); + _connection_created = false; + } + if (_database_created) { + const auto code = _driver->DatabaseRelease(&_database, error.get()); + if (code != ADBC_STATUS_OK && result.ok()) { + result = adbc_call_status("DatabaseRelease", code, error); + } + error.reset(); + _database_created = false; + } + // _driver itself is owned by AdbcDriverRegistry and is never released. + return result; + } + +private: + // Runs the statement FE generated. One statement per range, so this range is the whole query. + Status _execute_query(const std::string& query_sql, AdbcErrorGuard& error) { + RETURN_IF_ADBC_ERROR(_driver->StatementNew(&_connection, &_statement, error.get()), + "StatementNew", error); + _statement_created = true; + RETURN_IF_ADBC_ERROR( + _driver->StatementSetSqlQuery(&_statement, query_sql.c_str(), error.get()), + "StatementSetSqlQuery", error); + int64_t rows_affected = -1; + RETURN_IF_ADBC_ERROR(_driver->StatementExecuteQuery(&_statement, &_c_stream, &rows_affected, + error.get()), + "StatementExecuteQuery", error); + return Status::OK(); + } + + // Reads one partition of a query FE already had the source execute. No statement is created: + // ADBC reads a partition off a connection, and the whole point is that this can happen on a + // different machine from the one that planned it. + Status _read_partition(const std::string& base64_descriptor, AdbcErrorGuard& error) { + std::string descriptor; + if (!base64_decode(base64_descriptor, &descriptor)) { + return Status::InvalidArgument("ADBC: the '{}' parameter is not valid base64", + kParamPartitionDescriptor); + } + RETURN_IF_ADBC_ERROR( + _driver->ConnectionReadPartition( + &_connection, reinterpret_cast(descriptor.data()), + descriptor.size(), &_c_stream, error.get()), + "ConnectionReadPartition", error); + return Status::OK(); + } + + Status _set_database_options(const std::map& params, + const std::string& uri, AdbcErrorGuard& error) { + RETURN_IF_ADBC_ERROR( + _driver->DatabaseSetOption(&_database, ADBC_OPTION_URI, uri.c_str(), error.get()), + "DatabaseSetOption(uri)", error); + for (const auto* key : {kParamUsername, kParamPassword}) { + const auto* value = find_param(params, key); + if (value == nullptr || value->empty()) { + continue; + } + RETURN_IF_ADBC_ERROR( + _driver->DatabaseSetOption(&_database, key, value->c_str(), error.get()), + "DatabaseSetOption(credentials)", error); + } + for (const auto& [key, value] : params) { + if (!key.starts_with(kAdbcOptionPrefix)) { + continue; + } + RETURN_IF_ADBC_ERROR( + _driver->DatabaseSetOption(&_database, key.c_str(), value.c_str(), error.get()), + "DatabaseSetOption(passthrough)", error); + } + return Status::OK(); + } + + const TFileRangeDesc _range; + const AdbcDriver* _driver = nullptr; + AdbcDatabase _database {}; + AdbcConnection _connection {}; + AdbcStatement _statement {}; + ArrowArrayStream _c_stream {}; + std::shared_ptr _reader; + bool _database_created = false; + bool _connection_created = false; + bool _statement_created = false; +}; + +Status create_real_adbc_stream(const TFileRangeDesc& range, std::unique_ptr* out) { + DORIS_CHECK(out != nullptr); + auto stream = std::make_unique(range); + RETURN_IF_ERROR(stream->open()); + *out = std::move(stream); + return Status::OK(); +} + +ColumnDefinition adbc_child_definition(const std::string& name, DataTypePtr type, int32_t local_id); + +// Mirrors synthesize_remote_doris_children in remote_doris_reader.cpp. Both readers expose table +// slots as file columns, so complex columns still need structural children for TableColumnMapper. +// Kept separate rather than shared to avoid reshaping the already-shipped remote_doris reader. +std::vector synthesize_adbc_children(const DataTypePtr& type) { + std::vector children; + DORIS_CHECK(type != nullptr); + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + children.push_back(adbc_child_definition("element", array_type->get_nested_type(), 0)); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + children.push_back(adbc_child_definition("key", map_type->get_key_type(), 0)); + children.push_back(adbc_child_definition("value", map_type->get_value_type(), 1)); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + children.reserve(struct_type->get_elements().size()); + for (size_t idx = 0; idx < struct_type->get_elements().size(); ++idx) { + children.push_back(adbc_child_definition(struct_type->get_element_name(idx), + struct_type->get_element(idx), + cast_set(idx))); + } + break; + } + default: + break; + } + return children; +} + +ColumnDefinition adbc_child_definition(const std::string& name, DataTypePtr type, + int32_t local_id) { + ColumnDefinition child; + child.identifier = Field::create_field(name); + child.local_id = local_id; + child.name = name; + child.type = std::move(type); + child.children = synthesize_adbc_children(child.type); + return child; +} + +// A stream that forwards everything to the driver's and, on release, does the one thing some +// drivers forget: clear its own release callback. Heap-allocated because Arrow keeps only the +// ArrowArrayStream it was handed, and the delegate has to outlive this function. +struct DelegatingStream { + ArrowArrayStream inner; +}; + +int delegating_get_schema(ArrowArrayStream* self, ArrowSchema* out) { + auto& inner = static_cast(self->private_data)->inner; + return inner.get_schema(&inner, out); +} + +int delegating_get_next(ArrowArrayStream* self, ArrowArray* out) { + auto& inner = static_cast(self->private_data)->inner; + return inner.get_next(&inner, out); +} + +const char* delegating_get_last_error(ArrowArrayStream* self) { + auto& inner = static_cast(self->private_data)->inner; + return inner.get_last_error != nullptr ? inner.get_last_error(&inner) : nullptr; +} + +void delegating_release(ArrowArrayStream* self) { + auto* delegate = static_cast(self->private_data); + if (delegate->inner.release != nullptr) { + delegate->inner.release(&delegate->inner); + } + delete delegate; + self->private_data = nullptr; + // What the driver failed to do, and what Arrow aborts the process over. + self->release = nullptr; +} + +} // namespace + +void enforce_stream_release_contract(ArrowArrayStream* stream) { + DORIS_CHECK(stream != nullptr); + if (stream->release == nullptr) { + // Already released; nothing to delegate to, and wrapping it would hand Arrow a stream + // whose callbacks dereference a released delegate. + return; + } + auto* delegate = new DelegatingStream {.inner = *stream}; + *stream = ArrowArrayStream {.get_schema = delegating_get_schema, + .get_next = delegating_get_next, + .get_last_error = delegating_get_last_error, + .release = delegating_release, + .private_data = delegate}; +} + +AdbcFileReader::AdbcFileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileRangeDesc& range, + const std::vector& file_slot_descs, + AdbcStreamFactory stream_factory) + : FileReader(system_properties, file_description, std::move(io_ctx), profile), + _range(range), + _file_slot_descs(file_slot_descs), + _stream_factory(std::move(stream_factory)) { + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctz); +} + +AdbcFileReader::~AdbcFileReader() { + static_cast(close()); +} + +void AdbcFileReader::_init_profile() { + if (_profile == nullptr) { + return; + } + const auto hierarchy = file_scan_profile::ensure_hierarchy(_profile); + _io_time = hierarchy.io; + static const char* adbc_profile = "AdbcFileReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, adbc_profile, file_scan_profile::FILE_READER, 1); + _open_stream_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "AdbcOpenStreamTime", adbc_profile, 1); + _next_batch_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "AdbcNextBatchTime", adbc_profile, 1); + _normalize_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "AdbcNormalizeTime", adbc_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "AdbcMaterializeTime", adbc_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "AdbcFilterTime", adbc_profile, 1); +} + +Status AdbcFileReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); + _runtime_state = state; + RETURN_IF_ERROR(validate_adbc_range(_range)); + RETURN_IF_ERROR(_build_col_name_to_file_id()); + _eof = false; + return Status::OK(); +} + +Status AdbcFileReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); + DORIS_CHECK(file_schema != nullptr); + file_schema->clear(); + file_schema->reserve(_file_slot_descs.size()); + for (size_t idx = 0; idx < _file_slot_descs.size(); ++idx) { + const auto* slot = _file_slot_descs[idx]; + DORIS_CHECK(slot != nullptr); + file_schema->push_back({ + .identifier = Field::create_field(cast_set(idx)), + .local_id = cast_set(idx), + .name = slot->col_name(), + .type = slot->type(), + .children = synthesize_adbc_children(slot->type()), + }); + } + return Status::OK(); +} + +Status AdbcFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_stream_time); + RETURN_IF_ERROR(FileReader::open(std::move(request))); + RETURN_IF_ERROR(_open_stream()); + _eof = false; + return Status::OK(); +} + +Status AdbcFileReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + if (_stream == nullptr) { + return Status::InternalError("ADBC reader is not open"); + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + // Observe cancellation before entering a potentially blocking driver read. + RETURN_IF_ERROR(close()); + *rows = 0; + *eof = true; + return Status::OK(); + } + + *rows = 0; + *eof = false; + std::shared_ptr batch; + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_next_batch_time); + RETURN_IF_ERROR(_stream->next(&batch)); + } + if (batch == nullptr) { + *eof = true; + _eof = true; + return Status::OK(); + } + + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + } + _record_scan_rows(cast_set(*rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR( + apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + } + return Status::OK(); +} + +Status AdbcFileReader::close() { + SCOPED_TIMER(_total_time); + if (_stream != nullptr) { + RETURN_IF_ERROR(_stream->close()); + _stream.reset(); + } + _request.reset(); + _eof = true; + return Status::OK(); +} + +Status AdbcFileReader::_open_stream() { + DORIS_CHECK(_stream == nullptr); + if (_stream_factory) { + RETURN_IF_ERROR(_stream_factory(_range, &_stream)); + } else { + RETURN_IF_ERROR(create_real_adbc_stream(_range, &_stream)); + } + DORIS_CHECK(_stream != nullptr); + return Status::OK(); +} + +Status AdbcFileReader::_materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, + size_t* rows) const { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + if (_request == nullptr) { + return Status::InternalError("ADBC reader is not open"); + } + + if (_col_name_to_file_id.empty()) { + // A pushed-down COUNT(*) projects no columns at all: the scan wants rows counted, no values. + // Counting here rather than falling into the loop below is not an optimization -- every column + // the source returns is unrequested by definition in this state, so the loop's unknown-column + // check would reject the first one and fail a query that asked for nothing but a number. + // FE sends a one-constant-column statement for this case, so the batch is narrow. + // + // Only the empty case is special-cased. An unrequested column arriving alongside requested ones + // still fails: that means FE and this reader disagree about the projection, and it is the one + // signal that the disagreement exists. + *rows = cast_set(batch.num_rows()); + return Status::OK(); + } + + std::vector materialized_columns(file_block->columns(), false); + for (int arrow_idx = 0; arrow_idx < batch.num_columns(); ++arrow_idx) { + const std::string& column_name = batch.schema()->field(arrow_idx)->name(); + const auto file_id_it = _col_name_to_file_id.find(column_name); + if (file_id_it == _col_name_to_file_id.end()) { + return Status::InternalError("ADBC source returned unknown column {}", column_name); + } + const auto block_position_it = _request->local_positions.find(file_id_it->second); + if (block_position_it == _request->local_positions.end()) { + continue; + } + std::shared_ptr array; + { + SCOPED_TIMER(_normalize_time); + RETURN_IF_ERROR(normalize_arrow_array(batch.column(arrow_idx), &array)); + } + RETURN_IF_ERROR(_materialize_arrow_column(column_name, array, batch.num_rows(), + file_id_it->second, block_position_it->second, + file_block)); + materialized_columns[block_position_it->second.value()] = true; + } + + for (const auto& [file_column_id, block_position] : _request->local_positions) { + if (block_position.value() >= materialized_columns.size()) { + return Status::InternalError( + "ADBC requested block position {} out of range, block columns {}", + block_position.value(), materialized_columns.size()); + } + if (!materialized_columns[block_position.value()]) { + return Status::InternalError("ADBC source did not return requested file column id {}", + file_column_id.value()); + } + } + + *rows = cast_set(batch.num_rows()); + return Status::OK(); +} + +Status AdbcFileReader::_materialize_arrow_column(const std::string& column_name, + const std::shared_ptr& array, + int64_t num_rows, LocalColumnId file_column_id, + const LocalIndex& block_position, + Block* file_block) const { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(array != nullptr); + if (block_position.value() >= file_block->columns()) { + return Status::InternalError("ADBC block position {} out of range, block columns {}", + block_position.value(), file_block->columns()); + } + auto columns_guard = file_block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + const auto& target_type = columns_guard.get_datatype_by_position(block_position.value()); + + // An all-null column arrives with a type that says nothing about the column. + // + // A source that infers Arrow types from the VALUES it returns -- rather than from the declared + // column type -- has nothing to infer from when every value in the result is null, and picks + // whatever its default is. Measured against the SQLite driver: the same TEXT column comes back + // as utf8 for `SELECT id, name FROM t1` and as int64 for + // `SELECT id, name FROM t1 WHERE name IS NULL`, purely because the filter left only nulls. + // Handing that to the serde fails with "Unsupported arrow type for string column: 9", and no + // amount of care on the FE side avoids it: FE cannot know in advance which rows will survive. + // + // N nulls are what this array means whatever type it claims, so materialize them directly. The + // check is narrow on purpose -- a column with even one non-null value keeps its real type and + // still fails loudly on a genuine mismatch, which is the signal that FE and the source disagree + // about the schema rather than about one result set. + // + // Only for a nullable target: substituting defaults into a NOT NULL column would turn a source + // that wrongly sent nulls into silently wrong data, so that keeps failing in the serde. + if (array->null_count() == array->length() && target_type->is_nullable()) { + columns[block_position.value()]->insert_many_defaults(cast_set(num_rows)); + return Status::OK(); + } + + try { + RETURN_IF_ERROR(target_type->get_serde()->read_column_from_arrow( + *columns[block_position.value()], array.get(), 0, num_rows, _ctz)); + } catch (const Exception& e) { + return Status::InternalError( + "Failed to convert ADBC Arrow column '{}' (file_column_id={}, arrow type={}) to " + "Doris block: {}", + column_name, file_column_id.value(), array->type()->ToString(), e.what()); + } + return Status::OK(); +} + +Status AdbcFileReader::_build_col_name_to_file_id() { + _col_name_to_file_id.clear(); + _col_name_to_file_id.reserve(_file_slot_descs.size()); + for (size_t idx = 0; idx < _file_slot_descs.size(); ++idx) { + const auto* slot = _file_slot_descs[idx]; + DORIS_CHECK(slot != nullptr); + _col_name_to_file_id.emplace(slot->col_name(), LocalColumnId(cast_set(idx))); + } + return Status::OK(); +} + +AdbcReader::AdbcReader(AdbcStreamFactory stream_factory) + : _stream_factory(std::move(stream_factory)) {} + +Status AdbcReader::init(TableReadOptions&& options) { + if (options.file_slot_descs == nullptr) { + return Status::InvalidArgument("ADBC reader requires file slot descriptors"); + } + return TableReader::init(std::move(options)); +} + +Status AdbcReader::prepare_split(const SplitReadOptions& options) { + { + // Keep protocol validation visible while avoiding overlap with TableReader's own scopes. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(validate_adbc_range(options.current_range)); + } + return TableReader::prepare_split(options); +} + +Status AdbcReader::create_file_reader(std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(_file_slot_descs != nullptr); + *reader = std::make_unique(_system_properties, _current_task->data_file, + _io_ctx, _scanner_profile, _current_file_range_desc, + *_file_slot_descs, _stream_factory); + return Status::OK(); +} + +} // namespace doris::format::adbc diff --git a/be/src/format_v2/table/adbc_reader.h b/be/src/format_v2/table/adbc_reader.h new file mode 100644 index 00000000000000..8e184364e99d72 --- /dev/null +++ b/be/src/format_v2/table/adbc_reader.h @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/file_reader.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { +class Block; +class RuntimeProfile; +class RuntimeState; +class SlotDescriptor; +} // namespace doris + +namespace doris::format::adbc { + +// Replaces *stream with one that delegates to it and, on release, clears its own release callback +// the way the Arrow C data interface requires. *stream is left released, as a move leaves it. +// +// Not a nicety: Arrow C++ aborts the process (ArrowArrayStreamRelease's assertion) when a release +// callback returns without clearing itself, and the Flight SQL driver's stream does exactly that -- +// observed, and the reason a scan used to take the BE down. The ADBC driver manager's own wrapper +// hides the flaw, but this connector loads drivers through the registry that owns their lifetime and +// so calls their entry points directly. Applied to every driver on purpose: any of them may have the +// same flaw, and a third-party driver's bug must not be able to abort the BE. +void enforce_stream_release_contract(ArrowArrayStream* stream); + +// Small abstraction around the ADBC C API so the block materialization path stays unit-testable +// without a live database. Production uses the real ADBC stream; tests can drive the same path from +// plain RecordBatches. Mirrors RemoteDorisStream deliberately -- the two readers differ only in +// where the Arrow stream comes from. +class AdbcStream { +public: + virtual ~AdbcStream() = default; + // Sets *batch to nullptr at end of stream. + virtual Status next(std::shared_ptr* batch) = 0; + virtual Status close() = 0; +}; + +using AdbcStreamFactory = + std::function*)>; + +class AdbcFileReader final : public FileReader { +public: + AdbcFileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileRangeDesc& range, const std::vector& file_slot_descs, + AdbcStreamFactory stream_factory = {}); + ~AdbcFileReader() override; + + Status init(RuntimeState* state) override; + Status get_schema(std::vector* file_schema) const override; + Status open(std::shared_ptr request) override; + Status get_block(Block* file_block, size_t* rows, bool* eof) override; + Status close() override; + +private: + void _init_profile() override; + Status _open_stream(); + Status _materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, + size_t* rows) const; + // Takes the already-normalized array rather than the batch column: third-party drivers emit + // Arrow variants the serdes reject, so normalization has to happen before this point. + Status _materialize_arrow_column(const std::string& column_name, + const std::shared_ptr& array, int64_t num_rows, + LocalColumnId file_column_id, const LocalIndex& block_position, + Block* file_block) const; + Status _build_col_name_to_file_id(); + + const TFileRangeDesc _range; + const std::vector _file_slot_descs; + AdbcStreamFactory _stream_factory; + cctz::time_zone _ctz; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_stream_time = nullptr; + RuntimeProfile::Counter* _next_batch_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _normalize_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + RuntimeState* _runtime_state = nullptr; + std::unique_ptr _stream; + std::unordered_map _col_name_to_file_id; +}; + +class AdbcReader final : public TableReader { +public: + explicit AdbcReader(AdbcStreamFactory stream_factory = {}); + + Status init(TableReadOptions&& options) override; + Status prepare_split(const SplitReadOptions& options) override; + +protected: + Status create_file_reader(std::unique_ptr* reader) override; + +private: + AdbcStreamFactory _stream_factory; +}; + +} // namespace doris::format::adbc diff --git a/be/src/util/adbc_driver_registry.cpp b/be/src/util/adbc_driver_registry.cpp new file mode 100644 index 00000000000000..1eb71a726233cb --- /dev/null +++ b/be/src/util/adbc_driver_registry.cpp @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/adbc_driver_registry.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris { + +namespace { + +std::string resolve_path(const std::string& driver_path) { + // A driver that is not on disk yet still needs a stable cache key, so keep the original string. + std::unique_ptr resolved(realpath(driver_path.c_str(), nullptr), &free); + if (resolved == nullptr) { + return driver_path; + } + return {resolved.get()}; +} + +// Drivers own the strings they put in AdbcError, so every populated error must be released. +std::string take_error_message(AdbcError* error) { + DORIS_CHECK(error != nullptr); + std::string message = error->message != nullptr ? error->message : ""; + if (error->release != nullptr) { + error->release(error); + } + return message; +} + +} // namespace + +AdbcDriverRegistry& AdbcDriverRegistry::instance() { + // Allocated and never freed, on purpose. A plain function-local static is destroyed during + // static destruction, and this registry must outlive that: it hands out AdbcDriver pointers + // documented to stay valid for the life of the process, and it holds each driver's + // manager-side state, which only the driver's release callback frees -- a call this registry + // never makes, because dlclosing a driver that owns background threads is a use-after-free. + // Destroying the map would therefore drop the last reference to memory that stays live anyway, + // which is also exactly what LeakSanitizer reports at exit (its check runs after every static + // destructor). + static auto* registry = new AdbcDriverRegistry(); + return *registry; +} + +Status AdbcDriverRegistry::get_or_load(const std::string& driver_path, + const std::string& entrypoint, const AdbcDriver** out) { + DORIS_CHECK(out != nullptr); + if (driver_path.empty()) { + return Status::InvalidArgument("ADBC: driver path is empty"); + } + + const std::string key = resolve_path(driver_path); + + std::lock_guard lock(_mutex); + auto it = _drivers.find(key); + if (it != _drivers.end()) { + if (!it->second.loaded) { + return it->second.load_status; + } + *out = &it->second.driver; + return Status::OK(); + } + + Entry entry; + AdbcError error = ADBC_ERROR_INIT; + const AdbcStatusCode code = + AdbcLoadDriver(key.c_str(), entrypoint.empty() ? nullptr : entrypoint.c_str(), + ADBC_VERSION_1_1_0, &entry.driver, &error); + if (code != ADBC_STATUS_OK) { + const std::string message = take_error_message(&error); + // The path is what the user controls, so it has to be in the message for them to fix it. + // AdbcStatusCode is a uint8_t, so spell out the name as well as the number. + entry.load_status = Status::InternalError( + "ADBC: failed to load driver '{}' ({}, code {}): {}", driver_path, + AdbcStatusCodeMessage(code), static_cast(code), + message.empty() ? "no error message from the driver manager" : message); + LOG(WARNING) << entry.load_status; + return _drivers.emplace(key, std::move(entry)).first->second.load_status; + } + // A successful load can still leave a warning behind, and it is the driver's memory to free. + take_error_message(&error); + entry.loaded = true; + + *out = &_drivers.emplace(key, std::move(entry)).first->second.driver; + return Status::OK(); +} + +size_t AdbcDriverRegistry::loaded_count() const { + std::lock_guard lock(_mutex); + return _drivers.size(); +} + +} // namespace doris diff --git a/be/src/util/adbc_driver_registry.h b/be/src/util/adbc_driver_registry.h new file mode 100644 index 00000000000000..fae7439131e1e4 --- /dev/null +++ b/be/src/util/adbc_driver_registry.h @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris { + +/// Process-wide ADBC driver load cache. +/// +/// Each resolved path is passed to AdbcLoadDriver (which dlopens it) at most once, and is **never +/// dlclosed**: drivers carry global state and background threads -- Go runtimes especially -- so +/// unloading one is a use-after-free hazard. Handles stay live until the process exits. +/// +/// Failures are cached too, so a bad path does not retry the dlopen once per scan range. +/// +/// **The registry itself is never destroyed either** -- see instance(). A function-local static +/// would be torn down during static destruction, which would both dangle every AdbcDriver* already +/// handed out and orphan the driver manager's own per-driver state (it is freed only by the +/// driver's release callback, which this registry deliberately never calls). +class AdbcDriverRegistry { +public: + AdbcDriverRegistry(const AdbcDriverRegistry&) = delete; + AdbcDriverRegistry& operator=(const AdbcDriverRegistry&) = delete; + + static AdbcDriverRegistry& instance(); + + /// Loads the driver at `driver_path`, or returns the already-loaded one. An empty `entrypoint` + /// lets the driver manager search for one based on the driver name. The returned pointer stays + /// valid for the lifetime of the process. + Status get_or_load(const std::string& driver_path, const std::string& entrypoint, + const AdbcDriver** out); + + /// Test only: how many paths have been attempted, successes and failures alike. + size_t loaded_count() const; + +private: + AdbcDriverRegistry() = default; + ~AdbcDriverRegistry() = default; + + struct Entry { + AdbcDriver driver {}; + Status load_status; + bool loaded = false; + }; + + mutable std::mutex _mutex; + // Keyed by realpath(driver_path), falling back to the original string when it cannot be + // resolved. std::map keeps the entries pointer-stable, which the returned AdbcDriver* needs. + std::map _drivers; +}; + +} // namespace doris diff --git a/be/test/exec/operator/file_scan_operator_adbc_test.cpp b/be/test/exec/operator/file_scan_operator_adbc_test.cpp new file mode 100644 index 00000000000000..32ecd13625da6d --- /dev/null +++ b/be/test/exec/operator/file_scan_operator_adbc_test.cpp @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "exec/operator/file_scan_operator.h" +#include "exec/scan/file_scanner_v2.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::pipeline { +namespace { + +TFileScanRangeParams params_for_table_format(const std::string& table_format) { + TFileScanRangeParams params; + if (!table_format.empty()) { + TTableFormatFileDesc fmt; + fmt.__set_table_format_type(table_format); + params.__set_table_format_params(fmt); + } + return params; +} + +} // namespace + +// enable_file_scanner_v2 is fuzzy=true, so the regression harness turns it off at random. ADBC has +// no v1 reader, so honoring the flag would make ADBC queries fail on a coin flip. +TEST(FileScanOperatorAdbcTest, AdbcAlwaysUsesScannerV2EvenWhenDisabled) { + TQueryOptions opts; + opts.__set_enable_file_scanner_v2(false); + const auto params = params_for_table_format("adbc"); + + EXPECT_TRUE( + FileScanLocalState::TEST_should_use_file_scanner_v2(opts, /*is_load=*/false, params)); +} + +// The escape hatch must not drag any other format onto v2 with it. +TEST(FileScanOperatorAdbcTest, NonAdbcStillHonorsTheSessionVariable) { + TQueryOptions opts; + opts.__set_enable_file_scanner_v2(false); + + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(opts, /*is_load=*/false, + params_for_table_format(""))); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2( + opts, /*is_load=*/false, params_for_table_format("hive"))); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2( + opts, /*is_load=*/false, params_for_table_format("remote_doris"))); +} + +// Loads have no ADBC path at all; forcing them onto v2 would route them somewhere they cannot run. +TEST(FileScanOperatorAdbcTest, AdbcForcingDoesNotApplyToLoads) { + TQueryOptions opts; + opts.__set_enable_file_scanner_v2(false); + const auto params = params_for_table_format("adbc"); + + EXPECT_FALSE( + FileScanLocalState::TEST_should_use_file_scanner_v2(opts, /*is_load=*/true, params)); +} + +// is_supported gates which ranges FileScannerV2 accepts; without adbc there it would refuse the +// very ranges the operator forces onto it. +TEST(FileScanOperatorAdbcTest, ScannerV2SupportsAdbcArrowRanges) { + TFileScanRangeParams params; + + TTableFormatFileDesc fmt; + fmt.__set_table_format_type("adbc"); + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_ARROW); + range.__set_table_format_params(fmt); + EXPECT_TRUE(FileScannerV2::is_supported(params, range)); + + // Still only under FORMAT_ARROW: adbc never produces files in another format. + TFileRangeDesc parquet_range = range; + parquet_range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + EXPECT_FALSE(FileScannerV2::is_supported(params, parquet_range)); +} + +} // namespace doris::pipeline diff --git a/be/test/format/arrow/arrow_array_normalizer_test.cpp b/be/test/format/arrow/arrow_array_normalizer_test.cpp new file mode 100644 index 00000000000000..d83385981fbecf --- /dev/null +++ b/be/test/format/arrow/arrow_array_normalizer_test.cpp @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/arrow/arrow_array_normalizer.h" + +#include +#include +#include +#include + +#include +#include +#include + +namespace doris { + +namespace { + +std::shared_ptr build_large_string(const std::vector& vals) { + arrow::LargeStringBuilder b; + for (const auto& v : vals) { + EXPECT_TRUE(b.Append(v).ok()); + } + std::shared_ptr out; + EXPECT_TRUE(b.Finish(&out).ok()); + return out; +} + +} // namespace + +// Doris' own Arrow is already in the accepted shape; normalizing must not copy it. +TEST(ArrowArrayNormalizerTest, PlainStringIsAcceptedUnchanged) { + arrow::StringBuilder b; + ASSERT_TRUE(b.Append("doris").ok()); + std::shared_ptr in; + ASSERT_TRUE(b.Finish(&in).ok()); + + EXPECT_TRUE(is_serde_acceptable_arrow_type(*in->type())); + + std::shared_ptr out; + ASSERT_TRUE(normalize_arrow_array(in, &out).ok()); + EXPECT_EQ(in.get(), out.get()); +} + +// Go-based drivers emit large_*; the string serde only takes STRING/BINARY/FIXED_SIZE_BINARY. +TEST(ArrowArrayNormalizerTest, LargeStringIsConvertedToString) { + auto in = build_large_string({"a", "bb", "ccc"}); + EXPECT_FALSE(is_serde_acceptable_arrow_type(*in->type())); + + std::shared_ptr out; + ASSERT_TRUE(normalize_arrow_array(in, &out).ok()); + ASSERT_EQ(out->type_id(), arrow::Type::STRING); + ASSERT_EQ(out->length(), 3); + auto sa = std::static_pointer_cast(out); + EXPECT_EQ(sa->GetString(0), "a"); + EXPECT_EQ(sa->GetString(2), "ccc"); +} + +// Dictionary encoding is an encoding, not a type: decode it to the value type. +TEST(ArrowArrayNormalizerTest, DictionaryIsDecodedToValueType) { + arrow::StringBuilder dict_b; + ASSERT_TRUE(dict_b.AppendValues({"x", "y"}).ok()); + std::shared_ptr dict; + ASSERT_TRUE(dict_b.Finish(&dict).ok()); + + arrow::Int32Builder idx_b; + ASSERT_TRUE(idx_b.AppendValues({0, 1, 0}).ok()); + std::shared_ptr idx; + ASSERT_TRUE(idx_b.Finish(&idx).ok()); + + auto dict_type = arrow::dictionary(arrow::int32(), arrow::utf8()); + auto in = std::make_shared(dict_type, idx, dict); + EXPECT_FALSE(is_serde_acceptable_arrow_type(*in->type())); + + std::shared_ptr out; + ASSERT_TRUE(normalize_arrow_array(in, &out).ok()); + ASSERT_EQ(out->type_id(), arrow::Type::STRING); + auto sa = std::static_pointer_cast(out); + ASSERT_EQ(sa->length(), 3); + EXPECT_EQ(sa->GetString(0), "x"); + EXPECT_EQ(sa->GetString(1), "y"); + EXPECT_EQ(sa->GetString(2), "x"); +} + +// A conversion that drops nulls corrupts data silently, which is worse than failing. +TEST(ArrowArrayNormalizerTest, NullsArePreservedAcrossConversion) { + arrow::LargeStringBuilder b; + ASSERT_TRUE(b.Append("a").ok()); + ASSERT_TRUE(b.AppendNull().ok()); + std::shared_ptr in; + ASSERT_TRUE(b.Finish(&in).ok()); + + std::shared_ptr out; + ASSERT_TRUE(normalize_arrow_array(in, &out).ok()); + ASSERT_EQ(out->length(), 2); + EXPECT_FALSE(out->IsNull(0)); + EXPECT_TRUE(out->IsNull(1)); +} + +// Decoding a dictionary can expose another variant underneath; one pass is not enough. +TEST(ArrowArrayNormalizerTest, DictionaryOfLargeStringIsFullyNormalized) { + auto dict = build_large_string({"x", "y"}); + + arrow::Int32Builder idx_b; + ASSERT_TRUE(idx_b.AppendValues({1, 0}).ok()); + std::shared_ptr idx; + ASSERT_TRUE(idx_b.Finish(&idx).ok()); + + auto dict_type = arrow::dictionary(arrow::int32(), arrow::large_utf8()); + auto in = std::make_shared(dict_type, idx, dict); + + std::shared_ptr out; + ASSERT_TRUE(normalize_arrow_array(in, &out).ok()); + ASSERT_EQ(out->type_id(), arrow::Type::STRING); + auto sa = std::static_pointer_cast(out); + ASSERT_EQ(sa->length(), 2); + EXPECT_EQ(sa->GetString(0), "y"); + EXPECT_EQ(sa->GetString(1), "x"); +} + +// An unsupported type must name itself, otherwise the offending column cannot be found in prod. +TEST(ArrowArrayNormalizerTest, UnsupportedTypeFailsLoudWithTypeName) { + auto in = arrow::MakeArrayOfNull(arrow::month_interval(), 1).ValueOrDie(); + EXPECT_FALSE(is_serde_acceptable_arrow_type(*in->type())); + + std::shared_ptr out; + Status st = normalize_arrow_array(in, &out); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("interval"), std::string::npos); +} + +} // namespace doris diff --git a/be/test/format_v2/table/adbc_reader_test.cpp b/be/test/format_v2/table/adbc_reader_test.cpp new file mode 100644 index 00000000000000..0d463641cba455 --- /dev/null +++ b/be/test/format_v2/table/adbc_reader_test.cpp @@ -0,0 +1,809 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/adbc_reader.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/object_pool.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "format_v2/file_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/file_factory.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "testutil/adbc_sqlite_driver.h" +#include "testutil/desc_tbl_builder.h" +#include "util/adbc_driver_registry.h" + +namespace doris::format::adbc { +namespace { + +// Skipping is only correct when thirdparty predates arrow-adbc; it must say so out loud rather +// than surface as an unrelated-looking dlopen failure. +#define SKIP_WITHOUT_SQLITE_DRIVER() \ + do { \ + if (!adbc_sqlite_driver_available()) { \ + GTEST_SKIP() << "ADBC SQLite driver not found at " << adbc_sqlite_driver_path() \ + << "; run 'cd thirdparty && ./build-thirdparty.sh arrow_adbc' to build " \ + "it. End-to-end ADBC coverage is NOT being exercised."; \ + } \ + } while (0) + +class BatchAdbcStream final : public AdbcStream { +public: + BatchAdbcStream(std::vector> batches, + std::shared_ptr close_count) + : _batches(std::move(batches)), _close_count(std::move(close_count)) {} + + Status next(std::shared_ptr* batch) override { + DORIS_CHECK(batch != nullptr); + if (_next_batch >= _batches.size()) { + *batch = nullptr; + return Status::OK(); + } + *batch = _batches[_next_batch++]; + return Status::OK(); + } + + Status close() override { + ++(*_close_count); + return Status::OK(); + } + +private: + std::vector> _batches; + std::shared_ptr _close_count; + size_t _next_batch = 0; +}; + +TFileRangeDesc adbc_range(std::map params) { + TTableFormatFileDesc table_desc; + table_desc.__set_table_format_type("adbc"); + table_desc.__set_adbc_params(std::move(params)); + + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_ARROW); + range.__set_path("/dummyPath"); + range.__set_table_format_params(std::move(table_desc)); + return range; +} + +// A range that passes validation but is never actually connected to; group A injects a fake stream. +TFileRangeDesc fake_adbc_range() { + return adbc_range({{"driver_path", "/dummy/libadbc_driver_fake.so"}, + {"uri", "file:/dummy.db"}, + {"query_sql", "SELECT 1"}}); +} + +std::vector string_slot(ObjectPool* pool, DescriptorTbl** desc_tbl) { + DescriptorTblBuilder builder(pool); + builder.declare_tuple() << std::make_tuple(std::make_shared(), + std::string("c_str")); + *desc_tbl = builder.build(); + return (*desc_tbl)->get_tuple_descriptor(0)->slots(); +} + +std::vector sqlite_slots(ObjectPool* pool, DescriptorTbl** desc_tbl) { + DescriptorTblBuilder builder(pool); + // SQLite stores integers as 64-bit and reals as doubles; the ADBC driver reports them as such. + builder.declare_tuple() + << std::make_tuple(std::make_shared(), std::string("id")) + << std::make_tuple(std::make_shared(), std::string("v_int")) + << std::make_tuple(std::make_shared(), std::string("v_dbl")) + << std::make_tuple(std::make_shared(), std::string("v_txt")); + *desc_tbl = builder.build(); + return (*desc_tbl)->get_tuple_descriptor(0)->slots(); +} + +// large_utf8 is what Go-based drivers emit and what the string serde refuses; feeding it through +// proves the reader normalizes before materializing. +std::shared_ptr make_named_large_string_batch(const std::string& column_name) { + arrow::LargeStringBuilder b; + EXPECT_TRUE(b.Append("doris").ok()); + EXPECT_TRUE(b.AppendNull().ok()); + std::shared_ptr arr; + EXPECT_TRUE(b.Finish(&arr).ok()); + auto schema = arrow::schema({arrow::field(column_name, arrow::large_utf8())}); + return arrow::RecordBatch::Make(schema, 2, {arr}); +} + +std::shared_ptr make_large_string_batch() { + return make_named_large_string_batch("c_str"); +} + +std::unique_ptr create_reader(RuntimeProfile* profile, const TFileRangeDesc& range, + const std::vector& slots, + AdbcStreamFactory factory) { + auto system_properties = std::make_shared(); + auto file_description = std::make_unique(); + file_description->path = "/dummyPath"; + return std::make_unique(system_properties, file_description, nullptr, profile, + range, slots, std::move(factory)); +} + +Block make_request_block(const std::vector& schema, + const std::vector& local_ids) { + Block block; + for (const auto local_id : local_ids) { + const auto it = std::find_if(schema.begin(), schema.end(), [&](const auto& column) { + return column.local_id == local_id; + }); + DORIS_CHECK(it != schema.end()); + block.insert({it->type->create_column(), it->type, it->name}); + } + return block; +} + +std::string nullable_string_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data_at(row).to_string(); +} + +int64_t nullable_int64_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + return assert_cast(nullable.get_nested_column()).get_data()[row]; +} + +double nullable_double_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + return assert_cast(nullable.get_nested_column()).get_data()[row]; +} + +bool is_null_at(const IColumn& column, size_t row) { + return assert_cast(column).is_null_at(row); +} + +// Runs DDL/DML through ADBC so the fixture needs neither the sqlite3 CLI nor a sqlite dev package. +::testing::AssertionResult run_sqlite_ddl(const std::string& uri, const std::string& sql) { + const AdbcDriver* driver = nullptr; + Status st = AdbcDriverRegistry::instance().get_or_load(adbc_sqlite_driver_path(), "", &driver); + if (!st.ok()) { + return ::testing::AssertionFailure() << "load driver: " << st.to_string(); + } + + AdbcError error = ADBC_ERROR_INIT; + AdbcDatabase database {}; + AdbcConnection connection {}; + AdbcStatement statement {}; + auto fail = [&](const char* what, AdbcStatusCode code) { + std::string message = error.message != nullptr ? error.message : ""; + if (error.release != nullptr) { + error.release(&error); + } + return ::testing::AssertionFailure() << what << " failed (" << code << "): " << message; + }; + + if (auto code = driver->DatabaseNew(&database, &error); code != ADBC_STATUS_OK) { + return fail("DatabaseNew", code); + } + if (auto code = driver->DatabaseSetOption(&database, ADBC_OPTION_URI, uri.c_str(), &error); + code != ADBC_STATUS_OK) { + return fail("DatabaseSetOption", code); + } + if (auto code = driver->DatabaseInit(&database, &error); code != ADBC_STATUS_OK) { + return fail("DatabaseInit", code); + } + if (auto code = driver->ConnectionNew(&connection, &error); code != ADBC_STATUS_OK) { + return fail("ConnectionNew", code); + } + if (auto code = driver->ConnectionInit(&connection, &database, &error); + code != ADBC_STATUS_OK) { + return fail("ConnectionInit", code); + } + if (auto code = driver->StatementNew(&connection, &statement, &error); code != ADBC_STATUS_OK) { + return fail("StatementNew", code); + } + if (auto code = driver->StatementSetSqlQuery(&statement, sql.c_str(), &error); + code != ADBC_STATUS_OK) { + return fail("StatementSetSqlQuery", code); + } + int64_t rows_affected = -1; + if (auto code = driver->StatementExecuteQuery(&statement, nullptr, &rows_affected, &error); + code != ADBC_STATUS_OK) { + return fail("StatementExecuteQuery", code); + } + if (error.release != nullptr) { + error.release(&error); + } + static_cast(driver->StatementRelease(&statement, &error)); + static_cast(driver->ConnectionRelease(&connection, &error)); + static_cast(driver->DatabaseRelease(&database, &error)); + if (error.release != nullptr) { + error.release(&error); + } + return ::testing::AssertionSuccess(); +} + +} // namespace + +// Group A: the materialization path, driven from a RecordBatch so no database is involved. + +// The key case: without normalize_arrow_array the serde hits its unsupported-type branch and this +// fails. Everything else in the reader can be right and the data still would not land. +TEST(AdbcReaderTest, NormalizesLargeStringBeforeMaterializing) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = string_slot(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_normalize_test"); + auto close_count = std::make_shared(0); + + auto reader = + create_reader(&profile, fake_adbc_range(), slots, + [close_count](const TFileRangeDesc&, std::unique_ptr* out) { + *out = std::make_unique( + std::vector> { + make_large_string_batch()}, + close_count); + return Status::OK(); + }); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(rows, 2); + EXPECT_FALSE(eof); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "doris"); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 1)); + + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader->close().ok()); + EXPECT_EQ(*close_count, 1); +} + +// An all-null int64 array standing in for a TEXT column, which is what a source that infers Arrow +// types from the values it returns sends when a filter leaves only nulls. +std::shared_ptr make_all_null_int64_batch(const std::string& column_name) { + arrow::Int64Builder b; + EXPECT_TRUE(b.AppendNull().ok()); + EXPECT_TRUE(b.AppendNull().ok()); + std::shared_ptr arr; + EXPECT_TRUE(b.Finish(&arr).ok()); + auto schema = arrow::schema({arrow::field(column_name, arrow::int64())}); + return arrow::RecordBatch::Make(schema, 2, {arr}); +} + +// A column whose values are ALL null arrives with a type that says nothing about the column: a +// source inferring Arrow types from values has nothing to infer from. Measured on the SQLite driver, +// the same TEXT column is utf8 for `SELECT id, name FROM t1` and int64 for the same query plus +// `WHERE name IS NULL`. Without the all-null branch this reaches the string serde and fails with +// "Unsupported arrow type for string column: 9", and FE cannot prevent it -- it cannot know which +// rows a filter will leave. +TEST(AdbcReaderTest, MaterializesAnAllNullColumnWhateverTypeTheSourceClaims) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = string_slot(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_all_null_test"); + auto close_count = std::make_shared(0); + + auto reader = + create_reader(&profile, fake_adbc_range(), slots, + [close_count](const TFileRangeDesc&, std::unique_ptr* out) { + *out = std::make_unique( + std::vector> { + make_all_null_int64_batch("c_str")}, + close_count); + return Status::OK(); + }); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(rows, 2); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 1)); +} + +// The other side of that branch: a column carrying real values keeps its real type, so a genuine +// FE/source schema disagreement still fails rather than being papered over as nulls. +TEST(AdbcReaderTest, StillRejectsATypeMismatchOnAColumnThatHasValues) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = string_slot(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_type_mismatch_test"); + auto close_count = std::make_shared(0); + + arrow::Int64Builder values; + EXPECT_TRUE(values.Append(7).ok()); + EXPECT_TRUE(values.AppendNull().ok()); + std::shared_ptr arr; + EXPECT_TRUE(values.Finish(&arr).ok()); + auto batch = arrow::RecordBatch::Make(arrow::schema({arrow::field("c_str", arrow::int64())}), 2, + {arr}); + + auto reader = create_reader( + &profile, fake_adbc_range(), slots, + [close_count, batch](const TFileRangeDesc&, std::unique_ptr* out) { + *out = std::make_unique( + std::vector> {batch}, close_count); + return Status::OK(); + }); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + EXPECT_FALSE(reader->get_block(&block, &rows, &eof).ok()); +} + +// A pushed-down COUNT(*) projects nothing, so every column the source returns is unrequested by +// definition. Without the empty-projection branch the unknown-column check below rejects the first +// one and a query asking for nothing but a number fails. +TEST(AdbcReaderTest, CountsRowsWhenTheScanProjectsNoColumns) { + RuntimeState state; + RuntimeProfile profile("adbc_reader_count_only_test"); + auto close_count = std::make_shared(0); + const std::vector no_slots; + + auto reader = + create_reader(&profile, fake_adbc_range(), no_slots, + [close_count](const TFileRangeDesc&, std::unique_ptr* out) { + *out = std::make_unique( + std::vector> { + make_named_large_string_batch("1")}, + close_count); + return Status::OK(); + }); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_TRUE(schema.empty()); + + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + Block block; + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(rows, 2); + EXPECT_FALSE(eof); + EXPECT_EQ(block.columns(), 0); + + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader->close().ok()); +} + +// The other half of the branch above: an unrequested column arriving ALONGSIDE requested ones still +// fails. That state means FE and this reader disagree about the projection, and this check is the +// only signal the disagreement exists -- relaxing it to tolerate the count case would remove it. +TEST(AdbcReaderTest, RejectsAColumnTheScanDidNotRequest) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = string_slot(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_unknown_column_test"); + auto close_count = std::make_shared(0); + + auto reader = + create_reader(&profile, fake_adbc_range(), slots, + [close_count](const TFileRangeDesc&, std::unique_ptr* out) { + *out = std::make_unique( + std::vector> { + make_named_large_string_batch("not_requested")}, + close_count); + return Status::OK(); + }); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("not_requested"), std::string::npos) << status.to_string(); +} + +// A range missing a required parameter must be rejected up front, not at connect time where the +// error would come back as an opaque driver message. +TEST(AdbcReaderTest, RejectsIncompleteAdbcParams) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = string_slot(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_bad_range_test"); + + for (const auto* missing : {"driver_path", "uri", "query_sql"}) { + auto range = fake_adbc_range(); + range.table_format_params.adbc_params.erase(missing); + auto reader = create_reader(&profile, range, slots, + [](const TFileRangeDesc&, std::unique_ptr* out) { + *out = nullptr; + return Status::OK(); + }); + EXPECT_FALSE(reader->init(&state).ok()) << "missing " << missing << " was accepted"; + } + + auto range = fake_adbc_range(); + range.table_format_params.__isset.adbc_params = false; + auto reader = create_reader(&profile, range, slots, {}); + EXPECT_FALSE(reader->init(&state).ok()); +} + +// A range says either "run this statement" or "read this partition of a statement already run". +// Accepting one that says both would let this reader execute a query the source has already +// executed, depending only on which branch the code happened to take first. +TEST(AdbcReaderTest, RejectsARangeThatSaysBothOrNeitherKindOfWork) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = string_slot(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_exclusive_work_test"); + + auto both = fake_adbc_range(); + both.table_format_params.adbc_params["partition_descriptor"] = "Zm9vYmFy"; + auto both_reader = create_reader(&profile, both, slots, {}); + const auto both_status = both_reader->init(&state); + ASSERT_FALSE(both_status.ok()); + EXPECT_NE(both_status.to_string().find("both"), std::string::npos) << both_status.to_string(); + + auto neither = fake_adbc_range(); + neither.table_format_params.adbc_params.erase("query_sql"); + auto neither_reader = create_reader(&profile, neither, slots, {}); + const auto neither_status = neither_reader->init(&state); + ASSERT_FALSE(neither_status.ok()); + EXPECT_NE(neither_status.to_string().find("neither"), std::string::npos) + << neither_status.to_string(); +} + +// Group B: the real ADBC C API call sequence, against the SQLite driver thirdparty builds. + +class AdbcSqliteReaderTest : public ::testing::Test { +protected: + void SetUp() override { + SKIP_WITHOUT_SQLITE_DRIVER(); + _db_path = std::filesystem::temp_directory_path() / + ("doris_adbc_reader_test_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(_db_path); + const std::string uri = "file:" + _db_path.string(); + ASSERT_TRUE(run_sqlite_ddl( + uri, "CREATE TABLE t (id INTEGER, v_int INTEGER, v_dbl REAL, v_txt TEXT)")); + ASSERT_TRUE(run_sqlite_ddl(uri, + "INSERT INTO t VALUES (1, 10, 1.5, 'alpha'), " + "(2, NULL, NULL, NULL), (3, 30, 3.5, 'gamma')")); + } + + void TearDown() override { + if (!_db_path.empty()) { + std::filesystem::remove(_db_path); + } + } + + std::string uri() const { return "file:" + _db_path.string(); } + + std::filesystem::path _db_path; +}; + +// Group A cannot prove the ADBC call sequence is right, nor that the driver manager is usable at +// run time. SQLite makes that testable without a server or a container. +TEST_F(AdbcSqliteReaderTest, ReadsFromRealSqliteDriverEndToEnd) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = sqlite_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_sqlite_e2e_test"); + + auto range = adbc_range({ + {"driver_path", adbc_sqlite_driver_path()}, + {"uri", uri()}, + {"query_sql", "SELECT id, v_int, v_dbl, v_txt FROM t ORDER BY id"}, + }); + + // No injected factory: this goes through the real ADBC stream. + auto reader = create_reader(&profile, range, slots, {}); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + for (int32_t id = 0; id < 4; ++id) { + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(id)).ok()); + } + const auto open_status = reader->open(request); + ASSERT_TRUE(open_status.ok()) << open_status.to_string(); + + auto block = make_request_block(schema, {0, 1, 2, 3}); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(rows, 3); + + const auto& id_col = *block.get_by_position(0).column; + const auto& int_col = *block.get_by_position(1).column; + const auto& dbl_col = *block.get_by_position(2).column; + const auto& txt_col = *block.get_by_position(3).column; + + EXPECT_EQ(nullable_int64_at(id_col, 0), 1); + EXPECT_EQ(nullable_int64_at(id_col, 2), 3); + + EXPECT_EQ(nullable_int64_at(int_col, 0), 10); + EXPECT_TRUE(is_null_at(int_col, 1)); + EXPECT_EQ(nullable_int64_at(int_col, 2), 30); + + EXPECT_DOUBLE_EQ(nullable_double_at(dbl_col, 0), 1.5); + EXPECT_TRUE(is_null_at(dbl_col, 1)); + EXPECT_DOUBLE_EQ(nullable_double_at(dbl_col, 2), 3.5); + + EXPECT_EQ(nullable_string_at(txt_col, 0), "alpha"); + EXPECT_TRUE(is_null_at(txt_col, 1)); + EXPECT_EQ(nullable_string_at(txt_col, 2), "gamma"); + + ASSERT_TRUE(reader->close().ok()); +} + +// A driver path that is not there is the most likely user error, so it must not look like an +// internal failure. +TEST_F(AdbcSqliteReaderTest, MissingDriverFailsWithThePathInTheMessage) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = sqlite_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_missing_driver_test"); + + auto range = adbc_range({ + {"driver_path", "/nonexistent/libadbc_driver_nope.so"}, + {"uri", uri()}, + {"query_sql", "SELECT 1"}, + }); + auto reader = create_reader(&profile, range, slots, {}); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + const auto status = reader->open(request); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("/nonexistent/libadbc_driver_nope.so"), std::string::npos); +} + +// A bad query has to surface the driver's own message, otherwise SQL problems are undiagnosable. +TEST_F(AdbcSqliteReaderTest, InvalidQuerySurfacesTheDriverMessage) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = sqlite_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_bad_query_test"); + + auto range = adbc_range({ + {"driver_path", adbc_sqlite_driver_path()}, + {"uri", uri()}, + {"query_sql", "SELECT * FROM no_such_table"}, + }); + auto reader = create_reader(&profile, range, slots, {}); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + const auto status = reader->open(request); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("no_such_table"), std::string::npos) + << "driver message was lost: " << status.to_string(); +} + +// A partition descriptor is FE's base64 of driver-private bytes. Garbage there has to be named as +// such: handed to the driver undecoded it would come back as an opaque parse failure from inside a +// protobuf, with nothing pointing at the parameter that was wrong. +TEST_F(AdbcSqliteReaderTest, RejectsAPartitionDescriptorThatIsNotBase64) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = sqlite_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_bad_partition_test"); + + auto range = adbc_range({ + {"driver_path", adbc_sqlite_driver_path()}, + {"uri", uri()}, + {"partition_descriptor", "not base64 at all!!"}, + }); + auto reader = create_reader(&profile, range, slots, {}); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + const auto status = reader->open(request); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("partition_descriptor"), std::string::npos) + << status.to_string(); +} + +// The reader must actually take the partition branch, not fall through to running a statement it +// was not given. SQLite has no partitioned execution, so the proof that the call was made is the +// driver's own refusal of it -- naming ConnectionReadPartition, the entry point only this branch +// reaches. Reading a partition successfully needs a source that produces one, which is the Flight +// SQL regression suite, not a unit test. +TEST_F(AdbcSqliteReaderTest, ReadsAPartitionThroughTheDriverInsteadOfRunningAStatement) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = sqlite_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("adbc_reader_partition_branch_test"); + + auto range = adbc_range({ + {"driver_path", adbc_sqlite_driver_path()}, + {"uri", uri()}, + // Valid base64; the bytes are meaningless to the driver, which never gets to look at + // them because it has no partition support at all. + {"partition_descriptor", "Zm9vYmFy"}, + }); + auto reader = create_reader(&profile, range, slots, {}); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + const auto status = reader->open(request); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("ConnectionReadPartition"), std::string::npos) + << "the partition branch was not taken: " << status.to_string(); +} + +// ---- the release contract Arrow aborts the process over ---- + +namespace { + +// A stream shaped like the one the Flight SQL driver hands out: its release callback runs, but +// leaves `release` set. The Arrow C data interface forbids that, and Arrow C++ does not merely +// complain -- ArrowArrayStreamRelease calls abort(), taking the whole BE with it. +struct MisbehavingDriverStream { + int release_calls = 0; + int get_next_calls = 0; +}; + +MisbehavingDriverStream& state_of(ArrowArrayStream* self) { + return *static_cast(self->private_data); +} + +int misbehaving_get_schema(ArrowArrayStream* /*self*/, ArrowSchema* /*out*/) { + return 0; +} + +int misbehaving_get_next(ArrowArrayStream* self, ArrowArray* /*out*/) { + state_of(self).get_next_calls++; + return 0; +} + +const char* misbehaving_get_last_error(ArrowArrayStream* /*self*/) { + return "driver said so"; +} + +void misbehaving_release(ArrowArrayStream* self) { + state_of(self).release_calls++; + // Deliberately does NOT clear self->release. This is the bug being defended against. +} + +ArrowArrayStream misbehaving_stream(MisbehavingDriverStream* state) { + ArrowArrayStream stream {}; + stream.get_schema = misbehaving_get_schema; + stream.get_next = misbehaving_get_next; + stream.get_last_error = misbehaving_get_last_error; + stream.release = misbehaving_release; + stream.private_data = state; + return stream; +} + +} // namespace + +TEST(AdbcStreamReleaseContractTest, ClearsReleaseEvenWhenTheDriverDoesNot) { + MisbehavingDriverStream state; + ArrowArrayStream stream = misbehaving_stream(&state); + + enforce_stream_release_contract(&stream); + ASSERT_NE(stream.release, nullptr); + stream.release(&stream); + + // The invariant Arrow asserts on, and the one a scan against Flight SQL used to break. + EXPECT_EQ(stream.release, nullptr); + // The driver still gets released, exactly once: the wrapper must not leak the real stream. + EXPECT_EQ(state.release_calls, 1); +} + +TEST(AdbcStreamReleaseContractTest, StillDelegatesEveryCallbackToTheDriver) { + // A wrapper that swallowed calls would turn a crash into silently empty results, which is + // worse: the scan would report success on rows it never read. + MisbehavingDriverStream state; + ArrowArrayStream stream = misbehaving_stream(&state); + enforce_stream_release_contract(&stream); + + ArrowArray array {}; + EXPECT_EQ(stream.get_next(&stream, &array), 0); + EXPECT_EQ(state.get_next_calls, 1); + EXPECT_STREQ(stream.get_last_error(&stream), "driver said so"); + + stream.release(&stream); +} + +TEST(AdbcStreamReleaseContractTest, LeavesAnAlreadyReleasedStreamAlone) { + // Wrapping one would hand Arrow callbacks that dereference a delegate with nothing behind it. + ArrowArrayStream stream {}; + enforce_stream_release_contract(&stream); + EXPECT_EQ(stream.release, nullptr); + EXPECT_EQ(stream.private_data, nullptr); +} + +} // namespace doris::format::adbc diff --git a/be/test/testutil/adbc_sqlite_driver.cpp b/be/test/testutil/adbc_sqlite_driver.cpp new file mode 100644 index 00000000000000..5d4d672dfe7a0a --- /dev/null +++ b/be/test/testutil/adbc_sqlite_driver.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "testutil/adbc_sqlite_driver.h" + +#include + +#include +#include + +namespace doris { + +namespace { + +// run-be-ut.sh exports both, defaulting DORIS_THIRDPARTY to ${DORIS_HOME}/thirdparty. +std::string thirdparty_dir() { + if (const char* tp = std::getenv("DORIS_THIRDPARTY"); tp != nullptr && *tp != '\0') { + return tp; + } + if (const char* home = std::getenv("DORIS_HOME"); home != nullptr && *home != '\0') { + return std::string(home) + "/thirdparty"; + } + return "thirdparty"; +} + +} // namespace + +std::string adbc_sqlite_driver_path() { + return thirdparty_dir() + "/installed/lib64/libadbc_driver_sqlite.so"; +} + +bool adbc_sqlite_driver_available() { + struct stat st {}; + return ::stat(adbc_sqlite_driver_path().c_str(), &st) == 0; +} + +} // namespace doris diff --git a/be/test/testutil/adbc_sqlite_driver.h b/be/test/testutil/adbc_sqlite_driver.h new file mode 100644 index 00000000000000..d392aaff49fd54 --- /dev/null +++ b/be/test/testutil/adbc_sqlite_driver.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + +/// Absolute path of the SQLite ADBC driver that thirdparty builds for tests. +/// +/// It is the only real, self-contained ADBC driver available to unit tests: it needs no server and +/// no container, so it is what proves the ADBC C API call sequence rather than just our own mocks. +/// It is not shipped -- see the D8 exception in the design doc. +std::string adbc_sqlite_driver_path(); + +/// Whether that driver exists. Tests skip themselves when thirdparty has not been rebuilt since +/// arrow-adbc was added, rather than failing with an unrelated-looking dlopen error. +bool adbc_sqlite_driver_available(); + +} // namespace doris diff --git a/be/test/util/adbc_driver_registry_test.cpp b/be/test/util/adbc_driver_registry_test.cpp new file mode 100644 index 00000000000000..ea076d9b13d79a --- /dev/null +++ b/be/test/util/adbc_driver_registry_test.cpp @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/adbc_driver_registry.h" + +#include + +#include +#include + +#include "testutil/adbc_sqlite_driver.h" + +namespace doris { + +namespace { + +// Skipping is only correct when thirdparty predates arrow-adbc; it must say so out loud rather +// than surface as an unrelated-looking dlopen failure. +#define SKIP_WITHOUT_SQLITE_DRIVER() \ + do { \ + if (!adbc_sqlite_driver_available()) { \ + GTEST_SKIP() << "ADBC SQLite driver not found at " << adbc_sqlite_driver_path() \ + << "; run 'cd thirdparty && ./build-thirdparty.sh arrow_adbc' to build " \ + "it. ADBC coverage is NOT being exercised."; \ + } \ + } while (0) + +} // namespace + +// Loading a real third-party driver is the only way to prove the driver manager is linked in and +// that the entrypoint search works. +TEST(AdbcDriverRegistryTest, LoadsSqliteDriver) { + SKIP_WITHOUT_SQLITE_DRIVER(); + const AdbcDriver* drv = nullptr; + Status st = AdbcDriverRegistry::instance().get_or_load(adbc_sqlite_driver_path(), "", &drv); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_NE(drv, nullptr); + EXPECT_NE(drv->DatabaseNew, nullptr); +} + +// The second call must not dlopen again -- that is the whole premise of never dlclosing. +TEST(AdbcDriverRegistryTest, SamePathReturnsCachedInstance) { + SKIP_WITHOUT_SQLITE_DRIVER(); + const AdbcDriver* a = nullptr; + const AdbcDriver* b = nullptr; + ASSERT_TRUE(AdbcDriverRegistry::instance().get_or_load(adbc_sqlite_driver_path(), "", &a).ok()); + const size_t after_first = AdbcDriverRegistry::instance().loaded_count(); + ASSERT_TRUE(AdbcDriverRegistry::instance().get_or_load(adbc_sqlite_driver_path(), "", &b).ok()); + EXPECT_EQ(a, b); + EXPECT_EQ(after_first, AdbcDriverRegistry::instance().loaded_count()); +} + +// Without the path in the message the user cannot tell where the driver was expected. +TEST(AdbcDriverRegistryTest, MissingFileFailsWithPathInMessage) { + const AdbcDriver* drv = nullptr; + Status st = AdbcDriverRegistry::instance().get_or_load("/nonexistent/libnope.so", "", &drv); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("/nonexistent/libnope.so"), std::string::npos); +} + +// Remembering the failure keeps a bad path from retrying dlopen once per scan range. +TEST(AdbcDriverRegistryTest, LoadFailureIsCachedNotRetried) { + const AdbcDriver* drv = nullptr; + ASSERT_FALSE(AdbcDriverRegistry::instance().get_or_load("/nonexistent/x.so", "", &drv).ok()); + const size_t after_first = AdbcDriverRegistry::instance().loaded_count(); + ASSERT_FALSE(AdbcDriverRegistry::instance().get_or_load("/nonexistent/x.so", "", &drv).ok()); + EXPECT_EQ(after_first, AdbcDriverRegistry::instance().loaded_count()); +} + +// An empty path would otherwise reach the driver manager and come back as a confusing dlopen error. +TEST(AdbcDriverRegistryTest, EmptyPathIsRejectedWithoutCaching) { + const AdbcDriver* drv = nullptr; + const size_t before = AdbcDriverRegistry::instance().loaded_count(); + EXPECT_FALSE(AdbcDriverRegistry::instance().get_or_load("", "", &drv).ok()); + EXPECT_EQ(before, AdbcDriverRegistry::instance().loaded_count()); +} + +} // namespace doris diff --git a/build.sh b/build.sh index 50539178292547..c9cdb02e60b773 100755 --- a/build.sh +++ b/build.sh @@ -726,7 +726,10 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Connector API, SPI, and plugin modules (loaded at runtime as plugins) modules+=("fe-connector/fe-connector-api") modules+=("fe-connector/fe-connector-spi") - for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg; do + # Keep this list identical to the deploy loop's (search CONN_PLUGIN_DIR). A module missing here + # but present there is not a no-op: the deploy step unzips whatever archive is left in the + # module's target/ from some earlier build, so the plugin silently ships stale. + for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc; do if [[ -d "${DORIS_HOME}/fe/fe-connector/fe-connector-${_conn_mod}" ]]; then modules+=("fe-connector/fe-connector-${_conn_mod}") fi @@ -1040,6 +1043,16 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then mkdir -p "${DORIS_OUTPUT}/fe/doris-meta" mkdir -p "${DORIS_OUTPUT}/fe/conf/ssl" mkdir -p "${DORIS_OUTPUT}/fe/plugins/jdbc_drivers/" + # Drop point for ADBC driver shared libraries. Doris does not ship the drivers themselves; the + # same file must be placed here AND under be/plugins/adbc_drivers on every BE, because partition + # descriptors are driver-private bytes with no interoperability across driver implementations. + mkdir -p "${DORIS_OUTPUT}/fe/plugins/adbc_drivers/" + # The ADBC JNI shim, built by thirdparty. NOT the copy inside the adbc-driver-jni jar: the + # released one needs GLIBC 2.34 / GLIBCXX 3.4.31, which the supported build hosts do not have. + # conf/fe.conf points arrow.adbc.driver.jni.library.path at this directory. + if [[ -f "${DORIS_THIRDPARTY}/installed/lib64/libadbc_driver_jni.so" ]]; then + cp -p "${DORIS_THIRDPARTY}/installed/lib64/libadbc_driver_jni.so" "${DORIS_OUTPUT}/fe/lib/" + fi mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_udf/" # Drop point for the trino-connector's own Trino plugins. Deliberately NOT the legacy # plugins/connectors/: that name is still read as a fallback for deployments upgrading from @@ -1070,7 +1083,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Deploy connector provider plugins as independent plugin directories. # Each sub-directory is one connector backend loaded at runtime by ConnectorPluginManager. CONN_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/connector" - for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg; do + for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc; do conn_plugin_target="${CONN_PLUGIN_DIR}/${conn_module}" conn_module_dir="${DORIS_HOME}/fe/fe-connector/fe-connector-${conn_module}" if [ ! -d "${conn_module_dir}" ]; then @@ -1308,6 +1321,8 @@ EOF mkdir -p "${DORIS_OUTPUT}/be/log" mkdir -p "${DORIS_OUTPUT}/be/storage" mkdir -p "${DORIS_OUTPUT}/be/plugins/jdbc_drivers/" + # Mirrors the FE drop point above; every BE must hold the same ADBC driver file the FE holds. + mkdir -p "${DORIS_OUTPUT}/be/plugins/adbc_drivers/" mkdir -p "${DORIS_OUTPUT}/be/plugins/java_udf/" mkdir -p "${DORIS_OUTPUT}/be/plugins/python_udf/" # Mirrors the FE drop point above; the BE JNI scanner loads the same Trino plugins independently. diff --git a/conf/fe.conf b/conf/fe.conf index b90b6a44359404..52d4003ce2bdc3 100644 --- a/conf/fe.conf +++ b/conf/fe.conf @@ -27,7 +27,7 @@ CUR_DATE=`date +%Y%m%d-%H%M%S` LOG_DIR = ${DORIS_HOME}/log # For jdk 17, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" +JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false -Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" # Set your own JAVA_HOME # JAVA_HOME=/path/to/jdk/ diff --git a/fe/fe-connector/fe-connector-adbc/pom.xml b/fe/fe-connector/fe-connector-adbc/pom.xml new file mode 100644 index 00000000000000..e604ef8fa99720 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/pom.xml @@ -0,0 +1,150 @@ + + + + 4.0.0 + + + org.apache.doris + fe-connector + ${revision} + ../pom.xml + + + fe-connector-adbc + jar + Doris FE Connector - ADBC + + ADBC (Arrow Database Connectivity) connector plugin for Doris FE. + Reaches the remote source through the ADBC JNI bridge, which dlopens the very same + driver shared library BE loads natively -- partition descriptors are driver-private + bytes with no cross-implementation interoperability, so FE and BE must not use + different driver implementations. + + + + + ${project.groupId} + fe-connector-spi + ${project.version} + + + + + ${project.groupId} + fe-thrift + ${project.version} + provided + + + + + ${project.groupId} + fe-connector-cache + ${project.version} + + + + + com.github.ben-manes.caffeine + caffeine + 2.9.3 + + + + + org.apache.arrow.adbc + adbc-core + + + org.apache.arrow.adbc + adbc-driver-jni + + + + + org.apache.arrow + arrow-memory-netty + + + + org.apache.logging.log4j + log4j-api + + + + org.junit.jupiter + junit-jupiter + test + + + + + doris-fe-connector-adbc + + + org.apache.maven.plugins + maven-surefire-plugin + + + + --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED @{argLine} + + + + + maven-assembly-plugin + + false + + src/main/assembly/plugin-zip.xml + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/fe-connector/fe-connector-adbc/src/main/assembly/plugin-zip.xml b/fe/fe-connector/fe-connector-adbc/src/main/assembly/plugin-zip.xml new file mode 100644 index 00000000000000..8fa8ed7ec296d9 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/assembly/plugin-zip.xml @@ -0,0 +1,75 @@ + + + + + plugin + + zip + + false + + + + ${project.build.directory}/${project.build.finalName}.jar + / + + + + src/main/resources/adbc.conf.template + / + + + + + + /lib + false + runtime + + org.apache.doris:fe-connector-api + org.apache.doris:fe-connector-spi + org.apache.doris:fe-extension-spi + org.apache.doris:fe-filesystem-api + org.apache.doris:fe-thrift + org.apache.thrift:libthrift + + org.apache.logging.log4j:* + org.slf4j:* + + + + diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java new file mode 100644 index 00000000000000..2ff0dda83e89cc --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcDatabase; +import org.apache.arrow.adbc.core.AdbcDriver; +import org.apache.arrow.adbc.core.AdbcException; +import org.apache.arrow.adbc.driver.jni.JniDriver; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; + +import java.io.Closeable; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +/** + * Owns the ADBC objects for one catalog: the Arrow allocator, the {@code AdbcDatabase}, and the short-lived + * connections metadata operations borrow. + * + *

Why the JNI driver and not a pure-Java one. The JNI bridge wraps the same C driver manager BE + * links statically and dlopens the same driver library, so anything opaque that crosses between FE and BE + * (partition descriptors) is produced and consumed by one implementation. The pure-Java Flight SQL driver + * would be lighter on FE, but it serializes partition descriptors as a different protobuf message than the + * Go driver expects, and protobuf mis-parses it silently rather than failing -- and it does not implement + * {@code getTableSchema} at all, which would force FE to derive column types from XDBC type codes while BE + * reads real Arrow types. + * + *

Nothing is opened eagerly. Driver loading is deferred to first use rather than done in the + * constructor: an FE follower replaying the edit log builds every catalog, and its filesystem layout need + * not match the leader's, so a missing driver file would otherwise stop FE from starting instead of + * failing the one catalog that cannot work. + */ +public class AdbcClient implements Closeable { + + private final Path driverPath; + private final String driverUrl; + private final String entrypoint; + private final String uri; + private final String user; + private final String password; + private final Map driverOptions; + + private volatile BufferAllocator allocator; + private volatile AdbcDatabase database; + private volatile boolean closed; + + public AdbcClient(Path driverPath, String driverUrl, String entrypoint, String uri, + String user, String password, Map driverOptions) { + this.driverPath = driverPath; + this.driverUrl = driverUrl; + this.entrypoint = entrypoint; + this.uri = uri; + this.user = user; + this.password = password; + this.driverOptions = driverOptions; + } + + /** + * Runs {@code body} on a fresh connection and closes it afterwards. + * + *

One connection per operation, deliberately: connection reuse only starts paying off once scans + * hold connections across a query, and a pool that outlives a statement would have to answer for + * per-user authorization on borrowed connections. Metadata calls are infrequent enough that the open + * cost does not show. + */ + public T withConnection(AdbcConnectionCall body) { + AdbcDatabase db = getOrOpenDatabase(); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + // The plugin loads child-first and the ADBC/Arrow classes live in it; pin the context + // classloader so any name-based lookup underneath resolves the plugin's copies rather than a + // parent one (which would ClassCast against the child-loaded objects it is handed). + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + try (AdbcConnection connection = db.connect()) { + return body.apply(connection); + } + } catch (AdbcException e) { + throw translate(e, "ADBC operation failed"); + } catch (DorisConnectorException e) { + throw e; + } catch (Exception e) { + throw new DorisConnectorException("ADBC operation failed: " + e.getMessage(), e); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } + + private AdbcDatabase getOrOpenDatabase() { + if (closed) { + throw new DorisConnectorException("AdbcClient has been closed"); + } + AdbcDatabase db = database; + if (db != null) { + return db; + } + synchronized (this) { + if (closed) { + throw new DorisConnectorException("AdbcClient has been closed"); + } + if (database == null) { + AdbcDriverPathResolver.checkExists(driverPath, driverUrl); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + if (allocator == null) { + allocator = new RootAllocator(); + } + database = new JniDriver(allocator).open(buildParameters()); + } catch (AdbcException e) { + throw translate(e, "Failed to open the ADBC driver " + driverPath); + } catch (UnsatisfiedLinkError e) { + throw new DorisConnectorException("Failed to load the ADBC JNI bridge." + + " FE loads it from the directory named by the JVM property" + + " arrow.adbc.driver.jni.library.path (set in fe.conf, normally" + + " ${DORIS_HOME}/lib): " + e.getMessage(), e); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } + return database; + } + } + + private Map buildParameters() { + Map params = new HashMap<>(); + JniDriver.PARAM_DRIVER.set(params, driverPath.toString()); + AdbcDriver.PARAM_URI.set(params, uri); + if (user != null && !user.isEmpty()) { + AdbcDriver.PARAM_USERNAME.set(params, user); + } + if (password != null && !password.isEmpty()) { + AdbcDriver.PARAM_PASSWORD.set(params, password); + } + if (entrypoint != null && !entrypoint.trim().isEmpty()) { + // The C driver manager reads this key to pick the init symbol to dlsym; verified against the + // SQLite driver, where a bogus value fails with "dlsym(...) failed: undefined symbol". + params.put("entrypoint", entrypoint.trim()); + } + // Option names keep their "adbc." prefix; see AdbcConnectorProperties.DRIVER_OPTION_PREFIX. + params.putAll(driverOptions); + return params; + } + + /** + * Carries the ADBC status/SQLSTATE/vendor code into a Doris error. + * + *

The driver's own message is appended only when it says something: the SQLite driver answers + * {@code NOT_IMPLEMENTED} with the literal text {@code (unknown error)}, so a message built by + * forwarding {@code getMessage()} would tell a user nothing at all. + */ + static DorisConnectorException translate(AdbcException e, String context) { + StringBuilder sb = new StringBuilder(context); + sb.append(" [status=").append(e.getStatus()); + if (e.getSqlState() != null && !e.getSqlState().isEmpty()) { + sb.append(", sqlState=").append(e.getSqlState()); + } + if (e.getVendorCode() != 0) { + sb.append(", vendorCode=").append(e.getVendorCode()); + } + sb.append(']'); + String message = e.getMessage(); + if (message != null && !message.isEmpty() && !"(unknown error)".equals(message)) { + sb.append(": ").append(message); + } + return new DorisConnectorException(sb.toString(), e); + } + + @Override + public synchronized void close() { + closed = true; + try { + if (database != null) { + database.close(); + } + } catch (Exception e) { + throw new DorisConnectorException("Failed to close the ADBC database: " + e.getMessage(), e); + } finally { + database = null; + if (allocator != null) { + allocator.close(); + allocator = null; + } + } + } + + /** A body that runs against a borrowed ADBC connection. */ + @FunctionalInterface + public interface AdbcConnectionCall { + T apply(AdbcConnection connection) throws Exception; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java new file mode 100644 index 00000000000000..e9180065e47434 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTestResult; +import org.apache.doris.connector.api.ConnectorValidationContext; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.ConnectorConf; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * ADBC connector implementation. Created once per catalog lifecycle. + */ +public class AdbcConnector implements Connector { + + private static final Logger LOG = LogManager.getLogger(AdbcConnector.class); + + private final Map properties; + private final ConnectorContext context; + private final AdbcSchemaStrategy schemaStrategy = new AdbcSchemaStrategy(); + private final AdbcPartitionedReadSupport partitionedRead = new AdbcPartitionedReadSupport(); + private final AdbcDialectSelector dialectSelector; + private final AdbcMetadataCache metadataCache; + + private volatile AdbcClient client; + private volatile boolean closed; + + public AdbcConnector(Map properties, ConnectorContext context) { + this.properties = properties; + this.context = context; + // Both read properties only; the driver is not touched here (see getOrCreateClient). + this.dialectSelector = new AdbcDialectSelector(properties); + this.metadataCache = new AdbcMetadataCache(properties); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + return new AdbcConnectorMetadata(getOrCreateClient(), schemaStrategy, properties, + () -> dialectSelector.select(this::getOrCreateClient), metadataCache); + } + + /** + * The catalog's metadata memory, shared by every statement's {@link AdbcConnectorMetadata} and dropped + * by the REFRESH hooks below. + */ + AdbcMetadataCache metadataCache() { + return metadataCache; + } + + /** {@code REFRESH TABLE}. The names are the remote ones, which for ADBC are the ones Doris shows. */ + @Override + public void invalidateTable(String dbName, String tableName) { + metadataCache.invalidateTable(dbName, tableName); + } + + /** {@code REFRESH DATABASE}. */ + @Override + public void invalidateDb(String dbName) { + metadataCache.invalidateDb(dbName); + } + + /** + * {@code REFRESH CATALOG}. Note that this does NOT rebuild the connector, so without dropping the cache + * here the catalog would keep serving what it remembered until the TTL ran out -- the statement a user + * reaches for when metadata looks wrong would be the one statement that changed nothing. + */ + @Override + public void invalidateAll() { + metadataCache.invalidateAll(); + } + + @Override + public ConnectorTestResult testConnection(ConnectorSession session) { + try { + List databases = getMetadata(session).listDatabaseNames(session); + return ConnectorTestResult.success( + "Connected successfully, found " + databases.size() + " databases"); + } catch (Exception e) { + LOG.warn("ADBC connection test failed", e); + return ConnectorTestResult.failure("ADBC connection failed: " + e.getMessage()); + } + } + + /** + * Scan planning, built per call because it holds no state -- everything worth keeping across queries + * (the driver's dialect, the connection, whether the driver partitions) lives on the connector. + */ + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + return new AdbcScanPlanProvider(properties, resolveDriverPath(), dialectSelector, + this::getOrCreateClient, partitionedRead); + } + + /** + * Runs on CREATE and ALTER CATALOG only -- never on edit-log replay or at query time -- so it is the + * one place that may reach the filesystem and the remote source without risking FE startup. + */ + @Override + public void preCreateValidation(ConnectorValidationContext validationContext) throws Exception { + // Checked here rather than through the context's own checksum service: that one resolves against + // jdbc_drivers_dir and a .jar grammar, neither of which applies to an ADBC driver library. + AdbcDriverPathResolver.checkChecksum(resolveDriverPath(), + properties.get(AdbcConnectorProperties.DRIVER_CHECKSUM), + properties.get(AdbcConnectorProperties.DRIVER_URL)); + // Before the remote check, because a misspelled dialect is answerable without a source and its + // error should not be preceded by a connection failure the user cannot act on. + dialectSelector.validateConfiguredName(); + checkRemoteCatalogIsPinned(); + } + + /** + * Verifies that {@code uri} pins a remote catalog, which is what lets ADBC's three naming levels be + * shown as Doris's two without concatenating anything (see {@link AdbcNamespace}). + * + *

A driver that cannot answer is trusted, not refused. {@code getCurrentCatalog} is optional + * in ADBC and drivers implement these context getters one by one -- the SQLite driver answers + * {@code getCurrentCatalog} but rejects {@code getCurrentDbSchema} with {@code NOT_FOUND}, so the + * failure status does not even reliably say "not implemented". Refusing on any error would therefore + * lock out drivers over a missing self-check rather than a misconfiguration, and a genuinely unpinned + * uri still surfaces clearly at the first {@code SHOW DATABASES}. Only an answer that arrives and is + * empty is treated as a real "not pinned". + */ + private void checkRemoteCatalogIsPinned() { + String currentCatalog; + try { + currentCatalog = getOrCreateClient().withConnection( + connection -> connection.getCurrentCatalog()); + } catch (DorisConnectorException e) { + // AdbcClient already funnels every driver-side failure (any AdbcException, whatever its status) + // into this one type, so catching it covers "the driver could not answer" in full. + LOG.info("ADBC driver cannot report its current catalog, so the '{}' property is accepted" + + " as written: {}", AdbcConnectorProperties.URI, e.toString()); + return; + } + if (currentCatalog != null && currentCatalog.isEmpty()) { + throw new DorisConnectorException("The ADBC source reports no current catalog, so '" + + AdbcConnectorProperties.URI + "' does not pin one. Name the remote catalog in the" + + " uri (for example postgresql://host:5432/mydb) or through a driver option, because" + + " a Doris catalog maps exactly one remote catalog."); + } + } + + private Path resolveDriverPath() { + Path driverPath = AdbcDriverPathResolver.resolve( + properties.get(AdbcConnectorProperties.DRIVER_URL), + driversDir(), + ConnectorConf.get(context, AdbcConnectorProperties.CONF_DRIVER_SECURE_PATH, null, + AdbcConnectorProperties.DEFAULT_DRIVER_SECURE_PATH)); + AdbcDriverPathResolver.checkExists(driverPath, properties.get(AdbcConnectorProperties.DRIVER_URL)); + return driverPath; + } + + /** + * The directory a bare {@code driver_url} file name resolves under: adbc.conf's + * {@code drivers_dir}, else {@code /plugins/adbc_drivers}. + * + *

The default is computed here rather than declared as an fe.conf {@code @ConfField}, because a + * key in fe-core is an engine change per connector setting. Null when DORIS_HOME is unknown and the + * conf file says nothing -- {@link AdbcDriverPathResolver#resolve} then reports the bare name as + * unresolvable instead of quietly resolving it against the process working directory. + */ + private String driversDir() { + String dorisHome = context.getEnvironment().get(AdbcConnectorProperties.ENV_DORIS_HOME); + String defaultDir = dorisHome == null + ? null : dorisHome + AdbcConnectorProperties.DEFAULT_DRIVERS_SUBDIR; + return ConnectorConf.get(context, AdbcConnectorProperties.CONF_DRIVERS_DIR, null, defaultDir); + } + + private AdbcClient getOrCreateClient() { + if (closed) { + throw new DorisConnectorException("AdbcConnector has been closed"); + } + AdbcClient existing = client; + if (existing != null) { + return existing; + } + synchronized (this) { + if (closed) { + throw new DorisConnectorException("AdbcConnector has been closed"); + } + if (client == null) { + // Deliberately NOT in the constructor: an FE follower replaying the edit log builds every + // catalog, and its filesystem layout need not match the leader's, so resolving the driver + // here would let a missing file stop FE from starting. + client = new AdbcClient(resolveDriverPath(), + properties.get(AdbcConnectorProperties.DRIVER_URL), + properties.get(AdbcConnectorProperties.DRIVER_ENTRYPOINT), + AdbcConnectorProperties.require(properties, AdbcConnectorProperties.URI), + properties.get(AdbcConnectorProperties.USER), + properties.get(AdbcConnectorProperties.PASSWORD), + AdbcConnectorProperties.driverOptions(properties)); + } + return client; + } + } + + @Override + public synchronized void close() throws IOException { + closed = true; + if (client != null) { + client.close(); + client = null; + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorMetadata.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorMetadata.java new file mode 100644 index 00000000000000..02062a044b4ba1 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorMetadata.java @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.handle.NamedColumnHandle; +import org.apache.doris.thrift.THiveTable; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcException; +import org.apache.arrow.adbc.core.AdbcStatement; +import org.apache.arrow.adbc.core.AdbcStatusCode; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * Serves {@code SHOW DATABASES} / {@code SHOW TABLES} / {@code DESC} for an ADBC catalog. + * + *

Created fresh per statement by the engine's metadata funnel, so it holds no cross-statement state. + * Everything worth keeping longer lives on the connector and is reached from here: which call a driver + * answers schemas with (re-probing it per table would cost one failed remote call per table forever), and + * the answers themselves ({@link AdbcMetadataCache}). + */ +public class AdbcConnectorMetadata implements ConnectorMetadata { + + /** + * Only base tables are surfaced; ADBC catalogs expose no views in Doris. Asking for the type is not + * enough to get it -- a Doris source answers this filter with everything it has -- so what actually + * holds the guarantee is {@code AdbcObjectsReader}, which drops whatever the answer calls a view. + */ + private static final String[] TABLE_TYPES = {"table"}; + + private final AdbcClient client; + private final AdbcSchemaStrategy schemaStrategy; + private final Map properties; + private final Supplier dialect; + private final AdbcMetadataCache cache; + + /** + * @param dialect resolved lazily, because only the {@code executeSchema} fallback needs it and + * resolving it can cost a remote call + * @param cache the catalog's, shared with every other statement + */ + public AdbcConnectorMetadata(AdbcClient client, AdbcSchemaStrategy schemaStrategy, + Map properties, Supplier dialect, AdbcMetadataCache cache) { + this.client = client; + this.schemaStrategy = schemaStrategy; + this.properties = properties; + this.dialect = dialect; + this.cache = cache; + } + + // ========= ConnectorSchemaOps ========= + + /** + * Reads the source, never the cache, and refreshes the cache with what it read. + * + *

This is not just a report for {@code SHOW DATABASES}: the engine's own name cache is loaded from it + * and then decides whether a database exists at all -- including its last-chance re-list for a name it + * has never seen. Answering that from a remembered listing would make the re-list pointless and leave a + * database created a moment ago unreachable until an entry expired. + */ + @Override + public List listDatabaseNames(ConnectorSession session) { + List names = new ArrayList<>(); + for (AdbcNamespace namespace : cache.reloadNamespaces(this::readNamespaces)) { + names.add(namespace.dorisDatabaseName()); + } + return names; + } + + @Override + public boolean databaseExists(ConnectorSession session, String dbName) { + return findNamespace(dbName).isPresent(); + } + + // ========= ConnectorTableMetadataOps ========= + + /** Reads the source, never the cache. Same reason as {@link #listDatabaseNames}, one level down. */ + @Override + public List listTableNames(ConnectorSession session, String dbName) { + Optional namespace = findNamespace(dbName); + if (!namespace.isPresent()) { + return List.of(); + } + return cache.reloadTableNames(namespace.get(), () -> readTableNames(namespace.get())); + } + + @Override + public Optional getTableHandle( + ConnectorSession session, String dbName, String tableName) { + Optional namespace = findNamespace(dbName); + if (!namespace.isPresent()) { + return Optional.empty(); + } + if (!tableExists(namespace.get(), tableName)) { + return Optional.empty(); + } + return Optional.of(new AdbcTableHandle(namespace.get(), tableName)); + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + AdbcTableHandle adbcHandle = (AdbcTableHandle) handle; + Schema arrowSchema = arrowSchemaOf(session, adbcHandle); + + List columns = new ArrayList<>(arrowSchema.getFields().size()); + for (Field field : arrowSchema.getFields()) { + columns.add(new ConnectorColumn(field.getName(), + AdbcTypeMapper.toDorisType(field.getName(), field), + null, field.isNullable(), null, true)); + } + return new ConnectorTableSchema(adbcHandle.getRemoteTable(), columns, "ADBC", properties); + } + + @Override + public Map getColumnHandles( + ConnectorSession session, ConnectorTableHandle handle) { + Schema arrowSchema = arrowSchemaOf(session, (AdbcTableHandle) handle); + Map handles = new LinkedHashMap<>(arrowSchema.getFields().size()); + for (Field field : arrowSchema.getFields()) { + handles.put(field.getName(), new NamedColumnHandle(field.getName())); + } + return handles; + } + + /** + * ADBC has no descriptor of its own, so it borrows the Hive one, as the other connectors that read + * through the generic file-scan path do. + * + *

Leaving the SPI default (null) is not neutral: fe-core would fall back to + * {@code TTableType.SCHEMA_TABLE} and BE would build a {@code SchemaTableDescriptor} instead of the + * descriptor the scan path expects. + */ + @Override + public TTableDescriptor buildTableDescriptor(ConnectorSession session, + long tableId, String tableName, String dbName, + String remoteName, int numCols, long catalogId) { + THiveTable hiveTable = new THiveTable(dbName, tableName, new HashMap<>()); + TTableDescriptor descriptor = new TTableDescriptor( + tableId, TTableType.HIVE_TABLE, numCols, 0, tableName, dbName); + descriptor.setHiveTable(hiveTable); + return descriptor; + } + + // ========= internals ========= + + private List readNamespaces() { + return client.withConnection(connection -> { + try (ArrowReader reader = connection.getObjects( + AdbcConnection.GetObjectsDepth.DB_SCHEMAS, null, null, null, null, null)) { + return AdbcObjectsReader.readNamespaces(reader); + } + }); + } + + /** + * Resolves a Doris database name, asking the source again before deciding there is no such database. + * + *

This is the path a query takes -- every {@code getTableHandle} starts here -- so it reads what was + * remembered. The second read is what keeps that from being able to say "no": a database created since + * the listing was cached is found rather than denied, and the extra remote call falls only on the path + * that was about to fail anyway. + */ + private Optional findNamespace(String dbName) { + Optional found = match(cache.namespaces(this::readNamespaces), dbName); + if (found.isPresent()) { + return found; + } + return match(cache.reloadNamespaces(this::readNamespaces), dbName); + } + + private static Optional match(List namespaces, String dbName) { + for (AdbcNamespace namespace : namespaces) { + if (namespace.dorisDatabaseName().equals(dbName)) { + return Optional.of(namespace); + } + } + return Optional.empty(); + } + + /** The table-level counterpart of {@link #findNamespace}; same reason, same cost. */ + private boolean tableExists(AdbcNamespace namespace, String tableName) { + return cache.tableNames(namespace, () -> readTableNames(namespace)).contains(tableName) + || cache.reloadTableNames(namespace, () -> readTableNames(namespace)).contains(tableName); + } + + private List readTableNames(AdbcNamespace namespace) { + return client.withConnection(connection -> { + try (ArrowReader reader = connection.getObjects(AdbcConnection.GetObjectsDepth.TABLES, + emptyToNull(namespace.getRemoteCatalog()), emptyToNull(namespace.getRemoteDbSchema()), + null, TABLE_TYPES, null)) { + return AdbcObjectsReader.readTableNames(reader, namespace); + } + }); + } + + /** + * Two layers, and neither makes the other redundant: the statement scope folds the two SPI calls one + * statement makes for the same table ({@code getTableSchema} then {@code getColumnHandles}) into one + * lookup, while the catalog cache carries the answer to the next statement. + */ + private Schema arrowSchemaOf(ConnectorSession session, AdbcTableHandle handle) { + return AdbcStatementScope.sharedTableSchema(session, handle, + () -> cache.tableSchema(handle, () -> fetchArrowSchema(handle))); + } + + /** + * Resolves one table's Arrow schema, falling back across the ways a driver may offer it. + * + *

Neither call is guaranteed. {@code getTableSchema} has a default implementation in the ADBC API + * that throws {@code NOT_IMPLEMENTED}, and drivers do leave it there -- the Java Flight SQL driver, for + * one. {@code executeSchema} is no safer in the other direction: the SQLite driver answers it with + * {@code NOT_IMPLEMENTED} while implementing {@code getTableSchema} fine. So each is the other's + * fallback, and the strategy is remembered per catalog rather than re-probed per table. + * + *

Only {@code NOT_IMPLEMENTED} triggers the fallback. Any other status means the driver did + * try and the table is at fault -- a missing table answers {@code NOT_FOUND} -- and falling back there + * would swap a precise "no such table" for a misleading "this driver implements neither method". + * + *

The column layer of {@code getObjects} is deliberately not a third fallback: it reports XDBC + * integer type codes, not Arrow types, so it would answer with a different type system than the one BE + * reads the data in. + */ + private Schema fetchArrowSchema(AdbcTableHandle handle) { + return client.withConnection(connection -> { + if (schemaStrategy.get() == AdbcSchemaStrategy.Kind.EXECUTE_SCHEMA) { + return executeSchema(connection, handle); + } + try { + Schema schema = connection.getTableSchema( + emptyToNull(handle.getRemoteCatalog()), emptyToNull(handle.getRemoteDbSchema()), + handle.getRemoteTable()); + schemaStrategy.set(AdbcSchemaStrategy.Kind.GET_TABLE_SCHEMA); + return schema; + } catch (AdbcException e) { + if (e.getStatus() != AdbcStatusCode.NOT_IMPLEMENTED) { + throw AdbcClient.translate(e, "Failed to read the schema of " + + handle.getDorisDbName() + "." + handle.getRemoteTable()); + } + Schema schema = executeSchemaOrExplain(connection, handle, e); + schemaStrategy.set(AdbcSchemaStrategy.Kind.EXECUTE_SCHEMA); + return schema; + } + }); + } + + private Schema executeSchemaOrExplain(AdbcConnection connection, AdbcTableHandle handle, + AdbcException getTableSchemaFailure) { + try { + return executeSchema(connection, handle); + } catch (Exception e) { + throw new DorisConnectorException("Cannot determine the schema of " + + handle.getDorisDbName() + "." + handle.getRemoteTable() + + ": this ADBC driver implements neither getTableSchema (status=" + + getTableSchemaFailure.getStatus() + ") nor executeSchema (" + e.getMessage() + + "). Doris cannot map the table's columns without one of them.", e); + } + } + + /** + * Asks for the shape of a row without fetching any. {@code WHERE 1 = 0} is the most portable way to + * say "no rows"; the table name comes from the dialect so this path and a scan address the same table + * the same way -- two spellings of one name is exactly how a source ends up working for queries and + * failing for {@code DESC}. + */ + private Schema executeSchema(AdbcConnection connection, AdbcTableHandle handle) { + String sql = "SELECT * FROM " + dialect.get().qualifiedTableName(handle) + " WHERE 1 = 0"; + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + return statement.executeSchema(); + } catch (AdbcException e) { + throw AdbcClient.translate(e, "executeSchema failed for: " + sql); + } catch (Exception e) { + throw new DorisConnectorException("executeSchema failed for: " + sql, e); + } + } + + /** + * ADBC treats null as "any" for a catalog or schema filter, and a source without that level reports it + * as the empty string. Passing the empty string through would ask for a level literally named "". + */ + private static String emptyToNull(String value) { + return value == null || value.isEmpty() ? null : value; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorProperties.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorProperties.java new file mode 100644 index 00000000000000..4fab5accecfee7 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorProperties.java @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Property constants for ADBC catalogs, and the keys of this plugin's own settings file. + * + *

fe-core parses no connector properties, so everything a user writes in {@code CREATE CATALOG} is + * interpreted here. Settings that are one-per-FE rather than one-per-catalog live in the plugin's + * {@code adbc.conf} instead ({@link #CONF_DRIVERS_DIR} / {@link #CONF_DRIVER_SECURE_PATH}), read with + * {@code ConnectorConf.get}. They have no fe.conf half: this connector is new, so there is no + * deployment that configured them before {@code adbc.conf} existed, and adding a {@code @ConfField} + * would tie a key name of this plugin into fe-core for nothing. + */ +public final class AdbcConnectorProperties { + + private AdbcConnectorProperties() { + } + + // -- driver -- + + /** + * The ADBC driver shared library. Named after the JDBC catalog's property for continuity, but it + * accepts local references only: a bare file name (resolved under {@link #ENV_DRIVERS_DIR}), a + * {@code file://} URL, or an absolute path. Remote schemes are rejected -- see + * {@link AdbcDriverPathResolver} for why downloading it per node cannot be made safe. + */ + public static final String DRIVER_URL = "driver_url"; + + /** + * Optional MD5 of the driver library, verified at {@code CREATE CATALOG} against the file this FE + * resolves. It guards the mistake this catalog type invites -- a hand-placed file that is the wrong + * build, or a stale copy on one node -- which otherwise loads fine and surfaces much later as a query + * failure that says nothing about a file. It only sees the FE's copy; see + * {@link AdbcDriverPathResolver#checkChecksum}. + */ + public static final String DRIVER_CHECKSUM = "driver_checksum"; + /** Optional; empty lets the driver manager infer the entry point symbol. */ + public static final String DRIVER_ENTRYPOINT = "driver_entrypoint"; + + // -- connection -- + + /** + * The ADBC connection URI. It must pin the remote catalog (e.g. {@code postgresql://host:5432/mydb}), + * because Doris flattens ADBC's three-level catalog/db_schema/table namespace onto its own two-level + * database/table one. + */ + public static final String URI = "uri"; + public static final String USER = "user"; + public static final String PASSWORD = "password"; + + // -- SQL generation -- + + /** + * The SQL dialect to generate pushed-down queries in, by {@link AdbcDialect#name()}. Optional: when it + * is absent the connector asks the driver for its vendor and falls back to ANSI, which is what an + * unrecognized source gets. Set it when the source's vendor string is unhelpful or when its SQL differs + * from what the vendor name implies. + */ + public static final String SQL_DIALECT = "sql_dialect"; + + // -- partitioned read -- + + /** + * How a scan may be split into the driver's own partitions and spread over several backends: one of + * {@link PartitionedReadMode}, spelled as its lowercase name. + * + *

Three states rather than a switch, because "use partitions when you can" and "use partitions or + * fail" are different requirements and encoding them as two booleans admits a combination that + * contradicts itself. + */ + public static final String PARTITIONED_READ = "partitioned_read"; + + /** What a catalog asks of partitioned execution. */ + public enum PartitionedReadMode { + /** + * Split the scan when the driver can, read it as one statement when it cannot. The default, + * because parallelism is the point of reading through ADBC rather than one connection, and a + * driver without partitions must still be usable. + */ + AUTO, + /** + * Never ask for partitions. Asking is not free: on a Flight SQL source the call that returns them + * is the query's execution, so planning gains a remote round trip and the source starts + * working before Doris has committed to running the plan. This is the way back to the + * single-statement path for a source that pays badly for that, or whose partitions Doris then + * fails to read. + */ + DISABLED, + /** + * Split the scan, or fail the query saying why. + * + *

For anything that must not silently lose its parallelism. A test is the clearest + * case: under {@link #AUTO} a driver that stops partitioning turns the test green while quietly + * exercising the fallback instead of the path under test -- the failure looks exactly like a pass. + * A deployment sized for N backends has the same problem in slower motion, which is why this is a + * supported mode and not a test-only flag. + */ + REQUIRED + } + + /** + * The most partitions one scan may plan. A guard rail against a pathological source, not a tuning knob: + * each partition costs a scan range carrying an opaque descriptor of a few hundred bytes, so a million + * of them would exhaust FE. Exceeding it fails the query rather than falling back to a single range, + * because by then the source has already executed the query and a fallback would make it execute a + * second time while the first result set sits unread. + */ + public static final String MAX_PARTITIONS = "max_partitions"; + + private static final int DEFAULT_MAX_PARTITIONS = 1024; + + /** + * Prefix for options passed straight through to the driver. The prefix is PART OF THE OPTION NAME and + * is NOT stripped: ADBC's own option names already start with {@code adbc.} (e.g. + * {@code adbc.snowflake.sql.db}), so a user writes {@code "adbc.adbc.snowflake.sql.db"}. BE applies the + * same rule to its own parameter map; the two must not diverge. + */ + public static final String DRIVER_OPTION_PREFIX = "adbc."; + + // -- adbc.conf keys (not catalog properties) -- + + /** + * Directory a bare {@code driver_url} file name resolves under. Defaults to + * {@code /plugins/adbc_drivers}, which is where build.sh creates the directory. + */ + public static final String CONF_DRIVERS_DIR = "drivers_dir"; + + /** + * Semicolon-separated directories a driver may be loaded from; {@code *} allows any. Defaults to + * {@link #DEFAULT_DRIVER_SECURE_PATH}. + */ + public static final String CONF_DRIVER_SECURE_PATH = "driver_secure_path"; + + /** The subdirectory of DORIS_HOME {@link #CONF_DRIVERS_DIR} defaults to. */ + public static final String DEFAULT_DRIVERS_SUBDIR = "/plugins/adbc_drivers"; + + /** Allow any directory, matching the jdbc catalog's default. */ + public static final String DEFAULT_DRIVER_SECURE_PATH = "*"; + + /** Engine-wide rather than this connector's setting, so it keeps coming from the environment. */ + public static final String ENV_DORIS_HOME = "doris_home"; + + /** + * Returns the {@code adbc.}-prefixed entries with their names kept intact, in iteration order. + */ + public static Map driverOptions(Map properties) { + Map options = new LinkedHashMap<>(); + for (Map.Entry entry : properties.entrySet()) { + if (entry.getKey().startsWith(DRIVER_OPTION_PREFIX)) { + options.put(entry.getKey(), entry.getValue()); + } + } + return options; + } + + /** + * Reads {@link #PARTITIONED_READ}, defaulting to {@link PartitionedReadMode#AUTO}. + * + *

An unrecognized value fails rather than falling back to the default: a typo that silently meant + * AUTO would show up only as lost parallelism, or -- worse, for a catalog that asked for REQUIRED -- + * as the silent downgrade that mode exists to forbid. + */ + public static PartitionedReadMode partitionedReadMode(Map properties) { + String value = properties.get(PARTITIONED_READ); + if (value == null || value.trim().isEmpty()) { + return PartitionedReadMode.AUTO; + } + String normalized = value.trim(); + for (PartitionedReadMode mode : PartitionedReadMode.values()) { + if (mode.name().equalsIgnoreCase(normalized)) { + return mode; + } + } + throw new IllegalArgumentException("Property '" + PARTITIONED_READ + "' must be one of 'auto'," + + " 'disabled' or 'required', but is '" + value + "'"); + } + + /** Reads {@link #MAX_PARTITIONS}, defaulting to {@value #DEFAULT_MAX_PARTITIONS}. */ + public static int maxPartitions(Map properties) { + String value = properties.get(MAX_PARTITIONS); + if (value == null || value.trim().isEmpty()) { + return DEFAULT_MAX_PARTITIONS; + } + int parsed; + try { + parsed = Integer.parseInt(value.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Property '" + MAX_PARTITIONS + + "' must be a positive integer, but is '" + value + "'", e); + } + if (parsed < 1) { + throw new IllegalArgumentException("Property '" + MAX_PARTITIONS + + "' must be at least 1, but is '" + value + "'"); + } + return parsed; + } + + /** Returns a required property, or throws naming the property that is missing. */ + public static String require(Map properties, String key) { + String value = properties.get(key); + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException( + "Required property '" + key + "' is missing for an adbc catalog"); + } + return value.trim(); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorProvider.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorProvider.java new file mode 100644 index 00000000000000..42d9f948b4bff6 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorProvider.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; + +import java.util.Map; + +/** + * SPI entry point for the ADBC connector. + * Discovered via META-INF/services/org.apache.doris.connector.spi.ConnectorProvider. + * + *

Class-loading invariant (guarded by {@code AdbcConnectorProviderIsolationTest}): neither this + * class's static initializer nor its no-arg constructor may touch {@code org.apache.arrow.adbc.*}, + * directly or transitively. The plugin loader builds the plugin classloader and instantiates every + * discovered factory BEFORE it checks for a duplicate type name + * ({@code DirectoryPluginRuntimeManager#loadAll}), so when a classpath built-in and a directory plugin + * both claim "adbc" the directory one is still constructed once and only then discarded. Touching an ADBC + * class there would trigger a second {@code System.load} of the JNI shim from a different classloader and + * throw {@code UnsatisfiedLinkError}. Production never hits that shape; test and embedded setups do. + * + *

Practically: keep the fields, the constructor and {@link #getType()} free of ADBC types, and let the + * ADBC classes first appear inside {@link #create}, whose body is resolved only when it runs. + */ +public class AdbcConnectorProvider implements ConnectorProvider { + + @Override + public String getType() { + return "adbc"; + } + + /** + * Cheap presence checks only. Resolving {@code driver_url} to a path needs adbc.conf's + * {@code drivers_dir} and {@code driver_secure_path}, which arrive through the connector context + * rather than the property map, so that half runs in {@code AdbcConnector#preCreateValidation}. + */ + @Override + public void validateProperties(Map properties) { + AdbcConnectorProperties.require(properties, AdbcConnectorProperties.DRIVER_URL); + AdbcConnectorProperties.require(properties, AdbcConnectorProperties.URI); + // Parsed for its exceptions: an unreadable value has to fail here, at CREATE CATALOG, and not on + // the first query -- these two decide how a scan is planned, and a typo in either would otherwise + // change that silently. + AdbcConnectorProperties.partitionedReadMode(properties); + AdbcConnectorProperties.maxPartitions(properties); + checkMetaCacheProperties(properties); + } + + /** + * The cache knobs, which {@code CacheSpec} otherwise reads leniently -- an unparseable ttl or capacity + * falls back to the default instead of failing. That is the wrong shape for a value an operator typed: + * the catalog would come up caching for a duration nobody chose, and nothing would say so. Bounds match + * the other connectors': ttl {@code >= -1} (-1 is "never expire"), capacity {@code >= 0} (0 disables). + */ + private static void checkMetaCacheProperties(Map properties) { + CacheSpec.PropertySpec spec = AdbcMetadataCache.propertySpec(); + CacheSpec.checkBooleanProperty(properties.get(spec.getEnableKey()), spec.getEnableKey()); + CacheSpec.checkLongProperty(properties.get(spec.getTtlKey()), -1L, spec.getTtlKey()); + CacheSpec.checkLongProperty(properties.get(spec.getCapacityKey()), 0L, spec.getCapacityKey()); + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + return new AdbcConnector(properties, context); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialect.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialect.java new file mode 100644 index 00000000000000..e3d349517642ad --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialect.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; + +/** + * The SQL spelling differences between the sources an ADBC driver can point at. + * + *

Why an interface rather than an enum. The JDBC connector can enumerate its dialects because it + * enumerates its drivers; ADBC cannot -- any driver implementing the ADBC C ABI is a valid source, including + * ones written after this code ships. Adding one must therefore cost a class and a registration, and nothing + * else: {@link AdbcQueryBuilder} is written against this interface and must never grow a per-dialect branch. + * A test pins that by driving the builder with a dialect it defines itself. + * + *

The surface is deliberately only what generating {@code SELECT ... FROM ... WHERE ... LIMIT} needs. A + * method with no caller could not be gotten right, because nothing would show when it was wrong. + */ +public interface AdbcDialect { + + /** + * The name this dialect is selected by -- matched against the {@code sql_dialect} property and against + * the vendor an ADBC driver reports. Compared case-insensitively. + */ + String name(); + + /** + * Whether {@code vendorName}, as reported by the driver through {@code getInfo(VENDOR_NAME)}, means this + * dialect. The default matches the dialect's own name, which is what makes registration a one-liner for + * a dialect named after its vendor; a dialect covering several vendor spellings overrides it. + */ + default boolean matchesVendor(String vendorName) { + return name().equalsIgnoreCase(vendorName); + } + + /** Renders {@code name} as a quoted identifier, escaping whatever the quote character is. */ + String quoteIdentifier(String name); + + /** + * Renders the remote table name for a {@code FROM} clause. + * + *

Takes the handle rather than a joined string because the Doris database name is a display key that + * is never parsed back into remote levels ({@link AdbcNamespace}); how many of the three remote levels a + * source will accept in one name is exactly a dialect question (MySQL rejects a three-part name). + */ + String qualifiedTableName(AdbcTableHandle handle); + + /** + * Renders {@code literal} as SQL, or returns {@code null} when this dialect cannot render that type. + * + *

{@code null} is a real answer, not a failure. A predicate is pushed down whole or not at all, + * so an unrenderable literal has to be reportable without throwing -- the caller drops that conjunct and + * leaves it to Doris. Throwing would turn "we cannot speed this up" into a failed query. + */ + String renderLiteral(ConnectorLiteral literal); + + /** + * Whether {@code LIMIT n} is accepted. A dialect answering {@code false} gets no row limit pushed at all + * rather than a guessed alternative spelling: emitting {@code FETCH FIRST}/{@code ROWNUM} to a source + * that wanted the other one produces a syntax error at scan time, while not pushing it only costs rows + * over the wire. + */ + default boolean supportsLimitClause() { + return true; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialectRegistry.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialectRegistry.java new file mode 100644 index 00000000000000..049cef690d1780 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialectRegistry.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.DorisConnectorException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * The dialects this connector can speak, by name. + * + *

The point of the registry is the cost of the next dialect. Adding one must be a class plus a + * {@link #register} call, with no edit to {@link AdbcQueryBuilder} and none to the selection code here -- + * an ADBC source is any driver implementing the C ABI, so the set cannot be closed and a switch over it + * would have to be reopened for every source anyone connects. A test drives the query builder with a + * dialect it defines itself to keep that true. + * + *

Two ship: {@link AnsiDialect} and {@link DorisDialect}, one per source actually run against. The + * others named in the design wait for a source to verify them against: an unverified dialect is a set of + * guesses about someone else's SQL, and a wrong guess is a syntax error at scan time or, worse, a predicate + * that quietly selects different rows. + */ +public final class AdbcDialectRegistry { + + /** Case-insensitive so a name is matched the way a user or a driver spells it. */ + private static final Map DIALECTS = + new ConcurrentSkipListMap<>(String.CASE_INSENSITIVE_ORDER); + + private static final AdbcDialect DEFAULT = new AnsiDialect(); + + static { + register(DEFAULT); + register(new DorisDialect()); + } + + private AdbcDialectRegistry() { + } + + /** Registers {@code dialect} under its own name, replacing any dialect already using that name. */ + public static void register(AdbcDialect dialect) { + DIALECTS.put(dialect.name(), dialect); + } + + /** The dialect used when nothing else identifies the source. */ + public static AdbcDialect defaultDialect() { + return DEFAULT; + } + + /** + * Looks up an explicitly requested dialect, failing with the registered names when there is no such one. + * + *

Fails rather than falling back to the default: the user named a dialect because the default was + * wrong for their source, so silently using it anyway would produce SQL their source rejects, at scan + * time, with a message pointing at the SQL rather than at the misspelled property. + */ + public static AdbcDialect require(String name) { + AdbcDialect dialect = DIALECTS.get(name); + if (dialect == null) { + throw new DorisConnectorException("Unknown '" + AdbcConnectorProperties.SQL_DIALECT + "' value '" + + name + "' for an adbc catalog. Registered dialects: " + registeredNames()); + } + return dialect; + } + + /** + * Finds the dialect claiming {@code vendorName}, as an ADBC driver reports it through + * {@code getInfo(VENDOR_NAME)}. Empty when no dialect claims it, which is the normal outcome. + */ + public static Optional forVendor(String vendorName) { + if (vendorName == null || vendorName.trim().isEmpty()) { + return Optional.empty(); + } + String trimmed = vendorName.trim(); + for (AdbcDialect dialect : DIALECTS.values()) { + if (dialect.matchesVendor(trimmed)) { + return Optional.of(dialect); + } + } + return Optional.empty(); + } + + /** The registered names, sorted, for error messages. */ + public static List registeredNames() { + return new ArrayList<>(DIALECTS.keySet()); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialectSelector.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialectSelector.java new file mode 100644 index 00000000000000..bd67dd8369d6ce --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDialectSelector.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcInfoCode; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +/** + * Picks one catalog's SQL dialect, once, and keeps it. + * + *

Order: the {@code sql_dialect} property, then the vendor the driver reports, then ANSI. The property + * comes first because it is the user's escape hatch for a source whose vendor string is unhelpful or shared, + * and a probe that could override it would make the property advisory. + * + *

The probe runs at most once per catalog. It is a remote call on a query's planning path, and the + * answer is a property of the driver rather than of the query. It also has to be allowed to fail: the + * metadata phase measured drivers implementing these context calls one at a time -- the SQLite driver + * answers {@code getCurrentCatalog} but rejects {@code getCurrentDbSchema} -- so a failed probe means + * "unknown vendor", never a failed query. + */ +public final class AdbcDialectSelector { + + private static final Logger LOG = LogManager.getLogger(AdbcDialectSelector.class); + + private final String configuredName; + private final AtomicReference resolved = new AtomicReference<>(); + + public AdbcDialectSelector(Map properties) { + String value = properties.get(AdbcConnectorProperties.SQL_DIALECT); + this.configuredName = value == null || value.trim().isEmpty() ? null : value.trim(); + } + + /** + * Validates the configured name without touching the source, so a misspelled {@code sql_dialect} is + * rejected when the catalog is created rather than when a query first scans. + */ + public void validateConfiguredName() { + if (configuredName != null) { + AdbcDialectRegistry.require(configuredName); + } + } + + /** + * The dialect for this catalog. {@code connectionSource} supplies a connection for the vendor probe and + * is not used once the answer is known. + */ + public AdbcDialect select(Supplier connectionSource) { + AdbcDialect known = resolved.get(); + if (known != null) { + return known; + } + AdbcDialect selected = resolve(connectionSource); + // A racing probe would have reached the same answer, so whichever lands first wins. + resolved.compareAndSet(null, selected); + return resolved.get(); + } + + private AdbcDialect resolve(Supplier connectionSource) { + if (configuredName != null) { + return AdbcDialectRegistry.require(configuredName); + } + Optional vendor = probeVendorName(connectionSource); + if (vendor.isPresent()) { + Optional byVendor = AdbcDialectRegistry.forVendor(vendor.get()); + if (byVendor.isPresent()) { + LOG.info("ADBC catalog uses the '{}' SQL dialect, matched on vendor '{}'", + byVendor.get().name(), vendor.get()); + return byVendor.get(); + } + LOG.info("ADBC vendor '{}' has no registered SQL dialect; using '{}'", + vendor.get(), AdbcDialectRegistry.defaultDialect().name()); + } + return AdbcDialectRegistry.defaultDialect(); + } + + private Optional probeVendorName(Supplier connectionSource) { + try { + return connectionSource.get().withConnection(AdbcDialectSelector::readVendorName); + } catch (Exception e) { + // Includes the driver refusing getInfo outright. Not knowing the vendor is the normal case for + // an arbitrary ADBC source, and ANSI is exactly the answer for it. + LOG.info("ADBC driver did not report its vendor, so the default SQL dialect is used: {}", + e.toString()); + return Optional.empty(); + } + } + + /** + * Reads {@code VENDOR_NAME} out of the {@code getInfo} result. + * + *

The result is a two-column table -- an integer info code and a dense union holding the value in + * whichever of six types that code uses. Both columns are read through the generic + * {@code ValueVector.getObject} rather than by casting to a concrete vector class: the union's child + * layout is the driver's to choose, and a cast that assumed one would fail on a driver that laid it out + * differently, for a probe whose whole purpose is to tolerate not being answered. + * + *

The requested code is re-checked per row because a driver may answer with more rows than were + * asked for. + */ + private static Optional readVendorName(AdbcConnection connection) throws Exception { + try (ArrowReader reader = connection.getInfo(new AdbcInfoCode[] {AdbcInfoCode.VENDOR_NAME})) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + while (reader.loadNextBatch()) { + FieldVector codes = root.getVector("info_name"); + FieldVector values = root.getVector("info_value"); + if (codes == null || values == null) { + return Optional.empty(); + } + for (int row = 0; row < root.getRowCount(); row++) { + Object code = codes.getObject(row); + if (!(code instanceof Number) + || ((Number) code).intValue() != AdbcInfoCode.VENDOR_NAME.getValue()) { + continue; + } + Object value = values.getObject(row); + if (value != null) { + return Optional.of(String.valueOf(value)); + } + } + } + } + return Optional.empty(); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDriverPathResolver.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDriverPathResolver.java new file mode 100644 index 00000000000000..f8c44f05fa15c4 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcDriverPathResolver.java @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Turns the {@code driver_url} property into the absolute path of a local ADBC driver shared library. + * + *

Why local only, unlike the JDBC catalog's {@code driver_url}. ADBC partition descriptors are + * driver-private opaque bytes with no interoperability guarantee across driver implementations -- two + * official implementations of the SAME protocol already serialize incompatible protobuf messages and + * mis-parse each other silently rather than erroring. So FE and every BE must load the identical driver + * file, and a URL each node downloads for itself cannot promise that. Rejecting remote schemes outright + * keeps that failure -- which surfaces as an unreadable partition, far from its cause -- unreachable. + * + *

This does NOT reuse {@code ConnectorValidationContext#validateAndResolveDriverPath}: that one resolves + * against {@code jdbc_drivers_dir} and enforces a {@code .jar} grammar. + */ +public final class AdbcDriverPathResolver { + + /** + * A scheme-less driver_url must be a plain shared-library file name: letters, digits, dot, underscore, + * hyphen, ending in {@code .so} or a versioned {@code .so.N[.N...]}. Forbidding every path separator is + * what makes it impossible to escape the configured drivers directory. + */ + private static final Pattern SAFE_DRIVER_FILE_NAME = + Pattern.compile("^[A-Za-z0-9._-]+\\.so(\\.[0-9]+)*$"); + + private AdbcDriverPathResolver() { + } + + /** + * Resolves and authorizes {@code driverUrl}. Does not touch the filesystem -- see + * {@link #checkExists(Path, String)} for that half, which is deliberately separate so the grammar and + * the allow-list can be tested without laying down files. + * + * @param driverUrl the raw {@code driver_url} property value + * @param driversDir directory a bare file name resolves under (adbc.conf's {@code drivers_dir}) + * @param securePath semicolon-separated allowed directories, or {@code *} / blank to allow all + * @throws IllegalArgumentException on any rejected form, naming what is wrong + */ + public static Path resolve(String driverUrl, String driversDir, String securePath) { + if (driverUrl == null || driverUrl.trim().isEmpty()) { + throw new IllegalArgumentException("Required property '" + + AdbcConnectorProperties.DRIVER_URL + "' is missing for an adbc catalog"); + } + String raw = driverUrl.trim(); + Path candidate = toAbsolutePath(raw, driversDir); + rejectTraversal(raw, candidate); + checkSecurePath(raw, candidate, securePath); + return candidate; + } + + private static Path toAbsolutePath(String raw, String driversDir) { + int schemeEnd = raw.indexOf("://"); + if (schemeEnd > 0) { + String scheme = raw.substring(0, schemeEnd); + if (!"file".equalsIgnoreCase(scheme)) { + throw new IllegalArgumentException("Invalid '" + AdbcConnectorProperties.DRIVER_URL + + "': scheme '" + scheme + "' is not supported, only a local file is." + + " An ADBC driver is not downloaded per node: FE and every BE must load the" + + " identical driver library, because partition descriptors are driver-private" + + " bytes with no interoperability across driver builds." + + " Place the file on FE and on all BEs and reference it by path or by bare name." + + " (got: " + raw + ")"); + } + URI uri; + try { + uri = new URI(raw); + } catch (URISyntaxException e) { + // Fail closed: an unparsable URL must never be accepted, or the checks below would be + // validating a different string than the loader ends up using. + throw new IllegalArgumentException("Invalid '" + AdbcConnectorProperties.DRIVER_URL + + "': " + raw, e); + } + String authority = uri.getRawAuthority(); + if ((authority != null && !authority.isEmpty()) + || uri.getRawQuery() != null || uri.getRawFragment() != null) { + throw new IllegalArgumentException("Invalid '" + AdbcConnectorProperties.DRIVER_URL + + "': a file:// URL must carry no authority, query or fragment (got: " + raw + ")"); + } + String path = uri.getPath(); + if (path == null || path.isEmpty()) { + throw new IllegalArgumentException("Invalid '" + AdbcConnectorProperties.DRIVER_URL + + "': no path in " + raw); + } + return Paths.get(path); + } + if (raw.startsWith("/")) { + return Paths.get(raw); + } + if (!SAFE_DRIVER_FILE_NAME.matcher(raw).matches()) { + throw new IllegalArgumentException("Invalid '" + AdbcConnectorProperties.DRIVER_URL + + "': a bare driver file name must match [A-Za-z0-9._-]+.so (got: " + raw + ")." + + " Use an absolute path or a file:// URL to reference a driver outside " + + AdbcConnectorProperties.CONF_DRIVERS_DIR + " (adbc.conf)"); + } + if (driversDir == null || driversDir.trim().isEmpty()) { + throw new IllegalArgumentException("Cannot resolve the bare driver file name '" + raw + + "': " + AdbcConnectorProperties.CONF_DRIVERS_DIR + + " is not configured in adbc.conf and DORIS_HOME is unknown"); + } + return Paths.get(driversDir.trim(), raw); + } + + /** + * Rejects {@code ..} on the DECODED path, so a percent-encoded parent segment cannot survive to the + * loader. Checked before normalization, because normalization is exactly what would hide it. + */ + private static void rejectTraversal(String raw, Path candidate) { + for (Path segment : candidate) { + if ("..".equals(segment.toString())) { + throw new IllegalArgumentException("Invalid '" + AdbcConnectorProperties.DRIVER_URL + + "': path traversal ('..') is not allowed: " + raw); + } + } + } + + /** + * Component-based containment check, so neither prefix confusion ({@code /opt/drv} vs + * {@code /opt/drv-evil}) nor traversal can place a driver outside an allowed directory. + */ + private static void checkSecurePath(String raw, Path candidate, String securePath) { + if (securePath == null || securePath.trim().isEmpty() || "*".equals(securePath.trim())) { + return; + } + List allowed = new ArrayList<>(); + for (String entry : securePath.split(";")) { + String trimmed = entry.trim(); + if (!trimmed.isEmpty()) { + allowed.add(Paths.get(trimmed).normalize()); + } + } + Path normalized = candidate.normalize(); + for (Path base : allowed) { + if (normalized.startsWith(base)) { + return; + } + } + throw new IllegalArgumentException("Driver path does not match any path allowed by " + + AdbcConnectorProperties.CONF_DRIVER_SECURE_PATH + " in adbc.conf (" + + securePath + "): " + raw); + } + + /** + * Fails with a self-serviceable message when the driver file is absent. + * + *

Doris does not ship ADBC drivers, so "file not found" is the single most likely first experience + * of this catalog type. A bare {@code dlopen failed: No such file} would leave a user with no idea what + * to put where, so the message states all four things they need: the exact path, that FE and every BE + * need it, and both places to obtain it. + */ + public static void checkExists(Path driverPath, String driverUrl) { + if (Files.isReadable(driverPath)) { + return; + } + throw new IllegalArgumentException("ADBC driver library not found: " + driverPath + + " (from '" + AdbcConnectorProperties.DRIVER_URL + "' = " + driverUrl + ")." + + " Doris does not ship ADBC drivers. Place the driver shared library at this path on the" + + " FE and at the matching path on EVERY BE -- the same file, because partition descriptors" + + " do not carry across driver builds. Drivers are published on the arrow-adbc GitHub" + + " releases page, and can also be extracted from the PyPI wheels (for Flight SQL:" + + " adbc_driver_flightsql)."); + } + + /** + * Fails when the driver file is not the one the catalog declared, by MD5. + * + *

Optional, and the only property here a user has to compute themselves. It earns its place because + * of what this catalog type asks of an operator: place one file, by hand, on every node, and keep the + * copies identical. Nothing about a wrong or stale copy announces itself -- the library still loads, and + * whatever it does differently surfaces later as a query failure with no mention of a file. + * + *

Scope: the file on THIS FE. It cannot see what any BE has, so it does not verify that the + * copies match across nodes -- the failure that would hurt most. Declaring the same checksum on the + * catalog and deploying with it is what makes that check meaningful; Doris only holds up its end here. + */ + public static void checkChecksum(Path driverPath, String declaredChecksum, String driverUrl) { + if (declaredChecksum == null || declaredChecksum.trim().isEmpty()) { + return; + } + String actual = md5Of(driverPath, driverUrl); + if (actual.equalsIgnoreCase(declaredChecksum.trim())) { + return; + } + throw new IllegalArgumentException("The ADBC driver at " + driverPath + " has MD5 " + actual + + ", but '" + AdbcConnectorProperties.DRIVER_CHECKSUM + "' declares " + + declaredChecksum.trim() + " (from '" + AdbcConnectorProperties.DRIVER_URL + "' = " + + driverUrl + "). This FE is holding a different driver build from the one the catalog was" + + " written for; every BE must hold that same build too."); + } + + private static String md5Of(Path driverPath, String driverUrl) { + try (InputStream in = Files.newInputStream(driverPath)) { + MessageDigest digest = MessageDigest.getInstance("MD5"); + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) >= 0) { + digest.update(buffer, 0, read); + } + StringBuilder hex = new StringBuilder(32); + for (byte b : digest.digest()) { + hex.append(Character.forDigit((b >> 4) & 0xf, 16)).append(Character.forDigit(b & 0xf, 16)); + } + return hex.toString(); + } catch (IOException | NoSuchAlgorithmException e) { + // Fail closed: a checksum that could not be computed has verified nothing, and treating that as + // a pass would make the property quietly optional on exactly the nodes where reading fails. + throw new IllegalArgumentException("Cannot compute the MD5 of the ADBC driver at " + driverPath + + " (from '" + AdbcConnectorProperties.DRIVER_URL + "' = " + driverUrl + "): " + + e.getMessage(), e); + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java new file mode 100644 index 00000000000000..c083552da0ea50 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.MetaCacheEntry; + +import org.apache.arrow.vector.types.pojo.Schema; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ForkJoinPool; +import java.util.function.Supplier; + +/** + * What one ADBC catalog remembers about the remote source between statements. + * + *

The engine builds a fresh {@link AdbcConnectorMetadata} per statement, so without this every query paid + * three remote round trips before planning even started -- list the databases, list one database's tables, + * read one table's schema -- two of which list every object and therefore cost more the larger the + * source is. This lives on {@link AdbcConnector} instead, whose lifetime is the catalog's. + * + *

Never answers "there is no such object" from memory. "I just created it and Doris says it isn't + * there" is the one staleness a user cannot reason their way out of, so nothing here is allowed to produce + * it. Two rules keep that true, and both are load-bearing: + * + *

    + *
  • A lookup that fails re-reads the listing ({@link #reloadTableNames} / {@link #reloadNamespaces}) + * before concluding anything -- a remote call, but only on the path that was about to raise an error + * anyway.
    • + *
  • {@code listDatabaseNames} / {@code listTableNames} are served live and merely refresh what is + * remembered. They look like reports, but the engine loads its own name cache from them and then + * decides existence from that -- including the last-chance re-list it does for a name it has never + * seen ({@code ExternalDatabase.buildTableForInit}). Answering those from a cache would make the + * engine's re-check meaningless and is exactly how a newly created table becomes unreachable.
    • + *
+ * + *

What is left cached is what a query actually repeats: resolving a database and a table name, and + * reading a schema. The engine caches the listings itself, so nothing is paid twice for them. + * + *

Safe to share between users only because ADBC has no per-user identity. Every key here is a + * plain object name, so anything cached under one is served to whoever asks next. That is correct exactly + * while a catalog reaches the source as one fixed principal, which is what this connector does (it declares + * no {@code SUPPORTS_USER_SESSION}); the day it projects the querying user onto the connection, these keys + * must carry that identity or one user's metadata will be served to another. Pinned by + * {@code AdbcConnectorCacheHookTest}. + */ +public final class AdbcMetadataCache { + + private static final String ENGINE = "adbc"; + + /** + * All three entries share one set of knobs ({@code meta.cache.adbc.metadata.*}). They are read together, + * dropped together and describe the same thing -- the shape of the remote source -- so separate knobs + * would be three ways to spell one intent. + */ + static final String ENTRY = "metadata"; + + /** + * Ten minutes, where the shared framework's default is a day. An ADBC source is another live database, + * not a warehouse of immutable files: its tables are altered by other people's DDL at any time, and + * nothing tells Doris when. This is the ceiling on how long someone who forgot to REFRESH keeps seeing + * the old shape, and it is deliberately far below the framework value -- do not "correct" it to 86400. + */ + private static final long DEFAULT_TTL_SECOND = 600L; + + private static final long DEFAULT_CAPACITY = 1000L; + + /** The database listing is a single value, so it needs a single key. */ + private static final String THE_ONLY_KEY = ""; + + private final MetaCacheEntry> namespaces; + private final MetaCacheEntry> tableNames; + private final MetaCacheEntry tableSchemas; + + /** + * Built in {@link AdbcConnector}'s constructor, which also runs on an FE replaying the edit log, so this + * reads properties and nothing else -- no driver, no filesystem, no remote call. + */ + public AdbcMetadataCache(Map properties) { + CacheSpec spec = cacheSpec(properties); + this.namespaces = entry("adbc-namespaces", spec); + this.tableNames = entry("adbc-table-names", spec); + this.tableSchemas = entry("adbc-table-schema", spec); + } + + static CacheSpec cacheSpec(Map properties) { + return CacheSpec.fromProperties(properties, propertySpec()); + } + + /** + * The knobs this cache reads, named once so that whoever validates them at {@code CREATE CATALOG} cannot + * drift from whoever reads them at runtime -- a validator guarding a key nobody reads would pass every + * catalog and protect nothing. + */ + static CacheSpec.PropertySpec propertySpec() { + return CacheSpec.metaCachePropertySpec(ENGINE, ENTRY, + CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY)); + } + + /** + * Contextual-only with manual miss load, as the iceberg caches are: the remote read runs OUTSIDE + * Caffeine's compute lock, so a slow source does not stall unrelated keys, the driver's own exception + * arrives unwrapped, and a load that failed is not remembered as an answer. + */ + private static MetaCacheEntry entry(String name, CacheSpec spec) { + return new MetaCacheEntry<>(name, null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); + } + + // ========= reads ========= + + List namespaces(Supplier> loader) { + return namespaces.get(THE_ONLY_KEY, ignored -> loader.get()); + } + + /** Re-reads the database listing, replacing whatever was remembered. See the class note. */ + List reloadNamespaces(Supplier> loader) { + namespaces.invalidateKey(THE_ONLY_KEY); + return namespaces(loader); + } + + List tableNames(AdbcNamespace namespace, Supplier> loader) { + return tableNames.get(namespace, ignored -> loader.get()); + } + + /** Re-reads one database's table listing, replacing whatever was remembered. See the class note. */ + List reloadTableNames(AdbcNamespace namespace, Supplier> loader) { + tableNames.invalidateKey(namespace); + return tableNames(namespace, loader); + } + + Schema tableSchema(AdbcTableHandle handle, Supplier loader) { + return tableSchemas.get(new TableKey(handle), ignored -> loader.get()); + } + + // ========= what each REFRESH forgets ========= + + /** + * {@code REFRESH TABLE}: that table's schema, and its database's table listing. + * + *

Dropping the listing too is not collateral damage. A table created remotely after the listing was + * cached is absent from it, and REFRESH TABLE is precisely what a user reaches for to make Doris look at + * that table again -- if the listing survived, no statement short of REFRESH CATALOG could ever bring the + * new name in, and the one the user tried would appear to do nothing. + */ + void invalidateTable(String dbName, String tableName) { + tableSchemas.invalidateIf(key -> key.is(dbName, tableName)); + tableNames.invalidateIf(namespace -> namespace.dorisDatabaseName().equals(dbName)); + } + + /** {@code REFRESH DATABASE}: that database's table listing and every schema in it. */ + void invalidateDb(String dbName) { + tableSchemas.invalidateIf(key -> key.isIn(dbName)); + tableNames.invalidateIf(namespace -> namespace.dorisDatabaseName().equals(dbName)); + } + + /** + * {@code REFRESH CATALOG}: everything, including which databases exist -- the only statement that both + * names no database to invalidate and is reached for when the catalog's own shape changed. + */ + void invalidateAll() { + namespaces.invalidateAll(); + tableNames.invalidateAll(); + tableSchemas.invalidateAll(); + } + + /** + * One table, identified the way Doris addresses it. The namespace is carried whole rather than reduced to + * the Doris database name so that two remote namespaces which happen to present the same name cannot + * share a schema. + */ + private static final class TableKey { + + private final AdbcNamespace namespace; + private final String table; + // Derived from the namespace, so it takes no part in identity; kept because the invalidate hooks + // match on it and AdbcNamespace#dorisDatabaseName throws for a namespace with no name at all. + private final String dorisDbName; + + private TableKey(AdbcTableHandle handle) { + this.namespace = handle.getNamespace(); + this.table = handle.getRemoteTable(); + this.dorisDbName = handle.getDorisDbName(); + } + + private boolean is(String db, String tableName) { + return dorisDbName.equals(db) && table.equals(tableName); + } + + private boolean isIn(String db) { + return dorisDbName.equals(db); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TableKey)) { + return false; + } + TableKey other = (TableKey) o; + return namespace.equals(other.namespace) && table.equals(other.table); + } + + @Override + public int hashCode() { + return Objects.hash(namespace, table); + } + + @Override + public String toString() { + return namespace + "." + table; + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcNamespace.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcNamespace.java new file mode 100644 index 00000000000000..5cc65bac579292 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcNamespace.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.DorisConnectorException; + +import java.util.Objects; + +/** + * One remote {@code (catalog, db_schema)} pair, and the Doris database name it presents as. + * + *

ADBC names objects in three levels (catalog / db_schema / table) while a Doris external table has two + * (database / table), because the outermost name is already spent on the catalog a user created. Something + * has to give, and this is where. + * + *

The Doris database name is a display and lookup key only; it is never parsed back. A catalog + * name may legally contain a dot, so a name built by joining two remote levels could not be split again + * unambiguously -- which is why {@link AdbcTableHandle} carries the three remote parts separately instead of + * re-deriving them. The {@code uri} property is required to pin the remote catalog, which is what keeps the + * mapping a projection rather than a join: at most one of the two remote levels varies. + */ +public final class AdbcNamespace { + + private final String remoteCatalog; + private final String remoteDbSchema; + + public AdbcNamespace(String remoteCatalog, String remoteDbSchema) { + // Empty and null both mean "this source has no such level" -- SQLite reports db_schema as the empty + // string, other drivers report null, and the difference is not meaningful. + this.remoteCatalog = remoteCatalog == null ? "" : remoteCatalog; + this.remoteDbSchema = remoteDbSchema == null ? "" : remoteDbSchema; + } + + public String getRemoteCatalog() { + return remoteCatalog; + } + + public String getRemoteDbSchema() { + return remoteDbSchema; + } + + /** + * The name this namespace shows up as in {@code SHOW DATABASES}. + * + *

When both levels are populated the schema wins: the catalog is pinned by {@code uri}, so it is the + * same for every namespace in this Doris catalog and would add nothing but ambiguity to the name. + */ + public String dorisDatabaseName() { + if (!remoteDbSchema.isEmpty()) { + return remoteDbSchema; + } + if (!remoteCatalog.isEmpty()) { + return remoteCatalog; + } + throw new DorisConnectorException( + "The ADBC source reported an object with neither a catalog nor a database schema name," + + " so it cannot be addressed as a Doris database"); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AdbcNamespace)) { + return false; + } + AdbcNamespace other = (AdbcNamespace) o; + return remoteCatalog.equals(other.remoteCatalog) && remoteDbSchema.equals(other.remoteDbSchema); + } + + @Override + public int hashCode() { + return Objects.hash(remoteCatalog, remoteDbSchema); + } + + @Override + public String toString() { + return "AdbcNamespace{catalog='" + remoteCatalog + "', dbSchema='" + remoteDbSchema + "'}"; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcObjectsReader.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcObjectsReader.java new file mode 100644 index 00000000000000..9f088fdf4a8275 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcObjectsReader.java @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Reads the nested Arrow result of {@code AdbcConnection.getObjects}. + * + *

The shape is fixed by the ADBC standard schemas and was confirmed against a live driver: + * 

+ * catalog_name: Utf8
+ * catalog_db_schemas: List<Struct<
+ *     db_schema_name: Utf8,
+ *     db_schema_tables: List<Struct<table_name: Utf8, table_type: Utf8, ...>>>>
+ *
+ * + *

The column layer is deliberately not read here. Its fields are XDBC integer type codes + * ({@code xdbc_data_type}, {@code xdbc_type_name}, ...), not Arrow types, so deriving Doris column types + * from them would reintroduce exactly the two-step type translation ADBC exists to avoid -- and it would + * diverge from the real Arrow arrays BE reads. Column types come from {@code getTableSchema} instead. + * + *

Values are pulled with {@code getObject}, which materializes the nested structs as maps. That costs an + * allocation per row, and is the right trade here: metadata listings are small and infrequent, while hand + * walking list offsets into child vectors is where this kind of code goes wrong silently. + */ +public final class AdbcObjectsReader { + + private static final String CATALOG_NAME = "catalog_name"; + private static final String CATALOG_DB_SCHEMAS = "catalog_db_schemas"; + private static final String DB_SCHEMA_NAME = "db_schema_name"; + private static final String DB_SCHEMA_TABLES = "db_schema_tables"; + private static final String TABLE_NAME = "table_name"; + private static final String TABLE_TYPE = "table_type"; + + private AdbcObjectsReader() { + } + + /** + * Every {@code (catalog, db_schema)} pair the source reports, in source order and de-duplicated. + * + *

A catalog with no schema layer still yields one namespace: drivers report it as a single row whose + * schema list is null (depth CATALOGS) or holds one entry with an empty name. + */ + public static List readNamespaces(ArrowReader reader) { + LinkedHashSet namespaces = new LinkedHashSet<>(); + forEachCatalogRow(reader, (catalogName, schemas) -> { + if (schemas == null || schemas.isEmpty()) { + namespaces.add(new AdbcNamespace(catalogName, null)); + return; + } + for (Object schema : schemas) { + namespaces.add(new AdbcNamespace(catalogName, stringField(schema, DB_SCHEMA_NAME))); + } + }); + return new ArrayList<>(namespaces); + } + + /** + * Base table names inside {@code namespace}. Other namespaces in the same result are skipped, and so are + * objects the source itself calls something other than a table, because {@code getObjects} filters are + * advisory -- a driver may answer a narrower request with everything it has. Returning those would list + * tables under the wrong database, or offer a view that {@code DESC} and {@code SELECT} then fail on. + */ + public static List readTableNames(ArrowReader reader, AdbcNamespace namespace) { + List tables = new ArrayList<>(); + forEachCatalogRow(reader, (catalogName, schemas) -> { + if (schemas == null) { + return; + } + for (Object schema : schemas) { + AdbcNamespace current = new AdbcNamespace(catalogName, stringField(schema, DB_SCHEMA_NAME)); + if (!current.equals(namespace)) { + continue; + } + Object rawTables = mapOf(schema).get(DB_SCHEMA_TABLES); + if (!(rawTables instanceof List)) { + continue; + } + for (Object table : (List) rawTables) { + String name = stringField(table, TABLE_NAME); + if (name != null && !name.isEmpty() && isBaseTable(stringField(table, TABLE_TYPE))) { + tables.add(name); + } + } + } + }); + return tables; + } + + /** + * Whether an object the source reported is a base table, judged by the {@code table_type} it came with + * rather than by the type filter the request carried. + * + *

A Doris source is the reason this exists: its Flight SQL endpoint recognises only the literal + * {@code "VIEW"} as a type filter and answers every other value -- including the {@code "table"} ADBC + * asks with -- by returning ALL objects. Its {@code table_type} column is still right, so the filtering + * is done here. Any source that honours the filter simply has nothing left to drop. + * + *

A type this does not recognise is dropped, because the request asked for base tables and an object + * the source calls something else is not one. Keeping them would be the more forgiving rule and is the + * wrong one: a view leaked into the listing SCANS FINE through ADBC, so nothing ever looks broken and + * the catalog quietly offers objects it does not mean to. A source that spells its tables some third + * way instead lists nothing at all -- noticed within a minute, and fixed by one name in this method. + * + *

A missing type is not an unrecognised one, and is kept: it says nothing about the object, and a + * source that omits the column should stay as usable as it was before this filter existed. + */ + static boolean isBaseTable(String tableType) { + if (tableType == null || tableType.trim().isEmpty()) { + return true; + } + String normalized = tableType.trim().toUpperCase(Locale.ROOT); + // "BASE TABLE" is what a Doris source answers with, and it covers its materialized views too -- + // those are storage Doris can scan, unlike a view, whose rows exist only as a query. + return normalized.equals("TABLE") || normalized.equals("BASE TABLE"); + } + + private static void forEachCatalogRow(ArrowReader reader, CatalogRowConsumer consumer) { + try { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + FieldVector catalogVector = requireVector(root, CATALOG_NAME); + FieldVector schemasVector = requireVector(root, CATALOG_DB_SCHEMAS); + while (reader.loadNextBatch()) { + for (int row = 0; row < root.getRowCount(); row++) { + Object schemas = schemasVector.getObject(row); + consumer.accept(asString(catalogVector.getObject(row)), + schemas instanceof List ? (List) schemas : null); + } + } + } catch (DorisConnectorException e) { + throw e; + } catch (Exception e) { + throw new DorisConnectorException( + "Failed to read the ADBC getObjects result: " + e.getMessage(), e); + } + } + + private static FieldVector requireVector(VectorSchemaRoot root, String name) { + FieldVector vector = root.getVector(name); + if (vector == null) { + throw new DorisConnectorException("The ADBC driver returned a getObjects result without a '" + + name + "' column; it does not follow the ADBC standard schema. Columns present: " + + root.getSchema().getFields()); + } + return vector; + } + + private static Map mapOf(Object struct) { + if (!(struct instanceof Map)) { + throw new DorisConnectorException( + "The ADBC getObjects result has an unexpected nesting shape at " + struct); + } + return (Map) struct; + } + + private static String stringField(Object struct, String field) { + return asString(mapOf(struct).get(field)); + } + + /** Arrow hands back {@code Text} for utf8 columns, so the value is stringified rather than cast. */ + private static String asString(Object value) { + return value == null ? null : value.toString(); + } + + @FunctionalInterface + private interface CatalogRowConsumer { + void accept(String catalogName, List schemas); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcPartitionedReadSupport.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcPartitionedReadSupport.java new file mode 100644 index 00000000000000..c938904eb8b03c --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcPartitionedReadSupport.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Remembers, per catalog, that this driver has no partitioned execution. + * + *

Whether {@code executePartitioned} exists is a property of the driver, not of the query, so probing it + * once and keeping the answer is what stops every scan in a catalog from paying for a failed remote call. + * It lives on the connector because a scan plan provider is built per query; a memo held there would + * re-probe forever. Same reasoning, and the same lifetime, as {@link AdbcSchemaStrategy}. + * + *

Two states, not three. "Not probed yet" and "known to support it" both mean "ask the driver", + * so they are one state; only "known not to support it" changes what happens next. The flag never moves + * back: a driver does not gain a method while a catalog is alive, and replacing the driver file means + * restarting FE, which builds a new connector. + */ +public final class AdbcPartitionedReadSupport { + + private final AtomicBoolean unsupported = new AtomicBoolean(false); + /** + * What the driver said when it refused. Kept because a later scan under {@code REQUIRED} has to fail + * with the reason, and by then nobody is calling the driver again to ask. + */ + private volatile String refusal = ""; + + /** True once the driver has answered {@code NOT_IMPLEMENTED}; scans then plan a single range. */ + public boolean isKnownUnsupported() { + return unsupported.get(); + } + + /** The driver's own answer, for a message. Empty until {@link #markUnsupported} has run. */ + public String getRefusal() { + return refusal; + } + + /** + * Records that the driver has no partitioned execution, and what it said. + * + * @return true if this call is the one that recorded it, so the caller logs the downgrade once per + * catalog rather than once per query + */ + public boolean markUnsupported(String driverAnswer) { + // Written before the flag, so any thread that sees the flag also sees the reason. + refusal = driverAnswer == null ? "" : driverAnswer; + return unsupported.compareAndSet(false, true); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcQueryBuilder.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcQueryBuilder.java new file mode 100644 index 00000000000000..53f1222798f282 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcQueryBuilder.java @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.NamedColumnHandle; +import org.apache.doris.connector.api.pushdown.ConnectorAnd; +import org.apache.doris.connector.api.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.api.pushdown.ConnectorComparison; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.pushdown.ConnectorIn; +import org.apache.doris.connector.api.pushdown.ConnectorIsNull; +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; +import org.apache.doris.connector.api.pushdown.ConnectorNot; +import org.apache.doris.connector.api.pushdown.ConnectorOr; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.StringJoiner; + +/** + * Builds the one SQL statement a scan range asks its remote source to run. + * + *

The shape is fixed -- {@code SELECT cols FROM table [WHERE ...] [LIMIT n]} -- and every difference + * between sources goes through {@link AdbcDialect}. Adding a source must not require touching this class; + * a test drives it with a dialect defined inside the test to keep that honest. + * + *

Two rules here are not style, they decide whether the query returns the right rows: + * + *

    + *
  • The projection is always explicit. BE matches the columns the source returns against the + * query's slots by name and rejects any column it did not ask for, so {@code SELECT *} fails the scan + * outright ({@code ADBC source returned unknown column}). This is the opposite of the JDBC connector, + * which selects {@code *} when it has no columns.
    • + *
  • A row limit is pushed only when every predicate was. BE re-evaluates the predicates on + * whatever comes back, so a source that truncated to {@code n} rows BEFORE the predicates Doris still + * has to apply returns fewer rows than the query asked for. Pushing the predicates is otherwise pure + * speed-up, which is exactly why a predicate that cannot be translated is simply left behind.
    • + *
+ */ +public final class AdbcQueryBuilder { + + private AdbcQueryBuilder() { + } + + /** A generated statement, and whether the source will do all of the filtering. */ + public static final class BuiltQuery { + + private final String sql; + private final boolean allFiltersPushed; + + BuiltQuery(String sql, boolean allFiltersPushed) { + this.sql = sql; + this.allFiltersPushed = allFiltersPushed; + } + + public String getSql() { + return sql; + } + + /** Whether every conjunct became part of the {@code WHERE} clause. */ + public boolean isAllFiltersPushed() { + return allFiltersPushed; + } + } + + /** + * @param limit the row limit, or a non-positive value for none + */ + public static BuiltQuery build(AdbcDialect dialect, AdbcTableHandle handle, + List columns, Optional filter, long limit) { + List predicates = new ArrayList<>(); + boolean allFiltersPushed = true; + if (filter.isPresent()) { + allFiltersPushed = collectPredicates(dialect, filter.get(), predicates); + } + + StringBuilder sql = new StringBuilder("SELECT "); + sql.append(renderProjection(dialect, columns)); + sql.append(" FROM ").append(dialect.qualifiedTableName(handle)); + if (!predicates.isEmpty()) { + sql.append(" WHERE (").append(String.join(") AND (", predicates)).append(')'); + } + if (limit > 0 && allFiltersPushed && dialect.supportsLimitClause()) { + sql.append(" LIMIT ").append(limit); + } + return new BuiltQuery(sql.toString(), allFiltersPushed); + } + + /** + * Renders the select list. + * + *

An empty column list is not "select everything": it is Doris pushing a {@code COUNT(*)} down, where + * the scan needs row count and no values at all. {@code SELECT 1} asks for exactly that, one narrow + * column per row, instead of dragging the full table width across the wire to be discarded. BE's + * ADBC reader recognizes the no-columns-requested case and counts rows without materializing any. + */ + private static String renderProjection(AdbcDialect dialect, List columns) { + StringJoiner joiner = new StringJoiner(", "); + for (ConnectorColumnHandle column : columns) { + joiner.add(dialect.quoteIdentifier(columnName(column))); + } + return joiner.length() == 0 ? "1" : joiner.toString(); + } + + private static String columnName(ConnectorColumnHandle column) { + if (column instanceof NamedColumnHandle) { + return ((NamedColumnHandle) column).getName(); + } + throw new IllegalArgumentException( + "An adbc scan received a column handle it did not create: " + column.getClass().getName()); + } + + /** + * Translates the conjuncts it can and reports whether it got all of them. + * + *

Splitting only the top-level {@code AND} is deliberate: each conjunct is translated whole or + * dropped whole, so a partly translated conjunct -- which would be a DIFFERENT predicate, not a weaker + * one -- can never be emitted. + */ + private static boolean collectPredicates(AdbcDialect dialect, ConnectorExpression filter, + List out) { + List conjuncts = filter instanceof ConnectorAnd + ? ((ConnectorAnd) filter).getConjuncts() + : List.of(filter); + boolean all = true; + for (ConnectorExpression conjunct : conjuncts) { + String rendered = render(dialect, conjunct); + if (rendered == null) { + all = false; + } else { + out.add(rendered); + } + } + return all; + } + + /** + * Renders one expression, or returns {@code null} when this connector will not push it. + * + *

The accepted set is the conservative one from the design: comparisons, null tests, {@code IN}, and + * the boolean connectives over them. Everything else -- functions, arithmetic, {@code LIKE}, + * {@code BETWEEN} -- is refused, because each is a claim that some unknown source spells and evaluates + * it as Doris does, and a wrong claim here changes the result set rather than the speed. + */ + private static String render(AdbcDialect dialect, ConnectorExpression expression) { + if (expression instanceof ConnectorColumnRef) { + return dialect.quoteIdentifier(((ConnectorColumnRef) expression).getColumnName()); + } + if (expression instanceof ConnectorLiteral) { + return dialect.renderLiteral((ConnectorLiteral) expression); + } + if (expression instanceof ConnectorComparison) { + return renderComparison(dialect, (ConnectorComparison) expression); + } + if (expression instanceof ConnectorIsNull) { + ConnectorIsNull isNull = (ConnectorIsNull) expression; + String operand = render(dialect, isNull.getOperand()); + return operand == null ? null + : operand + (isNull.isNegated() ? " IS NOT NULL" : " IS NULL"); + } + if (expression instanceof ConnectorIn) { + return renderIn(dialect, (ConnectorIn) expression); + } + if (expression instanceof ConnectorAnd) { + return renderConnective(dialect, ((ConnectorAnd) expression).getConjuncts(), " AND "); + } + if (expression instanceof ConnectorOr) { + return renderConnective(dialect, ((ConnectorOr) expression).getDisjuncts(), " OR "); + } + if (expression instanceof ConnectorNot) { + String operand = render(dialect, ((ConnectorNot) expression).getOperand()); + return operand == null ? null : "NOT (" + operand + ")"; + } + return null; + } + + private static String renderComparison(AdbcDialect dialect, ConnectorComparison comparison) { + if (comparison.getOperator() == ConnectorComparison.Operator.EQ_FOR_NULL) { + // Doris's null-safe equality has no portable spelling: standard SQL's IS NOT DISTINCT FROM is + // not universally implemented, and every substitute (=, IS NULL OR =) differs from it on nulls. + // Getting it wrong changes which rows match rather than failing, so it is not pushed at all. + return null; + } + String left = render(dialect, comparison.getLeft()); + String right = render(dialect, comparison.getRight()); + if (left == null || right == null) { + return null; + } + return left + " " + comparison.getOperator().getSymbol() + " " + right; + } + + private static String renderIn(AdbcDialect dialect, ConnectorIn in) { + String value = render(dialect, in.getValue()); + if (value == null || in.getInList().isEmpty()) { + // An empty list has no SQL spelling; Doris folds it away long before a scan, so refusing it + // costs nothing. + return null; + } + StringJoiner items = new StringJoiner(", "); + for (ConnectorExpression item : in.getInList()) { + String rendered = render(dialect, item); + if (rendered == null) { + return null; + } + items.add(rendered); + } + return value + (in.isNegated() ? " NOT IN (" : " IN (") + items + ")"; + } + + /** + * Renders {@code AND}/{@code OR} over its children, all or nothing. + * + *

{@code OR} obviously cannot drop a branch -- the remaining branches accept fewer rows than the + * whole. Nested {@code AND} is treated the same way, because it is reached only from inside another + * expression (under a {@code NOT}, or as an {@code OR} branch) where dropping a branch is not a + * weakening either. + */ + private static String renderConnective(AdbcDialect dialect, List operands, + String separator) { + if (operands.isEmpty()) { + return null; + } + StringJoiner joiner = new StringJoiner(separator); + for (ConnectorExpression operand : operands) { + String rendered = render(dialect, operand); + if (rendered == null) { + return null; + } + joiner.add("(" + rendered + ")"); + } + return joiner.toString(); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcScanPlanProvider.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcScanPlanProvider.java new file mode 100644 index 00000000000000..bc7e8c019664ae --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcScanPlanProvider.java @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; + +import org.apache.arrow.adbc.core.AdbcException; +import org.apache.arrow.adbc.core.AdbcStatement; +import org.apache.arrow.adbc.core.AdbcStatusCode; +import org.apache.arrow.adbc.core.PartitionDescriptor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * Turns a scan into the remote work that serves it: either the driver's partitions, one per backend, or + * the single statement one backend runs. + * + *

The partitioned path is what makes an ADBC catalog read faster than one connection can. It costs a + * remote round trip at planning time, and on a Flight SQL source that round trip is the query's + * execution -- the descriptors it returns identify result sets the source is already producing. That is why + * nothing else in this class may take it: see {@link #getScanNodeProperties}. + */ +public class AdbcScanPlanProvider implements ConnectorScanPlanProvider { + + private static final Logger LOG = LogManager.getLogger(AdbcScanPlanProvider.class); + + /** Names the reader on BE. Kept next to {@link AdbcScanRange#getFileFormat()}, which must agree. */ + private static final String FILE_FORMAT = "arrow"; + + private final Map properties; + private final Path driverPath; + private final AdbcDialectSelector dialectSelector; + private final Supplier clientSupplier; + private final AdbcPartitionedReadSupport partitionedRead; + + public AdbcScanPlanProvider(Map properties, Path driverPath, + AdbcDialectSelector dialectSelector, Supplier clientSupplier, + AdbcPartitionedReadSupport partitionedRead) { + this.properties = properties; + this.driverPath = driverPath; + this.dialectSelector = dialectSelector; + this.clientSupplier = clientSupplier; + this.partitionedRead = partitionedRead; + } + + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + AdbcTableHandle handle = adbcHandle(request.getTableHandle()); + AdbcQueryBuilder.BuiltQuery query = buildQuery(handle, request.getColumns(), + request.getFilter(), request.getLimit()); + LOG.debug("ADBC scan of {}.{}: {}", handle.getDorisDbName(), handle.getRemoteTable(), + query.getSql()); + // EXPLAIN plans a scan for real, so this is reached while describing a query as well as while + // running one -- and asking the driver to partition a statement EXECUTES it on the source. An + // EXPLAIN must not do that, so it is shown the statement instead, whatever the mode: refusing to + // describe a query because the driver cannot partition would help nobody. The statement is what a + // partitioned scan splits, so what EXPLAIN shows still describes the real scan; only the range + // count differs, and a range count for a query that was never run means nothing anyway. + AdbcConnectorProperties.PartitionedReadMode mode = + AdbcConnectorProperties.partitionedReadMode(properties); + if (!request.isExplainOnly() && mode != AdbcConnectorProperties.PartitionedReadMode.DISABLED) { + if (partitionedRead.isKnownUnsupported()) { + // Already asked once, on this catalog, and the driver said no. + refuseIfRequired(mode, partitionedRead.getRefusal()); + } else { + List partitioned = planPartitions(query.getSql(), mode); + if (partitioned != null) { + return partitioned; + } + } + } + return Collections.singletonList(rangeBuilder().querySql(query.getSql()).build()); + } + + /** + * Fails when the catalog asked for partitioned execution and cannot have it. + * + *

Without this the loss is invisible: the scan still returns the right rows, just from one backend + * instead of many. A test written to exercise the partitioned path would go green while exercising the + * fallback -- the failure and the pass look identical -- and a deployment sized for the parallelism + * would simply be slow. + */ + private static void refuseIfRequired(AdbcConnectorProperties.PartitionedReadMode mode, + String driverAnswer) { + if (mode != AdbcConnectorProperties.PartitionedReadMode.REQUIRED) { + return; + } + throw new DorisConnectorException("This catalog sets '" + + AdbcConnectorProperties.PARTITIONED_READ + "'='required', but its ADBC driver does not" + + " implement partitioned execution, so the scan cannot be split across backends." + + " The driver answered: " + (driverAnswer == null || driverAnswer.isEmpty() + ? "(no message)" : driverAnswer) + + ". Use a driver that partitions, or set the property to 'auto' to allow reading through" + + " a single statement instead."); + } + + /** + * Asks the driver to split this statement, and returns one range per partition -- or null when the + * driver has no partitioned execution and the caller should plan the statement itself. + * + *

Only {@code NOT_IMPLEMENTED} means "this driver cannot do it". Any other status means the driver + * tried and the query or the source is at fault, and swallowing that into a single-range plan would + * hide a real failure behind a slower query -- and would run the statement remotely a second time. + */ + private List planPartitions(String sql, + AdbcConnectorProperties.PartitionedReadMode mode) { + // Set by the NOT_IMPLEMENTED branch so the downgrade can be logged with what the driver + // actually said. Without it the log says only that SOMETHING answered NOT_IMPLEMENTED, and + // the layer that did -- driver, driver manager, or JNI bridge -- has to be found by hand. + AdbcException[] refusal = new AdbcException[1]; + List descriptors = clientSupplier.get().withConnection(connection -> { + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + try { + return statement.executePartitioned().getPartitionDescriptors(); + } catch (AdbcException e) { + if (e.getStatus() != AdbcStatusCode.NOT_IMPLEMENTED) { + // Naming the dialect is the point of this message. The source answers in its own + // words -- typically a syntax error about a quote character -- and nothing in that + // hints that Doris wrote the statement, or that which SQL it writes is settable. + throw AdbcClient.translate(e, + "Failed to plan a partitioned ADBC scan of [" + sql + "], generated in the '" + + dialectSelector.select(clientSupplier).name() + "' dialect (set '" + + AdbcConnectorProperties.SQL_DIALECT + "' on this catalog if the" + + " source expects different SQL)"); + } + refusal[0] = e; + return null; + } + } + }); + if (descriptors == null) { + String answer = refusal[0] == null ? "" : refusal[0].toString(); + if (partitionedRead.markUnsupported(answer)) { + LOG.info("The ADBC driver does not implement partitioned execution, so scans of this" + + " catalog run as one range on one backend. Set '{}'='disabled' to stop asking," + + " or 'required' to fail instead. The driver answered: {}", + AdbcConnectorProperties.PARTITIONED_READ, + answer.isEmpty() ? "(no exception)" : answer); + } + refuseIfRequired(mode, answer); + return null; + } + if (descriptors.isEmpty()) { + // Not read as "no rows": the partition count reflects the source's parallelism, not its + // cardinality -- an empty table still yields a partition that returns nothing. Planning zero + // ranges for a driver that answered with nothing would report an empty result for a query that + // has one, which is the one failure a user cannot see. + throw new DorisConnectorException("The ADBC driver reported no partitions for [" + + sql + "]. Doris cannot tell that apart from a lost result set, so the query fails" + + " rather than returning no rows. Set '" + + AdbcConnectorProperties.PARTITIONED_READ + + "'='disabled' on this catalog to read it through a single statement instead."); + } + int limit = AdbcConnectorProperties.maxPartitions(properties); + if (descriptors.size() > limit) { + // Deliberately not a fallback to the single-range path: the source has already executed the + // statement to produce these descriptors, so re-planning it as a statement would execute it a + // second time while this result set sits unread until the source times it out. + throw new DorisConnectorException("The ADBC driver split [" + sql + "] into " + + descriptors.size() + " partitions, over the '" + + AdbcConnectorProperties.MAX_PARTITIONS + "' limit of " + limit + + ". Raise that property, narrow the query, or set '" + + AdbcConnectorProperties.PARTITIONED_READ + "'='disabled' on this catalog."); + } + List ranges = new ArrayList<>(descriptors.size()); + for (PartitionDescriptor descriptor : descriptors) { + ranges.add(rangeBuilder().partitionDescriptor(encode(descriptor)).build()); + } + LOG.debug("ADBC scan planned into {} partitions: {}", ranges.size(), sql); + return ranges; + } + + /** + * No {@code getHosts()} goes with these ranges: the descriptor is opaque bytes FE does not decode, so + * it has no location to prefer and Doris assigns them by its own policy. Decoding a Flight endpoint's + * location to schedule for affinity is a later optimization, not a correctness matter. + */ + private static String encode(PartitionDescriptor descriptor) { + ByteBuffer buffer = descriptor.getDescriptor(); + byte[] bytes = new byte[buffer.remaining()]; + // duplicate() so reading the bytes does not consume the descriptor's own buffer. + buffer.duplicate().get(bytes); + return Base64.getEncoder().encodeToString(bytes); + } + + /** + * The properties the scan node reads before it has any ranges: the reader to use, and the statement to + * show in {@code EXPLAIN}. + * + *

It regenerates the statement rather than reusing one, because {@code EXPLAIN} never calls + * {@link #planScan}. The two must agree or {@code EXPLAIN} describes a query that will not be run; a + * test pins that they do. + * + *

This path must never ask for partitions -- and neither may {@link #planScan} when it is + * planning an {@code EXPLAIN}, which it also is. On a Flight SQL source that call executes the query, + * so either one would run the very query the user asked only to have described. Tests pin both by + * planning with a client that refuses to be opened. + */ + @Override + public Map getScanNodeProperties(ConnectorSession session, + ConnectorTableHandle handle, List columns, + Optional filter) { + Map props = new HashMap<>(); + props.put(ScanNodePropertyKeys.FILE_FORMAT_TYPE, FILE_FORMAT); + // No row limit here: EXPLAIN has none to show, and planScan applies its own. + props.put(ScanNodePropertyKeys.REMOTE_QUERY, + buildQuery(adbcHandle(handle), columns, filter, -1L).getSql()); + return props; + } + + private AdbcQueryBuilder.BuiltQuery buildQuery(AdbcTableHandle handle, + List columns, Optional filter, long limit) { + return AdbcQueryBuilder.build(dialectSelector.select(clientSupplier), handle, columns, + filter, limit); + } + + /** Everything a range needs except the work itself; the caller adds a statement or a partition. */ + private AdbcScanRange.Builder rangeBuilder() { + return new AdbcScanRange.Builder() + .driverPath(driverPath.toString()) + .driverEntrypoint(properties.get(AdbcConnectorProperties.DRIVER_ENTRYPOINT)) + .uri(AdbcConnectorProperties.require(properties, AdbcConnectorProperties.URI)) + .username(properties.get(AdbcConnectorProperties.USER)) + .password(properties.get(AdbcConnectorProperties.PASSWORD)) + .driverOptions(AdbcConnectorProperties.driverOptions(properties)); + } + + private static AdbcTableHandle adbcHandle(ConnectorTableHandle handle) { + if (handle instanceof AdbcTableHandle) { + return (AdbcTableHandle) handle; + } + // Reached by a passthrough-SQL table handle (a table-valued function forwarding raw SQL), which + // this connector does not accept yet. Naming the handle keeps the message useful when it does. + throw new DorisConnectorException("An adbc catalog cannot scan through a " + + handle.getClass().getSimpleName() + "; only its own tables are readable."); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcScanRange.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcScanRange.java new file mode 100644 index 00000000000000..49ec3163997899 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcScanRange.java @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** + * One unit of ADBC work: the connection parameters plus either the statement to run or the one partition + * to read. + * + *

A scan plans one range carrying a statement when the driver has no partitioned execution, and + * otherwise one range per partition the driver reported -- which is how a single remote query ends up read + * by several backends at once. + * + *

The parameter names below are a contract with BE. Its reader looks them up literally + * ({@code be/src/format_v2/table/adbc_reader.cpp}, the {@code kParam*} constants), so a rename on either + * side has to happen on both. Two are easy to get wrong: the credentials travel as {@code username} / + * {@code password} -- ADBC's own option names, not this connector's {@code user} property -- and an + * {@code adbc.}-prefixed option keeps its prefix, because the prefix is part of the ADBC option name. + */ +public class AdbcScanRange implements ConnectorScanRange { + + private static final long serialVersionUID = 1L; + + /** Selects BE's ADBC reader. Must match the string BE's FileScannerV2 dispatches on. */ + private static final String TABLE_FORMAT_TYPE = "adbc"; + + /** + * There is no file. The path is carried anyway because the generic scan machinery expects one; a + * scheme-less placeholder is used rather than something like {@code adbc://...} so nothing tries to + * resolve it as a filesystem. Mirrors the remote_doris scan node, which reads Arrow the same way. + */ + private static final String VIRTUAL_PATH = "/dummyPath"; + + static final String PARAM_DRIVER_PATH = "driver_path"; + static final String PARAM_DRIVER_ENTRYPOINT = "driver_entrypoint"; + static final String PARAM_URI = "uri"; + static final String PARAM_USERNAME = "username"; + static final String PARAM_PASSWORD = "password"; + static final String PARAM_QUERY_SQL = "query_sql"; + static final String PARAM_PARTITION_DESCRIPTOR = "partition_descriptor"; + + private final Map properties; + + private AdbcScanRange(Map properties) { + this.properties = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + + @Override + public Optional getPath() { + return Optional.of(VIRTUAL_PATH); + } + + /** + * Arrow, because the rows arrive as Arrow record batches from the driver rather than from a file. This + * is what routes the scan to BE's Arrow reader path; the JNI default would land it in a scanner with no + * ADBC branch. + */ + @Override + public String getFileFormat() { + return "arrow"; + } + + @Override + public String getTableFormatType() { + return TABLE_FORMAT_TYPE; + } + + @Override + public Map getProperties() { + return properties; + } + + /** + * Writes the parameters into the ADBC slot of the range descriptor. + * + *

Overriding this is not optional: the inherited implementation writes to {@code jdbc_params}, which + * BE's ADBC reader never reads, so the scan would reach BE with no connection parameters at all and + * fail on a missing driver path. + */ + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + formatDesc.setAdbcParams(new LinkedHashMap<>(properties)); + } + + /** Collects the parameters for one range. */ + public static class Builder { + + private final Map props = new LinkedHashMap<>(); + + /** + * The absolute path of the driver library, as FE resolved it. + * + *

BE loads this path verbatim -- it has no drivers directory of its own to resolve a bare name + * against -- so FE and every BE must have the same driver file at the same path. That is what the + * deployment asks for, and it matches how a JDBC catalog's driver reference already works. + */ + public Builder driverPath(String path) { + props.put(PARAM_DRIVER_PATH, path); + return this; + } + + /** Optional; empty lets the driver manager infer the init symbol. */ + public Builder driverEntrypoint(String entrypoint) { + putIfPresent(PARAM_DRIVER_ENTRYPOINT, entrypoint); + return this; + } + + public Builder uri(String uri) { + props.put(PARAM_URI, uri); + return this; + } + + /** The connector's {@code user} property, under ADBC's name for it. */ + public Builder username(String user) { + putIfPresent(PARAM_USERNAME, user); + return this; + } + + public Builder password(String password) { + putIfPresent(PARAM_PASSWORD, password); + return this; + } + + public Builder querySql(String sql) { + props.put(PARAM_QUERY_SQL, sql); + return this; + } + + /** + * One partition of an already-executed remote query, base64 of the driver's opaque descriptor. + * + *

Base64 because the descriptor is arbitrary bytes (a serialized protobuf, for a Flight SQL + * driver) and the range parameters are a string map. It is opaque on this side on purpose: only the + * driver that produced it can read it, which is why FE and BE must load the same driver library. + */ + public Builder partitionDescriptor(String base64Descriptor) { + props.put(PARAM_PARTITION_DESCRIPTOR, base64Descriptor); + return this; + } + + /** Driver options, names unchanged -- the {@code adbc.} prefix is part of the option name. */ + public Builder driverOptions(Map options) { + props.putAll(options); + return this; + } + + /** + * @throws IllegalStateException if the range would carry both kinds of work or neither. BE fails + * the same way on the same condition; catching it here names the planning bug instead of + * letting one backend report it halfway through a query. + */ + public AdbcScanRange build() { + boolean hasStatement = props.containsKey(PARAM_QUERY_SQL); + boolean hasPartition = props.containsKey(PARAM_PARTITION_DESCRIPTOR); + if (hasStatement == hasPartition) { + throw new IllegalStateException("An ADBC scan range runs either a statement or one" + + " partition of an already-executed query, but this one carries " + + (hasStatement ? "both" : "neither")); + } + return new AdbcScanRange(props); + } + + private void putIfPresent(String key, String value) { + // An empty value is left out rather than sent as "": BE skips empty credentials but hands any + // present entrypoint to dlsym, where "" is not the same as "use the default". + if (value != null && !value.isEmpty()) { + props.put(key, value); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcSchemaStrategy.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcSchemaStrategy.java new file mode 100644 index 00000000000000..d3113a9af97c17 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcSchemaStrategy.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * Remembers, per catalog, which call this driver answers table schemas with. + * + *

A driver's answer is a property of the driver, not of the table, so probing it once and keeping the + * result is what stops every table in a catalog from paying for one failed remote call. It lives on the + * connector rather than the per-statement metadata object for the same reason -- a per-statement memo would + * re-probe on every statement. + * + *

Only ever moves away from {@link Kind#UNKNOWN}; there is no invalidation, because a driver does not + * gain or lose a method while a catalog is alive. Replacing the driver file means restarting FE, which + * builds a new connector. + */ +public final class AdbcSchemaStrategy { + + public enum Kind { + /** Not probed yet: try getTableSchema first, fall back to executeSchema. */ + UNKNOWN, + /** getTableSchema works; a later failure is about the table, not the driver. */ + GET_TABLE_SCHEMA, + /** getTableSchema is not implemented here; go straight to executeSchema. */ + EXECUTE_SCHEMA + } + + private final AtomicReference kind = new AtomicReference<>(Kind.UNKNOWN); + + public Kind get() { + return kind.get(); + } + + public void set(Kind newKind) { + kind.set(newKind); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcStatementScope.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcStatementScope.java new file mode 100644 index 00000000000000..d06d761c04ccc3 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcStatementScope.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScopes; + +import org.apache.arrow.vector.types.pojo.Schema; + +import java.util.function.Supplier; + +/** + * Shares one table's Arrow schema across the paths that each need it within a single statement. + * + *

{@code getTableSchema} and {@code getColumnHandles} are separate SPI calls that derive different + * products (Doris columns vs. name-to-handle map) from the same remote answer, and the engine may call both + * for one table in one statement. Routing them through the statement scope collapses that to one remote + * round trip while each keeps its own derivation. + * + *

Under a null session or a scope of {@code NONE} (offline, no live statement) the loader runs on every + * call, which is byte-identical to fetching every time. + */ +final class AdbcStatementScope { + + /** + * Namespace for the per-statement Arrow-schema memo. Prefixed with the connector's type name so it + * stays distinct from a sibling connector's memo inside a heterogeneous gateway; guarded by + * {@code AdbcStatementScopeTest}. + */ + static final String TABLE_SCHEMA_NAMESPACE = "adbc.table_schema"; + + private AdbcStatementScope() { + } + + static Schema sharedTableSchema(ConnectorSession session, AdbcTableHandle handle, + Supplier loader) { + // Keyed by the Doris database name rather than the remote parts: it is what the engine addresses + // the table by, and within one catalog it identifies the namespace uniquely. + return ConnectorStatementScopes.resolveInStatement( + session, TABLE_SCHEMA_NAMESPACE, handle.getDorisDbName(), handle.getRemoteTable(), loader); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcTableHandle.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcTableHandle.java new file mode 100644 index 00000000000000..fd46711e9a7f71 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcTableHandle.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +/** + * Table handle carrying the remote coordinates of one ADBC table. + * + *

All three remote levels are stored separately, and none is ever re-derived from the Doris name. + * Doris shows a two-level name, so a handle that kept only that would have to split it back apart to + * address the table remotely -- and that split has no correct implementation: a remote catalog or schema + * name may itself contain a dot, so {@code MY.DB.PUBLIC} has several readings and the wrong one addresses a + * table that does not exist. Keeping the parts is what makes the question never arise. + */ +public class AdbcTableHandle implements ConnectorTableHandle { + + private static final long serialVersionUID = 1L; + + private final String remoteCatalog; + private final String remoteDbSchema; + private final String remoteTable; + private final String dorisDbName; + + public AdbcTableHandle(AdbcNamespace namespace, String remoteTable) { + this.remoteCatalog = namespace.getRemoteCatalog(); + this.remoteDbSchema = namespace.getRemoteDbSchema(); + this.remoteTable = remoteTable; + this.dorisDbName = namespace.dorisDatabaseName(); + } + + public String getRemoteCatalog() { + return remoteCatalog; + } + + public String getRemoteDbSchema() { + return remoteDbSchema; + } + + public String getRemoteTable() { + return remoteTable; + } + + /** Display and lookup key only. Never parsed. */ + public String getDorisDbName() { + return dorisDbName; + } + + public AdbcNamespace getNamespace() { + return new AdbcNamespace(remoteCatalog, remoteDbSchema); + } + + @Override + public String toString() { + return "AdbcTableHandle{catalog='" + remoteCatalog + "', dbSchema='" + remoteDbSchema + + "', table='" + remoteTable + "'}"; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcTypeMapper.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcTypeMapper.java new file mode 100644 index 00000000000000..ddab1f6baff32e --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcTypeMapper.java @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * Maps an Arrow type onto the Doris type Doris will present for the column. + * + *

This runs on the FE side of a path whose BE side reads the very same Arrow arrays, so the mapping is + * the contract between the two. Two consequences shape it: + * + *

    + *
  • Unsupported types fail here, not later. A column Doris has no type for is rejected while the + * table is being described, naming the column and the Arrow type. Mapping it to something lossy would + * turn a clear error into wrong data, and deferring it to BE would produce an error with no column + * name in it.
    • + *
  • Unsigned integers widen. Doris has no unsigned types, so a {@code uint32} maps to BIGINT + * rather than INT: the narrow choice silently wraps every value above 2^31.
    • + *
+ * + *

Encoding variants map by their decoded value type, matching the BE normalizer that decodes them before + * handing arrays to the serde. Dictionary encoding needs no case of its own: an Arrow {@code Field} carries + * the dictionary's VALUE type as its own type and keeps the index type in its {@code DictionaryEncoding} + * metadata, so a dictionary column already arrives here as the type Doris will see. Run-end encoding does + * need one, because there the value type is a child. + */ +public final class AdbcTypeMapper { + + /** Doris DATETIMEV2/TIMESTAMPTZ scale is capped at 6 digits; Arrow nanosecond precision has 9. */ + private static final int MAX_DATETIME_SCALE = 6; + private static final int MAX_DECIMAL128_PRECISION = 38; + private static final int MAX_DECIMAL256_PRECISION = 76; + + private AdbcTypeMapper() { + } + + /** Maps one top-level column. {@code columnName} only ever appears in error messages. */ + public static ConnectorType toDorisType(String columnName, Field field) { + return toDorisType(columnName, field, field.getType()); + } + + private static ConnectorType toDorisType(String columnName, Field field, ArrowType arrowType) { + switch (arrowType.getTypeID()) { + case Bool: + return ConnectorType.of("BOOLEAN"); + case Int: + return intType(columnName, (ArrowType.Int) arrowType); + case FloatingPoint: + return floatType((ArrowType.FloatingPoint) arrowType); + case Decimal: + return decimalType(columnName, (ArrowType.Decimal) arrowType); + case Date: + // DATEDAY is a calendar date; DATEMILLI carries a time-of-day and would lose it as DATEV2. + return ((ArrowType.Date) arrowType).getUnit() == org.apache.arrow.vector.types.DateUnit.DAY + ? ConnectorType.of("DATEV2") + : ConnectorType.of("DATETIMEV2", 3, 0); + case Timestamp: + return timestampType((ArrowType.Timestamp) arrowType); + case Utf8: + case LargeUtf8: + case Utf8View: + return ConnectorType.of("STRING"); + case Binary: + case LargeBinary: + case BinaryView: + case FixedSizeBinary: + // Doris has no general binary column type on the external-table path. + return ConnectorType.of("STRING"); + case List: + case LargeList: + case FixedSizeList: + case ListView: + case LargeListView: + return ConnectorType.arrayOf(childType(columnName, field, 0)); + case Struct: + return structType(columnName, field); + case Map: + return mapType(columnName, field); + case RunEndEncoded: + // Children are (run_ends, values); only the value type reaches Doris. + return childType(columnName, field, field.getChildren().size() - 1); + default: + throw unsupported(columnName, arrowType); + } + } + + private static ConnectorType intType(String columnName, ArrowType.Int type) { + if (type.getIsSigned()) { + switch (type.getBitWidth()) { + case 8: + return ConnectorType.of("TINYINT"); + case 16: + return ConnectorType.of("SMALLINT"); + case 32: + return ConnectorType.of("INT"); + case 64: + return ConnectorType.of("BIGINT"); + default: + throw unsupported(columnName, type); + } + } + // Widen by one step: Doris has no unsigned integers, and the same-width type would wrap the upper + // half of the range into negatives without any error. + switch (type.getBitWidth()) { + case 8: + return ConnectorType.of("SMALLINT"); + case 16: + return ConnectorType.of("INT"); + case 32: + return ConnectorType.of("BIGINT"); + case 64: + return ConnectorType.of("LARGEINT"); + default: + throw unsupported(columnName, type); + } + } + + private static ConnectorType floatType(ArrowType.FloatingPoint type) { + switch (type.getPrecision()) { + case HALF: + case SINGLE: + return ConnectorType.of("FLOAT"); + default: + return ConnectorType.of("DOUBLE"); + } + } + + private static ConnectorType decimalType(String columnName, ArrowType.Decimal type) { + int limit = type.getBitWidth() > 128 ? MAX_DECIMAL256_PRECISION : MAX_DECIMAL128_PRECISION; + if (type.getPrecision() > limit) { + throw new DorisConnectorException("Column '" + columnName + "' has Arrow type " + + type + ", whose precision " + type.getPrecision() + + " exceeds the maximum Doris DECIMALV3 precision " + limit); + } + return ConnectorType.of("DECIMALV3", type.getPrecision(), type.getScale()); + } + + private static ConnectorType timestampType(ArrowType.Timestamp type) { + int scale; + switch (type.getUnit()) { + case SECOND: + scale = 0; + break; + case MILLISECOND: + scale = 3; + break; + case NANOSECOND: + // Truncated rather than rejected: sub-microsecond precision is rare in stored data, and + // refusing the column would make whole tables unreadable over a fractional digit. + scale = MAX_DATETIME_SCALE; + break; + default: + scale = MAX_DATETIME_SCALE; + break; + } + // A zoned Arrow timestamp is an instant, and TIMESTAMPTZ is the only Doris type that keeps one: + // DATETIMEV2 would drop the zone and leave a wall clock whose meaning depends on who reads it. + // + // This is where the catalog property enable.mapping.timestamp_tz would belong -- the JDBC + // catalog and the file formats consult it and default to DATETIMEV2 -- and it is deliberately + // NOT consulted: ExternalCatalog.setDefaultPropsIfMissing stamps the property as "false" into + // every external catalog that does not name it, so a connector reading it cannot tell a user + // who asked for wall clocks from one who said nothing at all, and honouring it would force + // DATETIMEV2 on every adbc catalog. This connector's default is TIMESTAMPTZ. Making the + // property settable here needs fe-core to let a connector supply its own default first. + boolean zoned = type.getTimezone() != null && !type.getTimezone().isEmpty(); + return ConnectorType.of(zoned ? "TIMESTAMPTZ" : "DATETIMEV2", scale, 0); + } + + private static ConnectorType structType(String columnName, Field field) { + List children = field.getChildren(); + if (children.isEmpty()) { + throw new DorisConnectorException("Column '" + columnName + + "' is an Arrow struct with no fields, which Doris cannot represent"); + } + List names = new ArrayList<>(children.size()); + List types = new ArrayList<>(children.size()); + for (Field child : children) { + // Lowercased level by level: BE indexes struct children by lowercase key, and a mixed-case + // child name crashes it rather than failing the query. + names.add(child.getName().toLowerCase(Locale.ROOT)); + types.add(toDorisType(columnName + "." + child.getName(), child, child.getType())); + } + return ConnectorType.structOf(names, types); + } + + private static ConnectorType mapType(String columnName, Field field) { + // Arrow models a map as list>, so the pair sits one level down. + List children = field.getChildren(); + if (children.size() != 1 || children.get(0).getChildren().size() != 2) { + throw new DorisConnectorException("Column '" + columnName + + "' is an Arrow map with an unexpected shape: " + field); + } + Field entries = children.get(0); + return ConnectorType.mapOf( + toDorisType(columnName + ".key", entries.getChildren().get(0), + entries.getChildren().get(0).getType()), + toDorisType(columnName + ".value", entries.getChildren().get(1), + entries.getChildren().get(1).getType())); + } + + private static ConnectorType childType(String columnName, Field field, int index) { + List children = field.getChildren(); + if (children.isEmpty() || index < 0 || index >= children.size()) { + throw new DorisConnectorException("Column '" + columnName + + "' has Arrow type " + field.getType() + " with no usable child type"); + } + Field child = children.get(index); + return toDorisType(columnName, child, child.getType()); + } + + private static DorisConnectorException unsupported(String columnName, ArrowType arrowType) { + return new DorisConnectorException("Column '" + columnName + "' has Arrow type " + arrowType + + ", which has no Doris equivalent. Cast or drop it on the remote side, or exclude the" + + " column from the query."); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AnsiDialect.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AnsiDialect.java new file mode 100644 index 00000000000000..af7db53aea69df --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AnsiDialect.java @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * The fallback dialect: standard SQL, and nothing a particular source added on top. + * + *

It is what an ADBC catalog gets when the driver's vendor is unknown or unrecognized, which is the + * common case -- any driver implementing the ADBC C ABI is a valid source. So this class is not a lowest + * common denominator chosen for tidiness; it is what most catalogs will actually run, and every choice in it + * is made for a source nobody has identified yet. + * + *

That is why {@link #renderLiteral} refuses more than it accepts. A literal it cannot render only costs + * a predicate that stays in Doris, whereas one it renders WRONGLY -- a type whose text spelling the source + * reads differently -- silently changes which rows come back. + */ +public class AnsiDialect implements AdbcDialect { + + public static final String NAME = "ansi"; + + private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + /** + * Fractional seconds are emitted only when present, and never beyond microseconds: Doris stores at most + * microsecond precision, and a source that parses fewer digits would round rather than reject, moving + * a boundary comparison onto the wrong side. + */ + private static final DateTimeFormatter TIMESTAMP_FORMAT = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss") + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true) + .optionalEnd() + .toFormatter(); + + /** + * The Doris types whose literal value arrives as a {@link String} AND that a source will read back as + * character data. The gate matters because the engine's expression converter falls back to + * {@code getStringValue()} for every type it has no branch for -- LARGEINT, IPV4, JSON -- so "the value + * is a String" on its own would quote a 128-bit integer and compare it as text. + */ + private static final Set CHARACTER_TYPE_NAMES = + new HashSet<>(Arrays.asList("CHAR", "VARCHAR", "STRING")); + + /** {@code PrimitiveType.TIMESTAMPTZ}'s name, as {@code ConnectorType} spells it. */ + private static final String TIMESTAMPTZ_TYPE_NAME = "TIMESTAMPTZ"; + + @Override + public String name() { + return NAME; + } + + @Override + public String quoteIdentifier(String name) { + return '"' + name.replace("\"", "\"\"") + '"'; + } + + /** + * Two parts at most, and never the catalog alongside the schema. + * + *

The {@code uri} property is required to pin one remote catalog, so every table reachable through + * this Doris catalog shares it and naming it again adds nothing -- while a three-part name is outright + * rejected by some sources. A source that has a catalog but no schema (SQLite reports {@code main} and + * an empty schema) uses the catalog as the one qualifier. + */ + @Override + public String qualifiedTableName(AdbcTableHandle handle) { + StringBuilder sb = new StringBuilder(); + if (!handle.getRemoteDbSchema().isEmpty()) { + sb.append(quoteIdentifier(handle.getRemoteDbSchema())).append('.'); + } else if (!handle.getRemoteCatalog().isEmpty()) { + sb.append(quoteIdentifier(handle.getRemoteCatalog())).append('.'); + } + return sb.append(quoteIdentifier(handle.getRemoteTable())).toString(); + } + + @Override + public String renderLiteral(ConnectorLiteral literal) { + Object value = literal.getValue(); + if (value == null) { + // A NULL literal is renderable in principle, and both sides would agree that a comparison + // against it matches nothing. It is refused anyway: the agreement rests on the source treating + // three-valued logic exactly as Doris does, which is not a property an unknown source has been + // shown to have, and the planner folds these away long before they reach a scan. + return null; + } + if (value instanceof Boolean) { + return ((Boolean) value) ? "TRUE" : "FALSE"; + } + if (value instanceof Long || value instanceof Integer || value instanceof Short + || value instanceof Byte || value instanceof BigDecimal) { + return value.toString(); + } + if (value instanceof Double || value instanceof Float) { + double d = ((Number) value).doubleValue(); + if (Double.isNaN(d) || Double.isInfinite(d)) { + // No portable spelling; the source would either reject it or read it as an identifier. + return null; + } + return value.toString(); + } + if (value instanceof LocalDate) { + return "DATE '" + DATE_FORMAT.format((LocalDate) value) + "'"; + } + if (value instanceof LocalDateTime) { + if (TIMESTAMPTZ_TYPE_NAME.equals(literal.getType().getTypeName())) { + // A zoned literal reaches here already converted to UTC, and standard SQL's + // TIMESTAMP 'yyyy-MM-dd HH:mm:ss' carries no zone -- so the source would read the UTC + // wall clock as ITS OWN local time and compare that against its column. On a source + // east of UTC that only widens the match and Doris re-filters what comes back; on one + // west of UTC it NARROWS it, and the rows the source drops are rows the query wanted. + // There is no portable spelling that says which instant is meant, so the comparison + // stays with Doris. + return null; + } + return "TIMESTAMP '" + TIMESTAMP_FORMAT.format((LocalDateTime) value) + "'"; + } + if (value instanceof String && CHARACTER_TYPE_NAMES.contains(literal.getType().getTypeName())) { + return renderStringLiteral((String) value); + } + return null; + } + + private static String renderStringLiteral(String value) { + if (value.indexOf('\0') >= 0) { + // Standard SQL has no escape for it inside a quoted string, and sources disagree on whether it + // terminates the value. Leave the predicate to Doris rather than send a string that may be + // silently truncated at the source. + return null; + } + return '\'' + value.replace("'", "''") + '\''; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/DorisDialect.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/DorisDialect.java new file mode 100644 index 00000000000000..1f7469918b3b3c --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/DorisDialect.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +/** + * Standard SQL as {@link AnsiDialect} renders it, with the one thing Doris spells differently: identifiers. + * + *

Doris reads a double-quoted name as a string literal, so ANSI quoting against a Doris source does not + * merely look foreign -- {@code SELECT "id" FROM "db"."t1"} does not parse at all, which is how the first + * Flight SQL scan against another Doris failed. Everything else ANSI produces (literals, the two-part table + * name, {@code LIMIT n}) Doris accepts unchanged, so this dialect changes one method. + * + *

Its scope is Doris alone. Backticks are the whole MySQL family's spelling and MySQL or StarRocks would + * very likely work, but neither has been run against, and a dialect that claims a source on family + * resemblance is a guess about someone else's SQL -- the failure mode being a syntax error at scan time, or + * a predicate that quietly matches different rows. Such a source can still ask for this dialect by name. + */ +public final class DorisDialect extends AnsiDialect { + + public static final String NAME = "doris"; + + /** + * What a Doris source calls itself through {@code getInfo(VENDOR_NAME)}. The value is its Flight SQL + * server name, {@code DorisFE} (see {@code SqlInfoBuilder.withFlightSqlServerName}), so matching the + * dialect's own name -- the default -- would never claim it. Matched as a prefix, case-insensitively, + * so a source that reports plain {@code Doris} is not left on a dialect it cannot parse. + */ + private static final String VENDOR_PREFIX = "doris"; + + @Override + public String name() { + return NAME; + } + + @Override + public boolean matchesVendor(String vendorName) { + return vendorName != null + && vendorName.regionMatches(true, 0, VENDOR_PREFIX, 0, VENDOR_PREFIX.length()); + } + + @Override + public String quoteIdentifier(String name) { + return '`' + name.replace("`", "``") + '`'; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider b/fe/fe-connector/fe-connector-adbc/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider new file mode 100644 index 00000000000000..3402d978ddea2d --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +org.apache.doris.connector.adbc.AdbcConnectorProvider diff --git a/fe/fe-connector/fe-connector-adbc/src/main/resources/adbc.conf.template b/fe/fe-connector/fe-connector-adbc/src/main/resources/adbc.conf.template new file mode 100644 index 00000000000000..ada016c00a100e --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/main/resources/adbc.conf.template @@ -0,0 +1,23 @@ +# ADBC connector plugin configuration. +# +# build.sh seeds adbc.conf from this file on first deploy and never overwrites it, so an upgrade that +# unzips a new plugin build over this directory keeps whatever you configured here. +# This file must exist on EVERY FE node -- it is not replicated through Doris metadata. +# Changes take effect after an FE restart. +# +# Every setting below is optional and falls back to the default named with it. Unlike the older +# connectors, neither has an fe.conf key: this connector was written after the plugin conf file +# existed, so there is no deployment that configured them anywhere else. + +# Directory a bare driver file name in a catalog's driver_url resolves under. +# Default: /plugins/adbc_drivers +# Whatever you set, the same driver library must also be placed under be/plugins/adbc_drivers on +# every BE: ADBC partition descriptors are driver-private bytes with no interoperability guarantee +# across driver builds, so FE and every BE must load the very same file. +# drivers_dir=/opt/doris/plugins/adbc_drivers + +# Directories a driver may be loaded from, separated by semicolons. +# Default: * (allow any); blank also means any. +# When set to concrete paths, a driver path is matched component-wise, so neither path traversal nor +# prefix confusion (/opt/drv vs /opt/drv-evil) can place a driver outside them. +# driver_secure_path=/opt/doris/plugins/adbc_drivers diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java new file mode 100644 index 00000000000000..6cd59606dc0153 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.arrow.adbc.core.AdbcException; +import org.apache.arrow.adbc.core.AdbcStatusCode; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Map; + +/** + * Exercises {@link AdbcClient} against the real JNI bridge and the real SQLite driver from thirdparty. + * Skips (loudly) when those libraries are absent -- see {@link AdbcNativeTestSupport}. + */ +class AdbcClientTest { + + private static AdbcClient sqliteClient(Path dbFile) { + return new AdbcClient(AdbcNativeTestSupport.sqliteDriver(), "libadbc_driver_sqlite.so", + null, "file:" + dbFile, null, null, Map.of()); + } + + @Test + void opensAConnectionThroughTheJniBridge(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("probe.db"))) { + String catalog = client.withConnection(connection -> connection.getCurrentCatalog()); + // SQLite's single catalog. Asserting the value (not just "no exception") is what proves the call + // reached the driver rather than stopping somewhere in the bridge. + Assertions.assertEquals("main", catalog); + } + } + + @Test + void reopensAfterTheFirstUseAndReleasesArrowMemoryOnClose(@TempDir Path tempDir) { + AdbcClient client = sqliteClient(tempDir.resolve("probe.db")); + try { + client.withConnection(connection -> connection.getCurrentCatalog()); + client.withConnection(connection -> connection.getCurrentCatalog()); + } finally { + // Closing must not throw; an allocator leak would surface here as an IllegalStateException from + // Arrow ("Memory was leaked by query"), which is precisely the failure a per-catalog allocator + // risks if a connection is left open. + Assertions.assertDoesNotThrow(client::close); + } + } + + @Test + void usingAClosedClientFailsLoud(@TempDir Path tempDir) { + AdbcClient client = sqliteClient(tempDir.resolve("probe.db")); + client.withConnection(connection -> connection.getCurrentCatalog()); + client.close(); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> client.withConnection(connection -> connection.getCurrentCatalog())); + Assertions.assertTrue(e.getMessage().contains("closed"), e.getMessage()); + } + + @Test + void missingDriverFileFailsAtFirstUseNotAtConstruction(@TempDir Path tempDir) { + // An FE follower replaying the edit log constructs every catalog; if construction reached the + // filesystem, one node missing a driver file would stop FE from starting instead of failing that + // one catalog. + AdbcClient client = new AdbcClient(tempDir.resolve("absent.so"), "absent.so", + null, "file:" + tempDir.resolve("x.db"), null, null, Map.of()); + + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> client.withConnection(connection -> connection.getCurrentCatalog())); + Assertions.assertTrue(e.getMessage().contains("EVERY BE"), e.getMessage()); + } + + @Test + void badEntrypointIsReportedWithTheDriverDetail(@TempDir Path tempDir) { + AdbcClient client = new AdbcClient(AdbcNativeTestSupport.sqliteDriver(), + "libadbc_driver_sqlite.so", "NoSuchSymbolXyz", + "file:" + tempDir.resolve("probe.db"), null, null, Map.of()); + try { + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> client.withConnection(connection -> connection.getCurrentCatalog())); + // Proves driver_entrypoint actually reaches the driver manager, and that a driver-side failure + // arrives with enough detail to act on. + Assertions.assertTrue(e.getMessage().contains("NoSuchSymbolXyz"), e.getMessage()); + } finally { + client.close(); + } + } + + @Test + void unhelpfulDriverMessagesAreNotForwardedAsTheWholeError() { + // The SQLite driver answers NOT_IMPLEMENTED with the literal text "(unknown error)". Forwarding it + // verbatim would produce an error that names neither the operation nor the cause, so the status has + // to carry the meaning instead. + AdbcException unhelpful = new AdbcException("(unknown error)", null, + AdbcStatusCode.NOT_IMPLEMENTED, null, 0); + DorisConnectorException translated = AdbcClient.translate(unhelpful, "getTableSchema failed"); + + Assertions.assertTrue(translated.getMessage().contains("getTableSchema failed"), + translated.getMessage()); + Assertions.assertTrue(translated.getMessage().contains("NOT_IMPLEMENTED"), translated.getMessage()); + Assertions.assertFalse(translated.getMessage().contains("(unknown error)"), + translated.getMessage()); + } + + @Test + void meaningfulDriverMessagesAreKept() { + AdbcException helpful = new AdbcException("relation \"t\" does not exist", null, + AdbcStatusCode.NOT_FOUND, "42P01", 7); + DorisConnectorException translated = AdbcClient.translate(helpful, "listTableNames failed"); + + String message = translated.getMessage(); + Assertions.assertTrue(message.contains("relation \"t\" does not exist"), message); + Assertions.assertTrue(message.contains("42P01"), message); + Assertions.assertTrue(message.contains("7"), message); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorCacheHookTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorCacheHookTest.java new file mode 100644 index 00000000000000..6fa1d6bcceec7d --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorCacheHookTest.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorCapability; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * What each REFRESH statement reaches once it arrives at the connector. + * + *

The three hooks were no-ops for as long as the connector remembered nothing, and a cache without them + * would be worse than no cache: metadata that no statement can refresh. Nothing here opens a driver -- the + * connector's constructor only reads properties, which is what lets an FE replaying the edit log build a + * catalog whose driver file it does not have. + */ +class AdbcConnectorCacheHookTest { + + private static final AdbcNamespace MAIN = new AdbcNamespace("main", ""); + private static final AdbcTableHandle MAIN_T1 = new AdbcTableHandle(MAIN, "t1"); + + private static AdbcConnector connector() { + return new AdbcConnector(Map.of(AdbcConnectorProperties.URI, "file:/tmp/does-not-matter.db", + AdbcConnectorProperties.DRIVER_URL, "libadbc_driver_sqlite.so"), context()); + } + + private static ConnectorContext context() { + return new ConnectorContext() { + @Override + public String getCatalogName() { + return "adbc_test"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }; + } + + /** Remembers one table's schema and answers how many times the source was asked for it since. */ + private static AtomicInteger rememberSchemaOf(AdbcMetadataCache cache, AdbcTableHandle handle) { + AtomicInteger reads = new AtomicInteger(); + cache.tableSchema(handle, () -> { + reads.incrementAndGet(); + return new Schema(Collections.emptyList()); + }); + return reads; + } + + private static AtomicInteger rememberTableNamesOf(AdbcMetadataCache cache, AdbcNamespace namespace) { + AtomicInteger reads = new AtomicInteger(); + cache.tableNames(namespace, () -> { + reads.incrementAndGet(); + return List.of("t1"); + }); + return reads; + } + + @Test + void refreshTableReachesTheCatalogsMemory() { + AdbcConnector connector = connector(); + AtomicInteger reads = rememberSchemaOf(connector.metadataCache(), MAIN_T1); + + connector.invalidateTable("main", "t1"); + + connector.metadataCache().tableSchema(MAIN_T1, () -> { + reads.incrementAndGet(); + return new Schema(Collections.emptyList()); + }); + Assertions.assertEquals(2, reads.get()); + } + + @Test + void refreshDatabaseReachesTheCatalogsMemory() { + AdbcConnector connector = connector(); + AtomicInteger reads = rememberTableNamesOf(connector.metadataCache(), MAIN); + + connector.invalidateDb("main"); + + connector.metadataCache().tableNames(MAIN, () -> { + reads.incrementAndGet(); + return List.of("t1"); + }); + Assertions.assertEquals(2, reads.get()); + } + + @Test + void refreshCatalogReachesTheCatalogsMemory() { + AdbcConnector connector = connector(); + AtomicInteger reads = new AtomicInteger(); + connector.metadataCache().namespaces(() -> { + reads.incrementAndGet(); + return List.of(MAIN); + }); + + connector.invalidateAll(); + + connector.metadataCache().namespaces(() -> { + reads.incrementAndGet(); + return List.of(MAIN); + }); + Assertions.assertEquals(2, reads.get()); + } + + /** + * The premise the cache rests on. Its keys are bare object names, so whatever one query caches is served + * to the next query whoever sends it -- correct only while a catalog reaches the source as one fixed + * principal. Declaring {@link ConnectorCapability#SUPPORTS_USER_SESSION} would make the connection carry + * the querying user's identity, and then these keys would hand one user's metadata to another. If this + * test ever fails, the capability is not the thing to revert: the cache keys have to carry the identity + * too, or the cache has to be per-session. + */ + @Test + void theCatalogReachesItsSourceAsOnePrincipalSoOneCacheCanServeEveryUser() { + Assertions.assertFalse(connector().getCapabilities().contains( + ConnectorCapability.SUPPORTS_USER_SESSION), + "ADBC now projects the querying user onto the connection, so AdbcMetadataCache's keys" + + " (database / table names) no longer identify what they cache"); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorConfTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorConfTest.java new file mode 100644 index 00000000000000..135d250775ddd5 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorConfTest.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.spi.ConnectorConf; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * How this connector's two deployment-level settings are resolved: adbc.conf, then the built-in + * default. There is no fe.conf half -- this connector is newer than the plugin conf file -- so these + * tests also pin that no fe.conf key is consulted behind the administrator's back. + * + *

Asserted rather than trusted because every failure direction here is silent: a template under the + * wrong name deploys a file the engine never opens, and a settings read that ignores adbc.conf makes an + * administrator's edit do nothing, with neither file showing why. + */ +public class AdbcConnectorConfTest { + + @Test + public void theConfTemplateIsNamedAfterTheProvider() { + // The engine reads .conf, so a template under any other name deploys a file nothing ever + // opens -- silently, with every setting in it ignored. Renaming name() must break here. + String expected = new AdbcConnectorProvider().name() + ".conf.template"; + Assertions.assertNotNull(getClass().getClassLoader().getResource(expected), + "the plugin must ship " + expected + " on its classpath"); + } + + @Test + public void driversDirComesFromThePluginConfThenFromDorisHome() { + Map dorisHome = Collections.singletonMap( + AdbcConnectorProperties.ENV_DORIS_HOME, "/opt/doris"); + + Assertions.assertEquals("/from/plugin/conf", ConnectorConf.get( + context(Collections.singletonMap(AdbcConnectorProperties.CONF_DRIVERS_DIR, + "/from/plugin/conf"), dorisHome), + AdbcConnectorProperties.CONF_DRIVERS_DIR, null, + "/opt/doris" + AdbcConnectorProperties.DEFAULT_DRIVERS_SUBDIR)); + + Assertions.assertEquals("/opt/doris/plugins/adbc_drivers", ConnectorConf.get( + context(Collections.emptyMap(), dorisHome), + AdbcConnectorProperties.CONF_DRIVERS_DIR, null, + "/opt/doris" + AdbcConnectorProperties.DEFAULT_DRIVERS_SUBDIR)); + } + + @Test + public void blankIsTreatedAsUnset() { + // An operator who writes "drivers_dir=" means "I have not configured this", so the default has + // to win. Reading it as the empty string would make the bare-name resolution fail with a + // "not configured" message that the conf file appears to contradict. + Assertions.assertEquals("/default", ConnectorConf.get( + context(Collections.singletonMap(AdbcConnectorProperties.CONF_DRIVERS_DIR, " "), + Collections.emptyMap()), + AdbcConnectorProperties.CONF_DRIVERS_DIR, null, "/default")); + } + + @Test + public void noFeConfKeyIsConsultedForTheseSettings() { + // The pre-plugin-conf shape of this connector read fe.conf's adbc_drivers_dir / + // adbc_driver_secure_path through the environment. Those @ConfFields are gone; an environment + // that still carries them must not resurrect a channel fe-core no longer feeds. + Map legacyEnv = new HashMap<>(); + legacyEnv.put("adbc_drivers_dir", "/from/fe/conf"); + legacyEnv.put("adbc_driver_secure_path", "/from/fe/conf"); + + Assertions.assertEquals("/default", ConnectorConf.get( + context(Collections.emptyMap(), legacyEnv), + AdbcConnectorProperties.CONF_DRIVERS_DIR, null, "/default")); + Assertions.assertEquals(AdbcConnectorProperties.DEFAULT_DRIVER_SECURE_PATH, ConnectorConf.get( + context(Collections.emptyMap(), legacyEnv), + AdbcConnectorProperties.CONF_DRIVER_SECURE_PATH, null, + AdbcConnectorProperties.DEFAULT_DRIVER_SECURE_PATH)); + } + + private static ConnectorContext context(Map conf, Map env) { + Map confCopy = new HashMap<>(conf); + Map envCopy = new HashMap<>(env); + return new ConnectorContext() { + @Override + public String getCatalogName() { + return "adbc_test"; + } + + @Override + public long getCatalogId() { + return 1L; + } + + @Override + public Map getConnectorConfig() { + return confCopy; + } + + @Override + public Map getEnvironment() { + return envCopy; + } + }; + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorMetadataTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorMetadataTest.java new file mode 100644 index 00000000000000..e87f7882051977 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorMetadataTest.java @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.arrow.adbc.core.AdbcStatement; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The metadata surface end to end against the real SQLite ADBC driver: the same Java -> JNI -> C driver + * manager -> driver .so path FE takes in production, with only the source swapped for a temporary file. + * + *

Skips loudly when thirdparty's native libraries are absent -- see {@link AdbcNativeTestSupport}. A + * skipped run has verified nothing here. + */ +class AdbcConnectorMetadataTest { + + private static AdbcClient sqliteClient(Path dbFile) { + return new AdbcClient(AdbcNativeTestSupport.sqliteDriver(), "libadbc_driver_sqlite.so", + null, "file:" + dbFile, null, null, Map.of()); + } + + /** + * A fresh cache per call, so each test here reads the source rather than the previous test's answers -- + * what the cache does across statements is {@link AdbcMetadataCacheNativeTest}'s subject, not this one's. + */ + private static AdbcConnectorMetadata metadataOn(AdbcClient client) { + return new AdbcConnectorMetadata(client, new AdbcSchemaStrategy(), Map.of(), + AdbcDialectRegistry::defaultDialect, new AdbcMetadataCache(Map.of())); + } + + /** + * The INSERT is load-bearing, not decoration. SQLite is dynamically typed and its ADBC driver derives + * the Arrow schema from the values present, not from the declared column types: on an empty t1 it + * reports all four columns as int64. So a fixture with no rows would assert a type mapping the driver + * never actually produced. + */ + private static void seed(AdbcClient client) { + client.withConnection(connection -> { + for (String sql : new String[] { + "CREATE TABLE IF NOT EXISTS t1 (c_int INTEGER, c_dbl REAL, c_txt TEXT, c_blob BLOB)", + "INSERT INTO t1 VALUES (1, 1.5, 'a', x'00ff')", + "CREATE TABLE IF NOT EXISTS t2 (a INTEGER)", + "CREATE VIEW IF NOT EXISTS v1 AS SELECT * FROM t1"}) { + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + statement.executeUpdate(); + } + } + return null; + }); + } + + @Test + void showDatabasesReturnsTheFlattenedNamespace(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + AdbcConnectorMetadata metadata = metadataOn(client); + + Assertions.assertEquals(List.of("main"), metadata.listDatabaseNames(null)); + Assertions.assertTrue(metadata.databaseExists(null, "main")); + Assertions.assertFalse(metadata.databaseExists(null, "no_such_db")); + } + } + + @Test + void showTablesExcludesViews(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + + // Doris presents no views for an ADBC catalog, so listing v1 would produce a name that DESC and + // SELECT would both then fail on. + Assertions.assertEquals(List.of("t1", "t2"), metadataOn(client).listTableNames(null, "main")); + } + } + + @Test + void listingTablesOfAnUnknownDatabaseIsEmptyNotAnError(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + Assertions.assertEquals(List.of(), metadataOn(client).listTableNames(null, "no_such_db")); + } + } + + @Test + void tableHandleCarriesTheRemoteCoordinates(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + + Optional handle = metadataOn(client).getTableHandle(null, "main", "t1"); + + Assertions.assertTrue(handle.isPresent()); + AdbcTableHandle adbc = (AdbcTableHandle) handle.get(); + // SQLite names a catalog but no schema, so the Doris database name came from the catalog level. + // The handle must still record which level it came from, or the pushed-down SQL later cannot + // qualify the table. + Assertions.assertEquals("main", adbc.getRemoteCatalog()); + Assertions.assertEquals("", adbc.getRemoteDbSchema()); + Assertions.assertEquals("t1", adbc.getRemoteTable()); + Assertions.assertEquals("main", adbc.getDorisDbName()); + } + } + + @Test + void unknownTableAndUnknownDatabaseBothYieldNoHandle(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + AdbcConnectorMetadata metadata = metadataOn(client); + + Assertions.assertEquals(Optional.empty(), metadata.getTableHandle(null, "main", "no_such")); + Assertions.assertEquals(Optional.empty(), metadata.getTableHandle(null, "no_such", "t1")); + // A view is not a table here; handing back a handle for it would defer the failure to DESC. + Assertions.assertEquals(Optional.empty(), metadata.getTableHandle(null, "main", "v1")); + } + } + + @Test + void descMapsTheRealArrowSchema(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + AdbcConnectorMetadata metadata = metadataOn(client); + ConnectorTableHandle handle = metadata.getTableHandle(null, "main", "t1").orElseThrow(); + + ConnectorTableSchema schema = metadata.getTableSchema(null, handle); + + // Expected values come from what the driver actually reports: SQLite INTEGER arrives as int64, + // REAL as float64, TEXT as utf8, BLOB as binary. Doris has no binary column type here, so BLOB + // lands on STRING. + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (ConnectorColumn column : schema.getColumns()) { + names.add(column.getName()); + types.add(column.getType()); + } + Assertions.assertEquals(List.of("c_int", "c_dbl", "c_txt", "c_blob"), names); + Assertions.assertEquals(List.of(ConnectorType.of("BIGINT"), ConnectorType.of("DOUBLE"), + ConnectorType.of("STRING"), ConnectorType.of("STRING")), types); + Assertions.assertEquals("t1", schema.getTableName()); + } + } + + @Test + void columnHandlesMatchTheSchemaAndItsOrder(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + AdbcConnectorMetadata metadata = metadataOn(client); + ConnectorTableHandle handle = metadata.getTableHandle(null, "main", "t1").orElseThrow(); + + Map handles = metadata.getColumnHandles(null, handle); + + // Order matters: the scan path pairs handles with schema columns positionally. + Assertions.assertEquals(List.of("c_int", "c_dbl", "c_txt", "c_blob"), + new ArrayList<>(handles.keySet())); + } + } + + @Test + void missingTableIsReportedAsSuchNotAsADriverGap(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + AdbcConnectorMetadata metadata = metadataOn(client); + // Build a handle for a table that does not exist, bypassing getTableHandle's existence check. + AdbcTableHandle ghost = new AdbcTableHandle(new AdbcNamespace("main", ""), "no_such_table"); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getTableSchema(null, ghost)); + + // The fallback to executeSchema must fire only on NOT_IMPLEMENTED. Falling back on every error + // would answer a plain missing table with "this driver implements neither method", sending the + // user to look at their driver instead of their table name. + Assertions.assertTrue(e.getMessage().contains("no_such_table"), e.getMessage()); + Assertions.assertFalse(e.getMessage().contains("implements neither"), e.getMessage()); + } + } + + @Test + void tableDescriptorIsTypedForTheScanPath(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("meta.db"))) { + seed(client); + + TTableDescriptor descriptor = metadataOn(client) + .buildTableDescriptor(null, 7L, "t1", "main", "t1", 4, 42L); + + // Returning null (the SPI default) would let fe-core fall back to SCHEMA_TABLE, and BE would + // then build a SchemaTableDescriptor rather than the one the file-scan path expects. + Assertions.assertEquals(TTableType.HIVE_TABLE, descriptor.getTableType()); + Assertions.assertTrue(descriptor.isSetHiveTable()); + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorPropertiesTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorPropertiesTest.java new file mode 100644 index 00000000000000..d35185fae8cd65 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorPropertiesTest.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +class AdbcConnectorPropertiesTest { + + private static Map minimalProperties() { + Map props = new LinkedHashMap<>(); + props.put(AdbcConnectorProperties.DRIVER_URL, "libadbc_driver_flightsql.so"); + props.put(AdbcConnectorProperties.URI, "grpc://remote-fe:9090"); + return props; + } + + @Test + void driverOptionsKeepTheAdbcPrefix() { + // The prefix is part of the option name, NOT a namespace to strip: ADBC's own option names already + // begin with "adbc." (e.g. adbc.snowflake.sql.db), so stripping would send the driver a name it + // does not know and the option would be silently ignored. BE applies the same rule; if the two + // sides ever disagree, a catalog would plan against different driver settings than it reads with. + Map props = minimalProperties(); + props.put("adbc.adbc.snowflake.sql.db", "MYDB"); + props.put("adbc.custom.option", "v"); + + Map options = AdbcConnectorProperties.driverOptions(props); + + Assertions.assertEquals(Map.of("adbc.adbc.snowflake.sql.db", "MYDB", "adbc.custom.option", "v"), + options); + } + + @Test + void nonPrefixedPropertiesAreNotPassedToTheDriver() { + Map props = minimalProperties(); + props.put("password", "secret"); + + Assertions.assertEquals(Map.of(), AdbcConnectorProperties.driverOptions(props)); + } + + @Test + void partitionedReadIsAutomaticUnlessTheCatalogSaysOtherwise() { + Assertions.assertEquals(AdbcConnectorProperties.PartitionedReadMode.AUTO, + AdbcConnectorProperties.partitionedReadMode(minimalProperties())); + + for (AdbcConnectorProperties.PartitionedReadMode mode + : AdbcConnectorProperties.PartitionedReadMode.values()) { + Map props = minimalProperties(); + // Spelled the way a user writes it, and case-insensitively. + props.put(AdbcConnectorProperties.PARTITIONED_READ, mode.name().toLowerCase()); + Assertions.assertEquals(mode, AdbcConnectorProperties.partitionedReadMode(props)); + } + } + + @Test + void anUnreadableModeFailsInsteadOfMeaningTheDefault() { + // Falling back to AUTO on a typo would be the worst answer for the one mode that exists to stop + // a silent downgrade: 'requred' would quietly permit exactly what 'required' forbids. + Map props = minimalProperties(); + props.put(AdbcConnectorProperties.PARTITIONED_READ, "requred"); + + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> AdbcConnectorProperties.partitionedReadMode(props)); + Assertions.assertTrue(e.getMessage().contains(AdbcConnectorProperties.PARTITIONED_READ), + e.getMessage()); + } + + @Test + void thePartitionLimitDefaultsAndRejectsValuesThatCannotBeOne() { + Assertions.assertEquals(1024, AdbcConnectorProperties.maxPartitions(minimalProperties())); + + Map raised = minimalProperties(); + raised.put(AdbcConnectorProperties.MAX_PARTITIONS, " 4096 "); + Assertions.assertEquals(4096, AdbcConnectorProperties.maxPartitions(raised)); + + for (String bad : new String[] {"0", "-1", "many"}) { + Map props = minimalProperties(); + props.put(AdbcConnectorProperties.MAX_PARTITIONS, bad); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> AdbcConnectorProperties.maxPartitions(props), bad); + Assertions.assertTrue(e.getMessage().contains(AdbcConnectorProperties.MAX_PARTITIONS), + e.getMessage()); + } + } + + @Test + void providerRejectsAnUnreadablePartitionSettingAtCreateTime() { + // These decide how every scan is planned, so a typo has to fail at CREATE CATALOG rather than + // changing the plan shape silently from the first query onwards. + AdbcConnectorProvider provider = new AdbcConnectorProvider(); + for (String[] bad : new String[][] { + {AdbcConnectorProperties.PARTITIONED_READ, "yes"}, + {AdbcConnectorProperties.MAX_PARTITIONS, "0"}}) { + Map props = minimalProperties(); + props.put(bad[0], bad[1]); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> provider.validateProperties(props)); + Assertions.assertTrue(e.getMessage().contains(bad[0]), e.getMessage()); + } + } + + /** + * A cache knob is read through {@code CacheSpec}, which answers an unparseable value with the default + * rather than an error. Silently caching for ten minutes when the operator wrote {@code ttl-second=6O0} + * is exactly the kind of thing nobody notices until the metadata is wrong, so the value has to be + * refused where the operator is still looking at it. + */ + @Test + void providerRejectsAnUnreadableCacheSettingAtCreateTime() { + AdbcConnectorProvider provider = new AdbcConnectorProvider(); + for (String[] bad : new String[][] { + {"meta.cache.adbc.metadata.enable", "no"}, + {"meta.cache.adbc.metadata.ttl-second", "6O0"}, + {"meta.cache.adbc.metadata.ttl-second", "-2"}, + {"meta.cache.adbc.metadata.capacity", "-1"}}) { + Map props = minimalProperties(); + props.put(bad[0], bad[1]); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> provider.validateProperties(props), bad[0] + "=" + bad[1]); + Assertions.assertTrue(e.getMessage().contains(bad[0]), e.getMessage()); + } + + // -1 is the framework's "never expire", not a typo. + Map noExpiry = minimalProperties(); + noExpiry.put("meta.cache.adbc.metadata.ttl-second", "-1"); + Assertions.assertDoesNotThrow(() -> provider.validateProperties(noExpiry)); + } + + @Test + void providerRejectsAMissingDriverUrlOrUri() { + AdbcConnectorProvider provider = new AdbcConnectorProvider(); + Assertions.assertDoesNotThrow(() -> provider.validateProperties(minimalProperties())); + + for (String required : new String[] { + AdbcConnectorProperties.DRIVER_URL, AdbcConnectorProperties.URI}) { + Map props = minimalProperties(); + props.remove(required); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> provider.validateProperties(props)); + Assertions.assertTrue(e.getMessage().contains(required), e.getMessage()); + + Map blank = minimalProperties(); + blank.put(required, " "); + IllegalArgumentException blankError = Assertions.assertThrows(IllegalArgumentException.class, + () -> provider.validateProperties(blank)); + Assertions.assertTrue(blankError.getMessage().contains(required), blankError.getMessage()); + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorProviderIsolationTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorProviderIsolationTest.java new file mode 100644 index 00000000000000..e2f155183af7c7 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorProviderIsolationTest.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.spi.ConnectorProvider; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Pins the class-loading invariant stated on {@link AdbcConnectorProvider}: discovering and instantiating + * the provider must not load any {@code org.apache.arrow.adbc.*} class. + * + *

WHY this matters, and why a plain "it constructs fine" test would not catch it: the plugin loader + * ({@code DirectoryPluginRuntimeManager#loadAll}) builds the plugin classloader and instantiates every + * discovered factory, and only afterwards rejects one whose type name is already claimed. So when the same + * connector exists both as a classpath built-in and as a directory plugin — the normal shape in tests and + * embedded setups — the directory copy IS constructed once, in its own classloader, before being discarded. + * If that construction reached an ADBC class, it would run a second {@code System.load} of the JNI shim from + * a second classloader, which the JVM answers with {@code UnsatisfiedLinkError}, and FE would lose the + * connector it had already loaded correctly. Deferring every ADBC reference into method bodies keeps the + * discarded instance inert. + * + *

The test reproduces that shape: it re-defines the whole connector package in a loader that refuses + * {@code org.apache.arrow.adbc.*}, then constructs the provider and asks it for its type. Verified by + * mutation: adding {@code private static final Object M = AdbcDriver.PARAM_URI;} to the provider makes this + * fail, naming {@code org.apache.arrow.adbc.core.AdbcDriver}. + * + *

One shape is invisible here and that is correct, not a gap: referencing a {@code static final String} + * of an ADBC class (e.g. {@code AdbcDriver.PARAM_URL}) is inlined by javac into a constant, leaving no + * symbolic reference — so it loads no class and cannot trigger the failure this guard exists to prevent. + */ +class AdbcConnectorProviderIsolationTest { + + @Test + void constructingTheProviderLoadsNoAdbcClass() throws Exception { + AdbcBlockingClassLoader loader = new AdbcBlockingClassLoader(getClass().getClassLoader()); + + Class providerClass = loader.loadClass(AdbcConnectorProvider.class.getName()); + Assertions.assertNotSame(AdbcConnectorProvider.class, providerClass, + "The test must exercise a provider defined by the blocking loader, not the one already" + + " loaded by the test classloader"); + + Object provider; + try { + provider = providerClass.getDeclaredConstructor().newInstance(); + } catch (Throwable t) { + throw new AssertionError("Instantiating AdbcConnectorProvider reached ADBC classes " + + loader.blockedNames() + "; keep the fields, static initializer and constructor free of" + + " org.apache.arrow.adbc types and let them first appear inside create()", t); + } + + Assertions.assertEquals("adbc", ((ConnectorProvider) provider).getType(), + "getType() must be answerable without any ADBC class"); + Assertions.assertTrue(((ConnectorProvider) provider).isStandaloneCatalogType(), + "adbc is a catalog type a user writes in CREATE CATALOG"); + + Assertions.assertEquals(List.of(), loader.blockedNames(), + "Provider discovery must not load any org.apache.arrow.adbc class"); + } + + /** + * Child-first for the connector's own package, and a hard stop for {@code org.apache.arrow.adbc.*}. + * Re-defining the whole package (not just the provider) is deliberate: it makes the check transitive, + * so a provider that stayed clean itself but constructed a connector class whose static initializer + * touched ADBC would still fail here. + */ + private static final class AdbcBlockingClassLoader extends ClassLoader { + + private static final String BLOCKED_PREFIX = "org.apache.arrow.adbc."; + private static final String OWNED_PREFIX = "org.apache.doris.connector.adbc."; + + private final List blocked = new ArrayList<>(); + + AdbcBlockingClassLoader(ClassLoader parent) { + super(parent); + } + + List blockedNames() { + return blocked; + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (name.startsWith(BLOCKED_PREFIX)) { + blocked.add(name); + throw new ClassNotFoundException("blocked by " + AdbcBlockingClassLoader.class.getSimpleName() + + ": " + name); + } + if (!name.startsWith(OWNED_PREFIX)) { + return super.loadClass(name, resolve); + } + synchronized (getClassLoadingLock(name)) { + Class loaded = findLoadedClass(name); + if (loaded == null) { + byte[] bytes = readClassBytes(name); + loaded = defineClass(name, bytes, 0, bytes.length); + } + if (resolve) { + resolveClass(loaded); + } + return loaded; + } + } + + private byte[] readClassBytes(String name) throws ClassNotFoundException { + String resource = name.replace('.', '/') + ".class"; + try (InputStream in = getParent().getResourceAsStream(resource)) { + if (in == null) { + throw new ClassNotFoundException(name); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } catch (IOException e) { + throw new ClassNotFoundException(name, e); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcDialectTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcDialectTest.java new file mode 100644 index 00000000000000..5ad450a16e26fd --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcDialectTest.java @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Map; + +/** + * What the default dialect will and will not say, and how one gets chosen. + * + *

The refusals carry the weight here. ANSI is what an ADBC catalog gets whenever the source is not + * recognized -- the common case -- so a literal it renders wrongly does not fail, it returns a different set + * of rows from a source nobody has tested against. + */ +class AdbcDialectTest { + + private static final AdbcDialect ANSI = AdbcDialectRegistry.defaultDialect(); + + private static ConnectorLiteral literal(String typeName, Object value) { + return new ConnectorLiteral(ConnectorType.of(typeName), value); + } + + private static AdbcTableHandle handle(String catalog, String schema, String table) { + return new AdbcTableHandle(new AdbcNamespace(catalog, schema), table); + } + + // ---------- identifiers ---------- + + @Test + void quotesIdentifiersAndDoublesAnEmbeddedQuote() { + Assertions.assertEquals("\"orders\"", ANSI.quoteIdentifier("orders")); + // Without doubling, a name containing a quote would close the identifier early and turn the rest of + // the name into stray SQL -- the injection shape, reached through a column name from the source. + Assertions.assertEquals("\"we\"\"ird\"", ANSI.quoteIdentifier("we\"ird")); + } + + @Test + void qualifiesWithTheSchemaWhenThereIsOne() { + Assertions.assertEquals("\"public\".\"orders\"", + ANSI.qualifiedTableName(handle("mydb", "public", "orders"))); + } + + @Test + void qualifiesWithTheCatalogOnlyWhenThereIsNoSchema() { + // SQLite's shape: catalog "main", empty schema. + Assertions.assertEquals("\"main\".\"t1\"", ANSI.qualifiedTableName(handle("main", "", "t1"))); + } + + @Test + void neverEmitsThreeParts() { + // The uri pins one remote catalog, so repeating it adds nothing -- and some sources reject a + // three-part name outright. + String name = ANSI.qualifiedTableName(handle("MYDB", "PUBLIC", "ORDERS")); + Assertions.assertEquals("\"PUBLIC\".\"ORDERS\"", name); + } + + @Test + void neverHasToNameAnUnqualifiedTable() { + // There is no "neither level" case to qualify: a namespace with neither is refused when it is + // built, because it has no Doris database name to be shown under either. So the dialect is never + // asked to render a bare table name, and does not have to decide what that would mean. + Assertions.assertThrows(DorisConnectorException.class, () -> handle("", "", "t1")); + } + + // ---------- literals it renders ---------- + + @Test + void rendersNumbersBooleansAndCharacterData() { + Assertions.assertEquals("42", ANSI.renderLiteral(literal("INT", 42L))); + Assertions.assertEquals("-1.5", ANSI.renderLiteral(literal("DOUBLE", -1.5d))); + Assertions.assertEquals("3.140", ANSI.renderLiteral( + literal("DECIMAL64", new BigDecimal("3.140")))); + Assertions.assertEquals("TRUE", ANSI.renderLiteral(literal("BOOLEAN", Boolean.TRUE))); + Assertions.assertEquals("FALSE", ANSI.renderLiteral(literal("BOOLEAN", Boolean.FALSE))); + Assertions.assertEquals("'abc'", ANSI.renderLiteral(literal("VARCHAR", "abc"))); + } + + @Test + void doublesAnEmbeddedQuoteInAStringLiteral() { + Assertions.assertEquals("'O''Brien'", ANSI.renderLiteral(literal("STRING", "O'Brien"))); + } + + @Test + void rendersDatesAndTimestampsWithTheirTypeKeyword() { + // Bare '2024-01-31' is read as character data by some sources and compared as text against a date + // column, which orders differently. The keyword is what makes the comparison a date comparison. + Assertions.assertEquals("DATE '2024-01-31'", + ANSI.renderLiteral(literal("DATEV2", LocalDate.of(2024, 1, 31)))); + Assertions.assertEquals("TIMESTAMP '2024-01-31 12:30:05'", + ANSI.renderLiteral(literal("DATETIMEV2", + LocalDateTime.of(2024, 1, 31, 12, 30, 5)))); + } + + @Test + void keepsSubSecondPrecisionButNoFurtherThanMicroseconds() { + Assertions.assertEquals("TIMESTAMP '2024-01-31 12:30:05.000123'", + ANSI.renderLiteral(literal("DATETIMEV2", + LocalDateTime.of(2024, 1, 31, 12, 30, 5, 123_000)))); + } + + // ---------- literals it refuses ---------- + + @Test + void refusesANullLiteral() { + Assertions.assertNull(ANSI.renderLiteral(ConnectorLiteral.ofNull(ConnectorType.of("INT")))); + } + + @Test + void refusesAZonedTimestampLiteral() { + // Same Java value as the DATETIMEV2 case above and a different answer, because the TYPE changes + // what the value means: the engine hands a TIMESTAMPTZ literal over already converted to UTC, + // and TIMESTAMP '...' has no zone in it, so the source would compare its own local wall clock + // against a UTC one. West of UTC that drops rows the query asked for -- rows a scan cannot get + // back, since the source never sends them -- so the comparison is left to Doris. + Assertions.assertNull(ANSI.renderLiteral(literal("TIMESTAMPTZ", + LocalDateTime.of(2024, 1, 31, 12, 30, 5)))); + Assertions.assertEquals("TIMESTAMP '2024-01-31 12:30:05'", + ANSI.renderLiteral(literal("DATETIMEV2", + LocalDateTime.of(2024, 1, 31, 12, 30, 5)))); + } + + @Test + void refusesAStringValueWhoseTypeIsNotCharacterData() { + // The engine's converter falls back to the text form for every type it has no branch for. Quoting + // that would compare a 128-bit integer, an IP or a JSON document as text, which orders and matches + // differently from how the source stores it. + Assertions.assertNull(ANSI.renderLiteral(literal("LARGEINT", "170141183460469231731687303715884105727"))); + Assertions.assertNull(ANSI.renderLiteral(literal("IPV4", "10.0.0.1"))); + Assertions.assertNull(ANSI.renderLiteral(literal("JSON", "{\"a\":1}"))); + } + + @Test + void refusesNonFiniteFloatingPointValues() { + Assertions.assertNull(ANSI.renderLiteral(literal("DOUBLE", Double.NaN))); + Assertions.assertNull(ANSI.renderLiteral(literal("DOUBLE", Double.POSITIVE_INFINITY))); + } + + @Test + void refusesAStringContainingANulCharacter() { + // Standard SQL cannot escape it inside a quoted string and sources disagree on whether it ends the + // value, so sending it risks a silently truncated comparison. + Assertions.assertNull(ANSI.renderLiteral(literal("STRING", "a\0b"))); + } + + // ---------- selection ---------- + + @Test + void selectsAnsiWhenNothingIdentifiesTheSource() { + AdbcDialectSelector selector = new AdbcDialectSelector(Map.of()); + // The probe cannot run without a client; supplying one that fails stands in for a driver that + // refuses getInfo, which must land on ANSI rather than fail the query. + Assertions.assertSame(ANSI, selector.select(() -> { + throw new DorisConnectorException("driver unavailable"); + })); + } + + @Test + void selectsTheConfiguredDialectWithoutAskingTheSource() { + AdbcDialect fake = new FakeDialect("fake-for-selection"); + AdbcDialectRegistry.register(fake); + AdbcDialectSelector selector = new AdbcDialectSelector( + Map.of(AdbcConnectorProperties.SQL_DIALECT, "fake-for-selection")); + + Assertions.assertSame(fake, selector.select(() -> { + // A probe here would make the property advisory, and would also cost a remote call on a + // question the user already answered. + throw new AssertionError("the source must not be probed when sql_dialect is set"); + })); + } + + @Test + void matchesTheConfiguredNameCaseInsensitively() { + AdbcDialectSelector selector = new AdbcDialectSelector( + Map.of(AdbcConnectorProperties.SQL_DIALECT, "ANSI")); + Assertions.assertSame(ANSI, selector.select(() -> { + throw new AssertionError("not probed"); + })); + } + + @Test + void rejectsAnUnknownConfiguredDialectAndNamesTheRegisteredOnes() { + AdbcDialectSelector selector = new AdbcDialectSelector( + Map.of(AdbcConnectorProperties.SQL_DIALECT, "postgres-typo")); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + selector::validateConfiguredName); + Assertions.assertTrue(failure.getMessage().contains("postgres-typo"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains(AnsiDialect.NAME), failure.getMessage()); + } + + @Test + void findsARegisteredDialectByTheVendorNameADriverReports() { + AdbcDialectRegistry.register(new FakeDialect("fake-vendor-match")); + Assertions.assertEquals("fake-vendor-match", + AdbcDialectRegistry.forVendor("Fake-Vendor-Match").map(AdbcDialect::name).orElse(null)); + Assertions.assertFalse(AdbcDialectRegistry.forVendor("Nothing-Claims-This").isPresent()); + Assertions.assertFalse(AdbcDialectRegistry.forVendor("").isPresent()); + Assertions.assertFalse(AdbcDialectRegistry.forVendor(null).isPresent()); + } + + /** A dialect defined outside the shipped set, to show that being one is all it takes. */ + private static final class FakeDialect implements AdbcDialect { + + private final String name; + + FakeDialect(String name) { + this.name = name; + } + + @Override + public String name() { + return name; + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + @Override + public String qualifiedTableName(AdbcTableHandle handle) { + return quoteIdentifier(handle.getRemoteTable()); + } + + @Override + public String renderLiteral(ConnectorLiteral value) { + return String.valueOf(value.getValue()); + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcDriverPathResolverTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcDriverPathResolverTest.java new file mode 100644 index 00000000000000..519d0be25df69b --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcDriverPathResolverTest.java @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Rules for turning {@code driver_url} into a driver library path. + * + *

Each rejection case below exists because letting it through has a concrete consequence, stated per + * test. The rejections matter more than the acceptances: this resolver is the only thing standing between + * a catalog property and a {@code dlopen} of an arbitrary file into the FE process. + */ +class AdbcDriverPathResolverTest { + + private static final String DRIVERS_DIR = "/opt/doris/plugins/adbc_drivers"; + private static final String ALLOW_ALL = "*"; + + private static Path resolve(String driverUrl) { + return AdbcDriverPathResolver.resolve(driverUrl, DRIVERS_DIR, ALLOW_ALL); + } + + private static String rejectionOf(String driverUrl, String securePath) { + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> AdbcDriverPathResolver.resolve(driverUrl, DRIVERS_DIR, securePath)); + return e.getMessage(); + } + + // ---- accepted forms ---- + + @Test + void bareFileNameResolvesUnderTheDriversDirectory() { + Assertions.assertEquals(Paths.get(DRIVERS_DIR, "libadbc_driver_flightsql.so"), + resolve("libadbc_driver_flightsql.so")); + } + + @Test + void versionedSonameIsAccepted() { + // Drivers extracted from a release tarball routinely carry an soname suffix; rejecting it would + // force users to rename the file for no reason. + Assertions.assertEquals(Paths.get(DRIVERS_DIR, "libadbc_driver_sqlite.so.112.0.0"), + resolve("libadbc_driver_sqlite.so.112.0.0")); + } + + @Test + void fileUrlResolvesToItsPath() { + Assertions.assertEquals(Paths.get("/data/drivers/libadbc_driver_postgresql.so"), + resolve("file:///data/drivers/libadbc_driver_postgresql.so")); + } + + @Test + void absolutePathIsTakenAsIs() { + Assertions.assertEquals(Paths.get("/data/drivers/libadbc_driver_postgresql.so"), + resolve("/data/drivers/libadbc_driver_postgresql.so")); + } + + @Test + void surroundingWhitespaceIsTrimmed() { + Assertions.assertEquals(Paths.get(DRIVERS_DIR, "libadbc_driver_flightsql.so"), + resolve(" libadbc_driver_flightsql.so ")); + } + + // ---- rejected forms ---- + + @Test + void remoteSchemesAreRejectedWithTheReason() { + // Downloading per node cannot guarantee FE and every BE end up with the identical library, and a + // mismatch surfaces as an unreadable partition descriptor -- far from its cause. So the message + // has to explain the constraint, not just say "unsupported". + for (String url : new String[] { + "http://example.com/libadbc_driver_flightsql.so", + "https://example.com/libadbc_driver_flightsql.so", + "s3://bucket/libadbc_driver_flightsql.so"}) { + String message = rejectionOf(url, ALLOW_ALL); + Assertions.assertTrue(message.contains("only a local file"), + "must say local-only: " + message); + Assertions.assertTrue(message.contains("every BE"), + "must explain that FE and every BE need the same file: " + message); + } + } + + @Test + void plainTraversalIsRejected() { + String message = rejectionOf("file:///opt/doris/plugins/adbc_drivers/../../../etc/evil.so", ALLOW_ALL); + Assertions.assertTrue(message.contains("path traversal"), message); + } + + @Test + void percentEncodedTraversalIsRejected() { + // The URL is decoded once before the check, exactly as the loader will decode it. Checking the raw + // string instead would let %2e%2e through and land the dlopen outside the allowed directory. + String message = rejectionOf( + "file:///opt/doris/plugins/adbc_drivers/%2e%2e/%2e%2e/etc/evil.so", ALLOW_ALL); + Assertions.assertTrue(message.contains("path traversal"), message); + } + + @Test + void bareNameWithASeparatorIsRejected() { + // A bare name is the one form resolved relative to a directory, so any separator in it would be an + // escape hatch out of that directory. + String message = rejectionOf("../../etc/evil.so", ALLOW_ALL); + Assertions.assertTrue(message.contains("bare driver file name"), message); + } + + @Test + void bareNameThatIsNotALibraryIsRejected() { + String message = rejectionOf("driver.jar", ALLOW_ALL); + Assertions.assertTrue(message.contains("bare driver file name"), message); + } + + @Test + void fileUrlWithAnAuthorityIsRejected() { + // "file://attacker/dir/x.so" carries a remote authority that URI.getPath() does not show, so + // validating the path alone would authorize an object the loader would fetch from elsewhere. + String message = rejectionOf("file://attacker/dir/libadbc_driver_flightsql.so", ALLOW_ALL); + Assertions.assertTrue(message.contains("no authority, query or fragment"), message); + } + + @Test + void fileUrlWithAQueryIsRejected() { + String message = rejectionOf("file:///dir/libadbc_driver_flightsql.so?x=1", ALLOW_ALL); + Assertions.assertTrue(message.contains("no authority, query or fragment"), message); + } + + @Test + void missingDriverUrlNamesTheProperty() { + for (String value : new String[] {null, "", " "}) { + String message = rejectionOf(value, ALLOW_ALL); + Assertions.assertTrue(message.contains("driver_url"), message); + } + } + + // ---- secure path ---- + + @Test + void securePathAllowsWhatIsUnderIt() { + Assertions.assertEquals(Paths.get("/opt/drv/libadbc_driver_flightsql.so"), + AdbcDriverPathResolver.resolve("/opt/drv/libadbc_driver_flightsql.so", + DRIVERS_DIR, "/opt/drv;/srv/drv")); + Assertions.assertEquals(Paths.get("/srv/drv/sub/libadbc_driver_flightsql.so"), + AdbcDriverPathResolver.resolve("/srv/drv/sub/libadbc_driver_flightsql.so", + DRIVERS_DIR, "/opt/drv;/srv/drv")); + } + + @Test + void securePathRejectsWhatIsOutsideIt() { + String message = rejectionOf("/etc/libadbc_driver_flightsql.so", "/opt/drv"); + Assertions.assertTrue(message.contains("driver_secure_path"), message); + } + + @Test + void securePathIsMatchedByComponentNotByStringPrefix() { + // "/opt/drv-evil" starts with the string "/opt/drv" but is a different directory. A raw prefix + // check would authorize it. + String message = rejectionOf("/opt/drv-evil/libadbc_driver_flightsql.so", "/opt/drv"); + Assertions.assertTrue(message.contains("driver_secure_path"), message); + } + + @Test + void starAndBlankSecurePathAllowEverything() { + for (String securePath : new String[] {"*", "", " ", null}) { + Assertions.assertEquals(Paths.get("/anywhere/libadbc_driver_flightsql.so"), + AdbcDriverPathResolver.resolve("/anywhere/libadbc_driver_flightsql.so", + DRIVERS_DIR, securePath)); + } + } + + // ---- existence ---- + + @Test + void missingDriverFileProducesASelfServiceableMessage(@TempDir Path tempDir) { + Path absent = tempDir.resolve("libadbc_driver_flightsql.so"); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> AdbcDriverPathResolver.checkExists(absent, "libadbc_driver_flightsql.so")); + String message = e.getMessage(); + // Doris ships no ADBC driver, so this is the most likely first experience of the catalog type; + // the message has to answer all of "which path", "who else needs it" and "where do I get it". + Assertions.assertTrue(message.contains(absent.toString()), message); + Assertions.assertTrue(message.contains("EVERY BE"), message); + Assertions.assertTrue(message.contains("arrow-adbc"), message); + Assertions.assertTrue(message.contains("adbc_driver_flightsql"), message); + } + + @Test + void presentDriverFilePasses(@TempDir Path tempDir) throws Exception { + Path present = Files.createFile(tempDir.resolve("libadbc_driver_flightsql.so")); + Assertions.assertDoesNotThrow( + () -> AdbcDriverPathResolver.checkExists(present, "libadbc_driver_flightsql.so")); + } + + // ---- checksum ---- + + /** {@code md5sum} of the three bytes below, which is what a user would paste into the property. */ + private static final String ABC_MD5 = "900150983cd24fb0d6963f7d28e17f72"; + + private static Path driverFileContaining(Path tempDir, String content) throws Exception { + Path file = tempDir.resolve("libadbc_driver_flightsql.so"); + Files.write(file, content.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + return file; + } + + @Test + void declaredChecksumIsCheckedAgainstTheFile(@TempDir Path tempDir) throws Exception { + Path driver = driverFileContaining(tempDir, "abc"); + Assertions.assertDoesNotThrow(() -> AdbcDriverPathResolver.checkChecksum( + driver, ABC_MD5, "libadbc_driver_flightsql.so")); + } + + @Test + void checksumIsComparedWithoutRegardToCase(@TempDir Path tempDir) throws Exception { + Path driver = driverFileContaining(tempDir, "abc"); + // md5sum prints lowercase, other tools print upper; rejecting one of them would only teach + // users that the property is unreliable. + Assertions.assertDoesNotThrow(() -> AdbcDriverPathResolver.checkChecksum( + driver, ABC_MD5.toUpperCase(java.util.Locale.ROOT), "libadbc_driver_flightsql.so")); + } + + @Test + void theWrongFileIsNamedAlongWithBothChecksums(@TempDir Path tempDir) throws Exception { + Path driver = driverFileContaining(tempDir, "not the driver you meant"); + + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> AdbcDriverPathResolver.checkChecksum(driver, ABC_MD5, "libadbc_driver_flightsql.so")); + + // Both values, because the useful next step is comparing them with the file the user meant to + // deploy, and the path, because on this catalog type the usual mistake is a stale copy. + String message = e.getMessage(); + Assertions.assertTrue(message.contains(ABC_MD5), message); + Assertions.assertTrue(message.contains(driver.toString()), message); + Assertions.assertTrue(message.contains(AdbcConnectorProperties.DRIVER_CHECKSUM), message); + } + + @Test + void noChecksumMeansNoCheck(@TempDir Path tempDir) throws Exception { + Path driver = driverFileContaining(tempDir, "abc"); + for (String absent : new String[] {null, "", " "}) { + Assertions.assertDoesNotThrow(() -> AdbcDriverPathResolver.checkChecksum( + driver, absent, "libadbc_driver_flightsql.so"), String.valueOf(absent)); + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheNativeTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheNativeTest.java new file mode 100644 index 00000000000000..cd0e975c499c7e --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheNativeTest.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import org.apache.arrow.adbc.core.AdbcStatement; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The metadata path with a catalog-level cache in front of it, against the real SQLite driver. + * + *

Each test changes the source behind Doris's back and then asks what Doris sees. That is the only + * evidence that says whether an answer came from memory or from the driver, and unlike a call counter it + * cannot be satisfied by a cache that stores things it never reads. + * + *

Every {@code metadata()} call stands for one statement: the engine builds a fresh + * {@link AdbcConnectorMetadata} per statement, and the cache is what they share. + * + *

Skips loudly when thirdparty's native libraries are absent -- see {@link AdbcNativeTestSupport}. + */ +class AdbcMetadataCacheNativeTest { + + private final AdbcMetadataCache cache = new AdbcMetadataCache(Map.of()); + + private static AdbcClient sqliteClient(Path dbFile) { + return new AdbcClient(AdbcNativeTestSupport.sqliteDriver(), "libadbc_driver_sqlite.so", + null, "file:" + dbFile, null, null, Map.of()); + } + + /** One statement's view of the catalog. Separate objects, one shared cache -- as in production. */ + private AdbcConnectorMetadata metadata(AdbcClient client) { + return new AdbcConnectorMetadata(client, new AdbcSchemaStrategy(), Map.of(), + AdbcDialectRegistry::defaultDialect, cache); + } + + private static void execute(AdbcClient client, String... statements) { + client.withConnection(connection -> { + for (String sql : statements) { + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + statement.executeUpdate(); + } + } + return null; + }); + } + + /** SQLite derives its Arrow types from the values present, so a row is needed for the types to be real. */ + private static void seed(AdbcClient client) { + execute(client, + "CREATE TABLE t1 (c_int INTEGER, c_txt TEXT)", + "INSERT INTO t1 VALUES (1, 'a')"); + } + + private static List columnNames(ConnectorTableSchema schema) { + List names = new ArrayList<>(); + for (ConnectorColumn column : schema.getColumns()) { + names.add(column.getName()); + } + return names; + } + + private List columnsOf(AdbcClient client, String table) { + ConnectorTableHandle handle = metadata(client).getTableHandle(null, "main", table).orElseThrow(); + return columnNames(metadata(client).getTableSchema(null, handle)); + } + + @Test + void theNextStatementReadsTheSchemaTheLastOneAlreadyPaidFor(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("cache.db"))) { + seed(client); + Assertions.assertEquals(List.of("c_int", "c_txt"), columnsOf(client, "t1")); + + execute(client, "ALTER TABLE t1 ADD COLUMN c_added INTEGER"); + + // The column really is there now -- the source changed and Doris was not told. Serving the + // remembered shape is the whole point; noticing the change here would mean nothing was cached. + Assertions.assertEquals(List.of("c_int", "c_txt"), columnsOf(client, "t1")); + } + } + + @Test + void refreshTableIsWhatMakesTheAlteredColumnsVisible(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("cache.db"))) { + seed(client); + columnsOf(client, "t1"); + execute(client, "ALTER TABLE t1 ADD COLUMN c_added INTEGER"); + + cache.invalidateTable("main", "t1"); + + Assertions.assertEquals(List.of("c_int", "c_txt", "c_added"), columnsOf(client, "t1")); + } + } + + /** + * Decision C. Reading the listing from memory is fine; concluding from memory that a name does not exist + * is not. A user who just created a table and is told it is not there has no way to tell that from a + * typo, and no reason to suspect a cache. + */ + @Test + void tableCreatedAfterTheListingWasCachedIsStillFound(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("cache.db"))) { + seed(client); + metadata(client).listTableNames(null, "main"); + + execute(client, "CREATE TABLE t_new (a INTEGER)", "INSERT INTO t_new VALUES (1)"); + + Optional handle = metadata(client).getTableHandle(null, "main", "t_new"); + Assertions.assertTrue(handle.isPresent(), "a table created after the listing was cached must" + + " still be reachable by name"); + Assertions.assertEquals(List.of("a"), columnNames(metadata(client) + .getTableSchema(null, handle.get()))); + } + } + + @Test + void missingTableIsStillMissingAfterTheListingIsReRead(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("cache.db"))) { + seed(client); + metadata(client).listTableNames(null, "main"); + + // Re-reading the listing is a last chance to find the name, not a way to accept any name. + Assertions.assertEquals(Optional.empty(), + metadata(client).getTableHandle(null, "main", "no_such_table")); + } + } + + /** + * The listing methods stay live however much is remembered. They read like reports, but the engine loads + * its own name cache from them and then decides from that whether a table exists at all -- including the + * re-list it does as a last chance for a name it has never seen. A cached answer here would turn that + * re-check into a formality and leave a table created a moment ago unreachable. + */ + @Test + void listingTheTablesAlwaysAsksTheSource(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("cache.db"))) { + seed(client); + Assertions.assertEquals(List.of("t1"), metadata(client).listTableNames(null, "main")); + + execute(client, "CREATE TABLE t_new (a INTEGER)"); + + Assertions.assertEquals(List.of("t1", "t_new"), metadata(client).listTableNames(null, "main")); + } + } + + @Test + void listingTheDatabasesAlwaysAsksTheSource(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("cache.db"))) { + seed(client); + // Plant a database the source does not have. Creating one behind Doris's back is what this + // stands in for -- SQLite gains a catalog only by ATTACH, which does not outlive the connection + // it ran on -- and it fails the same way: anything answered from memory shows the ghost. + cache.namespaces(() -> List.of(new AdbcNamespace("ghost", ""))); + + Assertions.assertEquals(List.of("main"), metadata(client).listDatabaseNames(null)); + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java new file mode 100644 index 00000000000000..331869772011de --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java @@ -0,0 +1,334 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.cache.CacheSpec; + +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * What the catalog remembers between statements, and what each REFRESH is supposed to forget. + * + *

Every test counts how often the source was asked, because the point of this cache is not that the + * answers are right -- they were right before it existed -- but that the remote calls stop happening. A test + * that only compared returned values would pass against a cache that never caches. + */ +class AdbcMetadataCacheTest { + + private static final AdbcNamespace MAIN = new AdbcNamespace("main", ""); + private static final AdbcNamespace OTHER = new AdbcNamespace("other", ""); + private static final AdbcTableHandle MAIN_T1 = new AdbcTableHandle(MAIN, "t1"); + private static final AdbcTableHandle MAIN_T2 = new AdbcTableHandle(MAIN, "t2"); + private static final AdbcTableHandle OTHER_T1 = new AdbcTableHandle(OTHER, "t1"); + + private static final Schema SCHEMA = new Schema(Collections.emptyList()); + + private static AdbcMetadataCache cacheWith(String... keysAndValues) { + Map properties = new java.util.HashMap<>(); + for (int i = 0; i < keysAndValues.length; i += 2) { + properties.put(keysAndValues[i], keysAndValues[i + 1]); + } + return new AdbcMetadataCache(properties); + } + + // ========= what is remembered ========= + + @Test + void theDatabaseListingIsReadFromTheSourceOnce() { + AdbcMetadataCache cache = cacheWith(); + Counting> source = new Counting<>(List.of(MAIN)); + + Assertions.assertEquals(List.of(MAIN), cache.namespaces(source)); + Assertions.assertEquals(List.of(MAIN), cache.namespaces(source)); + + Assertions.assertEquals(1, source.calls); + } + + @Test + void tableListingsAreRememberedPerDatabase() { + AdbcMetadataCache cache = cacheWith(); + Counting> main = new Counting<>(List.of("t1")); + Counting> other = new Counting<>(List.of("t9")); + + Assertions.assertEquals(List.of("t1"), cache.tableNames(MAIN, main)); + Assertions.assertEquals(List.of("t9"), cache.tableNames(OTHER, other)); + cache.tableNames(MAIN, main); + cache.tableNames(OTHER, other); + + // One key per database: a shared key would serve one database's tables under another's name. + Assertions.assertEquals(1, main.calls); + Assertions.assertEquals(1, other.calls); + } + + @Test + void schemasAreRememberedPerTable() { + AdbcMetadataCache cache = cacheWith(); + Counting t1 = new Counting<>(SCHEMA); + Counting t2 = new Counting<>(SCHEMA); + + cache.tableSchema(MAIN_T1, t1); + cache.tableSchema(MAIN_T2, t2); + cache.tableSchema(MAIN_T1, t1); + cache.tableSchema(MAIN_T2, t2); + + Assertions.assertEquals(1, t1.calls); + Assertions.assertEquals(1, t2.calls); + } + + @Test + void twoTablesOfTheSameNameInDifferentDatabasesAreNotConfused() { + AdbcMetadataCache cache = cacheWith(); + Counting main = new Counting<>(SCHEMA); + Counting other = new Counting<>(SCHEMA); + + cache.tableSchema(MAIN_T1, main); + cache.tableSchema(OTHER_T1, other); + + // Keying on the bare table name would serve main.t1's columns for other.t1. + Assertions.assertEquals(1, main.calls); + Assertions.assertEquals(1, other.calls); + } + + // ========= what a REFRESH forgets ========= + + @Test + void refreshTableForgetsThatTablesSchema() { + AdbcMetadataCache cache = cacheWith(); + Counting t1 = new Counting<>(SCHEMA); + cache.tableSchema(MAIN_T1, t1); + + cache.invalidateTable("main", "t1"); + cache.tableSchema(MAIN_T1, t1); + + Assertions.assertEquals(2, t1.calls); + } + + /** + * Decision D. A user typing REFRESH TABLE means "get to know this table again", and a table that was + * created remotely after the listing was cached cannot be got to know at all while the listing that + * omits it survives -- REFRESH TABLE would have no way to ever make it appear. + */ + @Test + void refreshTableAlsoForgetsItsDatabaseTableListing() { + AdbcMetadataCache cache = cacheWith(); + Counting> listing = new Counting<>(List.of("t1")); + cache.tableNames(MAIN, listing); + + cache.invalidateTable("main", "t1"); + cache.tableNames(MAIN, listing); + + Assertions.assertEquals(2, listing.calls); + } + + @Test + void refreshTableLeavesTheRestOfTheCatalogAlone() { + AdbcMetadataCache cache = cacheWith(); + Counting sibling = new Counting<>(SCHEMA); + Counting otherDb = new Counting<>(SCHEMA); + Counting> otherListing = new Counting<>(List.of("t9")); + Counting> databases = new Counting<>(List.of(MAIN, OTHER)); + cache.tableSchema(MAIN_T2, sibling); + cache.tableSchema(OTHER_T1, otherDb); + cache.tableNames(OTHER, otherListing); + cache.namespaces(databases); + + cache.invalidateTable("main", "t1"); + + cache.tableSchema(MAIN_T2, sibling); + cache.tableSchema(OTHER_T1, otherDb); + cache.tableNames(OTHER, otherListing); + cache.namespaces(databases); + Assertions.assertEquals(1, sibling.calls); + Assertions.assertEquals(1, otherDb.calls); + Assertions.assertEquals(1, otherListing.calls); + Assertions.assertEquals(1, databases.calls); + } + + @Test + void refreshDatabaseForgetsItsListingAndEverySchemaInIt() { + AdbcMetadataCache cache = cacheWith(); + Counting> listing = new Counting<>(List.of("t1", "t2")); + Counting t1 = new Counting<>(SCHEMA); + Counting t2 = new Counting<>(SCHEMA); + cache.tableNames(MAIN, listing); + cache.tableSchema(MAIN_T1, t1); + cache.tableSchema(MAIN_T2, t2); + + cache.invalidateDb("main"); + + cache.tableNames(MAIN, listing); + cache.tableSchema(MAIN_T1, t1); + cache.tableSchema(MAIN_T2, t2); + Assertions.assertEquals(2, listing.calls); + Assertions.assertEquals(2, t1.calls); + Assertions.assertEquals(2, t2.calls); + } + + @Test + void refreshDatabaseLeavesAnotherDatabaseAlone() { + AdbcMetadataCache cache = cacheWith(); + Counting otherDb = new Counting<>(SCHEMA); + Counting> otherListing = new Counting<>(List.of("t9")); + cache.tableSchema(OTHER_T1, otherDb); + cache.tableNames(OTHER, otherListing); + + cache.invalidateDb("main"); + + cache.tableSchema(OTHER_T1, otherDb); + cache.tableNames(OTHER, otherListing); + Assertions.assertEquals(1, otherDb.calls); + Assertions.assertEquals(1, otherListing.calls); + } + + /** + * Only REFRESH CATALOG forgets which databases exist: it is the one statement a user reaches for when + * the shape of the catalog itself changed, and it is also the only one that names no database to + * invalidate. + */ + @Test + void refreshCatalogForgetsEverythingIncludingWhichDatabasesExist() { + AdbcMetadataCache cache = cacheWith(); + Counting> databases = new Counting<>(List.of(MAIN)); + Counting> listing = new Counting<>(List.of("t1")); + Counting t1 = new Counting<>(SCHEMA); + cache.namespaces(databases); + cache.tableNames(MAIN, listing); + cache.tableSchema(MAIN_T1, t1); + + cache.invalidateAll(); + + cache.namespaces(databases); + cache.tableNames(MAIN, listing); + cache.tableSchema(MAIN_T1, t1); + Assertions.assertEquals(2, databases.calls); + Assertions.assertEquals(2, listing.calls); + Assertions.assertEquals(2, t1.calls); + } + + // ========= never answering "no such object" from memory (decision C) ========= + + @Test + void tableListingCanBeReReadOnDemand() { + AdbcMetadataCache cache = cacheWith(); + Counting> listing = new Counting<>(List.of("t1")); + cache.tableNames(MAIN, listing); + + cache.reloadTableNames(MAIN, listing); + + // The caller reaches for this having failed to find a name, so a cached answer is exactly what it + // must not get; the fresh answer then replaces the stale one for everybody else. + Assertions.assertEquals(2, listing.calls); + cache.tableNames(MAIN, listing); + Assertions.assertEquals(2, listing.calls); + } + + @Test + void theDatabaseListingCanBeReReadOnDemand() { + AdbcMetadataCache cache = cacheWith(); + Counting> databases = new Counting<>(List.of(MAIN)); + cache.namespaces(databases); + + cache.reloadNamespaces(databases); + + Assertions.assertEquals(2, databases.calls); + cache.namespaces(databases); + Assertions.assertEquals(2, databases.calls); + } + + // ========= configuration ========= + + /** + * Decision B, and the one number here that differs from every other connector: the framework default is + * 24 hours, ADBC uses 10 minutes. An ADBC source is another live database rather than a warehouse of + * files, so its tables change under Doris far more often, and this bounds how long a user who forgot to + * REFRESH stays wrong. It is a decision, not an oversight -- do not "fix" it back to the framework value. + */ + @Test + void metadataIsForgottenAfterTenMinutesByDefault() { + CacheSpec spec = AdbcMetadataCache.cacheSpec(Map.of()); + + Assertions.assertTrue(spec.isEnable()); + Assertions.assertEquals(600L, spec.getTtlSecond()); + Assertions.assertEquals(1000L, spec.getCapacity()); + } + + @Test + void everyCacheKnobIsReadFromTheCatalogProperties() { + CacheSpec spec = AdbcMetadataCache.cacheSpec(Map.of( + "meta.cache.adbc.metadata.enable", "false", + "meta.cache.adbc.metadata.ttl-second", "42", + "meta.cache.adbc.metadata.capacity", "7")); + + Assertions.assertFalse(spec.isEnable()); + Assertions.assertEquals(42L, spec.getTtlSecond()); + Assertions.assertEquals(7L, spec.getCapacity()); + } + + @Test + void turningTheCacheOffSendsEveryReadBackToTheSource() { + AdbcMetadataCache cache = cacheWith("meta.cache.adbc.metadata.enable", "false"); + Counting> databases = new Counting<>(List.of(MAIN)); + Counting> listing = new Counting<>(List.of("t1")); + Counting t1 = new Counting<>(SCHEMA); + + for (int i = 0; i < 2; i++) { + cache.namespaces(databases); + cache.tableNames(MAIN, listing); + cache.tableSchema(MAIN_T1, t1); + } + + Assertions.assertEquals(2, databases.calls); + Assertions.assertEquals(2, listing.calls); + Assertions.assertEquals(2, t1.calls); + } + + @Test + void zeroTtlAlsoMeansNoCaching() { + AdbcMetadataCache cache = cacheWith("meta.cache.adbc.metadata.ttl-second", "0"); + Counting t1 = new Counting<>(SCHEMA); + + cache.tableSchema(MAIN_T1, t1); + cache.tableSchema(MAIN_T1, t1); + + Assertions.assertEquals(2, t1.calls); + } + + /** A supplier that answers the same thing every time and records how often it was asked. */ + private static final class Counting implements Supplier { + + private final T value; + private int calls; + + private Counting(T value) { + this.value = value; + } + + @Override + public T get() { + calls++; + return value; + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcNamespaceTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcNamespaceTest.java new file mode 100644 index 00000000000000..91bc30901bdfb4 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcNamespaceTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * The three-level to two-level projection. Each case below is a real source shape: PostgreSQL populates + * both levels, MySQL only the schema, SQLite only the catalog. + */ +class AdbcNamespaceTest { + + @Test + void sourceWithOnlyASchemaLevelUsesTheSchema() { + Assertions.assertEquals("testdb", new AdbcNamespace("", "testdb").dorisDatabaseName()); + Assertions.assertEquals("testdb", new AdbcNamespace(null, "testdb").dorisDatabaseName()); + } + + @Test + void sourceWithOnlyACatalogLevelUsesTheCatalog() { + Assertions.assertEquals("main", new AdbcNamespace("main", "").dorisDatabaseName()); + Assertions.assertEquals("main", new AdbcNamespace("main", null).dorisDatabaseName()); + } + + @Test + void whenBothLevelsExistTheSchemaWins() { + // uri pins the remote catalog, so it is identical for every namespace in this Doris catalog: + // including it in the name would add nothing and cost a delimiter that cannot be parsed back. + Assertions.assertEquals("public", new AdbcNamespace("mydb", "public").dorisDatabaseName()); + } + + @Test + void nullAndEmptyAreTheSameAbsentLevel() { + // Drivers disagree: SQLite reports the missing schema level as "", others as null. If the two were + // treated differently, the same source would map to a different database name depending on which + // driver build served it. + Assertions.assertEquals(new AdbcNamespace("main", null), new AdbcNamespace("main", "")); + Assertions.assertEquals(new AdbcNamespace("main", null).hashCode(), + new AdbcNamespace("main", "").hashCode()); + } + + @Test + void namespaceWithNeitherLevelFailsLoud() { + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> new AdbcNamespace("", "").dorisDatabaseName()); + Assertions.assertTrue(e.getMessage().contains("Doris database"), e.getMessage()); + } + + @Test + void theRemotePartsSurviveACatalogNameContainingADot() { + // The display name is never parsed back, which is what lets a remote catalog contain a dot. If any + // code path re-derived the remote parts from the Doris name, "MY.DB"/"PUBLIC" would split wrong and + // the pushed-down SQL would address a table that does not exist. + AdbcNamespace namespace = new AdbcNamespace("MY.DB", "PUBLIC"); + + Assertions.assertEquals("PUBLIC", namespace.dorisDatabaseName()); + Assertions.assertEquals("MY.DB", namespace.getRemoteCatalog()); + Assertions.assertEquals("PUBLIC", namespace.getRemoteDbSchema()); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcNativeTestSupport.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcNativeTestSupport.java new file mode 100644 index 00000000000000..72d05bd4478b3e --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcNativeTestSupport.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.junit.jupiter.api.Assumptions; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Locates the ADBC native libraries thirdparty builds, so the connector can be exercised against a real + * driver instead of a stand-in. + * + *

The SQLite driver is not the source this connector targets, but it travels the identical path: + * Java -> JNI bridge -> C driver manager -> a driver {@code .so}. That path is the whole reason FE uses the + * JNI driver at all, so a test that stubs it out would leave the one risky part unexercised. What SQLite + * cannot cover (it answers {@code NOT_IMPLEMENTED} to {@code executePartitioned} and {@code executeSchema}) + * is exactly the set of driver capability gaps the connector must degrade around, so it doubles as the + * negative sample for those. + * + *

When the libraries are absent the tests SKIP, and say so loudly: a skipped run has verified nothing + * about the native path and must not be read as a pass. + */ +final class AdbcNativeTestSupport { + + static final String JNI_LIBRARY_PATH_PROPERTY = "arrow.adbc.driver.jni.library.path"; + + private static final String JNI_LIBRARY = "libadbc_driver_jni.so"; + private static final String SQLITE_DRIVER = "libadbc_driver_sqlite.so"; + + private AdbcNativeTestSupport() { + } + + /** + * Returns the directory holding the native libraries, or skips the calling test. + * + *

Note the JNI property names a DIRECTORY, not a file: the resolver inside adbc-driver-jni appends + * {@code System.mapLibraryName("adbc_driver_jni")} to it. Pointing it at the {@code .so} itself yields a + * path like {@code .../libadbc_driver_jni.so/libadbc_driver_jni.so} and a load failure. + */ + static Path requireNativeLibraryDir() { + Path dir = findNativeLibraryDir(); + Assumptions.assumeTrue(dir != null, + "SKIPPED: " + JNI_LIBRARY + " / " + SQLITE_DRIVER + " were not found. Build them with" + + " 'cd thirdparty && ./build-thirdparty.sh arrow_adbc', or set DORIS_THIRDPARTY." + + " THE ADBC NATIVE PATH IS NOT BEING EXERCISED BY THIS RUN."); + System.setProperty(JNI_LIBRARY_PATH_PROPERTY, dir.toString()); + return dir; + } + + static Path sqliteDriver() { + return requireNativeLibraryDir().resolve(SQLITE_DRIVER); + } + + private static Path findNativeLibraryDir() { + String fromEnv = System.getenv("DORIS_THIRDPARTY"); + if (fromEnv != null && !fromEnv.isEmpty()) { + Path candidate = Paths.get(fromEnv, "installed", "lib64"); + if (hasBothLibraries(candidate)) { + return candidate; + } + } + // Walk up from the module directory to the repository root, which is where thirdparty sits after a + // local './build-thirdparty.sh' run. + Path here = Paths.get("").toAbsolutePath(); + while (here != null) { + Path candidate = here.resolve("thirdparty").resolve("installed").resolve("lib64"); + if (hasBothLibraries(candidate)) { + return candidate; + } + here = here.getParent(); + } + return null; + } + + private static boolean hasBothLibraries(Path dir) { + return Files.isReadable(dir.resolve(JNI_LIBRARY)) && Files.isReadable(dir.resolve(SQLITE_DRIVER)); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java new file mode 100644 index 00000000000000..450d4a5222d485 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcStatement; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.channels.Channels; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Reads {@code getObjects} results produced by the real SQLite driver, so the nested-Arrow parsing is + * checked against a shape a driver actually emits rather than one this test invented. A hand-built result + * covers the case a real driver will not produce on demand: a non-standard schema. + */ +class AdbcObjectsReaderTest { + + private static AdbcClient sqliteClient(Path dbFile) { + return new AdbcClient(AdbcNativeTestSupport.sqliteDriver(), "libadbc_driver_sqlite.so", + null, "file:" + dbFile, null, null, Map.of()); + } + + private static void seed(AdbcClient client) { + client.withConnection(connection -> { + for (String sql : new String[] { + "CREATE TABLE IF NOT EXISTS t1 (c_int INTEGER, c_txt TEXT)", + "CREATE TABLE IF NOT EXISTS t2 (a INTEGER)", + "CREATE VIEW IF NOT EXISTS v1 AS SELECT * FROM t1"}) { + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + statement.executeUpdate(); + } + } + return null; + }); + } + + @Test + void readsTheNamespaceASourceWithNoSchemaLayerReports(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("objects.db"))) { + seed(client); + + List namespaces = client.withConnection(connection -> { + try (ArrowReader reader = connection.getObjects( + AdbcConnection.GetObjectsDepth.DB_SCHEMAS, null, null, null, null, null)) { + return AdbcObjectsReader.readNamespaces(reader); + } + }); + + // SQLite has catalogs but no schema layer, and reports the missing level as an EMPTY STRING + // rather than null -- treating only null as "absent" would produce a Doris database named "". + Assertions.assertEquals(1, namespaces.size(), namespaces.toString()); + Assertions.assertEquals("main", namespaces.get(0).getRemoteCatalog()); + Assertions.assertEquals("", namespaces.get(0).getRemoteDbSchema()); + Assertions.assertEquals("main", namespaces.get(0).dorisDatabaseName()); + } + } + + @Test + void readsTableNamesAndHonoursTheTableTypeFilter(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("objects.db"))) { + seed(client); + AdbcNamespace main = new AdbcNamespace("main", ""); + + List tables = client.withConnection(connection -> { + try (ArrowReader reader = connection.getObjects( + AdbcConnection.GetObjectsDepth.TABLES, null, null, null, + new String[] {"table"}, null)) { + return AdbcObjectsReader.readTableNames(reader, main); + } + }); + + // v1 is a view. Doris presents no views for ADBC catalogs, so listing it would produce a table + // that DESC and SELECT both fail on. + Assertions.assertEquals(List.of("t1", "t2"), tables); + } + } + + @Test + void viewsAreDroppedEvenWhenTheSourceIgnoresTheTypeFilter(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("objects.db"))) { + seed(client); + AdbcNamespace main = new AdbcNamespace("main", ""); + + // Asking with no type filter reproduces, through a real driver, what a Doris source does to the + // filter the connector does send: its Flight SQL endpoint recognises only the literal "VIEW" and + // answers "table" with everything. The guarantee has to survive that, so it cannot live in the + // request -- v1 must be gone because of the table_type that came back with it. + List tables = client.withConnection(connection -> { + try (ArrowReader reader = connection.getObjects( + AdbcConnection.GetObjectsDepth.TABLES, null, null, null, null, null)) { + return AdbcObjectsReader.readTableNames(reader, main); + } + }); + + Assertions.assertEquals(List.of("t1", "t2"), tables); + } + } + + @Test + void tablesOfOtherNamespacesAreNotListed(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("objects.db"))) { + seed(client); + // getObjects filters are advisory: a driver may answer with everything it has. Returning rows + // that belong to a different namespace would list those tables under the wrong database. + AdbcNamespace other = new AdbcNamespace("someother", "public"); + + List tables = client.withConnection(connection -> { + try (ArrowReader reader = connection.getObjects( + AdbcConnection.GetObjectsDepth.TABLES, null, null, null, + new String[] {"table"}, null)) { + return AdbcObjectsReader.readTableNames(reader, other); + } + }); + + Assertions.assertEquals(List.of(), tables); + } + } + + @Test + void theTypeNamesRealSourcesUseAreClassifiedTheWayTheyMean() { + // A Doris source spells a table "BASE TABLE" -- getting that one wrong does not hide a view, it + // hides EVERY table of every ADBC catalog pointed at Doris, and the catalog just looks empty. + Assertions.assertTrue(AdbcObjectsReader.isBaseTable("BASE TABLE")); + // ...and its materialized views come back under the same name, which is right: those are storage + // that can be scanned, not a query wearing a table's name. + Assertions.assertTrue(AdbcObjectsReader.isBaseTable("table")); + + Assertions.assertFalse(AdbcObjectsReader.isBaseTable("VIEW")); + Assertions.assertFalse(AdbcObjectsReader.isBaseTable("view")); + // What Doris calls its information_schema tables. Reading one through ADBC is not supported either. + Assertions.assertFalse(AdbcObjectsReader.isBaseTable("SYSTEM VIEW")); + + // Dropped, deliberately. The forgiving rule -- keep what is not recognised -- is the wrong one here: + // a leaked view scans fine through ADBC, so it never announces itself. A source whose tables land + // here lists nothing instead, which does. + Assertions.assertFalse(AdbcObjectsReader.isBaseTable("OLAP")); + + // Saying nothing is not the same as saying something unrecognised: a source that omits the column + // stays exactly as usable as it was before this filter existed. + Assertions.assertTrue(AdbcObjectsReader.isBaseTable(null)); + Assertions.assertTrue(AdbcObjectsReader.isBaseTable("")); + } + + @Test + void resultWithoutTheStandardColumnsIsRejectedByName() throws Exception { + // A driver that answers getObjects with its own shape must fail with something that says so; the + // alternative is a NullPointerException from deep inside the reader. + Schema schema = new Schema(List.of( + new Field("something_else", FieldType.nullable(ArrowType.Utf8.INSTANCE), null))); + try (BufferAllocator allocator = new RootAllocator(); + ArrowReader reader = oneRowReader(allocator, schema)) { + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> AdbcObjectsReader.readNamespaces(reader)); + Assertions.assertTrue(e.getMessage().contains("catalog_name"), e.getMessage()); + Assertions.assertTrue(e.getMessage().contains("something_else"), e.getMessage()); + } + } + + private static ArrowReader oneRowReader(BufferAllocator allocator, Schema schema) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(bytes))) { + VarCharVector vector = (VarCharVector) root.getVector(0); + vector.allocateNew(1); + vector.setSafe(0, "x".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + root.setRowCount(1); + writer.start(); + writer.writeBatch(); + writer.end(); + } + return new ArrowStreamReader(new ByteArrayInputStream(bytes.toByteArray()), allocator); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcPartitionedReadNativeTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcPartitionedReadNativeTest.java new file mode 100644 index 00000000000000..1d33a88176884b --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcPartitionedReadNativeTest.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.NamedColumnHandle; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.apache.arrow.adbc.core.AdbcStatement; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Puts the partitioned-read downgrade in front of a real driver. + * + *

The unit tests script a driver that answers {@code NOT_IMPLEMENTED}, which proves the connector reacts + * correctly to that status -- but the status itself is an assumption those tests cannot check. SQLite is a + * driver that genuinely has no partitioned execution, so this is the one place that shows the connector + * recognizes a real refusal, arriving through the real JNI bridge and C driver manager, rather than a + * refusal shaped the way the test author expected. + * + *

The other half -- a driver that DOES partition -- needs a live Flight SQL source and lives in the + * regression suite, not here. + * + *

Skips loudly without the native libraries; a skipped run has verified nothing. + */ +class AdbcPartitionedReadNativeTest { + + private static final AdbcTableHandle T1 = + new AdbcTableHandle(new AdbcNamespace("main", ""), "t1"); + + private static AdbcClient sqliteClient(Path dbFile) { + return new AdbcClient(AdbcNativeTestSupport.sqliteDriver(), "libadbc_driver_sqlite.so", + null, "file:" + dbFile, null, null, Map.of()); + } + + private static void seed(AdbcClient client) { + client.withConnection(connection -> { + for (String sql : new String[] { + "CREATE TABLE t1 (id INTEGER, name TEXT)", + "INSERT INTO t1 VALUES (1, 'a')", + "INSERT INTO t1 VALUES (2, 'b')"}) { + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + statement.executeUpdate(); + } + } + return null; + }); + } + + @Test + void realDriverWithoutPartitionedExecutionPlansOneStatementInstead(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("partition.db"))) { + seed(client); + AdbcPartitionedReadSupport support = new AdbcPartitionedReadSupport(); + AdbcScanPlanProvider planner = new AdbcScanPlanProvider( + Map.of(AdbcConnectorProperties.URI, "file:" + tempDir.resolve("partition.db")), + AdbcNativeTestSupport.sqliteDriver(), + new AdbcDialectSelector( + Map.of(AdbcConnectorProperties.SQL_DIALECT, AnsiDialect.NAME)), + () -> client, support); + List columns = List.of(new NamedColumnHandle("id")); + + List ranges = planner.planScan(null, + ConnectorScanRequest.builder(T1, columns).build()); + + Assertions.assertEquals(1, ranges.size()); + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + ranges.get(0).populateRangeParams(formatDesc, new TFileRangeDesc()); + Assertions.assertEquals("SELECT \"id\" FROM \"main\".\"t1\"", + formatDesc.getAdbcParams().get("query_sql")); + // The refusal was recognized as "this driver cannot", not merely as some failure, so the next + // scan of this catalog goes straight to a statement. + Assertions.assertTrue(support.isKnownUnsupported(), + "a real NOT_IMPLEMENTED must be remembered, or every scan pays for the round trip"); + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcQueryBuilderNativeTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcQueryBuilderNativeTest.java new file mode 100644 index 00000000000000..cb25ffabb3a69a --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcQueryBuilderNativeTest.java @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.NamedColumnHandle; +import org.apache.doris.connector.api.pushdown.ConnectorAnd; +import org.apache.doris.connector.api.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.api.pushdown.ConnectorComparison; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.pushdown.ConnectorIn; +import org.apache.doris.connector.api.pushdown.ConnectorIsNull; +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; + +import org.apache.arrow.adbc.core.AdbcStatement; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Runs the generated SQL through the real SQLite ADBC driver. + * + *

Asserting the text of a statement only proves it is the text that was intended. Whether a source + * ACCEPTS it -- the quoting, the literal spellings, the placement of {@code LIMIT} -- is a different + * question, and one the ANSI dialect answers on behalf of sources nobody has tried. This is the cheapest + * available source that answers it for real. + * + *

Skips loudly without the native libraries; a skipped run has verified nothing about real SQL + * acceptance, only about string building. + */ +class AdbcQueryBuilderNativeTest { + + private static final AdbcDialect ANSI = AdbcDialectRegistry.defaultDialect(); + private static final AdbcTableHandle T1 = + new AdbcTableHandle(new AdbcNamespace("main", ""), "t1"); + + private static AdbcClient sqliteClient(Path dbFile) { + return new AdbcClient(AdbcNativeTestSupport.sqliteDriver(), "libadbc_driver_sqlite.so", + null, "file:" + dbFile, null, null, Map.of()); + } + + /** + * Values chosen so every literal form the dialect renders appears in a predicate below, and so the + * quoting matters: a column named with a reserved word and a string holding a quote. + */ + private static void seed(AdbcClient client) { + client.withConnection(connection -> { + for (String sql : new String[] { + "CREATE TABLE t1 (id INTEGER, \"select\" REAL, name TEXT)", + "INSERT INTO t1 VALUES (1, 1.5, 'a')", + "INSERT INTO t1 VALUES (2, 2.5, 'O''Brien')", + "INSERT INTO t1 VALUES (3, 3.5, NULL)"}) { + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + statement.executeUpdate(); + } + } + return null; + }); + } + + private static List columns(String... names) { + List handles = new ArrayList<>(names.length); + for (String name : names) { + handles.add(new NamedColumnHandle(name)); + } + return handles; + } + + private static ConnectorColumnRef col(String name, String type) { + return new ConnectorColumnRef(name, ConnectorType.of(type)); + } + + /** Runs the generated statement and returns its column names and row count. */ + private static Result run(AdbcClient client, List cols, + ConnectorExpression filter, long limit) { + String sql = AdbcQueryBuilder.build(ANSI, T1, cols, Optional.ofNullable(filter), limit).getSql(); + return client.withConnection(connection -> { + try (AdbcStatement statement = connection.createStatement()) { + statement.setSqlQuery(sql); + try (AdbcStatement.QueryResult queryResult = statement.executeQuery()) { + ArrowReader reader = queryResult.getReader(); + List names = new ArrayList<>(); + reader.getVectorSchemaRoot().getSchema().getFields() + .forEach(field -> names.add(field.getName())); + int rows = 0; + while (reader.loadNextBatch()) { + rows += reader.getVectorSchemaRoot().getRowCount(); + } + return new Result(sql, names, rows); + } + } + }); + } + + @Test + void theSourceAcceptsAProjectionOfQuotedIdentifiers(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("scan.db"))) { + seed(client); + // "select" is a reserved word; unquoted it is a syntax error, which is what makes this a real + // test of the quoting rather than of the string building. + Result result = run(client, columns("id", "select"), null, -1); + + Assertions.assertEquals(List.of("id", "select"), result.columnNames, result.sql); + Assertions.assertEquals(3, result.rows, result.sql); + } + } + + @Test + void theSourceReturnsOnlyTheRequestedColumns(@TempDir Path tempDir) { + // BE rejects any column it did not ask for, so this is the property the scan depends on -- not + // merely that the query runs. + try (AdbcClient client = sqliteClient(tempDir.resolve("proj.db"))) { + seed(client); + Assertions.assertEquals(List.of("id"), run(client, columns("id"), null, -1).columnNames); + } + } + + @Test + void theSourceAcceptsTheNumericAndStringLiteralsTheDialectWrites(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("lit.db"))) { + seed(client); + + Assertions.assertEquals(2, run(client, columns("id"), + new ConnectorComparison(ConnectorComparison.Operator.GT, + col("id", "INT"), ConnectorLiteral.ofLong(1)), -1).rows); + Assertions.assertEquals(1, run(client, columns("id"), + new ConnectorComparison(ConnectorComparison.Operator.LT, + col("select", "DOUBLE"), ConnectorLiteral.ofDouble(2.0d)), -1).rows); + // The escaped quote survives the round trip to the source, rather than ending the literal. + Assertions.assertEquals(1, run(client, columns("id"), + new ConnectorComparison(ConnectorComparison.Operator.EQ, + col("name", "STRING"), ConnectorLiteral.ofString("O'Brien")), -1).rows); + } + } + + @Test + void theSourceAcceptsNullTestsInListsAndConjunctions(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("pred.db"))) { + seed(client); + + Assertions.assertEquals(1, run(client, columns("id"), + new ConnectorIsNull(col("name", "STRING"), false), -1).rows); + Assertions.assertEquals(2, run(client, columns("id"), + new ConnectorIn(col("id", "INT"), + List.of(ConnectorLiteral.ofLong(1), ConnectorLiteral.ofLong(3)), false), + -1).rows); + Assertions.assertEquals(1, run(client, columns("id"), new ConnectorAnd(List.of( + new ConnectorComparison(ConnectorComparison.Operator.GE, + col("id", "INT"), ConnectorLiteral.ofLong(2)), + new ConnectorIsNull(col("name", "STRING"), true))), -1).rows); + } + } + + @Test + void theSourceAcceptsTheLimitClauseWhereTheBuilderPutsIt(@TempDir Path tempDir) { + try (AdbcClient client = sqliteClient(tempDir.resolve("limit.db"))) { + seed(client); + Assertions.assertEquals(2, run(client, columns("id"), null, 2).rows); + Assertions.assertEquals(1, run(client, columns("id"), + new ConnectorComparison(ConnectorComparison.Operator.GT, + col("id", "INT"), ConnectorLiteral.ofLong(1)), 1).rows); + } + } + + @Test + void theSourceAcceptsTheCountOnlyProjection(@TempDir Path tempDir) { + // What a pushed-down COUNT(*) sends: one narrow column per row, no table values. + try (AdbcClient client = sqliteClient(tempDir.resolve("count.db"))) { + seed(client); + Result result = run(client, columns(), null, -1); + Assertions.assertEquals(3, result.rows, result.sql); + Assertions.assertEquals(1, result.columnNames.size(), result.sql); + } + } + + private static final class Result { + + private final String sql; + private final List columnNames; + private final int rows; + + Result(String sql, List columnNames, int rows) { + this.sql = sql; + this.columnNames = columnNames; + this.rows = rows; + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcQueryBuilderTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcQueryBuilderTest.java new file mode 100644 index 00000000000000..379234bfca7982 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcQueryBuilderTest.java @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.NamedColumnHandle; +import org.apache.doris.connector.api.pushdown.ConnectorAnd; +import org.apache.doris.connector.api.pushdown.ConnectorBetween; +import org.apache.doris.connector.api.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.api.pushdown.ConnectorComparison; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.pushdown.ConnectorFunctionCall; +import org.apache.doris.connector.api.pushdown.ConnectorIn; +import org.apache.doris.connector.api.pushdown.ConnectorIsNull; +import org.apache.doris.connector.api.pushdown.ConnectorLike; +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; +import org.apache.doris.connector.api.pushdown.ConnectorNot; +import org.apache.doris.connector.api.pushdown.ConnectorOr; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * The statement a scan sends to its source. + * + *

Two of these assertions are about rows rather than text, and are the reason this class exists: + * + *

    + *
  • the select list is never {@code *} -- BE rejects any column the query did not ask for, so a + * widened projection does not waste bandwidth, it fails the scan;
    • + *
  • a row limit is emitted only when every predicate was, because BE applies the predicates again to + * whatever comes back, and a source that truncated first would leave too few rows to filter.
    • + *
+ */ +class AdbcQueryBuilderTest { + + private static final AdbcDialect ANSI = AdbcDialectRegistry.defaultDialect(); + private static final AdbcTableHandle T1 = + new AdbcTableHandle(new AdbcNamespace("main", ""), "t1"); + + private static List columns(String... names) { + List handles = new ArrayList<>(names.length); + for (String name : names) { + handles.add(new NamedColumnHandle(name)); + } + return handles; + } + + private static ConnectorColumnRef col(String name) { + return new ConnectorColumnRef(name, ConnectorType.of("INT")); + } + + private static ConnectorLiteral num(long value) { + return ConnectorLiteral.ofLong(value); + } + + private static String sql(List cols, ConnectorExpression filter, long limit) { + return AdbcQueryBuilder.build(ANSI, T1, cols, Optional.ofNullable(filter), limit).getSql(); + } + + // ---------- projection ---------- + + @Test + void selectsExactlyTheRequestedColumns() { + Assertions.assertEquals("SELECT \"a\", \"b\" FROM \"main\".\"t1\"", + sql(columns("a", "b"), null, -1)); + } + + @Test + void neverSelectsStar() { + // BE matches returned columns to query slots by name and errors on one it did not request, so a + // star would fail the scan outright rather than merely over-read. + Assertions.assertFalse(sql(columns("a"), null, -1).contains("*")); + } + + @Test + void selectsAConstantWhenNoColumnsAreRequested() { + // An empty projection is COUNT(*) pushed down: the scan wants rows counted, no values. A constant + // is one narrow column per row instead of the whole table width, and BE counts without + // materializing when it asked for no columns. + Assertions.assertEquals("SELECT 1 FROM \"main\".\"t1\"", sql(columns(), null, -1)); + } + + // ---------- predicates that are pushed ---------- + + @Test + void pushesComparisons() { + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" > 10)", + sql(columns("a"), + new ConnectorComparison(ConnectorComparison.Operator.GT, col("a"), num(10)), -1)); + } + + @Test + void pushesNullTestsInAndOut() { + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" IS NULL)", + sql(columns("a"), new ConnectorIsNull(col("a"), false), -1)); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" IS NOT NULL)", + sql(columns("a"), new ConnectorIsNull(col("a"), true), -1)); + } + + @Test + void pushesInLists() { + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" IN (1, 2))", + sql(columns("a"), new ConnectorIn(col("a"), List.of(num(1), num(2)), false), -1)); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" NOT IN (1))", + sql(columns("a"), new ConnectorIn(col("a"), List.of(num(1)), true), -1)); + } + + @Test + void pushesBooleanConnectives() { + ConnectorExpression either = new ConnectorOr(List.of( + new ConnectorComparison(ConnectorComparison.Operator.EQ, col("a"), num(1)), + new ConnectorComparison(ConnectorComparison.Operator.EQ, col("a"), num(2)))); + Assertions.assertEquals( + "SELECT \"a\" FROM \"main\".\"t1\" WHERE ((\"a\" = 1) OR (\"a\" = 2))", + sql(columns("a"), either, -1)); + Assertions.assertEquals( + "SELECT \"a\" FROM \"main\".\"t1\" WHERE (NOT ((\"a\" = 1) OR (\"a\" = 2)))", + sql(columns("a"), new ConnectorNot(either), -1)); + } + + @Test + void splitsTopLevelConjunctsIntoSeparatePredicates() { + ConnectorExpression both = new ConnectorAnd(List.of( + new ConnectorComparison(ConnectorComparison.Operator.GT, col("a"), num(1)), + new ConnectorIsNull(col("b"), true))); + Assertions.assertEquals( + "SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" > 1) AND (\"b\" IS NOT NULL)", + sql(columns("a"), both, -1)); + } + + // ---------- predicates that are refused ---------- + + @Test + void keepsATranslatablePredicateAndDropsTheOneBesideIt() { + // All-or-nothing is per conjunct, not per query: dropping one conjunct still leaves a superset of + // the rows, and BE applies the dropped one itself. + ConnectorExpression both = new ConnectorAnd(List.of( + new ConnectorComparison(ConnectorComparison.Operator.GT, col("a"), num(1)), + new ConnectorLike(ConnectorLike.Operator.LIKE, col("b"), + ConnectorLiteral.ofString("x%")))); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" > 1)", + sql(columns("a"), both, -1)); + } + + @Test + void refusesAConjunctWholeWhenAnyPartOfItIsUntranslatable() { + // Half of a predicate is a DIFFERENT predicate, not a weaker one: emitting "a > 1" for + // "a > 1 OR f(b)" would drop the rows the function would have matched. + ConnectorExpression either = new ConnectorOr(List.of( + new ConnectorComparison(ConnectorComparison.Operator.GT, col("a"), num(1)), + new ConnectorFunctionCall("some_udf", ConnectorType.of("BOOLEAN"), List.of(col("b"))))); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", sql(columns("a"), either, -1)); + } + + @Test + void refusesTheConstructsOutsideTheConservativeSet() { + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", + sql(columns("a"), new ConnectorLike(ConnectorLike.Operator.LIKE, col("b"), + ConnectorLiteral.ofString("x%")), -1)); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", + sql(columns("a"), new ConnectorBetween(col("a"), num(1), num(9)), -1)); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", + sql(columns("a"), new ConnectorFunctionCall("abs", ConnectorType.of("INT"), + List.of(col("a"))), -1)); + } + + @Test + void refusesNullSafeEquality() { + // Standard SQL has no portable spelling for it, and every substitute differs from it on nulls -- + // which changes which rows match instead of failing. + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", + sql(columns("a"), new ConnectorComparison( + ConnectorComparison.Operator.EQ_FOR_NULL, col("a"), num(1)), -1)); + } + + @Test + void refusesAComparisonWhoseLiteralTheDialectCannotRender() { + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", + sql(columns("a"), new ConnectorComparison(ConnectorComparison.Operator.EQ, col("a"), + ConnectorLiteral.ofNull(ConnectorType.of("INT"))), -1)); + } + + // ---------- the row limit ---------- + + @Test + void pushesTheLimitWhenEveryPredicateWentWithIt() { + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" > 1) LIMIT 5", + sql(columns("a"), + new ConnectorComparison(ConnectorComparison.Operator.GT, col("a"), num(1)), 5)); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" LIMIT 5", sql(columns("a"), null, 5)); + } + + @Test + void withholdsTheLimitWhenAPredicateStayedBehind() { + // THE row-count bug this file exists to prevent: the source would return 5 rows, BE would then + // apply the predicate it kept, and the query would answer with fewer than 5. + String generated = sql(columns("a"), new ConnectorAnd(List.of( + new ConnectorComparison(ConnectorComparison.Operator.GT, col("a"), num(1)), + new ConnectorLike(ConnectorLike.Operator.LIKE, col("b"), + ConnectorLiteral.ofString("x%")))), 5); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" > 1)", generated); + Assertions.assertFalse(AdbcQueryBuilder.build(ANSI, T1, columns("a"), + Optional.of(new ConnectorLike(ConnectorLike.Operator.LIKE, col("b"), + ConnectorLiteral.ofString("x%"))), 5).isAllFiltersPushed()); + } + + @Test + void emitsNoLimitClauseWhenThereIsNoLimit() { + Assertions.assertFalse(sql(columns("a"), null, -1).contains("LIMIT")); + Assertions.assertFalse(sql(columns("a"), null, 0).contains("LIMIT")); + } + + @Test + void withholdsTheLimitFromADialectThatHasNoLimitClause() { + AdbcDialect noLimit = new BacktickDialect() { + @Override + public boolean supportsLimitClause() { + return false; + } + }; + Assertions.assertFalse( + AdbcQueryBuilder.build(noLimit, T1, columns("a"), Optional.empty(), 5) + .getSql().contains("LIMIT")); + } + + // ---------- the extension point ---------- + + @Test + void speaksAnyRegisteredDialectWithoutTheBuilderKnowingIt() { + // The invariant the dialect layer exists for: a dialect defined entirely outside this connector + // changes every part of the statement, and the builder has no branch for it. If this ever needs a + // change inside AdbcQueryBuilder, the layer has stopped doing its job. + Assertions.assertEquals("SELECT `a` FROM `t1` WHERE (`a` > <10>) LIMIT 5", + AdbcQueryBuilder.build(new BacktickDialect(), T1, columns("a"), + Optional.of(new ConnectorComparison( + ConnectorComparison.Operator.GT, col("a"), num(10))), 5).getSql()); + } + + /** Quotes, qualifies and spells literals differently from every shipped dialect. */ + private static class BacktickDialect implements AdbcDialect { + + @Override + public String name() { + return "backtick"; + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + @Override + public String qualifiedTableName(AdbcTableHandle handle) { + return quoteIdentifier(handle.getRemoteTable()); + } + + @Override + public String renderLiteral(ConnectorLiteral value) { + return "<" + value.getValue() + ">"; + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcScanPlanProviderTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcScanPlanProviderTest.java new file mode 100644 index 00000000000000..244e0fe0287f6b --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcScanPlanProviderTest.java @@ -0,0 +1,541 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.handle.NamedColumnHandle; +import org.apache.doris.connector.api.handle.PassthroughQueryTableHandle; +import org.apache.doris.connector.api.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.api.pushdown.ConnectorComparison; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcException; +import org.apache.arrow.adbc.core.AdbcStatement; +import org.apache.arrow.adbc.core.AdbcStatusCode; +import org.apache.arrow.adbc.core.PartitionDescriptor; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * Planning one scan: the remote work it produces, and the properties the scan node reads before there are + * any ranges. + * + *

Two shapes are planned. Without partitioned execution a scan is one range carrying a statement, the + * shape the JDBC connector has always had. With it, the driver splits the query and each partition becomes + * its own range for its own backend -- and the split call has already run the query remotely, which is what + * makes the failure modes here worth pinning individually. + * + *

Where a provider is built with a client supplier that throws, that is not a shortcut: it pins that the + * path under test needs no remote call at all. + */ +class AdbcScanPlanProviderTest { + + private static final AdbcTableHandle T1 = + new AdbcTableHandle(new AdbcNamespace("main", ""), "t1"); + + private static final String DRIVER = + "/opt/doris/plugins/adbc_drivers/libadbc_driver_sqlite.so"; + + /** Plans statements: partitioned execution off, so no client is needed at all. */ + private static AdbcScanPlanProvider statementProvider() { + return statementProvider(Map.of(AdbcConnectorProperties.URI, "file:/tmp/x.db")); + } + + private static AdbcScanPlanProvider statementProvider(Map properties) { + Map withoutPartitions = new LinkedHashMap<>(properties); + withoutPartitions.put(AdbcConnectorProperties.PARTITIONED_READ, "disabled"); + return provider(withoutPartitions, () -> { + throw new AssertionError("planning a statement must not open a connection"); + }, new AdbcPartitionedReadSupport()); + } + + private static AdbcScanPlanProvider provider(Map properties, + Supplier clientSupplier, AdbcPartitionedReadSupport partitionedRead) { + AdbcDialectSelector selector = new AdbcDialectSelector( + Map.of(AdbcConnectorProperties.SQL_DIALECT, AnsiDialect.NAME)); + return new AdbcScanPlanProvider(properties, Paths.get(DRIVER), selector, clientSupplier, + partitionedRead); + } + + private static List columns(String... names) { + List handles = new ArrayList<>(names.length); + for (String name : names) { + handles.add(new NamedColumnHandle(name)); + } + return handles; + } + + private static ConnectorExpression greaterThan(long value) { + return new ConnectorComparison(ConnectorComparison.Operator.GT, + new ConnectorColumnRef("a", ConnectorType.of("INT")), ConnectorLiteral.ofLong(value)); + } + + private static Map paramsOf(ConnectorScanRange range) { + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + range.populateRangeParams(formatDesc, new TFileRangeDesc()); + return formatDesc.getAdbcParams(); + } + + private static String querySqlOf(ConnectorScanRange range) { + return paramsOf(range).get("query_sql"); + } + + private static List planAll(AdbcScanPlanProvider provider) { + return provider.planScan(null, ConnectorScanRequest.builder(T1, columns("a")).build()); + } + + // -- statement planning -------------------------------------------------------------------------- + + @Test + void plansOneRangeCarryingTheStatementWhenTheDriverCannotPartition() { + List ranges = statementProvider().planScan(null, + ConnectorScanRequest.builder(T1, columns("a", "b")) + .filter(Optional.of(greaterThan(10))).limit(7).build()); + + Assertions.assertEquals(1, ranges.size()); + Assertions.assertEquals("SELECT \"a\", \"b\" FROM \"main\".\"t1\" WHERE (\"a\" > 10) LIMIT 7", + querySqlOf(ranges.get(0))); + } + + @Test + void carriesTheCatalogConnectionOntoTheRange() { + Map properties = new LinkedHashMap<>(); + properties.put(AdbcConnectorProperties.URI, "file:/tmp/x.db"); + properties.put(AdbcConnectorProperties.USER, "alice"); + properties.put(AdbcConnectorProperties.PASSWORD, "secret"); + properties.put("adbc.snowflake.sql.db", "MYDB"); + + Map params = paramsOf(planAll(statementProvider(properties)).get(0)); + + Assertions.assertEquals(DRIVER, params.get("driver_path")); + Assertions.assertEquals("file:/tmp/x.db", params.get("uri")); + Assertions.assertEquals("alice", params.get("username")); + Assertions.assertEquals("secret", params.get("password")); + Assertions.assertEquals("MYDB", params.get("adbc.snowflake.sql.db")); + } + + @Test + void refusesAHandleItDidNotCreate() { + // A table-valued function forwarding raw SQL. Not supported yet, and the message has to say which + // handle arrived or the failure reads as an internal cast error. + ConnectorTableHandle passthrough = new PassthroughQueryTableHandle("SELECT 1"); + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> statementProvider().planScan(null, + ConnectorScanRequest.builder(passthrough, columns("a")).build())); + Assertions.assertTrue( + failure.getMessage().contains(PassthroughQueryTableHandle.class.getSimpleName()), + failure.getMessage()); + } + + // -- partition planning -------------------------------------------------------------------------- + + @Test + void plansOneRangePerPartitionTheDriverReports() { + // Three partitions, three ranges: this is what lets one remote query be read by three backends. + FakeClient client = FakeClient.partitioning("p0", "p1", "p2"); + List ranges = planAll(provider( + Map.of(AdbcConnectorProperties.URI, "grpc://remote:9090"), () -> client, + new AdbcPartitionedReadSupport())); + + Assertions.assertEquals(3, ranges.size()); + List descriptors = new ArrayList<>(); + for (ConnectorScanRange range : ranges) { + Map params = paramsOf(range); + descriptors.add(params.get("partition_descriptor")); + // Each range still carries the connection: a partition is read on a fresh connection made by + // whichever backend gets it, not on the one that planned. + Assertions.assertEquals("grpc://remote:9090", params.get("uri")); + Assertions.assertNull(params.get("query_sql")); + } + Assertions.assertEquals(List.of(encode("p0"), encode("p1"), encode("p2")), descriptors); + } + + @Test + void splitsTheSameStatementItWouldOtherwiseHaveSent() { + // The pushed-down statement is what gets partitioned. Splitting an unfiltered query instead would + // make the source materialize the whole table and Doris filter it afterwards. + FakeClient client = FakeClient.partitioning("p0"); + provider(Map.of(AdbcConnectorProperties.URI, "grpc://remote:9090"), () -> client, + new AdbcPartitionedReadSupport()).planScan(null, + ConnectorScanRequest.builder(T1, columns("a")) + .filter(Optional.of(greaterThan(10))).limit(7).build()); + + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\" WHERE (\"a\" > 10) LIMIT 7", + client.lastSql); + } + + @Test + void fallsBackToOneStatementAndStopsAskingADriverThatCannotPartition() { + FakeClient client = FakeClient.notImplemented(); + AdbcPartitionedReadSupport support = new AdbcPartitionedReadSupport(); + AdbcScanPlanProvider planner = provider( + Map.of(AdbcConnectorProperties.URI, "file:/tmp/x.db"), () -> client, support); + + List first = planAll(planner); + Assertions.assertEquals(1, first.size()); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", querySqlOf(first.get(0))); + Assertions.assertTrue(support.isKnownUnsupported()); + + // The answer is a property of the driver, so a second scan must not pay for the same round trip. + Assertions.assertEquals(1, planAll(planner).size()); + Assertions.assertEquals(1, client.attempts); + } + + @Test + void letsADriverFailureThatIsNotAMissingMethodThrough() { + // Only NOT_IMPLEMENTED means "this driver cannot". Anything else means the driver tried and the + // query or the source is at fault; planning a statement instead would hide that behind a slower + // query AND run the statement on the source a second time. + FakeClient client = FakeClient.failing(AdbcStatusCode.IO); + AdbcPartitionedReadSupport support = new AdbcPartitionedReadSupport(); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> planAll(provider(Map.of(AdbcConnectorProperties.URI, "grpc://remote:9090"), + () -> client, support))); + + Assertions.assertTrue(failure.getMessage().contains("SELECT \"a\""), failure.getMessage()); + Assertions.assertFalse(support.isKnownUnsupported()); + + // A source that rejects the statement is the first thing anyone pointing this connector at + // something other than Doris will hit, and the rejection arrives in the source's own words -- + // usually a syntax error about a quote character. Nothing in that says Doris generated the SQL, + // let alone that which SQL it generates is a property the user can set. So the message has to. + Assertions.assertTrue(failure.getMessage().contains(AdbcConnectorProperties.SQL_DIALECT), + failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains(AnsiDialect.NAME), failure.getMessage()); + } + + @Test + void failsRatherThanPlanningNothingWhenTheDriverReportsNoPartitions() { + // Zero ranges would be a scan that returns no rows, silently. The partition count reflects the + // source's parallelism, not its cardinality, so it is never a legitimate way to say "empty". + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> planAll(provider(Map.of(AdbcConnectorProperties.URI, "grpc://remote:9090"), + FakeClient::partitioningNothing, new AdbcPartitionedReadSupport()))); + Assertions.assertTrue(failure.getMessage().contains("no partitions"), failure.getMessage()); + } + + @Test + void failsWhenThereAreMorePartitionsThanTheLimitAllows() { + // Not a fallback to one statement: the source has already executed the query to produce these + // descriptors, so re-planning it as a statement would execute it a second time. + Map properties = new LinkedHashMap<>(); + properties.put(AdbcConnectorProperties.URI, "grpc://remote:9090"); + properties.put(AdbcConnectorProperties.MAX_PARTITIONS, "2"); + FakeClient client = FakeClient.partitioning("p0", "p1", "p2"); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> planAll(provider(properties, () -> client, new AdbcPartitionedReadSupport()))); + + Assertions.assertTrue(failure.getMessage().contains("3 partitions"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains(AdbcConnectorProperties.MAX_PARTITIONS), + failure.getMessage()); + } + + @Test + void skipsThePartitionRoundTripWhenTheCatalogTurnedItOff() { + Map properties = new LinkedHashMap<>(); + properties.put(AdbcConnectorProperties.URI, "grpc://remote:9090"); + properties.put(AdbcConnectorProperties.PARTITIONED_READ, "disabled"); + FakeClient client = FakeClient.partitioning("p0", "p1"); + + List ranges = planAll( + provider(properties, () -> client, new AdbcPartitionedReadSupport())); + + Assertions.assertEquals(1, ranges.size()); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", querySqlOf(ranges.get(0))); + Assertions.assertEquals(0, client.attempts); + } + + // -- scan node properties (the EXPLAIN path) ----------------------------------------------------- + + @Test + void namesTheArrowReaderInTheScanNodeProperties() { + // The scan node reads the format from here, not from the range. Leaving it out routes the scan to + // the JNI scanner, which has no ADBC branch. + Assertions.assertEquals("arrow", + statementProvider().getScanNodeProperties(null, T1, columns("a"), Optional.empty()) + .get(ScanNodePropertyKeys.FILE_FORMAT_TYPE)); + } + + @Test + void explainsTheSameStatementItWillRun() { + // EXPLAIN never calls planScan, so the statement it shows is generated a second time. If the two + // ever diverge, EXPLAIN describes a query that is not the one executed. + List cols = columns("a", "b"); + Optional filter = Optional.of(greaterThan(10)); + + String planned = querySqlOf(statementProvider().planScan(null, + ConnectorScanRequest.builder(T1, cols).filter(filter).build()).get(0)); + String explained = statementProvider().getScanNodeProperties(null, T1, cols, filter) + .get(ScanNodePropertyKeys.REMOTE_QUERY); + + Assertions.assertEquals(planned, explained); + } + + @Test + void explainsWithoutARowLimitBecauseItHasNoneToShow() { + String explained = statementProvider() + .getScanNodeProperties(null, T1, columns("a"), Optional.empty()) + .get(ScanNodePropertyKeys.REMOTE_QUERY); + Assertions.assertFalse(explained.contains("LIMIT"), explained); + } + + @Test + void requiredModeFailsRatherThanDowngradingWhenTheDriverCannotPartition() { + // The whole point of the mode: a downgrade is invisible in the result -- the same rows arrive, + // from one backend instead of many -- so a test written for the partitioned path would pass while + // exercising the fallback, and the pass would be indistinguishable from the real thing. + Map properties = new LinkedHashMap<>(); + properties.put(AdbcConnectorProperties.URI, "grpc://remote:9090"); + properties.put(AdbcConnectorProperties.PARTITIONED_READ, "required"); + FakeClient client = FakeClient.notImplemented(); + AdbcPartitionedReadSupport support = new AdbcPartitionedReadSupport(); + AdbcScanPlanProvider planner = provider(properties, () -> client, support); + + DorisConnectorException failure = + Assertions.assertThrows(DorisConnectorException.class, () -> planAll(planner)); + Assertions.assertTrue(failure.getMessage().contains("required"), failure.getMessage()); + + // And it keeps failing without asking the driver again -- the answer is a property of the driver, + // so a second scan must neither pay for the round trip nor quietly succeed. + Assertions.assertThrows(DorisConnectorException.class, () -> planAll(planner)); + Assertions.assertEquals(1, client.attempts); + } + + @Test + void requiredModeCarriesTheDriversOwnAnswerIntoTheFailure() { + // Without it the message says only that partitioning is unavailable, and which layer refused -- + // driver, driver manager, or the JNI bridge -- has to be found by hand. + Map properties = new LinkedHashMap<>(); + properties.put(AdbcConnectorProperties.URI, "grpc://remote:9090"); + properties.put(AdbcConnectorProperties.PARTITIONED_READ, "required"); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> planAll(provider(properties, FakeClient::notImplemented, + new AdbcPartitionedReadSupport()))); + + Assertions.assertTrue(failure.getMessage().contains("scripted failure"), failure.getMessage()); + } + + @Test + void requiredModePlansPartitionsWhenTheDriverHasThem() { + Map properties = new LinkedHashMap<>(); + properties.put(AdbcConnectorProperties.URI, "grpc://remote:9090"); + properties.put(AdbcConnectorProperties.PARTITIONED_READ, "required"); + + Assertions.assertEquals(2, planAll(provider(properties, + () -> FakeClient.partitioning("p0", "p1"), new AdbcPartitionedReadSupport())).size()); + } + + @Test + void explainStillWorksUnderRequiredBecauseItNeverPartitions() { + // EXPLAIN deliberately does not ask for partitions, so it must not be failed for not having any: + // refusing to describe a query would help nobody, and describing it costs the source nothing. + Map properties = new LinkedHashMap<>(); + properties.put(AdbcConnectorProperties.URI, "grpc://remote:9090"); + properties.put(AdbcConnectorProperties.PARTITIONED_READ, "required"); + AdbcScanPlanProvider planner = provider(properties, () -> { + throw new AssertionError("EXPLAIN must not reach the ADBC driver"); + }, new AdbcPartitionedReadSupport()); + + List ranges = planner.planScan(null, + ConnectorScanRequest.builder(T1, columns("a")).explainOnly(true).build()); + Assertions.assertEquals(1, ranges.size()); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", querySqlOf(ranges.get(0))); + } + + @Test + void planningAnExplainAsksForNoPartitionsAndShowsTheStatement() { + // EXPLAIN reaches planScan too -- that is where its inputSplitNum comes from -- and asking the + // driver to partition IS executing the query on a Flight SQL source. An EXPLAIN that ran the + // query it was asked to describe would double the work with nothing on screen to show for it. + FakeClient client = FakeClient.partitioning("p0", "p1"); + List ranges = provider( + Map.of(AdbcConnectorProperties.URI, "grpc://remote:9090"), () -> client, + new AdbcPartitionedReadSupport()).planScan(null, + ConnectorScanRequest.builder(T1, columns("a")).explainOnly(true).build()); + + Assertions.assertEquals(0, client.attempts, "EXPLAIN executed the query on the source"); + // Still the statement a real scan would send, so what EXPLAIN shows is not fiction. + Assertions.assertEquals(1, ranges.size()); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", querySqlOf(ranges.get(0))); + } + + @Test + void neverAsksForPartitionsWhileExplaining() { + // Asking for partitions IS executing the query on a Flight SQL source, so an EXPLAIN that took + // this path would run the very query the user asked only to have described. Partitioned read is + // ON here, and the client still must not be touched. + AdbcScanPlanProvider planner = provider(Map.of(AdbcConnectorProperties.URI, "grpc://remote:9090"), + () -> { + throw new AssertionError("EXPLAIN must not reach the ADBC driver"); + }, new AdbcPartitionedReadSupport()); + + Assertions.assertDoesNotThrow( + () -> planner.getScanNodeProperties(null, T1, columns("a"), Optional.empty())); + } + + private static String encode(String descriptor) { + return Base64.getEncoder().encodeToString(descriptor.getBytes(StandardCharsets.UTF_8)); + } + + /** + * An {@link AdbcClient} that opens nothing and answers {@code executePartitioned} from a script. + * + *

Hand-written rather than mocked because it also has to enforce the negatives: any call planning + * is not allowed to make -- executing a query, updating -- fails the test where it happens. + */ + private static final class FakeClient extends AdbcClient { + + private final List descriptors; + private final AdbcStatusCode failure; + private String lastSql; + private int attempts; + + private FakeClient(List descriptors, AdbcStatusCode failure) { + super(Paths.get(DRIVER), "libadbc_driver_sqlite.so", null, "file:/tmp/x.db", null, null, + Map.of()); + this.descriptors = descriptors; + this.failure = failure; + } + + static FakeClient partitioning(String... descriptors) { + return new FakeClient(List.of(descriptors), null); + } + + static FakeClient partitioningNothing() { + return new FakeClient(Collections.emptyList(), null); + } + + static FakeClient notImplemented() { + return new FakeClient(null, AdbcStatusCode.NOT_IMPLEMENTED); + } + + static FakeClient failing(AdbcStatusCode status) { + return new FakeClient(null, status); + } + + @Override + public T withConnection(AdbcConnectionCall body) { + try { + return body.apply(new FakeConnection(this)); + } catch (AdbcException e) { + throw AdbcClient.translate(e, "ADBC operation failed"); + } catch (DorisConnectorException e) { + throw e; + } catch (Exception e) { + throw new DorisConnectorException("ADBC operation failed: " + e.getMessage(), e); + } + } + } + + private static final class FakeConnection implements AdbcConnection { + + private final FakeClient client; + + private FakeConnection(FakeClient client) { + this.client = client; + } + + @Override + public AdbcStatement createStatement() { + return new FakeStatement(client); + } + + @Override + public ArrowReader getInfo(int[] infoCodes) { + throw new AssertionError("planning a scan must not probe the driver for info"); + } + + @Override + public void close() { + } + } + + private static final class FakeStatement implements AdbcStatement { + + private final FakeClient client; + + private FakeStatement(FakeClient client) { + this.client = client; + } + + @Override + public void setSqlQuery(String query) { + client.lastSql = query; + } + + @Override + public PartitionResult executePartitioned() throws AdbcException { + client.attempts++; + if (client.failure != null) { + throw new AdbcException("scripted failure", null, client.failure, null, 0); + } + List result = new ArrayList<>(client.descriptors.size()); + for (String descriptor : client.descriptors) { + result.add(new PartitionDescriptor( + ByteBuffer.wrap(descriptor.getBytes(StandardCharsets.UTF_8)))); + } + return new PartitionResult(new Schema(Collections.emptyList()), -1, result); + } + + @Override + public QueryResult executeQuery() { + throw new AssertionError("planning a scan must not execute a query on FE"); + } + + @Override + public UpdateResult executeUpdate() { + throw new AssertionError("planning a scan must not write to the source"); + } + + @Override + public void prepare() { + throw new AssertionError("planning a scan must not prepare a statement"); + } + + @Override + public void close() { + } + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcScanRangeTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcScanRangeTest.java new file mode 100644 index 00000000000000..c25c2d0f4f2056 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcScanRangeTest.java @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * What a scan range puts on the wire. + * + *

Every assertion here is one half of a contract whose other half is C++ in + * {@code be/src/format_v2/table/adbc_reader.cpp}. Nothing checks the two agree at build time, so a key + * renamed on one side surfaces as a scan that fails at run time complaining about a missing parameter -- + * which is why the names are asserted literally rather than through the constants that produced them. + */ +class AdbcScanRangeTest { + + private static AdbcScanRange.Builder minimal() { + return new AdbcScanRange.Builder() + .driverPath("/opt/doris/plugins/adbc_drivers/libadbc_driver_sqlite.so") + .uri("file:/tmp/x.db") + .querySql("SELECT \"a\" FROM \"main\".\"t1\""); + } + + private static Map adbcParams(AdbcScanRange range) { + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + range.populateRangeParams(formatDesc, new TFileRangeDesc()); + return formatDesc.getAdbcParams(); + } + + @Test + void routesToTheAdbcReaderThroughTheArrowPath() { + // Both are required: BE enters its Arrow scanner on the format and picks the ADBC reader on the + // table format. Either one alone lands the scan in a reader with no ADBC branch. + Assertions.assertEquals("arrow", minimal().build().getFileFormat()); + Assertions.assertEquals("adbc", minimal().build().getTableFormatType()); + } + + @Test + void carriesAPlaceholderPathWithNoScheme() { + // There is no file. A scheme-bearing placeholder risks being resolved as a filesystem, so the + // shape follows the remote_doris scan node, which reads Arrow the same way. + String path = minimal().build().getPath().orElse(null); + Assertions.assertNotNull(path); + Assertions.assertFalse(path.contains("://"), path); + } + + @Test + void writesTheParametersIntoTheAdbcSlotAndNotTheJdbcOne() { + // The inherited implementation writes jdbc_params, which BE's ADBC reader never reads: the scan + // would arrive with no driver path at all. + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + minimal().build().populateRangeParams(formatDesc, new TFileRangeDesc()); + + Assertions.assertTrue(formatDesc.isSetAdbcParams()); + Assertions.assertFalse(formatDesc.isSetJdbcParams()); + } + + @Test + void usesTheParameterNamesBeLooksUp() { + Map params = adbcParams(minimal() + .driverEntrypoint("AdbcDriverInit") + .username("alice") + .password("secret") + .build()); + + Assertions.assertEquals("/opt/doris/plugins/adbc_drivers/libadbc_driver_sqlite.so", + params.get("driver_path")); + Assertions.assertEquals("AdbcDriverInit", params.get("driver_entrypoint")); + Assertions.assertEquals("file:/tmp/x.db", params.get("uri")); + Assertions.assertEquals("SELECT \"a\" FROM \"main\".\"t1\"", params.get("query_sql")); + } + + @Test + void carriesAPartitionDescriptorInsteadOfAStatement() { + Map params = adbcParams(new AdbcScanRange.Builder() + .driverPath("/opt/doris/plugins/adbc_drivers/libadbc_driver_flightsql.so") + .uri("grpc://remote:9090") + .partitionDescriptor("Zm9vYmFy") + .build()); + + Assertions.assertEquals("Zm9vYmFy", params.get("partition_descriptor")); + // No statement travels with a partition: the source already ran it, and a BE that found both + // would have to guess which one the plan meant. + Assertions.assertFalse(params.containsKey("query_sql")); + } + + @Test + void refusesToCarryBothKindsOfWorkOrNeither() { + // The two are alternatives, and BE rejects a range that says both or neither. Failing while + // planning names the bug; failing on BE reports it as one backend's problem, mid-query. + IllegalStateException both = Assertions.assertThrows(IllegalStateException.class, + () -> minimal().partitionDescriptor("Zm9vYmFy").build()); + Assertions.assertTrue(both.getMessage().contains("both"), both.getMessage()); + + IllegalStateException neither = Assertions.assertThrows(IllegalStateException.class, + () -> new AdbcScanRange.Builder() + .driverPath("/opt/doris/plugins/adbc_drivers/libadbc_driver_sqlite.so") + .uri("file:/tmp/x.db") + .build()); + Assertions.assertTrue(neither.getMessage().contains("neither"), neither.getMessage()); + } + + @Test + void sendsTheUserPropertyUnderAdbcsNameForIt() { + // The catalog property is 'user'; the ADBC option is 'username'. Sending the property name would + // leave the source unauthenticated with no complaint from either side. + Map params = adbcParams(minimal().username("alice").password("secret").build()); + Assertions.assertEquals("alice", params.get("username")); + Assertions.assertEquals("secret", params.get("password")); + Assertions.assertFalse(params.containsKey("user")); + } + + @Test + void omitsCredentialsAndEntrypointWhenThereAreNone() { + Map params = adbcParams(minimal() + .driverEntrypoint(null).username(null).password("").build()); + Assertions.assertFalse(params.containsKey("username")); + Assertions.assertFalse(params.containsKey("password")); + // An empty entrypoint is not "use the default": BE hands whatever is present to dlsym. + Assertions.assertFalse(params.containsKey("driver_entrypoint")); + } + + @Test + void passesDriverOptionsThroughWithTheirPrefixIntact() { + // The "adbc." prefix is part of the ADBC option name, not a namespace this connector added, so + // stripping it would name an option no driver knows. + Map options = new LinkedHashMap<>(); + options.put("adbc.snowflake.sql.db", "MYDB"); + Map params = adbcParams(minimal().driverOptions(options).build()); + + Assertions.assertEquals("MYDB", params.get("adbc.snowflake.sql.db")); + Assertions.assertFalse(params.containsKey("snowflake.sql.db")); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcStatementScopeTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcStatementScopeTest.java new file mode 100644 index 00000000000000..b428dc7f45434e --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcStatementScopeTest.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Guards this connector's statement-scope namespace prefix. + * + *

Statement-scoped values are keyed by (catalog, db, table, queryId) plus a connector-owned namespace + * string. Inside a heterogeneous gateway two connectors serve tables in one statement, so a namespace that + * did not start with the connector's own type name could collide with a sibling's -- and the failure would + * be one connector reading the other's memoized object, i.e. a ClassCastException or, worse, plausible + * metadata from the wrong source. The type prefix makes the namespaces distinct by construction. + */ +class AdbcStatementScopeTest { + + @Test + void namespaceIsPrefixedWithTheConnectorType() { + String type = new AdbcConnectorProvider().getType(); + + Assertions.assertTrue(AdbcStatementScope.TABLE_SCHEMA_NAMESPACE.startsWith(type + "."), + "Namespace '" + AdbcStatementScope.TABLE_SCHEMA_NAMESPACE + + "' must start with the connector type '" + type + ".'"); + } + + @Test + void nullSessionRunsTheLoaderEveryTime() { + // Offline callers (no live statement) must still get an answer, and it must be freshly loaded: + // silently memoizing outside a statement would leak one statement's view into the next. + int[] loads = {0}; + for (int i = 0; i < 3; i++) { + AdbcStatementScope.sharedTableSchema(null, + new AdbcTableHandle(new AdbcNamespace("main", ""), "t1"), + () -> { + loads[0]++; + return null; + }); + } + Assertions.assertEquals(3, loads[0]); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcTableHandleTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcTableHandleTest.java new file mode 100644 index 00000000000000..9376f4a54255f6 --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcTableHandleTest.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * The handle keeps all three remote levels, so nothing ever has to recover them from the Doris name. + */ +class AdbcTableHandleTest { + + @Test + void remotePartsSurviveACatalogNameContainingADot() { + // The case that makes re-derivation impossible rather than merely awkward: with catalog "MY.DB" and + // schema "PUBLIC", a joined name reads equally well as ("MY", "DB.PUBLIC") or ("MY.DB", "PUBLIC"), + // and the wrong reading produces SQL against a table that does not exist. Storing the parts is what + // keeps the ambiguity out of the code. + AdbcTableHandle handle = new AdbcTableHandle(new AdbcNamespace("MY.DB", "PUBLIC"), "ORDERS"); + + Assertions.assertEquals("MY.DB", handle.getRemoteCatalog()); + Assertions.assertEquals("PUBLIC", handle.getRemoteDbSchema()); + Assertions.assertEquals("ORDERS", handle.getRemoteTable()); + Assertions.assertEquals("PUBLIC", handle.getDorisDbName()); + } + + @Test + void tableNameContainingADotIsAlsoKeptWhole() { + AdbcTableHandle handle = new AdbcTableHandle(new AdbcNamespace("", "sales"), "q1.orders"); + + Assertions.assertEquals("q1.orders", handle.getRemoteTable()); + Assertions.assertEquals("sales", handle.getDorisDbName()); + } + + @Test + void namespaceRoundTripsThroughTheHandle() { + AdbcNamespace namespace = new AdbcNamespace("main", ""); + Assertions.assertEquals(namespace, new AdbcTableHandle(namespace, "t1").getNamespace()); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcTypeMapperTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcTypeMapperTest.java new file mode 100644 index 00000000000000..67f15da97e726d --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcTypeMapperTest.java @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +class AdbcTypeMapperTest { + + private static Field leaf(String name, ArrowType type) { + return new Field(name, FieldType.nullable(type), null); + } + + private static Field nested(String name, ArrowType type, List children) { + return new Field(name, FieldType.nullable(type), children); + } + + private static ConnectorType map(ArrowType type) { + return AdbcTypeMapper.toDorisType("c", leaf("c", type)); + } + + private static String rejectionOf(String columnName, Field field) { + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> AdbcTypeMapper.toDorisType(columnName, field)); + return e.getMessage(); + } + + @Test + void booleanMapsToBoolean() { + Assertions.assertEquals(ConnectorType.of("BOOLEAN"), map(ArrowType.Bool.INSTANCE)); + } + + @Test + void signedIntegersKeepTheirWidth() { + Assertions.assertEquals(ConnectorType.of("TINYINT"), map(new ArrowType.Int(8, true))); + Assertions.assertEquals(ConnectorType.of("SMALLINT"), map(new ArrowType.Int(16, true))); + Assertions.assertEquals(ConnectorType.of("INT"), map(new ArrowType.Int(32, true))); + Assertions.assertEquals(ConnectorType.of("BIGINT"), map(new ArrowType.Int(64, true))); + } + + @Test + void unsignedIntegersWidenByOneStep() { + // Doris has no unsigned integers. Keeping the width would wrap every value above the signed + // maximum into a negative number with no error anywhere -- silent data corruption, which is the + // one outcome this mapper exists to prevent. + Assertions.assertEquals(ConnectorType.of("SMALLINT"), map(new ArrowType.Int(8, false))); + Assertions.assertEquals(ConnectorType.of("INT"), map(new ArrowType.Int(16, false))); + Assertions.assertEquals(ConnectorType.of("BIGINT"), map(new ArrowType.Int(32, false))); + Assertions.assertEquals(ConnectorType.of("LARGEINT"), map(new ArrowType.Int(64, false))); + } + + @Test + void floatsMapByPrecision() { + Assertions.assertEquals(ConnectorType.of("FLOAT"), + map(new ArrowType.FloatingPoint(FloatingPointPrecision.HALF))); + Assertions.assertEquals(ConnectorType.of("FLOAT"), + map(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE))); + Assertions.assertEquals(ConnectorType.of("DOUBLE"), + map(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))); + } + + @Test + void decimalsKeepPrecisionAndScale() { + Assertions.assertEquals(ConnectorType.of("DECIMALV3", 18, 4), + map(new ArrowType.Decimal(18, 4, 128))); + Assertions.assertEquals(ConnectorType.of("DECIMALV3", 60, 10), + map(new ArrowType.Decimal(60, 10, 256))); + } + + @Test + void decimalsBeyondTheDorisLimitAreRejected() { + // 128-bit decimals cap at 38 digits and 256-bit ones at 76. Clamping instead would truncate the + // high-order digits of every value in the column. + String narrow = rejectionOf("amount", leaf("amount", new ArrowType.Decimal(39, 2, 128))); + Assertions.assertTrue(narrow.contains("amount"), narrow); + Assertions.assertTrue(narrow.contains("38"), narrow); + + String wide = rejectionOf("amount", leaf("amount", new ArrowType.Decimal(77, 2, 256))); + Assertions.assertTrue(wide.contains("76"), wide); + } + + @Test + void datesMapByUnit() { + Assertions.assertEquals(ConnectorType.of("DATEV2"), map(new ArrowType.Date(DateUnit.DAY))); + // DATEMILLI carries a time of day; DATEV2 would drop it. + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), + map(new ArrowType.Date(DateUnit.MILLISECOND))); + } + + @Test + void timestampScaleFollowsTheArrowUnit() { + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 0, 0), + map(new ArrowType.Timestamp(TimeUnit.SECOND, null))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), + map(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), + map(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null))); + } + + @Test + void nanosecondTimestampsTruncateRatherThanFail() { + // Doris tops out at 6 fractional digits. Rejecting the column would make whole tables unreadable + // over a precision difference that stored data rarely uses. + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), + map(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null))); + } + + @Test + void zonedTimestampsMapToTimestamptz() { + Assertions.assertEquals(ConnectorType.of("TIMESTAMPTZ", 6, 0), + map(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"))); + } + + @Test + void allStringAndBinaryVariantsMapToString() { + // The large_/view variants exist only as physical layouts; BE normalizes them before the serde, so + // FE must present them identically or DESC would disagree with what a query returns. + for (ArrowType type : Arrays.asList( + ArrowType.Utf8.INSTANCE, ArrowType.LargeUtf8.INSTANCE, ArrowType.Utf8View.INSTANCE, + ArrowType.Binary.INSTANCE, ArrowType.LargeBinary.INSTANCE, ArrowType.BinaryView.INSTANCE, + new ArrowType.FixedSizeBinary(16))) { + Assertions.assertEquals(ConnectorType.of("STRING"), map(type), "for " + type); + } + } + + @Test + void listVariantsMapToArrayOfTheElementType() { + for (ArrowType listType : Arrays.asList( + ArrowType.List.INSTANCE, ArrowType.LargeList.INSTANCE, new ArrowType.FixedSizeList(4))) { + Field field = nested("c", listType, + List.of(leaf("item", ArrowType.LargeUtf8.INSTANCE))); + Assertions.assertEquals(ConnectorType.arrayOf(ConnectorType.of("STRING")), + AdbcTypeMapper.toDorisType("c", field), "for " + listType); + } + } + + @Test + void structFieldNamesAreLowercasedAtEveryLevel() { + // BE indexes struct children by lowercase key, so a mixed-case child name crashes it rather than + // producing a query error. The guard has to apply at every level, not just the top one. + Field inner = nested("Inner", ArrowType.Struct.INSTANCE, + List.of(leaf("DeepField", new ArrowType.Int(32, true)))); + Field outer = nested("c", ArrowType.Struct.INSTANCE, + List.of(leaf("OuterField", ArrowType.Utf8.INSTANCE), inner)); + + ConnectorType mapped = AdbcTypeMapper.toDorisType("c", outer); + + Assertions.assertEquals(List.of("outerfield", "inner"), mapped.getFieldNames()); + Assertions.assertEquals(List.of("deepfield"), mapped.getChildren().get(1).getFieldNames()); + } + + @Test + void mapsUnwrapTheEntriesStruct() { + // Arrow models a map as list>; reading the pair from the top level would give the + // entries struct instead of the key and value types. + Field entries = nested("entries", ArrowType.Struct.INSTANCE, + List.of(leaf("key", ArrowType.Utf8.INSTANCE), leaf("value", new ArrowType.Int(64, true)))); + Field field = nested("c", new ArrowType.Map(false), List.of(entries)); + + Assertions.assertEquals( + ConnectorType.mapOf(ConnectorType.of("STRING"), ConnectorType.of("BIGINT")), + AdbcTypeMapper.toDorisType("c", field)); + } + + @Test + void runEndEncodedMapsToItsValueType() { + Field field = nested("c", new ArrowType.RunEndEncoded(), + List.of(leaf("run_ends", new ArrowType.Int(32, true)), + leaf("values", ArrowType.Utf8.INSTANCE))); + Assertions.assertEquals(ConnectorType.of("STRING"), AdbcTypeMapper.toDorisType("c", field)); + } + + @Test + void typesWithNoDorisEquivalentAreRejectedByName() { + // The message must carry the column name: without it a user facing a wide table has no way to tell + // which column to cast or drop on the remote side. + for (ArrowType type : Arrays.asList( + new ArrowType.Time(TimeUnit.MICROSECOND, 64), + new ArrowType.Duration(TimeUnit.SECOND), + new ArrowType.Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), + ArrowType.Null.INSTANCE)) { + String message = rejectionOf("weird_col", leaf("weird_col", type)); + Assertions.assertTrue(message.contains("weird_col"), message); + Assertions.assertTrue(message.contains(type.toString()), message); + } + } + + @Test + void anUnsupportedTypeNestedInsideAStructNamesThePath() { + Field field = nested("c", ArrowType.Struct.INSTANCE, + List.of(leaf("ok", ArrowType.Utf8.INSTANCE), + leaf("bad", new ArrowType.Time(TimeUnit.MICROSECOND, 64)))); + String message = rejectionOf("c", field); + Assertions.assertTrue(message.contains("c.bad"), message); + } +} diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/DorisDialectTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/DorisDialectTest.java new file mode 100644 index 00000000000000..668d77db19dd8d --- /dev/null +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/DorisDialectTest.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.adbc; + +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; + +/** + * The dialect a Doris source gets, written from a real failure. + * + *

The first scan against another Doris over Flight SQL sent {@code SELECT "id" FROM "db"."t1"} and the + * source answered {@code no viable alternative at input 'FROM "db"'}. Doris reads a double-quoted name as a + * string literal, so ANSI quoting there is not merely unidiomatic: the statement does not parse at all, and + * the connector cannot read the one source phase one exists to replace. + * + *

These tests are written against the registry rather than the class, because being reachable is the + * half that was broken: an unclaimed vendor name falls back to ANSI silently. + */ +class DorisDialectTest { + + private static final String NAME = "doris"; + + private static AdbcDialect doris() { + return AdbcDialectRegistry.require(NAME); + } + + private static AdbcTableHandle handle(String catalog, String schema, String table) { + return new AdbcTableHandle(new AdbcNamespace(catalog, schema), table); + } + + private static ConnectorLiteral literal(String typeName, Object value) { + return new ConnectorLiteral(ConnectorType.of(typeName), value); + } + + // ---------- being chosen ---------- + + @Test + void isClaimedByTheVendorNameDorisActuallyReports() { + // Measured, not assumed: Doris answers getInfo(VENDOR_NAME) with "DorisFE", from + // SqlInfoBuilder.withFlightSqlServerName in DorisFlightSqlProducer. Matching only the dialect's own + // name -- the default -- would leave "DorisFE" unclaimed, which is exactly how the ANSI fallback + // produced unparseable SQL. + Assertions.assertEquals(NAME, + AdbcDialectRegistry.forVendor("DorisFE").map(AdbcDialect::name).orElse(null)); + Assertions.assertEquals(NAME, + AdbcDialectRegistry.forVendor("Doris").map(AdbcDialect::name).orElse(null)); + } + + @Test + void doesNotClaimSourcesItWasNeverVerifiedAgainst() { + // Backticks are the MySQL family's spelling, but only Doris has been run against. Claiming a vendor + // on family resemblance would hand unverified sources a default they never asked for; they can still + // ask for this dialect by name. + Assertions.assertFalse(AdbcDialectRegistry.forVendor("SQLite").isPresent()); + Assertions.assertFalse(AdbcDialectRegistry.forVendor("MySQL").isPresent()); + } + + // ---------- what it spells differently ---------- + + @Test + void quotesIdentifiersWithBackticks() { + Assertions.assertEquals("`id`", doris().quoteIdentifier("id")); + } + + @Test + void qualifiesTheTableWithBackticksToo() { + // The exact token the parser choked on in the failing run. + Assertions.assertEquals("`test_db`.`t1`", + doris().qualifiedTableName(handle("internal", "test_db", "t1"))); + } + + @Test + void doublesAnEmbeddedBacktick() { + // Without doubling, a backtick inside a name closes the identifier early and turns the rest into + // stray SQL -- the injection shape, reached through a column name coming from the source. + Assertions.assertEquals("`we``ird`", doris().quoteIdentifier("we`ird")); + } + + // ---------- what it must not change ---------- + + @Test + void rendersLiteralsExactlyAsAnsiDoes() { + // Only the quoting character differs. A literal rendered differently here would not fail loudly like + // a syntax error does; it would return a different set of rows. + AdbcDialect ansi = AdbcDialectRegistry.defaultDialect(); + Assertions.assertEquals(ansi.renderLiteral(literal("STRING", "O'Brien")), + doris().renderLiteral(literal("STRING", "O'Brien"))); + Assertions.assertEquals(ansi.renderLiteral(literal("DATEV2", LocalDate.of(2024, 1, 31))), + doris().renderLiteral(literal("DATEV2", LocalDate.of(2024, 1, 31)))); + Assertions.assertEquals(ansi.renderLiteral(literal("INT", 42L)), + doris().renderLiteral(literal("INT", 42L))); + // Including the refusals: a type whose text spelling the source reads differently stays in Doris. + Assertions.assertNull(doris().renderLiteral(literal("LARGEINT", "170141183460469231731687303715884105727"))); + } +} diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanRequest.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanRequest.java index db377ac99ee0cb..9dff5781aa5743 100644 --- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanRequest.java +++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanRequest.java @@ -49,16 +49,18 @@ public final class ConnectorScanRequest { private final long limit; private final List requiredPartitions; private final boolean countPushdown; + private final boolean explainOnly; private ConnectorScanRequest(ConnectorTableHandle tableHandle, List columns, Optional filter, long limit, List requiredPartitions, - boolean countPushdown) { + boolean countPushdown, boolean explainOnly) { this.tableHandle = tableHandle; this.columns = columns; this.filter = filter; this.limit = limit; this.requiredPartitions = requiredPartitions; this.countPushdown = countPushdown; + this.explainOnly = explainOnly; } /** @@ -114,10 +116,26 @@ public boolean isCountPushdown() { return countPushdown; } + /** + * Whether this plan is being built only to be shown ({@code EXPLAIN}), never run. + * + *

{@code EXPLAIN} plans a scan for real — that is where its {@code inputSplitNum} comes from — so a + * connector whose planning has a SIDE EFFECT on the source must read this and take a cheaper route. The + * ADBC connector is one: asking a driver to partition a query executes that query on the source, and an + * {@code EXPLAIN} that ran the query it was asked to describe would be a surprise the user cannot see. + * A connector that only lists files or builds a string ignores this and plans identically either way.

+ * + *

It is not permission to plan something DIFFERENT from what would run: whatever is planned here is + * what {@code EXPLAIN} shows the user, so it must still describe the real scan.

+ */ + public boolean isExplainOnly() { + return explainOnly; + } + /** This request with the partition set replaced — the batched scan's per-batch request. */ public ConnectorScanRequest withRequiredPartitions(List partitions) { return new ConnectorScanRequest(tableHandle, columns, filter, limit, - normalizePartitions(partitions), countPushdown); + normalizePartitions(partitions), countPushdown, explainOnly); } private static List normalizePartitions(List partitions) { @@ -133,6 +151,7 @@ public static final class Builder { private long limit = -1; private List requiredPartitions = Collections.emptyList(); private boolean countPushdown; + private boolean explainOnly; private Builder(ConnectorTableHandle tableHandle, List columns) { this.tableHandle = Objects.requireNonNull(tableHandle, "tableHandle"); @@ -160,9 +179,15 @@ public Builder countPushdown(boolean countPushdown) { return this; } + /** Defaults to false: a plan that will be run. */ + public Builder explainOnly(boolean explainOnly) { + this.explainOnly = explainOnly; + return this; + } + public ConnectorScanRequest build() { return new ConnectorScanRequest(tableHandle, columns, filter, limit, - requiredPartitions, countPushdown); + requiredPartitions, countPushdown, explainOnly); } } } diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderBatchScanTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderBatchScanTest.java index 8aa6656b007966..8892ee4995c03f 100644 --- a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderBatchScanTest.java +++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderBatchScanTest.java @@ -99,6 +99,7 @@ public void testPlanScanForPartitionBatchRescopesTheRequestToTheBatch() { .filter(Optional.of(filter)) .limit(7L) .countPushdown(true) + .explainOnly(true) .build(); List result = provider.planScanForPartitionBatch(null, request, batch); @@ -111,6 +112,10 @@ public void testPlanScanForPartitionBatchRescopesTheRequestToTheBatch() { Assertions.assertSame(filter, forwarded.getFilter().orElse(null)); Assertions.assertEquals(7L, forwarded.getLimit()); Assertions.assertTrue(forwarded.isCountPushdown()); + // Dropping this one would silently make a batched EXPLAIN plan the way a real scan does -- + // which for a connector whose planning has a side effect on the source means EXPLAIN runs the + // query. Losing it is invisible in the plan output. + Assertions.assertTrue(forwarded.isExplainOnly()); } @Test @@ -125,6 +130,9 @@ public void testRequestDefaultsAskForNothingSpecial() { Assertions.assertEquals(-1L, request.getLimit()); Assertions.assertTrue(request.getRequiredPartitions().isEmpty()); Assertions.assertFalse(request.isCountPushdown()); + // Default false = "this plan will be run": a connector that reads it takes its normal path + // unless the engine says otherwise. + Assertions.assertFalse(request.isExplainOnly()); // null is accepted for the partition set and means the same as empty: scan everything. Assertions.assertTrue(ConnectorScanRequest.builder(HANDLE, Collections.emptyList()) .requiredPartitions(null).build().getRequiredPartitions().isEmpty()); diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index edffeaf74bf578..a99a981c7d28b6 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -73,6 +73,7 @@ under the License. fe-connector-trino fe-connector-maxcompute fe-connector-jdbc + fe-connector-adbc diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java index 910aa25e85d610..9eedf9d41fbb42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java @@ -168,6 +168,10 @@ public class PluginDrivenScanNode extends FileQueryScanNode { // Populated from ConnectorScanPlanProvider.getScanNodePropertiesResult() private ScanNodePropertiesResult cachedPropertiesResult; private Map scanNodeProperties; + // The tuple's slot ids and the filter as they were when cachedPropertiesResult was built, so the + // EXPLAIN path can tell that the projection has narrowed since. See remoteQueryForExplain(). + private List propertiesSlotIds; + private Optional propertiesFilter; // Maps filtered conjunct indices (after CAST removal) back to original conjunct indices private List filteredToOriginalIndex; @@ -436,7 +440,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { output.append(prefix).append("QUERY: ").append(query).append("\n"); } else { Map props = getOrLoadScanNodeProperties(); - String query = props.get(ScanNodePropertyKeys.REMOTE_QUERY); + String query = remoteQueryForExplain(props); output.append(prefix).append("TABLE: ") .append(desc.getTable().getNameWithFullQualifiers()).append("\n"); // Surface the backing connector/catalog type (e.g. es, jdbc, max_compute) so the @@ -1399,6 +1403,11 @@ public List getSplits(int numBackends) throws UserException { .limit(sourceLimit) .requiredPartitions(requiredPartitions) .countPushdown(countPushdown) + // EXPLAIN plans the scan for real -- that is where its inputSplitNum comes from -- so a + // connector whose planning has a side effect on the source (ADBC: asking the driver to + // partition a query EXECUTES it) needs to know the plan is only going to be shown. + // Connectors that just list files are unaffected: they never read this. + .explainOnly(isExplainOnly()) .build(); List ranges = onPluginClassLoader(scanProvider, () -> scanProvider.planScan(connectorSession, request)); @@ -1486,6 +1495,21 @@ static List sampleSplits(List splits, TableSample tableSample, lon return splits.subList(0, index); } + /** + * Whether the statement being planned is an {@code EXPLAIN}, so this plan will be shown and never run. + * + *

Read from the executor's parsed statement, which {@code ExplainCommand} marks before it plans. Any + * path without a live executor answers false, which is the safe way round: a connector then plans what + * it would have planned anyway.

+ */ + private static boolean isExplainOnly() { + ConnectContext ctx = ConnectContext.get(); + if (ctx == null || ctx.getExecutor() == null || ctx.getExecutor().getParsedStmt() == null) { + return false; + } + return ctx.getExecutor().getParsedStmt().isExplain(); + } + /** * Counts the scan ranges read by BE's native (ORC/Parquet) reader (vs JNI), via the generic * {@link ConnectorScanRange#isNativeReadRange()} (default false). Drives the EXPLAIN @@ -1988,6 +2012,8 @@ private ScanNodePropertiesResult getOrLoadPropertiesResult() { // rebuilds it over the full schema (no-op for every non-lazy-mat read / every connector // without a pruned dictionary). pinTopnLazyMaterialize(); + propertiesSlotIds = currentSlotIds(); + propertiesFilter = filter; cachedPropertiesResult = onPluginClassLoader(scanProvider, () -> scanProvider.getScanNodePropertiesResult( connectorSession, currentHandle, columns, filter)); @@ -1999,6 +2025,59 @@ private ScanNodePropertiesResult getOrLoadPropertiesResult() { return cachedPropertiesResult; } + /** The tuple's slot ids, in order, as the projection currently stands. */ + private List currentSlotIds() { + List ids = new ArrayList<>(desc.getSlots().size()); + for (SlotDescriptor slot : desc.getSlots()) { + ids.add(slot.getId().asInt()); + } + return ids; + } + + /** + * The remote statement to SHOW, which is not always the one in the cached properties. + * + *

The cache is warmed in {@code init()} -- {@code initSchemaParams()} asks for the path + * partition keys, which come from these properties -- and at that moment the scan tuple still + * carries every column of the table. Nereids prunes it to the query's slots afterwards, when the + * project above the scan is translated, and only then does {@code planScan} build the statement + * that actually runs. So a connector that renders its remote SQL from the column list (adbc, + * jdbc) has a cached statement that over-projects against the one BE will execute, and EXPLAIN + * would describe a query that is not the one run. + * + *

Re-asked with the CURRENT columns but the ORIGINAL filter: by now the pushed-down conjuncts + * have been removed from {@code conjuncts}, so rebuilding the filter here would drop the pushed + * predicates out of the displayed WHERE clause. Everything else -- what goes to BE, the pins, + * the conjunct pruning -- keeps using the cached result, so this affects the display only. + * + *

Costs nothing when the projection did not narrow, which is the common case: the slot ids + * are compared first and the connector is only re-asked when they differ. + */ + private String remoteQueryForExplain(Map cachedProps) { + String cached = cachedProps.get(ScanNodePropertyKeys.REMOTE_QUERY); + if (cached == null || propertiesSlotIds == null + || propertiesSlotIds.equals(currentSlotIds())) { + return cached; + } + ConnectorScanPlanProvider scanProvider = resolveScanProvider(); + if (scanProvider == null) { + return cached; + } + List columns; + try { + columns = buildColumnHandles(); + } catch (UserException e) { + // Same unchecked channel getOrLoadPropertiesResult uses: this method is reached from + // getNodeExplainString, which cannot declare UserException. + throw new RuntimeException("Failed to build column handles for plugin-driven scan", e); + } + Map props = onPluginClassLoader(scanProvider, + () -> scanProvider.getScanNodeProperties( + connectorSession, currentHandle, columns, propertiesFilter)); + String pruned = props == null ? null : props.get(ScanNodePropertyKeys.REMOTE_QUERY); + return pruned == null ? cached : pruned; + } + /** * Lazily loads scan node properties from the connector's scan plan provider. */ @@ -2015,7 +2094,8 @@ private Map getOrLoadScanNodeProperties() { /** * Maps a file format name string to the corresponding TFileFormatType. */ - private static TFileFormatType mapFileFormatType(String format) { + /** Package-visible and static so the mapping is unit-testable without a planner. */ + static TFileFormatType mapFileFormatType(String format) { switch (format.toLowerCase()) { case "parquet": return TFileFormatType.FORMAT_PARQUET; @@ -2033,6 +2113,11 @@ private static TFileFormatType mapFileFormatType(String format) { return TFileFormatType.FORMAT_AVRO; case "es_http": return TFileFormatType.FORMAT_ES_HTTP; + case "arrow": + // A connector whose reader hands BE Arrow record batches rather than a file (adbc, and the + // remote_doris scan node already outside this switch). Without this the format falls to + // FORMAT_JNI below and BE never enters the Arrow reader path. + return TFileFormatType.FORMAT_ARROW; default: return TFileFormatType.FORMAT_JNI; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeFileFormatTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeFileFormatTest.java new file mode 100644 index 00000000000000..98e7457f5e1680 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeFileFormatTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.scan; + +import org.apache.doris.thrift.TFileFormatType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Pins the connector-agnostic file-format name to the thrift enum BE selects its reader with. + * + *

WHY this matters beyond the switch being obvious: the fall-through is {@code FORMAT_JNI}, which is a + * WORKING reader rather than an error. A format name the switch does not know therefore produces no failure + * here at all — the scan simply lands in the JNI scanner and fails later, in BE, with a message about the + * wrong reader. Every name a connector may return has to be listed for that reason, not for coverage.

+ */ +public class PluginDrivenScanNodeFileFormatTest { + + @Test + public void mapsArrowToTheArrowReader() { + // A connector that hands BE Arrow record batches instead of a file (adbc). BE gates entry to its + // Arrow table-format path on FORMAT_ARROW, so mapping this to the JNI default would route the scan + // to a reader that has no ADBC branch. + Assertions.assertEquals(TFileFormatType.FORMAT_ARROW, + PluginDrivenScanNode.mapFileFormatType("arrow")); + } + + @Test + public void mapsTheFileBackedFormatsToTheirNativeReaders() { + Assertions.assertEquals(TFileFormatType.FORMAT_PARQUET, + PluginDrivenScanNode.mapFileFormatType("parquet")); + Assertions.assertEquals(TFileFormatType.FORMAT_ORC, + PluginDrivenScanNode.mapFileFormatType("orc")); + Assertions.assertEquals(TFileFormatType.FORMAT_TEXT, + PluginDrivenScanNode.mapFileFormatType("text")); + Assertions.assertEquals(TFileFormatType.FORMAT_CSV_PLAIN, + PluginDrivenScanNode.mapFileFormatType("csv")); + Assertions.assertEquals(TFileFormatType.FORMAT_JSON, + PluginDrivenScanNode.mapFileFormatType("json")); + Assertions.assertEquals(TFileFormatType.FORMAT_AVRO, + PluginDrivenScanNode.mapFileFormatType("avro")); + Assertions.assertEquals(TFileFormatType.FORMAT_ES_HTTP, + PluginDrivenScanNode.mapFileFormatType("es_http")); + } + + @Test + public void isCaseInsensitiveBecauseTheNameComesFromAConnector() { + // The name arrives as a free-form string in the connector's scan-node properties, so the casing is + // the connector's choice, not the engine's. + Assertions.assertEquals(TFileFormatType.FORMAT_ARROW, + PluginDrivenScanNode.mapFileFormatType("ARROW")); + } + + @Test + public void fallsBackToJniForAnUnknownName() { + Assertions.assertEquals(TFileFormatType.FORMAT_JNI, + PluginDrivenScanNode.mapFileFormatType("jni")); + Assertions.assertEquals(TFileFormatType.FORMAT_JNI, + PluginDrivenScanNode.mapFileFormatType("not-a-format")); + } +} diff --git a/fe/pom.xml b/fe/pom.xml index 4fa658e448b4e9..3ac2adb15d2055 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -351,6 +351,10 @@ under the License. 0.53.2-public 19.0.0 + + 0.24.0 2.7.4-11 3.0.0-8 @@ -1852,6 +1856,31 @@ under the License. arrow-jdbc ${arrow.version} + + + org.apache.arrow + arrow-c-data + ${arrow.version} + + + + org.apache.arrow.adbc + adbc-core + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-manager + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-jni + ${adbc.version} + org.immutables value diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index bbf2c64b05d77e..5da8e55cdc785d 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -478,6 +478,11 @@ struct TTableFormatFileDesc { // ES per-shard parameters (used when table_format_type == "es") // Contains: index, type, shard_id, host_port, es_hosts 13: optional map es_params + // ADBC connection and query parameters (used when table_format_type == "adbc"). + // Keys: driver_path, driver_entrypoint, uri, username, password, + // adbc.