From 38bdd3f82dc3f38ed17c0a6c27648484502d8e89 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:23 +0200 Subject: [PATCH 01/29] Target 7.2.5 and mirror the new server constants --- ravendb/http/request_executor.py | 2 +- ravendb/primitives/constants.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index a9402629..360da4dc 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -56,7 +56,7 @@ class RequestExecutor: __INITIAL_TOPOLOGY_ETAG = -2 __GLOBAL_APPLICATION_IDENTIFIER = uuid.uuid4() - CLIENT_VERSION = "7.2.3" + CLIENT_VERSION = "7.2.5" logger = logging.getLogger("request_executor") # todo: initializer should take also cryptography certificates diff --git a/ravendb/primitives/constants.py b/ravendb/primitives/constants.py index a391e8d9..d0bca7bb 100644 --- a/ravendb/primitives/constants.py +++ b/ravendb/primitives/constants.py @@ -27,6 +27,7 @@ class QueryString: class Headers: REQUEST_TIME = "Request-Time" + X_FORWARDED_FOR = "X-Forwarded-For" SERVER_STARTUP_TIME = "Server-Startup-Time" REFRESH_TOPOLOGY = "Refresh-Topology" TOPOLOGY_ETAG = "Topology-Etag" @@ -312,6 +313,9 @@ class Obsolete: class DatabaseRecord: class SupportedFeatures: THROW_REVISION_KEY_TOO_BIG_FIX = "ThrowRevisionKeyTooBigFix" + HASHED_REVISION_PK = "HashedRevisionPk" + PULL_REPLICATION_COMPOSITE_CHANGE_VECTORS = "PullReplicationCompositeChangeVectors" + THROW_CONTROL_CHARACTERS_IN_IDENTIFIER = "ThrowControlCharactersInIdentifier" class VectorSearch: From e6d98d07fdd09bb40457ea6867a5e40607b227b9 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:23 +0200 Subject: [PATCH 02/29] Let S3 settings turn off checksum validation --- .../operations/attachments/__init__.py | 4 ++ .../documents/operations/backups/settings.py | 4 ++ .../operations_tests/test_s3_settings.py | 69 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 ravendb/tests/operations_tests/test_s3_settings.py diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index 1dfdcd40..b4924446 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -403,6 +403,7 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, + disable_checksum_validation: bool = None, storage_class: Optional[S3StorageClass] = None, ): self.aws_access_key = aws_access_key @@ -413,6 +414,7 @@ def __init__( self.bucket_name = bucket_name self.custom_server_url = custom_server_url self.force_path_style = force_path_style + self.disable_checksum_validation = disable_checksum_validation self.storage_class = storage_class @classmethod @@ -427,6 +429,7 @@ def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: json_dict.get("BucketName"), json_dict.get("CustomServerUrl"), json_dict.get("ForcePathStyle"), + json_dict.get("DisableChecksumValidation"), S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, ) @@ -440,6 +443,7 @@ def to_json(self) -> dict: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } if self.storage_class is not None: result["StorageClass"] = self.storage_class.value diff --git a/ravendb/documents/operations/backups/settings.py b/ravendb/documents/operations/backups/settings.py index 565e3df8..0a242863 100644 --- a/ravendb/documents/operations/backups/settings.py +++ b/ravendb/documents/operations/backups/settings.py @@ -130,6 +130,7 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, + disable_checksum_validation: bool = None, ): super().__init__( disabled, @@ -143,6 +144,7 @@ def __init__( self.bucket_name = bucket_name self.custom_server_url = custom_server_url self.force_path_style = force_path_style + self.disable_checksum_validation = disable_checksum_validation @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: @@ -157,6 +159,7 @@ def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: json_dict["BucketName"], json_dict["CustomServerUrl"], json_dict["ForcePathStyle"], + json_dict.get("DisableChecksumValidation"), ) def to_json(self) -> Dict[str, Any]: @@ -171,6 +174,7 @@ def to_json(self) -> Dict[str, Any]: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } diff --git a/ravendb/tests/operations_tests/test_s3_settings.py b/ravendb/tests/operations_tests/test_s3_settings.py new file mode 100644 index 00000000..f4a93300 --- /dev/null +++ b/ravendb/tests/operations_tests/test_s3_settings.py @@ -0,0 +1,69 @@ +""" +Tests for DisableChecksumValidation, added in 7.2.5 to both S3 settings classes for +S3-compatible storage that does not support modern object integrity checks. +""" + +import unittest + +from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings +from ravendb.documents.operations.backups.settings import GetBackupConfigurationScript, S3Settings + + +class TestBackupS3Settings(unittest.TestCase): + def _settings(self, **kwargs) -> S3Settings: + return S3Settings( + disabled=False, + get_backup_configuration_script=GetBackupConfigurationScript(), + aws_access_key="key", + aws_secret_key="secret", + aws_session_token=None, + aws_region_name="eu-central-1", + remote_folder_name="backups", + bucket_name="rvn", + custom_server_url="https://minio.local", + force_path_style=True, + **kwargs, + ) + + def test_checksum_validation_is_left_alone_by_default(self): + self.assertIsNone(self._settings().disable_checksum_validation) + + def test_the_flag_is_written(self): + self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + + def test_the_flag_round_trips(self): + serialized = self._settings(disable_checksum_validation=True).to_json() + + self.assertEqual(serialized, S3Settings.from_json(serialized).to_json()) + + def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): + serialized = self._settings().to_json() + del serialized["DisableChecksumValidation"] + + self.assertIsNone(S3Settings.from_json(serialized).disable_checksum_validation) + + +class TestRemoteAttachmentsS3Settings(unittest.TestCase): + def _settings(self, **kwargs) -> RemoteAttachmentsS3Settings: + return RemoteAttachmentsS3Settings( + aws_access_key="key", + aws_secret_key="secret", + aws_region_name="eu-central-1", + bucket_name="rvn", + force_path_style=True, + **kwargs, + ) + + def test_checksum_validation_is_left_alone_by_default(self): + self.assertIsNone(self._settings().disable_checksum_validation) + + def test_the_flag_is_written(self): + self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + + def test_the_flag_round_trips(self): + serialized = self._settings(disable_checksum_validation=True).to_json() + + self.assertEqual(serialized, RemoteAttachmentsS3Settings.from_json(serialized).to_json()) + + def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): + self.assertIsNone(RemoteAttachmentsS3Settings.from_json({"BucketName": "rvn"}).disable_checksum_validation) From 03f97a3c6b58fbdfe181dcae8c0303912eebb11a Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:24 +0200 Subject: [PATCH 03/29] Add the Azure Service Bus queue broker --- .../azure_service_bus_connection_settings.py | 85 +++++++++++ .../operations/etl/queue/connection.py | 14 ++ ...est_azure_service_bus_connection_string.py | 137 ++++++++++++++++++ 3 files changed, 236 insertions(+) create mode 100644 ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py create mode 100644 ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py diff --git a/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py new file mode 100644 index 00000000..6e8529f2 --- /dev/null +++ b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py @@ -0,0 +1,85 @@ +from typing import Optional, Dict, Any + + +class AzureServiceBusEntraId: + """Microsoft Entra ID (service principal) credentials for an Azure Service Bus namespace.""" + + def __init__( + self, + namespace: Optional[str] = None, + tenant_id: Optional[str] = None, + client_id: Optional[str] = None, + client_secret: Optional[str] = None, + ): + # Fully qualified namespace, e.g. 'mynamespace.servicebus.windows.net'. + self.namespace = namespace + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + + def to_json(self) -> Dict[str, Any]: + return { + "Namespace": self.namespace, + "TenantId": self.tenant_id, + "ClientId": self.client_id, + "ClientSecret": self.client_secret, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AzureServiceBusEntraId": + return cls( + namespace=json_dict.get("Namespace"), + tenant_id=json_dict.get("TenantId"), + client_id=json_dict.get("ClientId"), + client_secret=json_dict.get("ClientSecret"), + ) + + +class AzureServiceBusPasswordless: + """Machine authentication (Managed Identity) against an Azure Service Bus namespace.""" + + def __init__(self, namespace: Optional[str] = None): + # Fully qualified namespace, e.g. 'mynamespace.servicebus.windows.net'. + self.namespace = namespace + + def to_json(self) -> Dict[str, Any]: + return {"Namespace": self.namespace} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AzureServiceBusPasswordless": + return cls(namespace=json_dict.get("Namespace")) + + +class AzureServiceBusConnectionSettings: + """ + Azure Service Bus connection settings. Exactly one authentication method may be set: + a connection string, Entra ID credentials, or passwordless (Managed Identity). + The server rejects a configuration that sets none or more than one. + """ + + def __init__( + self, + connection_string: Optional[str] = None, + entra_id: Optional[AzureServiceBusEntraId] = None, + passwordless: Optional[AzureServiceBusPasswordless] = None, + ): + self.connection_string = connection_string + self.entra_id = entra_id + self.passwordless = passwordless + + def to_json(self) -> Dict[str, Any]: + return { + "ConnectionString": self.connection_string, + "EntraId": self.entra_id.to_json() if self.entra_id else None, + "Passwordless": self.passwordless.to_json() if self.passwordless else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AzureServiceBusConnectionSettings": + entra_id_dict = json_dict.get("EntraId") + passwordless_dict = json_dict.get("Passwordless") + return cls( + connection_string=json_dict.get("ConnectionString"), + entra_id=AzureServiceBusEntraId.from_json(entra_id_dict) if entra_id_dict else None, + passwordless=AzureServiceBusPasswordless.from_json(passwordless_dict) if passwordless_dict else None, + ) diff --git a/ravendb/documents/operations/etl/queue/connection.py b/ravendb/documents/operations/etl/queue/connection.py index 985d65d0..02709c9a 100644 --- a/ravendb/documents/operations/etl/queue/connection.py +++ b/ravendb/documents/operations/etl/queue/connection.py @@ -6,6 +6,9 @@ from ravendb.documents.operations.etl.queue.azure_queue_storage_connection_settings import ( AzureQueueStorageConnectionSettings, ) +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, +) from ravendb.documents.operations.etl.queue.kafka_connection_settings import KafkaConnectionSettings from ravendb.documents.operations.etl.queue.rabbit_mq_connection_settings import RabbitMqConnectionSettings @@ -16,6 +19,7 @@ class QueueBrokerType(Enum): RABBIT_MQ = "RabbitMq" AZURE_QUEUE_STORAGE = "AzureQueueStorage" AMAZON_SQS = "AmazonSqs" + AZURE_SERVICE_BUS = "AzureServiceBus" class QueueConnectionString(ConnectionString): @@ -27,6 +31,7 @@ def __init__( rabbit_mq_settings: RabbitMqConnectionSettings = None, azure_queue_storage_settings: AzureQueueStorageConnectionSettings = None, amazon_sqs_settings: AmazonSqsConnectionSettings = None, + azure_service_bus_settings: AzureServiceBusConnectionSettings = None, ): super().__init__(name) self.broker_type = broker_type @@ -34,6 +39,7 @@ def __init__( self.rabbit_mq_settings = rabbit_mq_settings self.azure_queue_storage_settings = azure_queue_storage_settings self.amazon_sqs_settings = amazon_sqs_settings + self.azure_service_bus_settings = azure_service_bus_settings @property def get_type(self): @@ -49,6 +55,9 @@ def to_json(self): self.azure_queue_storage_settings.to_json() if self.azure_queue_storage_settings else None ), "AmazonSqsConnectionSettings": self.amazon_sqs_settings.to_json() if self.amazon_sqs_settings else None, + "AzureServiceBusConnectionSettings": ( + self.azure_service_bus_settings.to_json() if self.azure_service_bus_settings else None + ), "Type": ravendb.serverwide.server_operation_executor.ConnectionStringType.QUEUE, } @@ -77,4 +86,9 @@ def from_json(cls, json_dict: dict) -> "QueueConnectionString": if json_dict["AmazonSqsConnectionSettings"] else None ), + azure_service_bus_settings=( + AzureServiceBusConnectionSettings.from_json(json_dict["AzureServiceBusConnectionSettings"]) + if json_dict.get("AzureServiceBusConnectionSettings") + else None + ), ) diff --git a/ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py b/ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py new file mode 100644 index 00000000..78238b20 --- /dev/null +++ b/ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py @@ -0,0 +1,137 @@ +""" +Tests for the Azure Service Bus queue broker added in 7.2.5. +""" + +import unittest + +from ravendb.documents.operations.connection_string.get_connection_string_operation import ( + GetConnectionStringsOperation, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.connection_string.remove_connection_string_operation import ( + RemoveConnectionStringOperation, +) +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, + AzureServiceBusEntraId, + AzureServiceBusPasswordless, +) +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.tests.test_base import TestBase + +CONNECTION_STRING = "Endpoint=sb://rvn-test.servicebus.windows.net/;SharedAccessKeyName=k;SharedAccessKey=v" + + +class TestAzureServiceBusConnectionSettings(unittest.TestCase): + def test_azure_service_bus_is_a_known_broker(self): + self.assertEqual("AzureServiceBus", QueueBrokerType.AZURE_SERVICE_BUS.value) + + def test_connection_string_authentication_round_trips(self): + settings = AzureServiceBusConnectionSettings(connection_string=CONNECTION_STRING) + serialized = settings.to_json() + + self.assertEqual(CONNECTION_STRING, serialized["ConnectionString"]) + self.assertIsNone(serialized["EntraId"]) + self.assertIsNone(serialized["Passwordless"]) + self.assertEqual(serialized, AzureServiceBusConnectionSettings.from_json(serialized).to_json()) + + def test_entra_id_authentication_round_trips(self): + settings = AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId( + namespace="rvn-test.servicebus.windows.net", + tenant_id="tenant", + client_id="client", + client_secret="secret", + ) + ) + parsed = AzureServiceBusConnectionSettings.from_json(settings.to_json()) + + self.assertEqual("rvn-test.servicebus.windows.net", parsed.entra_id.namespace) + self.assertEqual("tenant", parsed.entra_id.tenant_id) + self.assertEqual("client", parsed.entra_id.client_id) + self.assertEqual("secret", parsed.entra_id.client_secret) + + def test_passwordless_authentication_round_trips(self): + settings = AzureServiceBusConnectionSettings( + passwordless=AzureServiceBusPasswordless(namespace="rvn-test.servicebus.windows.net") + ) + parsed = AzureServiceBusConnectionSettings.from_json(settings.to_json()) + + self.assertEqual("rvn-test.servicebus.windows.net", parsed.passwordless.namespace) + self.assertIsNone(parsed.entra_id) + + def test_the_settings_hang_off_the_queue_connection_string(self): + connection_string = QueueConnectionString( + name="asb1", + broker_type=QueueBrokerType.AZURE_SERVICE_BUS, + azure_service_bus_settings=AzureServiceBusConnectionSettings(connection_string=CONNECTION_STRING), + ) + serialized = connection_string.to_json() + + self.assertEqual("AzureServiceBus", serialized["BrokerType"]) + self.assertEqual(CONNECTION_STRING, serialized["AzureServiceBusConnectionSettings"]["ConnectionString"]) + + def test_a_queue_connection_string_from_an_older_server_has_no_service_bus_settings(self): + parsed = QueueConnectionString.from_json( + { + "Name": "kafka1", + "BrokerType": "Kafka", + "KafkaConnectionSettings": {"BootstrapServers": "localhost:9092"}, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + } + ) + + self.assertIsNone(parsed.azure_service_bus_settings) + + +class TestAzureServiceBusConnectionStringLifecycle(TestBase): + def setUp(self): + super().setUp() + + def test_the_server_stores_and_returns_an_azure_service_bus_connection_string(self): + connection_string = QueueConnectionString( + name="asb1", + broker_type=QueueBrokerType.AZURE_SERVICE_BUS, + azure_service_bus_settings=AzureServiceBusConnectionSettings(connection_string=CONNECTION_STRING), + ) + + put_result = self.store.maintenance.send(PutConnectionStringOperation(connection_string)) + self.assertGreater(put_result.raft_command_index, 0) + + get_result = self.store.maintenance.send(GetConnectionStringsOperation("asb1", ConnectionStringType.QUEUE)) + stored = get_result.queue_connection_strings["asb1"] + self.assertEqual(QueueBrokerType.AZURE_SERVICE_BUS, stored.broker_type) + self.assertEqual(CONNECTION_STRING, stored.azure_service_bus_settings.connection_string) + + remove_result = self.store.maintenance.send(RemoveConnectionStringOperation(connection_string)) + self.assertGreater(remove_result.raft_command_index, 0) + + after_delete = self.store.maintenance.send(GetConnectionStringsOperation("asb1", ConnectionStringType.QUEUE)) + self.assertFalse(after_delete.queue_connection_strings) + + def test_entra_id_credentials_survive_a_server_round_trip(self): + connection_string = QueueConnectionString( + name="asb2", + broker_type=QueueBrokerType.AZURE_SERVICE_BUS, + azure_service_bus_settings=AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId( + namespace="rvn-test.servicebus.windows.net", + tenant_id="tenant", + client_id="client", + client_secret="secret", + ) + ), + ) + + self.store.maintenance.send(PutConnectionStringOperation(connection_string)) + stored = self.store.maintenance.send( + GetConnectionStringsOperation("asb2", ConnectionStringType.QUEUE) + ).queue_connection_strings["asb2"] + + self.assertEqual("rvn-test.servicebus.windows.net", stored.azure_service_bus_settings.entra_id.namespace) + self.assertEqual("client", stored.azure_service_bus_settings.entra_id.client_id) From 3a3fc940415714996a16163995a9df0e490c1639 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:34 +0200 Subject: [PATCH 04/29] Let a conversation cancel the tool calls it is still waiting on --- ravendb/documents/ai/ai_conversation.py | 6 ++++ ravendb/documents/ai/ai_operations.py | 36 +++++++++++++++++-- .../ai/agents/run_conversation_operation.py | 9 +++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/ravendb/documents/ai/ai_conversation.py b/ravendb/documents/ai/ai_conversation.py index 50ea0fef..ea625616 100644 --- a/ravendb/documents/ai/ai_conversation.py +++ b/ravendb/documents/ai/ai_conversation.py @@ -43,6 +43,7 @@ def __init__( conversation_id: str = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): self._store = store self._agent_id = agent_id @@ -50,6 +51,9 @@ def __init__( self._conversation_id = conversation_id self._change_vector = change_vector self._debug = debug + # One-shot: the server drops the tool calls still awaiting a response on the next + # run, and the flag clears itself once that run succeeds. + self._cancel_pending_action_tools = cancel_pending_action_tools self._prompt_parts: List[ContentPart] = [] self._action_responses: Dict[str, AiAgentActionResponse] = {} @@ -194,6 +198,7 @@ def _run_internal( streamed_chunks_callback=streamed_chunks_callback, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, ) try: @@ -203,6 +208,7 @@ def _run_internal( self._change_vector = result.change_vector self._conversation_id = result.conversation_id + self._cancel_pending_action_tools = False self._action_requests = result.action_requests or [] return AiAnswer( diff --git a/ravendb/documents/ai/ai_operations.py b/ravendb/documents/ai/ai_operations.py index 77bf1ad8..93b1d84d 100644 --- a/ravendb/documents/ai/ai_operations.py +++ b/ravendb/documents/ai/ai_operations.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Any, Optional, Type +from typing import TYPE_CHECKING, Dict, Any, Optional, Type, Union import warnings @@ -11,7 +11,9 @@ from ravendb.documents.operations.ai.agents import ( AiAgentConfiguration, AiAgentConfigurationResult, + AiConversationMessagesResult, GetAiAgentsResponse, + GetConversationMessagesOptions, ) @@ -71,6 +73,25 @@ def get_agents(self, agent_id: str = None) -> GetAiAgentsResponse: operation = GetAiAgentOperation(agent_id) return self._store.maintenance.send(operation) + def get_conversation_messages( + self, conversation_id_or_parameters: Union[str, "GetConversationMessagesOptions"] + ) -> "AiConversationMessagesResult": + """ + Reads messages from an AI conversation. Returns the most recent messages by default. + + Args: + conversation_id_or_parameters: The conversation document ID, or a + GetConversationMessagesOptions for full control over paging + (before/after timestamps), page size, and detail level + + Returns: + The conversation's messages, cumulative usage, and paging state + """ + from ravendb.documents.operations.ai.agents import GetConversationMessagesOperation + + operation = GetConversationMessagesOperation(conversation_id_or_parameters) + return self._store.maintenance.send(operation) + def conversation( self, agent_id: str, @@ -78,6 +99,7 @@ def conversation( creation_options: "AiConversationCreationOptions" = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ) -> AiConversation: """ Creates a new conversation with the specified AI agent. @@ -88,12 +110,22 @@ def conversation( creation_options: Optional creation options for the conversation change_vector: Optional change vector for concurrency control debug: Optional flag enabling server-side conversation debugging + cancel_pending_action_tools: Drop the tool calls the conversation is still waiting on + instead of answering them, on the next run. Cleared once that run succeeds. Returns: Conversation operations interface for managing the conversation """ - return AiConversation(self._store, agent_id, creation_options, conversation_id, change_vector, debug) + return AiConversation( + self._store, + agent_id, + creation_options, + conversation_id, + change_vector, + debug, + cancel_pending_action_tools, + ) def conversation_with_id(self, conversation_id: str, change_vector: str = None) -> AiConversation: """ diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index b830284e..b73add25 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -295,6 +295,7 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): if not agent_id or (isinstance(agent_id, str) and agent_id.isspace()): raise ValueError("agent_id cannot be None or empty") @@ -314,6 +315,7 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._attachments_commands = attachments_commands or [] self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools def get_command(self, conventions: DocumentConventions) -> RavenCommand[ConversationResult[TSchema]]: return RunConversationCommand( @@ -329,6 +331,7 @@ def get_command(self, conventions: DocumentConventions) -> RavenCommand[Conversa conventions=conventions, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, ) @@ -347,6 +350,7 @@ def __init__( conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): from ravendb.util.util import RaftIdGenerator from ravendb.documents.commands.batches import PutAttachmentCommandData @@ -363,6 +367,7 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._conventions = conventions self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools self._attachments_commands = attachments_commands or [] # Raft id pinned at construction so retries keep the same id. @@ -409,6 +414,10 @@ def create_request(self, node: ServerNode) -> requests.Request: if self._debug is not None: url += f"&debug={self._debug}" + # Always sent: the server distinguishes "cancel the tool calls still pending" from + # "answer them", and has no default of its own. + url += f"&cancelPendingActionTools={self._cancel_pending_action_tools}" + request_body = ConversationRequestBody( action_responses=self._action_responses, artificial_actions=self._artificial_actions, From 5c4df301b6f9241439cfedd1c9895c4c132ca52f Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:35 +0200 Subject: [PATCH 05/29] Read an AI agent conversation back with GetConversationMessagesOperation --- .../operations/ai/agents/__init__.py | 17 ++ .../get_conversation_messages_operation.py | 284 ++++++++++++++++++ .../test_ai_conversation_messages.py | 218 ++++++++++++++ 3 files changed, 519 insertions(+) create mode 100644 ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py create mode 100644 ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py diff --git a/ravendb/documents/operations/ai/agents/__init__.py b/ravendb/documents/operations/ai/agents/__init__.py index d8099d41..5ba05db2 100644 --- a/ravendb/documents/operations/ai/agents/__init__.py +++ b/ravendb/documents/operations/ai/agents/__init__.py @@ -39,6 +39,16 @@ AiConversationParameterOptions, ) +from .get_conversation_messages_operation import ( + GetConversationMessagesOperation, + GetConversationMessagesOptions, + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, +) + __all__ = [ "AiAgentConfiguration", "AiAgentConfigurationResult", @@ -68,4 +78,11 @@ "GetAiAgentsResponse", "AddOrUpdateAiAgentOperation", "DeleteAiAgentOperation", + "GetConversationMessagesOperation", + "GetConversationMessagesOptions", + "AiConversationDetailLevel", + "AiConversationMessage", + "AiConversationMessagesResult", + "AiMessageRole", + "AiToolCallResult", ] diff --git a/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py new file mode 100644 index 00000000..20249dec --- /dev/null +++ b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import enum +import json +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Union + +import requests + +from ravendb.documents.conventions import DocumentConventions +from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants +from ravendb.tools.utils import Utils + + +class AiConversationDetailLevel(enum.Enum): + """Controls how much of a conversation the server returns.""" + + # User messages and assistant messages that have content. System prompts, tool calls, + # summaries and internal messages are excluded. + SIMPLE = "Simple" + # Adds system messages, tool calls with their results, and per-message usage. + DETAILED = "Detailed" + # No filtering at all, summaries and internal messages included. + FULL = "Full" + + def __str__(self) -> str: + return self.value + + +class AiMessageRole(enum.Enum): + SYSTEM = "System" + USER = "User" + ASSISTANT = "Assistant" + SUMMARY = "Summary" + INTERNAL = "Internal" + + def __str__(self) -> str: + return self.value + + +class AiToolCallResult: + """A tool call the model initiated, with the tool's response inlined.""" + + def __init__( + self, + id_: Optional[str] = None, + name: Optional[str] = None, + arguments: Optional[str] = None, + result: Optional[str] = None, + sub_conversation_id: Optional[str] = None, + ): + self.id_ = id_ + self.name = name + # Arguments the model passed, as a JSON string. + self.arguments = arguments + # None while the call is still pending (ActionRequired). + self.result = result + # Set when the call was a sub-agent invocation - the spawned conversation can be + # read on its own with GetConversationMessagesOperation. + self.sub_conversation_id = sub_conversation_id + + def to_json(self) -> Dict[str, Any]: + return { + "Id": self.id_, + "Name": self.name, + "Arguments": self.arguments, + "Result": self.result, + "SubConversationId": self.sub_conversation_id, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiToolCallResult: + return cls( + id_=json_dict.get("Id"), + name=json_dict.get("Name"), + arguments=json_dict.get("Arguments"), + result=json_dict.get("Result"), + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + +class AiConversationMessage: + """A single message in an AI agent conversation.""" + + def __init__( + self, + role: Optional[AiMessageRole] = None, + content: Optional[str] = None, + attachments: Optional[List[str]] = None, + timestamp: Optional[datetime] = None, + tool_calls: Optional[List[AiToolCallResult]] = None, + usage: Optional[AiUsage] = None, + sub_conversation_id: Optional[str] = None, + ): + self.role = role + # Multiple stored text parts arrive joined with line breaks. None for assistant + # messages that only initiated tool calls. + self.content = content + self.attachments = attachments + # Unique and monotonic within a conversation, so it is safe as a paging cursor. + self.timestamp = timestamp + self.tool_calls = tool_calls + self.usage = usage + # For Internal messages: the sub-conversation this message relates to. + self.sub_conversation_id = sub_conversation_id + + def to_json(self) -> Dict[str, Any]: + return { + "Role": self.role.value if self.role else None, + "Content": self.content, + "Attachments": self.attachments, + "Timestamp": Utils.datetime_to_string(self.timestamp), + "ToolCalls": [tool_call.to_json() for tool_call in self.tool_calls] if self.tool_calls else None, + "Usage": self.usage.to_json() if self.usage else None, + "SubConversationId": self.sub_conversation_id, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessage: + role = json_dict.get("Role") + tool_calls = json_dict.get("ToolCalls") + usage = json_dict.get("Usage") + return cls( + role=AiMessageRole(role) if role else None, + content=json_dict.get("Content"), + attachments=json_dict.get("Attachments"), + timestamp=Utils.string_to_datetime(json_dict.get("Timestamp")), + tool_calls=[AiToolCallResult.from_json(tool_call) for tool_call in tool_calls] if tool_calls else None, + usage=AiUsage.from_json(usage) if usage else None, + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + +class AiConversationMessagesResult: + """The result of reading an AI agent conversation.""" + + def __init__( + self, + conversation_id: Optional[str] = None, + agent: Optional[str] = None, + parameters: Optional[Dict[str, Any]] = None, + total_usage: Optional[AiUsage] = None, + last_message_at: Optional[datetime] = None, + messages: Optional[List[AiConversationMessage]] = None, + has_more_messages: bool = False, + sub_conversation_ids: Optional[List[str]] = None, + attachments: Optional[List[str]] = None, + ): + self.conversation_id = conversation_id + self.agent = agent + # Conversation parameters as a name -> value map. Values are heterogeneous + # (primitives and arrays), so they are handed back as they arrive. + self.parameters = parameters + # Cumulative token usage across every turn of the conversation. + self.total_usage = total_usage + self.last_message_at = last_message_at + # Chronological, oldest first. + self.messages = messages + # Older messages exist for backward/default paging, newer ones for `after` paging. + self.has_more_messages = has_more_messages + self.sub_conversation_ids = sub_conversation_ids + self.attachments = attachments + + def to_json(self) -> Dict[str, Any]: + return { + "ConversationId": self.conversation_id, + "Agent": self.agent, + "Parameters": self.parameters, + "TotalUsage": self.total_usage.to_json() if self.total_usage else None, + "LastMessageAt": Utils.datetime_to_string(self.last_message_at), + "HasMoreMessages": self.has_more_messages, + "SubConversationIds": self.sub_conversation_ids, + "Attachments": self.attachments, + "Messages": [message.to_json() for message in self.messages] if self.messages else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessagesResult: + total_usage = json_dict.get("TotalUsage") + messages = json_dict.get("Messages") + return cls( + conversation_id=json_dict.get("ConversationId"), + agent=json_dict.get("Agent"), + parameters=json_dict.get("Parameters"), + total_usage=AiUsage.from_json(total_usage) if total_usage else None, + last_message_at=Utils.string_to_datetime(json_dict.get("LastMessageAt")), + messages=[AiConversationMessage.from_json(message) for message in messages] if messages else None, + has_more_messages=json_dict.get("HasMoreMessages", False), + sub_conversation_ids=json_dict.get("SubConversationIds"), + attachments=json_dict.get("Attachments"), + ) + + +class GetConversationMessagesOptions: + """Paging and filtering for GetConversationMessagesOperation.""" + + def __init__( + self, + conversation_id: Optional[str] = None, + before: Optional[datetime] = None, + after: Optional[datetime] = None, + page_size: int = constants.int_max, + detail_level: AiConversationDetailLevel = AiConversationDetailLevel.SIMPLE, + ): + self.conversation_id = conversation_id + # Messages older than this timestamp - backward paging, scrolling up in a chat UI. + self.before = before + # Messages newer than this timestamp - catching up after a Changes() notification. + self.after = after + self.page_size = page_size + self.detail_level = detail_level + + def validate(self) -> None: + if not self.conversation_id: + raise ValueError("conversation_id cannot be None or empty") + + if self.before is not None and self.after is not None: + raise ValueError("before and after cannot both be specified.") + + if self.page_size is None or self.page_size <= 0: + raise ValueError("page_size must be greater than 0.") + + +class GetConversationMessagesOperation(MaintenanceOperation[AiConversationMessagesResult]): + """ + Reads messages from an AI agent conversation, with optional timestamp-based paging + and view filtering. Returns the most recent messages by default. + """ + + def __init__(self, conversation_id_or_parameters: Union[str, GetConversationMessagesOptions]): + if conversation_id_or_parameters is None: + raise ValueError("conversation_id_or_parameters cannot be None") + + if isinstance(conversation_id_or_parameters, GetConversationMessagesOptions): + self._parameters = conversation_id_or_parameters + else: + self._parameters = GetConversationMessagesOptions(conversation_id=conversation_id_or_parameters) + + self._parameters.validate() + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[AiConversationMessagesResult]: + return self._GetConversationMessagesCommand(self._parameters) + + class _GetConversationMessagesCommand(RavenCommand[AiConversationMessagesResult]): + def __init__(self, parameters: GetConversationMessagesOptions): + super().__init__(AiConversationMessagesResult) + self._parameters = parameters + + def is_read_request(self) -> bool: + return True + + @staticmethod + def _to_utc_string(value: datetime) -> str: + # Naive datetimes are taken as UTC, aware ones are converted, matching the + # server's EnsureUtc before it formats the cursor. + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + return Utils.datetime_to_string(value) + + def create_request(self, node: ServerNode) -> requests.Request: + url = ( + f"{node.url}/databases/{node.database}/ai/agent/conversation/messages" + f"?conversationId={Utils.quote_key(self._parameters.conversation_id)}" + ) + + if self._parameters.before is not None: + url += f"&before={Utils.quote_key(self._to_utc_string(self._parameters.before))}" + if self._parameters.after is not None: + url += f"&after={Utils.quote_key(self._to_utc_string(self._parameters.after))}" + + url += f"&pageSize={self._parameters.page_size}" + url += f"&detailLevel={self._parameters.detail_level.value}" + + return requests.Request("GET", url) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + return # 404 - conversation not found + + self.result = AiConversationMessagesResult.from_json(json.loads(response)) diff --git a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py new file mode 100644 index 00000000..00adeb5d --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py @@ -0,0 +1,218 @@ +""" +Tests for GetConversationMessagesOperation, the 7.2.5 way to read an AI agent +conversation back, and for the cancel-pending-action-tools flag on a run. +""" + +import json +import unittest +from datetime import datetime, timedelta, timezone + +from ravendb.documents.operations.ai.agents import ( + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, + AiUsage, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + RunConversationOperation, +) +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants + + +class TestGetConversationMessagesOptions(unittest.TestCase): + def test_defaults_to_the_whole_conversation_at_simple_detail(self): + options = GetConversationMessagesOptions("chats/1-A") + + self.assertEqual(constants.int_max, options.page_size) + self.assertEqual(AiConversationDetailLevel.SIMPLE, options.detail_level) + self.assertIsNone(options.before) + self.assertIsNone(options.after) + + def test_a_conversation_id_is_required(self): + with self.assertRaises(ValueError): + GetConversationMessagesOperation("") + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions(None)) + with self.assertRaises(ValueError): + GetConversationMessagesOperation(None) + + def test_paging_backwards_and_forwards_at_once_is_rejected(self): + options = GetConversationMessagesOptions("chats/1-A", before=datetime(2026, 6, 16), after=datetime(2026, 6, 15)) + + with self.assertRaises(ValueError): + GetConversationMessagesOperation(options) + + def test_a_non_positive_page_size_is_rejected(self): + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions("chats/1-A", page_size=0)) + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions("chats/1-A", page_size=-1)) + + +class TestGetConversationMessagesCommand(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def _url(self, conversation_id_or_options): + command = GetConversationMessagesOperation(conversation_id_or_options).get_command(None) + return command.create_request(self.node).url + + def test_the_conversation_id_is_escaped_into_the_query_string(self): + url = self._url("chats/1-A") + + self.assertTrue( + url.startswith("http://localhost:8080/databases/db/ai/agent/conversation/messages?conversationId="), + url, + ) + self.assertIn("conversationId=chats%2F1-A", url) + + def test_page_size_and_detail_level_are_always_sent(self): + url = self._url(GetConversationMessagesOptions("chats/1-A", page_size=25)) + + self.assertIn("&pageSize=25", url) + self.assertIn("&detailLevel=Simple", url) + + def test_detail_level_is_sent_by_its_server_side_name(self): + url = self._url(GetConversationMessagesOptions("chats/1-A", detail_level=AiConversationDetailLevel.FULL)) + + self.assertIn("&detailLevel=Full", url) + + def test_a_naive_cursor_is_sent_as_is(self): + # Naive datetimes count as UTC here, the same rule the rest of the client follows. + url = self._url(GetConversationMessagesOptions("chats/1-A", before=datetime(2026, 6, 16, 10, 30, 0))) + + self.assertIn("&before=2026-06-16T10%3A30%3A00.0000000", url) + + def test_an_aware_cursor_is_converted_to_utc(self): + warsaw = datetime(2026, 6, 16, 10, 30, 0, tzinfo=timezone(timedelta(hours=2))) + url = self._url(GetConversationMessagesOptions("chats/1-A", after=warsaw)) + + self.assertIn("&after=2026-06-16T08%3A30%3A00.0000000", url) + + def test_reading_messages_is_a_read_request(self): + command = GetConversationMessagesOperation("chats/1-A").get_command(None) + + self.assertTrue(command.is_read_request()) + + def test_a_missing_conversation_leaves_the_result_unset(self): + # The endpoint answers 404 for an unknown conversation, which reaches us as None. + command = GetConversationMessagesOperation("chats/1-A").get_command(None) + command.set_response(None, False) + + self.assertIsNone(command.result) + + +class TestAiConversationMessagesResult(unittest.TestCase): + RESPONSE = { + "ConversationId": "chats/1-A", + "Agent": "agents/support", + "Parameters": {"customer": "ALFKI", "tags": ["vip", "eu"], "retries": 2}, + "TotalUsage": {"PromptTokens": 10, "CompletionTokens": 5, "TotalTokens": 15, "ReasoningTokens": 3}, + "LastMessageAt": "2026-06-16T10:30:05.0000000", + "HasMoreMessages": True, + "SubConversationIds": ["chats/1-A/sub/1"], + "Attachments": ["invoice.pdf"], + "Messages": [ + {"Role": "User", "Content": "where is my order?", "Timestamp": "2026-06-16T10:30:00.0000000"}, + { + "Role": "Assistant", + "Content": None, + "Timestamp": "2026-06-16T10:30:05.0000000", + "Usage": {"PromptTokens": 10, "CompletionTokens": 5, "TotalTokens": 15}, + "ToolCalls": [ + { + "Id": "call_1", + "Name": "lookup_order", + "Arguments": '{"id":"orders/1-A"}', + "Result": '{"status":"Shipped"}', + "SubConversationId": "chats/1-A/sub/1", + } + ], + }, + ], + } + + def test_result_parses_the_conversation_envelope(self): + result = AiConversationMessagesResult.from_json(self.RESPONSE) + + self.assertEqual("chats/1-A", result.conversation_id) + self.assertEqual("agents/support", result.agent) + self.assertTrue(result.has_more_messages) + self.assertEqual(["chats/1-A/sub/1"], result.sub_conversation_ids) + self.assertEqual(["invoice.pdf"], result.attachments) + self.assertEqual(datetime(2026, 6, 16, 10, 30, 5), result.last_message_at) + + def test_heterogeneous_parameters_come_back_as_they_are(self): + # Parameter values mix primitives and arrays, so they are handed over untouched. + parameters = AiConversationMessagesResult.from_json(self.RESPONSE).parameters + + self.assertEqual({"customer": "ALFKI", "tags": ["vip", "eu"], "retries": 2}, parameters) + + def test_usage_includes_reasoning_tokens(self): + result = AiConversationMessagesResult.from_json(self.RESPONSE) + + self.assertIsInstance(result.total_usage, AiUsage) + self.assertEqual(3, result.total_usage.reasoning_tokens) + self.assertEqual(15, result.total_usage.total_tokens) + + def test_messages_keep_their_role_and_timestamp(self): + messages = AiConversationMessagesResult.from_json(self.RESPONSE).messages + + self.assertEqual(2, len(messages)) + self.assertEqual(AiMessageRole.USER, messages[0].role) + self.assertEqual(AiMessageRole.ASSISTANT, messages[1].role) + self.assertEqual(datetime(2026, 6, 16, 10, 30, 0), messages[0].timestamp) + + def test_tool_calls_carry_their_result_and_sub_conversation(self): + tool_call = AiConversationMessagesResult.from_json(self.RESPONSE).messages[1].tool_calls[0] + + self.assertIsInstance(tool_call, AiToolCallResult) + self.assertEqual("lookup_order", tool_call.name) + self.assertEqual('{"status":"Shipped"}', tool_call.result) + self.assertEqual("chats/1-A/sub/1", tool_call.sub_conversation_id) + + def test_an_assistant_message_that_only_called_tools_has_no_content(self): + messages = AiConversationMessagesResult.from_json(self.RESPONSE).messages + + self.assertIsNone(messages[1].content) + + def test_result_survives_a_json_round_trip(self): + result = AiConversationMessagesResult.from_json(self.RESPONSE) + round_tripped = AiConversationMessagesResult.from_json(result.to_json()) + + self.assertEqual(result.to_json(), round_tripped.to_json()) + + def test_a_pending_tool_call_has_no_result_yet(self): + tool_call = AiToolCallResult.from_json({"Id": "call_1", "Name": "lookup_order", "Arguments": "{}"}) + + self.assertIsNone(tool_call.result) + + def test_an_empty_message_list_stays_empty(self): + result = AiConversationMessagesResult.from_json({"ConversationId": "chats/1-A", "Messages": []}) + + self.assertIsNone(result.messages) + self.assertFalse(result.has_more_messages) + + def test_message_to_json_writes_the_role_by_name(self): + message = AiConversationMessage(role=AiMessageRole.SUMMARY, content="short") + + self.assertEqual("Summary", message.to_json()["Role"]) + + +class TestCancelPendingActionTools(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def _url(self, **kwargs): + operation = RunConversationOperation(agent_id="agents/support", conversation_id="chats/1-A", **kwargs) + return operation.get_command(None).create_request(self.node).url + + def test_the_flag_is_sent_on_every_run(self): + # The server has no default of its own, so the client always states its intent. + self.assertIn("&cancelPendingActionTools=False", self._url()) + + def test_the_flag_is_sent_when_it_is_set(self): + self.assertIn("&cancelPendingActionTools=True", self._url(cancel_pending_action_tools=True)) From 1253a7f2fffb375c7cb94cfe6152bfa345aa73f1 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:35 +0200 Subject: [PATCH 06/29] Carry SSO configuration on client certificates --- ravendb/serverwide/operations/certificates.py | 143 +++++++++++--- .../operations_tests/test_certificates_sso.py | 178 ++++++++++++++++++ .../test_certificate_disabled_flag.py | 9 +- 3 files changed, 298 insertions(+), 32 deletions(-) create mode 100644 ravendb/tests/operations_tests/test_certificates_sso.py diff --git a/ravendb/serverwide/operations/certificates.py b/ravendb/serverwide/operations/certificates.py index 29a22cba..cc92d62e 100644 --- a/ravendb/serverwide/operations/certificates.py +++ b/ravendb/serverwide/operations/certificates.py @@ -39,6 +39,53 @@ def __str__(self): return self.value +class SsoProvider(enum.Enum): + GITHUB = "Github" + GOOGLE = "Google" + MICROSOFT = "Microsoft" + WINDOWS = "Windows" + + def __str__(self): + return self.value + + +class CertificateUsage(enum.Enum): + RAVEN_SERVER = "RavenServer" + RAVEN_SERVER_FOR_COMMUNICATION = "RavenServerForCommunication" + CLIENT = "Client" + SSO_SERVER = "SsoServer" + SSO_CLIENT = "SsoClient" + WELL_KNOWN_ISSUER = "WellKnownIssuer" + + def __str__(self): + return self.value + + +class SsoIdentifier: + """Identifies an SSO user with the provider that authenticated them.""" + + def __init__(self, provider: SsoProvider = None, identifier: str = None, domain: str = None): + self.provider = provider + self.identifier = identifier + self.domain = domain + + def to_json(self) -> dict: + return { + "Provider": self.provider.value if self.provider else None, + "Domain": self.domain, + "Identifier": self.identifier, + } + + @classmethod + def from_json(cls, json_dict: dict) -> SsoIdentifier: + provider = json_dict.get("Provider") + return cls( + provider=SsoProvider(provider) if provider else None, + identifier=json_dict.get("Identifier"), + domain=json_dict.get("Domain"), + ) + + class CertificateRawData: def __init__(self, raw_data: bytes = None): self.raw_data = raw_data @@ -57,6 +104,10 @@ def __init__( public_key_pinning_hash: str = None, not_before: datetime = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): self.name = name self.security_clearance = security_clearance @@ -68,6 +119,24 @@ def __init__( self.public_key_pinning_hash = public_key_pinning_hash self.not_before = not_before self.disabled = disabled + self.usage = usage + # The SSO servers allowed to authenticate this user, by public-key pinning hash. + self.sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes or [] + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers or [] + + @staticmethod + def _sso_fields_from_json(json_dict: dict) -> dict: + usage = json_dict.get("Usage") + sso_identifiers = json_dict.get("SsoIdentifiers") + return { + "usage": CertificateUsage(usage) if usage else None, + "sso_server_public_key_pinning_hashes": json_dict.get("SsoServerPublicKeyPinningHashes"), + "allow_any_sso_server": json_dict.get("AllowAnySsoServer", False), + "sso_identifiers": ( + [SsoIdentifier.from_json(identifier) for identifier in sso_identifiers] if sso_identifiers else None + ), + } @classmethod def from_json(cls, json_dict: dict) -> CertificateMetadata: @@ -82,6 +151,7 @@ def from_json(cls, json_dict: dict) -> CertificateMetadata: json_dict.get("PublicKeyPinningHash", None), Utils.string_to_datetime(json_dict["NotBefore"]) if "NotBefore" in json_dict else None, json_dict.get("Disabled", False), + **cls._sso_fields_from_json(json_dict), ) @@ -99,6 +169,10 @@ def __init__( collection_primary_key: str = None, public_key_pinning_hash: str = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): super().__init__( name, @@ -110,6 +184,10 @@ def __init__( collection_primary_key, public_key_pinning_hash, disabled=disabled, + usage=usage, + sso_server_public_key_pinning_hashes=sso_server_public_key_pinning_hashes, + allow_any_sso_server=allow_any_sso_server, + sso_identifiers=sso_identifiers, ) self.certificate = certificate self.password = password @@ -126,6 +204,10 @@ def to_json(self) -> dict: "Certificate": self.certificate, "Password": self.password, "Disabled": self.disabled, + "Usage": self.usage.value if self.usage else None, + "SsoServerPublicKeyPinningHashes": self.sso_server_public_key_pinning_hashes, + "AllowAnySsoServer": self.allow_any_sso_server, + "SsoIdentifiers": [identifier.to_json() for identifier in self.sso_identifiers], } if self.not_after: json_dict.update({"NotAfter": Utils.datetime_to_string(self.not_after)}) @@ -145,6 +227,7 @@ def from_json(cls, json_dict: dict) -> CertificateDefinition: json_dict["CollectionPrimaryKey"], json_dict["PublicKeyPinningHash"], disabled=json_dict.get("Disabled", False), + **cls._sso_fields_from_json(json_dict), ) @@ -456,12 +539,22 @@ def __init__( name: str, clearance: SecurityClearance, disabled: bool = False, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = None, + sso_identifiers: List[SsoIdentifier] = None, ): self.thumbprint = thumbprint self.permissions = permissions self.name = name self.clearance = clearance self.disabled = disabled + # SSO configuration is opt-in: leave these None to keep the stored settings + # untouched, which is what a plain certificate edit or a disabled-only toggle + # wants. Setting any of them - even to an empty list - replaces the stored value, + # which is how an SSO user's authorizing servers or identifiers get cleared. + self.sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers def __init__(self, parameters: Parameters): if parameters is None: @@ -476,32 +569,15 @@ def __init__(self, parameters: Parameters): if parameters.permissions is None: raise ValueError("permissions cannot be None") - self.__name = parameters.name - self.__thumbprint = parameters.thumbprint - self.__permissions = parameters.permissions - self.__clearance = parameters.clearance - self.__disabled = parameters.disabled + self.__parameters = parameters def get_command(self, conventions: "DocumentConventions") -> "VoidRavenCommand": - return self.__EditCertificateClientCommand( - self.__thumbprint, self.__name, self.__permissions, self.__clearance, self.__disabled - ) + return self.__EditCertificateClientCommand(self.__parameters) class __EditCertificateClientCommand(VoidRavenCommand, RaftCommand): - def __init__( - self, - thumbprint: str, - name: str, - permissions: Dict[str, DatabaseAccess], - clearance: SecurityClearance, - disabled: bool, - ): + def __init__(self, parameters: "EditClientCertificateOperation.Parameters"): super().__init__() - self.__thumbprint = thumbprint - self.__name = name - self.__permissions = permissions - self.__clearance = clearance - self.__disabled = disabled + self.__parameters = parameters def is_read_request(self) -> bool: return False @@ -509,15 +585,26 @@ def is_read_request(self) -> bool: def create_request(self, node: ServerNode) -> requests.Request: url = f"{node.url}/admin/certificates/edit" - definition = CertificateDefinition() - definition.thumbprint = self.__thumbprint - definition.permissions = self.__permissions - definition.security_clearance = self.__clearance - definition.name = self.__name - definition.disabled = self.__disabled + parameters = self.__parameters + definition = { + "Thumbprint": parameters.thumbprint, + "Name": parameters.name, + "SecurityClearance": parameters.clearance, + "Disabled": parameters.disabled, + "Permissions": parameters.permissions, + } + + # Written only when explicitly provided, so the server leaves the existing SSO + # configuration alone on a partial edit and clears it when an empty list is sent. + if parameters.sso_server_public_key_pinning_hashes is not None: + definition["SsoServerPublicKeyPinningHashes"] = parameters.sso_server_public_key_pinning_hashes + if parameters.allow_any_sso_server is not None: + definition["AllowAnySsoServer"] = parameters.allow_any_sso_server + if parameters.sso_identifiers is not None: + definition["SsoIdentifiers"] = [identifier.to_json() for identifier in parameters.sso_identifiers] request = requests.Request("POST", url) - request.data = definition.to_json() + request.data = definition return request diff --git a/ravendb/tests/operations_tests/test_certificates_sso.py b/ravendb/tests/operations_tests/test_certificates_sso.py new file mode 100644 index 00000000..c3fdd601 --- /dev/null +++ b/ravendb/tests/operations_tests/test_certificates_sso.py @@ -0,0 +1,178 @@ +""" +Tests for the certificate SSO fields added in 7.2.5: the metadata a certificate now +carries, and the opt-in write semantics of EditClientCertificateOperation. +""" + +import unittest + +from ravendb.documents.conventions import DocumentConventions +from ravendb.http.server_node import ServerNode +from ravendb.serverwide.operations.certificates import ( + CertificateDefinition, + CertificateMetadata, + CertificateUsage, + DatabaseAccess, + EditClientCertificateOperation, + SecurityClearance, + SsoIdentifier, + SsoProvider, +) + + +class TestSsoIdentifier(unittest.TestCase): + def test_identifier_round_trips(self): + payload = {"Provider": "Google", "Domain": "ravendb.net", "Identifier": "gracjan@ravendb.net"} + identifier = SsoIdentifier.from_json(payload) + + self.assertEqual(SsoProvider.GOOGLE, identifier.provider) + self.assertEqual("ravendb.net", identifier.domain) + self.assertEqual(payload, identifier.to_json()) + + def test_a_domainless_identifier_is_allowed(self): + identifier = SsoIdentifier.from_json({"Provider": "Github", "Identifier": "gracjan"}) + + self.assertEqual(SsoProvider.GITHUB, identifier.provider) + self.assertIsNone(identifier.domain) + + def test_every_provider_the_server_knows_is_mapped(self): + for name in ("Github", "Google", "Microsoft", "Windows"): + self.assertEqual(name, SsoProvider(name).value) + + +class TestCertificateSsoMetadata(unittest.TestCase): + PAYLOAD = { + "Name": "sso-user", + "SecurityClearance": "ValidUser", + "Thumbprint": "TP", + "Permissions": {}, + "Disabled": False, + "Usage": "SsoClient", + "AllowAnySsoServer": True, + "SsoServerPublicKeyPinningHashes": ["hash-a", "hash-b"], + "SsoIdentifiers": [{"Provider": "Microsoft", "Identifier": "gracjan@ravendb.net", "Domain": "ravendb.net"}], + } + + def test_metadata_parses_the_sso_fields(self): + metadata = CertificateMetadata.from_json(self.PAYLOAD) + + self.assertEqual(CertificateUsage.SSO_CLIENT, metadata.usage) + self.assertTrue(metadata.allow_any_sso_server) + self.assertEqual(["hash-a", "hash-b"], metadata.sso_server_public_key_pinning_hashes) + self.assertEqual(SsoProvider.MICROSOFT, metadata.sso_identifiers[0].provider) + + def test_a_certificate_from_an_older_server_gets_the_neutral_defaults(self): + metadata = CertificateMetadata.from_json({"Name": "n", "SecurityClearance": "Operator", "Permissions": {}}) + + self.assertIsNone(metadata.usage) + self.assertFalse(metadata.allow_any_sso_server) + self.assertEqual([], metadata.sso_server_public_key_pinning_hashes) + self.assertEqual([], metadata.sso_identifiers) + + def test_definition_parses_and_writes_the_sso_fields(self): + definition = CertificateDefinition.from_json( + { + **self.PAYLOAD, + "Certificate": "cert", + "NotAfter": None, + "CollectionSecondaryKeys": [], + "CollectionPrimaryKey": "", + "PublicKeyPinningHash": "h", + } + ) + serialized = definition.to_json() + + self.assertEqual("SsoClient", serialized["Usage"]) + self.assertTrue(serialized["AllowAnySsoServer"]) + self.assertEqual(["hash-a", "hash-b"], serialized["SsoServerPublicKeyPinningHashes"]) + self.assertEqual("gracjan@ravendb.net", serialized["SsoIdentifiers"][0]["Identifier"]) + + def test_a_definition_with_no_usage_writes_none(self): + self.assertIsNone(CertificateDefinition().to_json()["Usage"]) + + def test_every_usage_the_server_knows_is_mapped(self): + for name in ( + "RavenServer", + "RavenServerForCommunication", + "Client", + "SsoServer", + "SsoClient", + "WellKnownIssuer", + ): + self.assertEqual(name, CertificateUsage(name).value) + + +class TestEditClientCertificateSsoPayload(unittest.TestCase): + NODE = ServerNode("http://localhost:8080", "db") + + def _payload(self, **kwargs): + parameters = EditClientCertificateOperation.Parameters( + thumbprint="TP", + permissions={"orders": DatabaseAccess.READ_WRITE}, + name="sso-user", + clearance=SecurityClearance.VALID_USER, + **kwargs, + ) + request = EditClientCertificateOperation(parameters).get_command(None).create_request(self.NODE) + return request.data + + def test_the_base_payload_carries_only_what_the_server_edits(self): + payload = self._payload() + + self.assertEqual({"Thumbprint", "Name", "SecurityClearance", "Disabled", "Permissions"}, set(payload)) + self.assertEqual("TP", payload["Thumbprint"]) + self.assertFalse(payload["Disabled"]) + + def test_permissions_and_clearance_serialize_by_their_server_side_names(self): + payload = self._payload() + encoded = DocumentConventions.json_default(payload["Permissions"]["orders"]) + + self.assertEqual("ReadWrite", encoded) + self.assertEqual("ValidUser", DocumentConventions.json_default(payload["SecurityClearance"])) + + def test_a_plain_edit_leaves_the_stored_sso_configuration_alone(self): + # Omitting the SSO fields is how a regular edit avoids clobbering an SSO user. + payload = self._payload(disabled=True) + + self.assertNotIn("SsoServerPublicKeyPinningHashes", payload) + self.assertNotIn("AllowAnySsoServer", payload) + self.assertNotIn("SsoIdentifiers", payload) + self.assertTrue(payload["Disabled"]) + + def test_an_empty_list_is_sent_so_it_can_clear_the_stored_value(self): + payload = self._payload(sso_server_public_key_pinning_hashes=[], sso_identifiers=[]) + + self.assertEqual([], payload["SsoServerPublicKeyPinningHashes"]) + self.assertEqual([], payload["SsoIdentifiers"]) + + def test_setting_the_sso_fields_writes_them_all(self): + payload = self._payload( + sso_server_public_key_pinning_hashes=["hash-a"], + allow_any_sso_server=True, + sso_identifiers=[SsoIdentifier(SsoProvider.GITHUB, "gracjan")], + ) + + self.assertEqual(["hash-a"], payload["SsoServerPublicKeyPinningHashes"]) + self.assertTrue(payload["AllowAnySsoServer"]) + self.assertEqual({"Provider": "Github", "Domain": None, "Identifier": "gracjan"}, payload["SsoIdentifiers"][0]) + + def test_allow_any_sso_server_can_be_turned_off_explicitly(self): + payload = self._payload(allow_any_sso_server=False) + + self.assertIn("AllowAnySsoServer", payload) + self.assertFalse(payload["AllowAnySsoServer"]) + + def test_the_operation_still_validates_its_required_arguments(self): + with self.assertRaises(ValueError): + EditClientCertificateOperation(None) + with self.assertRaises(ValueError): + EditClientCertificateOperation( + EditClientCertificateOperation.Parameters("TP", {}, None, SecurityClearance.VALID_USER) + ) + with self.assertRaises(ValueError): + EditClientCertificateOperation( + EditClientCertificateOperation.Parameters(None, {}, "n", SecurityClearance.VALID_USER) + ) + with self.assertRaises(ValueError): + EditClientCertificateOperation( + EditClientCertificateOperation.Parameters("TP", None, "n", SecurityClearance.VALID_USER) + ) diff --git a/ravendb/tests/session_tests/test_certificate_disabled_flag.py b/ravendb/tests/session_tests/test_certificate_disabled_flag.py index bfe74d5a..e22d54e9 100644 --- a/ravendb/tests/session_tests/test_certificate_disabled_flag.py +++ b/ravendb/tests/session_tests/test_certificate_disabled_flag.py @@ -5,6 +5,7 @@ import unittest +from ravendb.http.server_node import ServerNode from ravendb.serverwide.operations.certificates import ( CertificateDefinition, CertificateMetadata, @@ -82,10 +83,10 @@ def test_edit_operation_parameters_carries_disabled(self): disabled=True, ) op = EditClientCertificateOperation(params) - # The disabled flag flows through to the command and into the request body - # via the definition's to_json. Verify by reaching through the private - # field on the operation instance. - self.assertTrue(op._EditClientCertificateOperation__disabled) + request = op.get_command(None).create_request(ServerNode("http://localhost:8080", "db")) + + # The disabled flag flows through the command into the request body. + self.assertTrue(request.data["Disabled"]) if __name__ == "__main__": From a00307ab0e8339dc4b572754c0edae738d095909 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:36 +0200 Subject: [PATCH 07/29] Add the CDC Sink client surface --- .../documents/operations/cdc_sink/__init__.py | 127 +++++ .../operations/cdc_sink/configuration.py | 486 ++++++++++++++++++ ravendb/documents/operations/ongoing_tasks.py | 111 ++++ ravendb/serverwide/database_record.py | 4 + .../tests/operations_tests/test_cdc_sink.py | 284 ++++++++++ 5 files changed, 1012 insertions(+) create mode 100644 ravendb/documents/operations/cdc_sink/__init__.py create mode 100644 ravendb/documents/operations/cdc_sink/configuration.py create mode 100644 ravendb/tests/operations_tests/test_cdc_sink.py diff --git a/ravendb/documents/operations/cdc_sink/__init__.py b/ravendb/documents/operations/cdc_sink/__init__.py new file mode 100644 index 00000000..e76c8a87 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/__init__.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import requests + +from ravendb.documents.operations.cdc_sink.configuration import ( + AddCdcSinkOperationResult, + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkProcessState, + CdcSinkRelationType, + CdcSinkTableConfig, + CdcSinkTableLoadState, + CdcSinkTaskState, + UpdateCdcSinkOperationResult, +) +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class AddCdcSinkOperation(MaintenanceOperation[AddCdcSinkOperationResult]): + """Adds a new CDC Sink task, streaming an SQL source into RavenDB collections.""" + + def __init__(self, configuration: CdcSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[AddCdcSinkOperationResult]: + return self._AddCdcSinkCommand(self._configuration) + + class _AddCdcSinkCommand(RavenCommand[AddCdcSinkOperationResult], RaftCommand): + def __init__(self, configuration: CdcSinkConfiguration): + super().__init__(AddCdcSinkOperationResult) + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = AddCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +class UpdateCdcSinkOperation(MaintenanceOperation[UpdateCdcSinkOperationResult]): + """Updates an existing CDC Sink task.""" + + def __init__(self, task_id: int, configuration: CdcSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._task_id = task_id + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[UpdateCdcSinkOperationResult]: + return self._UpdateCdcSinkCommand(self._task_id, self._configuration) + + class _UpdateCdcSinkCommand(RavenCommand[UpdateCdcSinkOperationResult], RaftCommand): + def __init__(self, task_id: int, configuration: CdcSinkConfiguration): + super().__init__(UpdateCdcSinkOperationResult) + self._task_id = task_id + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink?id={self._task_id}" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = UpdateCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +__all__ = [ + "AddCdcSinkOperation", + "AddCdcSinkOperationResult", + "CdcColumnMapping", + "CdcColumnType", + "CdcSinkConfiguration", + "CdcSinkEmbeddedTableConfig", + "CdcSinkLinkedTableConfig", + "CdcSinkOnDeleteConfig", + "CdcSinkPostgresSettings", + "CdcSinkProcessState", + "CdcSinkRelationType", + "CdcSinkTableConfig", + "CdcSinkTableLoadState", + "CdcSinkTaskState", + "UpdateCdcSinkOperation", + "UpdateCdcSinkOperationResult", +] diff --git a/ravendb/documents/operations/cdc_sink/configuration.py b/ravendb/documents/operations/cdc_sink/configuration.py new file mode 100644 index 00000000..291757f7 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/configuration.py @@ -0,0 +1,486 @@ +from __future__ import annotations + +import enum +from typing import Any, Dict, List, Optional + + +class CdcColumnType(enum.Enum): + """Controls how a CDC Sink column is stored in the target RavenDB document.""" + + # Document property with standard type conversion. JSON/JSONB columns land as plain + # strings unless explicitly marked as JSON. + DEFAULT = "Default" + # Parse the string value as its native JSON type: objects, arrays, strings, numbers, + # booleans and null. + JSON = "Json" + # Store as a RavenDB attachment instead of a document property. + ATTACHMENT = "Attachment" + + def __str__(self) -> str: + return self.value + + +class CdcSinkRelationType(enum.Enum): + """How an embedded table's rows are stored inside the parent document.""" + + # One-to-many, as a JSON array. + ARRAY = "Array" + # One-to-many, as a JSON object keyed by primary key value(s). Composite keys are "pk1,pk2". + MAP = "Map" + # Many-to-one, as a single value/object. + VALUE = "Value" + + def __str__(self) -> str: + return self.value + + +class CdcColumnMapping: + """Maps a single SQL column to a RavenDB document property or attachment.""" + + def __init__( + self, + column: str = None, + name: str = None, + type_: CdcColumnType = CdcColumnType.DEFAULT, + ): + self.column = column + # Document property name for Default and Json, attachment name for Attachment. + self.name = name + self.type_ = type_ + + def to_json(self) -> Dict[str, Any]: + json_dict = { + "Column": self.column, + "Name": self.name, + } + # The server treats an absent Type as Default, so it is only written when it differs. + if self.type_ is not None and self.type_ != CdcColumnType.DEFAULT: + json_dict["Type"] = self.type_.value + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcColumnMapping: + type_ = json_dict.get("Type") + return cls( + column=json_dict.get("Column"), + name=json_dict.get("Name"), + type_=CdcColumnType(type_) if type_ else CdcColumnType.DEFAULT, + ) + + +class CdcSinkOnDeleteConfig: + """ + Controls how DELETE events are handled for a CDC Sink table, root or embedded. + Leave it unset to have deletes processed normally: root documents are deleted and + embedded items are removed from their parent. + """ + + def __init__(self, patch: str = None, ignore_deletes: bool = False): + # JavaScript patch that runs when a DELETE event arrives, before the delete is + # applied. For root tables `this` is the document, for embedded tables the parent; + # `$row` is the raw CDC row of the DELETE event. + self.patch = patch + # When True the DELETE is not applied - the patch (if any) still runs first. + self.ignore_deletes = ignore_deletes + + def to_json(self) -> Dict[str, Any]: + return { + "Patch": self.patch, + "IgnoreDeletes": self.ignore_deletes, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkOnDeleteConfig: + return cls( + patch=json_dict.get("Patch"), + ignore_deletes=json_dict.get("IgnoreDeletes", False), + ) + + +class CdcSinkPostgresSettings: + """ + PostgreSQL-specific settings. Optional on creation - the server auto-fills generated + names when omitted - and immutable once set. + """ + + def __init__(self, publication_name: str = None, slot_name: str = None): + # Publication used for logical replication, auto-filled as rvn_cdc_p_{guid}. + self.publication_name = publication_name + # Logical replication slot, auto-filled as rvn_cdc_s_{guid}. + self.slot_name = slot_name + + def to_json(self) -> Dict[str, Any]: + return { + "PublicationName": self.publication_name, + "SlotName": self.slot_name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkPostgresSettings: + return cls( + publication_name=json_dict.get("PublicationName"), + slot_name=json_dict.get("SlotName"), + ) + + +class CdcSinkLinkedTableConfig: + """A table referenced by document ID link rather than embedded.""" + + def __init__( + self, + source_table_schema: str = None, + source_table_name: str = None, + property_name: str = None, + join_columns: List[str] = None, + linked_collection_name: str = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + # Property name in the document, e.g. "Customer". + self.property_name = property_name + # Foreign key columns used to resolve the link. + self.join_columns = join_columns or [] + # Target collection used for document ID generation, e.g. "Customers" -> "Customers/ALFKI". + self.linked_collection_name = linked_collection_name + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "JoinColumns": self.join_columns, + "LinkedCollectionName": self.linked_collection_name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkLinkedTableConfig: + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + join_columns=json_dict.get("JoinColumns"), + linked_collection_name=json_dict.get("LinkedCollectionName"), + ) + + +class CdcSinkEmbeddedTableConfig: + """A table stored as a nested object, array or map inside its parent's documents.""" + + def __init__( + self, + source_table_schema: str = None, + source_table_name: str = None, + property_name: str = None, + columns: List[CdcColumnMapping] = None, + primary_key_columns: List[str] = None, + join_columns: List[str] = None, + type_: CdcSinkRelationType = CdcSinkRelationType.ARRAY, + patch: str = None, + on_delete: CdcSinkOnDeleteConfig = None, + case_sensitive_keys: bool = False, + embedded_tables: List[CdcSinkEmbeddedTableConfig] = None, + linked_tables: List[CdcSinkLinkedTableConfig] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + # Property name in the parent document, e.g. "Lines". + self.property_name = property_name + self.columns = columns or [] + # Used to match items within arrays and maps on update and delete. + self.primary_key_columns = primary_key_columns or [] + # Foreign key columns joining this table to its parent. + self.join_columns = join_columns or [] + self.type_ = type_ + # JavaScript patch that runs on the parent document after the embedded operation + # has been applied. `this` is the parent, `$row` the raw CDC row, `$old` the item + # as it was before this event (None for inserts). + self.patch = patch + self.on_delete = on_delete + # When False (default) string primary key values and map keys compare + # case-insensitively. + self.case_sensitive_keys = case_sensitive_keys + # Deep nesting. Requires the nested table to carry a denormalized FK to the root. + self.embedded_tables = embedded_tables or [] + self.linked_tables = linked_tables or [] + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "JoinColumns": self.join_columns, + "Type": self.type_.value if self.type_ else None, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete else None, + "CaseSensitiveKeys": self.case_sensitive_keys, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkEmbeddedTableConfig: + type_ = json_dict.get("Type") + on_delete = json_dict.get("OnDelete") + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + columns=[CdcColumnMapping.from_json(column) for column in json_dict.get("Columns") or []], + primary_key_columns=json_dict.get("PrimaryKeyColumns"), + join_columns=json_dict.get("JoinColumns"), + type_=CdcSinkRelationType(type_) if type_ else CdcSinkRelationType.ARRAY, + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete) if on_delete else None, + case_sensitive_keys=json_dict.get("CaseSensitiveKeys", False), + embedded_tables=[cls.from_json(embedded) for embedded in json_dict.get("EmbeddedTables") or []], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in json_dict.get("LinkedTables") or [] + ], + ) + + +class CdcSinkTableConfig: + """Maps a source SQL table to a RavenDB collection.""" + + def __init__( + self, + collection_name: str = None, + source_table_schema: str = None, + source_table_name: str = None, + columns: List[CdcColumnMapping] = None, + primary_key_columns: List[str] = None, + patch: str = None, + on_delete: CdcSinkOnDeleteConfig = None, + disabled: bool = False, + embedded_tables: List[CdcSinkEmbeddedTableConfig] = None, + linked_tables: List[CdcSinkLinkedTableConfig] = None, + ): + self.collection_name = collection_name + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.columns = columns or [] + # Used for document ID generation. + self.primary_key_columns = primary_key_columns or [] + # JavaScript patch that runs after column mapping and embedded operations. + # `this` is the mapped document, `$row` the raw CDC row, `$old` the document as + # stored before this event (None for inserts). + self.patch = patch + self.on_delete = on_delete + # When True the table is skipped entirely: no initial load and no change capture. + self.disabled = disabled + self.embedded_tables = embedded_tables or [] + self.linked_tables = linked_tables or [] + + def to_json(self) -> Dict[str, Any]: + return { + "CollectionName": self.collection_name, + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete else None, + "Disabled": self.disabled, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkTableConfig: + on_delete = json_dict.get("OnDelete") + return cls( + collection_name=json_dict.get("CollectionName"), + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + columns=[CdcColumnMapping.from_json(column) for column in json_dict.get("Columns") or []], + primary_key_columns=json_dict.get("PrimaryKeyColumns"), + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete) if on_delete else None, + disabled=json_dict.get("Disabled", False), + embedded_tables=[ + CdcSinkEmbeddedTableConfig.from_json(embedded) for embedded in json_dict.get("EmbeddedTables") or [] + ], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in json_dict.get("LinkedTables") or [] + ], + ) + + +class CdcSinkConfiguration: + """A CDC Sink task: an SQL source streamed into RavenDB collections.""" + + def __init__( + self, + name: str = None, + connection_string_name: str = None, + tables: List[CdcSinkTableConfig] = None, + task_id: int = 0, + disabled: bool = False, + mentor_node: str = None, + pin_to_mentor_node: bool = False, + postgres: CdcSinkPostgresSettings = None, + skip_initial_load: bool = False, + ): + self.name = name + self.connection_string_name = connection_string_name + self.tables = tables or [] + self.task_id = task_id + self.disabled = disabled + self.mentor_node = mentor_node + self.pin_to_mentor_node = pin_to_mentor_node + # PostgreSQL only. None for SQL Server, auto-filled on creation if omitted. + self.postgres = postgres + # Skip the initial full-table load and start streaming changes straight away. + # For a target database that is already populated, e.g. from a prior migration. + self.skip_initial_load = skip_initial_load + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "TaskId": self.task_id, + "Disabled": self.disabled, + "ConnectionStringName": self.connection_string_name, + "MentorNode": self.mentor_node, + "PinToMentorNode": self.pin_to_mentor_node, + "Tables": [table.to_json() for table in self.tables], + "Postgres": self.postgres.to_json() if self.postgres else None, + "SkipInitialLoad": self.skip_initial_load, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkConfiguration: + postgres = json_dict.get("Postgres") + return cls( + name=json_dict.get("Name"), + connection_string_name=json_dict.get("ConnectionStringName"), + tables=[CdcSinkTableConfig.from_json(table) for table in json_dict.get("Tables") or []], + task_id=json_dict.get("TaskId", 0), + disabled=json_dict.get("Disabled", False), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode", False), + postgres=CdcSinkPostgresSettings.from_json(postgres) if postgres else None, + skip_initial_load=json_dict.get("SkipInitialLoad", False), + ) + + +class CdcSinkProcessState: + """Which node a CDC Sink task last ran on.""" + + def __init__(self, node_tag: str = None, configuration_name: str = None): + self.node_tag = node_tag + self.configuration_name = configuration_name + + def to_json(self) -> Dict[str, Any]: + return { + "ConfigurationName": self.configuration_name, + "NodeTag": self.node_tag, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkProcessState: + return cls( + node_tag=json_dict.get("NodeTag"), + configuration_name=json_dict.get("ConfigurationName"), + ) + + @staticmethod + def generate_item_name(database_name: str, configuration_name: str) -> str: + return f"values/{database_name}/cdcsink/{configuration_name}" + + +class CdcSinkTableLoadState: + """Per-table initial-load progress inside a CDC Sink task state document.""" + + def __init__( + self, + initial_load_completed: bool = False, + last_key_values: List[str] = None, + key_columns: List[str] = None, + ): + self.initial_load_completed = initial_load_completed + # The last primary key values loaded, in primary key column order, so an + # interrupted initial load can resume. + self.last_key_values = last_key_values + self.key_columns = key_columns + + def to_json(self) -> Dict[str, Any]: + return { + "InitialLoadCompleted": self.initial_load_completed, + "LastKeyValues": self.last_key_values, + "KeyColumns": self.key_columns, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkTableLoadState: + return cls( + initial_load_completed=json_dict.get("InitialLoadCompleted", False), + last_key_values=json_dict.get("LastKeyValues"), + key_columns=json_dict.get("KeyColumns"), + ) + + +class CdcSinkTaskState: + """ + The state document of a CDC Sink task, stored in the @cdc-states collection. + Tracks the last processed LSN and per-table initial load progress. + """ + + COLLECTION_NAME = "@cdc-states" + + def __init__( + self, + configuration_name: str = None, + last_lsn: str = None, + tables: Dict[str, CdcSinkTableLoadState] = None, + ): + self.configuration_name = configuration_name + # The last successfully processed Log Sequence Number, used to resume streaming. + self.last_lsn = last_lsn + # Keyed by "schema.tableName". + self.tables = tables or {} + + def to_json(self) -> Dict[str, Any]: + return { + "ConfigurationName": self.configuration_name, + "LastLsn": self.last_lsn, + "Tables": {name: state.to_json() for name, state in self.tables.items()}, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkTaskState: + return cls( + configuration_name=json_dict.get("ConfigurationName"), + last_lsn=json_dict.get("LastLsn"), + tables={ + name: CdcSinkTableLoadState.from_json(state) for name, state in (json_dict.get("Tables") or {}).items() + }, + ) + + @classmethod + def get_document_id(cls, configuration_name: str) -> str: + # Configuration names compare case-insensitively, but the document ID keeps the + # casing it was created with. + return f"{cls.COLLECTION_NAME}/{configuration_name}" + + +class AddCdcSinkOperationResult: + def __init__(self, raft_command_index: Optional[int] = None, task_id: Optional[int] = None): + self.raft_command_index = raft_command_index + self.task_id = task_id + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index, "TaskId": self.task_id} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AddCdcSinkOperationResult: + return cls( + raft_command_index=json_dict.get("RaftCommandIndex"), + task_id=json_dict.get("TaskId"), + ) + + +class UpdateCdcSinkOperationResult(AddCdcSinkOperationResult): + # Same shape as the add result; kept as its own name so callers read what they got back. + pass diff --git a/ravendb/documents/operations/ongoing_tasks.py b/ravendb/documents/operations/ongoing_tasks.py index 62b869c6..cbbb28cf 100644 --- a/ravendb/documents/operations/ongoing_tasks.py +++ b/ravendb/documents/operations/ongoing_tasks.py @@ -1,6 +1,7 @@ from __future__ import annotations import json from enum import Enum +from datetime import datetime from typing import Optional, TYPE_CHECKING, Union import requests @@ -16,6 +17,7 @@ if TYPE_CHECKING: from ravendb.documents.conventions import DocumentConventions + from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration from ravendb.documents.operations.ai.gen_ai_configuration import GenAiConfiguration from ravendb.documents.operations.ai.embeddings_generation_configuration import EmbeddingsGenerationConfiguration @@ -33,6 +35,7 @@ class OngoingTaskType(Enum): PULL_REPLICATION_AS_HUB = "PullReplicationAsHub" PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" QUEUE_SINK = "QueueSink" + CDC_SINK = "CdcSink" EMBEDDINGS_GENERATION = "EmbeddingsGeneration" GEN_AI = "GenAi" @@ -392,6 +395,8 @@ def __init__( access_name: Optional[str] = None, allowed_hub_to_sink_paths: Optional[list] = None, allowed_sink_to_hub_paths: Optional[list] = None, + hub_cursor: Optional[str] = None, + sink_cursor: Optional[str] = None, ): super().__init__( task_id=task_id, @@ -414,6 +419,9 @@ def __init__( self.access_name = access_name self.allowed_hub_to_sink_paths = allowed_hub_to_sink_paths self.allowed_sink_to_hub_paths = allowed_sink_to_hub_paths + # Composite change vectors of the last item each side sent. + self.hub_cursor = hub_cursor + self.sink_cursor = sink_cursor def to_json(self) -> dict: result = super().to_json() @@ -427,6 +435,8 @@ def to_json(self) -> dict: result["AccessName"] = self.access_name result["AllowedHubToSinkPaths"] = self.allowed_hub_to_sink_paths result["AllowedSinkToHubPaths"] = self.allowed_sink_to_hub_paths + result["HubCursor"] = self.hub_cursor + result["SinkCursor"] = self.sink_cursor return result @classmethod @@ -457,6 +467,105 @@ def from_json(cls, json_dict: dict) -> Optional["OngoingTaskPullReplicationAsSin access_name=json_dict.get("AccessName"), allowed_hub_to_sink_paths=json_dict.get("AllowedHubToSinkPaths"), allowed_sink_to_hub_paths=json_dict.get("AllowedSinkToHubPaths"), + hub_cursor=json_dict.get("HubCursor"), + sink_cursor=json_dict.get("SinkCursor"), + ) + + +class OngoingTaskCdcSink(OngoingTask): + """Ongoing task information for a CDC Sink task.""" + + def __init__( + self, + task_id: Optional[int] = None, + responsible_node: Optional[NodeId] = None, + task_state: Optional[OngoingTaskState] = None, + task_connection_status: Optional[OngoingTaskConnectionStatus] = None, + task_name: Optional[str] = None, + error: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: Optional[bool] = None, + configuration: Optional["CdcSinkConfiguration"] = None, + connection_string_name: Optional[str] = None, + factory_name: Optional[str] = None, + last_batch_time: Optional[datetime] = None, + last_checkpoint: Optional[str] = None, + seconds_since_last_batch: Optional[float] = None, + last_activity_time: Optional[datetime] = None, + seconds_since_last_activity: Optional[float] = None, + health_issue: Optional[str] = None, + ): + super().__init__( + task_id=task_id, + task_type=OngoingTaskType.CDC_SINK, + responsible_node=responsible_node, + task_state=task_state, + task_connection_status=task_connection_status, + task_name=task_name, + error=error, + mentor_node=mentor_node, + pin_to_mentor_node=pin_to_mentor_node, + ) + self.configuration = configuration + self.connection_string_name = connection_string_name + self.factory_name = factory_name + # UTC time of the last successfully completed batch, None before the first one. + self.last_batch_time = last_batch_time + # The last persisted checkpoint (LSN/GTID). + self.last_checkpoint = last_checkpoint + self.seconds_since_last_batch = seconds_since_last_batch + # UTC time of the last activity from the source: a poll iteration (SQL Server), + # replication message (PostgreSQL) or binlog event (MySQL). Recent activity with + # a stale last_batch_time means the connection is alive but idle; both stale + # suggests the connection is dead. + self.last_activity_time = last_activity_time + self.seconds_since_last_activity = seconds_since_last_activity + # None when healthy, otherwise a diagnostic from the process itself. + self.health_issue = health_issue + + def to_json(self) -> dict: + result = super().to_json() + result["ConnectionStringName"] = self.connection_string_name + result["FactoryName"] = self.factory_name + result["Configuration"] = self.configuration.to_json() if self.configuration else None + result["LastBatchTime"] = Utils.datetime_to_string(self.last_batch_time) + result["LastCheckpoint"] = self.last_checkpoint + result["SecondsSinceLastBatch"] = self.seconds_since_last_batch + result["LastActivityTime"] = Utils.datetime_to_string(self.last_activity_time) + result["SecondsSinceLastActivity"] = self.seconds_since_last_activity + result["HealthIssue"] = self.health_issue + return result + + @classmethod + def from_json(cls, json_dict: dict) -> Optional["OngoingTaskCdcSink"]: + if json_dict is None: + return None + + from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration + + task_state_str = json_dict.get("TaskState") + task_connection_status_str = json_dict.get("TaskConnectionStatus") + configuration = json_dict.get("Configuration") + return cls( + task_id=json_dict.get("TaskId"), + responsible_node=NodeId.from_json(json_dict.get("ResponsibleNode")), + task_state=OngoingTaskState(task_state_str) if task_state_str else None, + task_connection_status=( + OngoingTaskConnectionStatus(task_connection_status_str) if task_connection_status_str else None + ), + task_name=json_dict.get("TaskName"), + error=json_dict.get("Error"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode"), + configuration=CdcSinkConfiguration.from_json(configuration) if configuration else None, + connection_string_name=json_dict.get("ConnectionStringName"), + factory_name=json_dict.get("FactoryName"), + last_batch_time=Utils.string_to_datetime(json_dict.get("LastBatchTime")), + last_checkpoint=json_dict.get("LastCheckpoint"), + seconds_since_last_batch=json_dict.get("SecondsSinceLastBatch"), + last_activity_time=Utils.string_to_datetime(json_dict.get("LastActivityTime")), + seconds_since_last_activity=json_dict.get("SecondsSinceLastActivity"), + health_issue=json_dict.get("HealthIssue"), ) @@ -637,6 +746,8 @@ def _deserialize_task( return OngoingTaskEmbeddingsGeneration.from_json(json_dict) elif self._task_type == OngoingTaskType.PULL_REPLICATION_AS_SINK: return OngoingTaskPullReplicationAsSink.from_json(json_dict) + elif self._task_type == OngoingTaskType.CDC_SINK: + return OngoingTaskCdcSink.from_json(json_dict) else: # todo: handle more types of tasks return OngoingTask.from_json(json_dict) diff --git a/ravendb/serverwide/database_record.py b/ravendb/serverwide/database_record.py index 955b1e84..580a0544 100644 --- a/ravendb/serverwide/database_record.py +++ b/ravendb/serverwide/database_record.py @@ -11,6 +11,7 @@ AutoIndexDefinition, ) from ravendb.documents.operations.backups.settings import PeriodicBackupConfiguration +from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration from ravendb.documents.operations.etl.configuration import RavenConnectionString, RavenEtlConfiguration from ravendb.documents.operations.etl.olap.connection import OlapConnectionString, OlapEtlConfiguration from ravendb.documents.operations.etl.sql import SqlConnectionString, SqlEtlConfiguration @@ -69,6 +70,7 @@ def __init__(self, database_name: Optional[str] = None): self.raven_etls: List[RavenEtlConfiguration] = [] self.sql_etls: List[SqlEtlConfiguration] = [] self.olap_etls: List[OlapEtlConfiguration] = [] + self.cdc_sinks: List[CdcSinkConfiguration] = [] self.embeddings_generations: List = [] self.client: Optional[ClientConfiguration] = None self.studio: Optional[StudioConfiguration] = None @@ -119,6 +121,7 @@ def to_json(self): "RavenEtls": self.raven_etls, "SqlEtls": self.sql_etls, "OlapEtls": self.olap_etls, + "CdcSinks": [cdc_sink.to_json() for cdc_sink in self.cdc_sinks or []], "Client": self.client, "Studio": self.studio, "TruncatedClusterTransactionCommand": self.truncated_cluster_transaction_commands_count, @@ -167,6 +170,7 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.raven_etls = json_dict.get("RavenEtls", None) record.sql_etls = json_dict.get("SqlEtls", None) record.olap_etls = json_dict.get("OlapEtls", None) + record.cdc_sinks = [CdcSinkConfiguration.from_json(cdc_sink) for cdc_sink in json_dict.get("CdcSinks") or []] embeddings_generations_data = json_dict.get("EmbeddingsGenerations", []) if embeddings_generations_data: from ravendb.documents.operations.ai.embeddings_generation_configuration import ( diff --git a/ravendb/tests/operations_tests/test_cdc_sink.py b/ravendb/tests/operations_tests/test_cdc_sink.py new file mode 100644 index 00000000..9bc02d2e --- /dev/null +++ b/ravendb/tests/operations_tests/test_cdc_sink.py @@ -0,0 +1,284 @@ +""" +Tests for the CDC Sink client surface added in 7.2.5: the configuration tree, +AddCdcSinkOperation / UpdateCdcSinkOperation, and the CdcSink ongoing task. +""" + +import json +import unittest + +from ravendb.documents.operations.cdc_sink import ( + AddCdcSinkOperation, + AddCdcSinkOperationResult, + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkProcessState, + CdcSinkRelationType, + CdcSinkTableConfig, + CdcSinkTaskState, + UpdateCdcSinkOperation, +) +from ravendb.documents.operations.ongoing_tasks import ( + GetOngoingTaskInfoOperation, + OngoingTaskCdcSink, + OngoingTaskType, +) +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.database_record import DatabaseRecord + + +def _configuration() -> CdcSinkConfiguration: + return CdcSinkConfiguration( + name="orders-cdc", + connection_string_name="pg", + postgres=CdcSinkPostgresSettings(publication_name="pub", slot_name="slot"), + skip_initial_load=True, + tables=[ + CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="public", + source_table_name="orders", + columns=[ + CdcColumnMapping("id", "Id"), + CdcColumnMapping("payload", "Payload", CdcColumnType.JSON), + CdcColumnMapping("blob", "file.bin", CdcColumnType.ATTACHMENT), + ], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True), + embedded_tables=[ + CdcSinkEmbeddedTableConfig( + source_table_name="order_lines", + property_name="Lines", + columns=[CdcColumnMapping("line_id", "LineId")], + primary_key_columns=["line_id"], + join_columns=["order_id"], + type_=CdcSinkRelationType.MAP, + case_sensitive_keys=True, + embedded_tables=[ + CdcSinkEmbeddedTableConfig( + source_table_name="line_notes", + property_name="Notes", + columns=[CdcColumnMapping("note_id", "NoteId")], + primary_key_columns=["note_id"], + join_columns=["line_id"], + ) + ], + ) + ], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ) + ], + ) + + +class TestCdcSinkConfiguration(unittest.TestCase): + def test_configuration_survives_a_json_round_trip(self): + configuration = _configuration() + serialized = configuration.to_json() + + self.assertEqual(serialized, CdcSinkConfiguration.from_json(serialized).to_json()) + + def test_configuration_serializes_the_task_level_fields(self): + serialized = _configuration().to_json() + + self.assertEqual("orders-cdc", serialized["Name"]) + self.assertEqual("pg", serialized["ConnectionStringName"]) + self.assertTrue(serialized["SkipInitialLoad"]) + self.assertEqual({"PublicationName": "pub", "SlotName": "slot"}, serialized["Postgres"]) + self.assertEqual(0, serialized["TaskId"]) + self.assertFalse(serialized["Disabled"]) + + def test_column_type_is_written_only_when_it_is_not_default(self): + # The server reads a missing Type as Default, and so does the C# client. + self.assertEqual({"Column": "c", "Name": "N"}, CdcColumnMapping("c", "N").to_json()) + self.assertEqual( + {"Column": "c", "Name": "N", "Type": "Json"}, + CdcColumnMapping("c", "N", CdcColumnType.JSON).to_json(), + ) + + def test_column_without_a_type_deserializes_as_default(self): + self.assertEqual(CdcColumnType.DEFAULT, CdcColumnMapping.from_json({"Column": "c", "Name": "N"}).type_) + + def test_embedded_tables_nest_to_any_depth(self): + configuration = CdcSinkConfiguration.from_json(_configuration().to_json()) + lines = configuration.tables[0].embedded_tables[0] + + self.assertEqual(CdcSinkRelationType.MAP, lines.type_) + self.assertTrue(lines.case_sensitive_keys) + self.assertEqual("Notes", lines.embedded_tables[0].property_name) + self.assertEqual(["line_id"], lines.embedded_tables[0].join_columns) + + def test_on_delete_round_trips(self): + on_delete = CdcSinkConfiguration.from_json(_configuration().to_json()).tables[0].on_delete + + self.assertEqual("this.Archived = true;", on_delete.patch) + self.assertTrue(on_delete.ignore_deletes) + + def test_a_table_without_an_on_delete_keeps_it_unset(self): + table = CdcSinkTableConfig.from_json({"CollectionName": "Orders"}) + + self.assertIsNone(table.on_delete) + self.assertEqual([], table.columns) + self.assertEqual([], table.embedded_tables) + + def test_collections_default_to_empty_rather_than_none(self): + configuration = CdcSinkConfiguration() + + self.assertEqual([], configuration.tables) + self.assertEqual([], CdcSinkEmbeddedTableConfig().columns) + self.assertEqual([], CdcSinkLinkedTableConfig().join_columns) + + +class TestCdcSinkTaskState(unittest.TestCase): + def test_task_state_round_trips(self): + payload = { + "ConfigurationName": "orders-cdc", + "LastLsn": "0/16B3748", + "Tables": { + "public.orders": { + "InitialLoadCompleted": True, + "LastKeyValues": ["10"], + "KeyColumns": ["id"], + } + }, + } + state = CdcSinkTaskState.from_json(payload) + + self.assertEqual("0/16B3748", state.last_lsn) + self.assertTrue(state.tables["public.orders"].initial_load_completed) + self.assertEqual(["id"], state.tables["public.orders"].key_columns) + self.assertEqual(payload, state.to_json()) + + def test_document_id_is_built_from_the_state_collection(self): + self.assertEqual("@cdc-states", CdcSinkTaskState.COLLECTION_NAME) + self.assertEqual("@cdc-states/orders-cdc", CdcSinkTaskState.get_document_id("orders-cdc")) + + def test_process_state_item_name(self): + self.assertEqual("values/db/cdcsink/orders-cdc", CdcSinkProcessState.generate_item_name("db", "orders-cdc")) + + +class TestCdcSinkOperations(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def test_add_sends_the_configuration_to_the_admin_endpoint(self): + configuration = _configuration() + command = AddCdcSinkOperation(configuration).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink", request.url) + self.assertEqual(configuration.to_json(), request.data) + self.assertFalse(command.is_read_request()) + + def test_update_addresses_the_task_by_id(self): + command = UpdateCdcSinkOperation(42, _configuration()).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink?id=42", request.url) + + def test_both_operations_are_raft_commands(self): + # The server applies CDC Sink changes through Raft, so retries must reuse one id. + add = AddCdcSinkOperation(_configuration()).get_command(None) + update = UpdateCdcSinkOperation(1, _configuration()).get_command(None) + + self.assertIsInstance(add, RaftCommand) + self.assertIsInstance(update, RaftCommand) + self.assertTrue(add.get_raft_unique_request_id()) + + def test_a_missing_configuration_is_rejected_client_side(self): + with self.assertRaises(ValueError): + AddCdcSinkOperation(None) + with self.assertRaises(ValueError): + UpdateCdcSinkOperation(1, None) + + def test_add_reads_the_task_id_off_the_response(self): + command = AddCdcSinkOperation(_configuration()).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 17, "TaskId": 5}), False) + + self.assertIsInstance(command.result, AddCdcSinkOperationResult) + self.assertEqual(17, command.result.raft_command_index) + self.assertEqual(5, command.result.task_id) + + +class TestCdcSinkOngoingTask(unittest.TestCase): + RESPONSE = { + "TaskId": 7, + "TaskType": "CdcSink", + "TaskName": "orders-cdc", + "TaskState": "Enabled", + "ResponsibleNode": {"NodeTag": "A", "NodeUrl": "http://localhost:8080"}, + "ConnectionStringName": "pg", + "FactoryName": "Npgsql", + "LastCheckpoint": "0/16B3748", + "LastBatchTime": "2026-06-16T10:30:00.0000000", + "SecondsSinceLastBatch": 12.5, + "LastActivityTime": "2026-06-16T10:30:05.0000000", + "SecondsSinceLastActivity": 7.5, + "HealthIssue": None, + "Configuration": {"Name": "orders-cdc", "Tables": [{"CollectionName": "Orders"}]}, + } + + def test_cdc_sink_is_a_known_ongoing_task_type(self): + self.assertEqual("CdcSink", OngoingTaskType.CDC_SINK.value) + + def test_ongoing_task_carries_the_health_and_lag_fields(self): + task = OngoingTaskCdcSink.from_json(self.RESPONSE) + + self.assertEqual(OngoingTaskType.CDC_SINK, task.task_type) + self.assertEqual("Npgsql", task.factory_name) + self.assertEqual("0/16B3748", task.last_checkpoint) + self.assertEqual(12.5, task.seconds_since_last_batch) + self.assertEqual(7.5, task.seconds_since_last_activity) + self.assertIsNone(task.health_issue) + self.assertEqual("2026-06-16T10:30:00.0000000", task.to_json()["LastBatchTime"]) + + def test_ongoing_task_parses_the_nested_configuration(self): + task = OngoingTaskCdcSink.from_json(self.RESPONSE) + + self.assertIsInstance(task.configuration, CdcSinkConfiguration) + self.assertEqual("Orders", task.configuration.tables[0].collection_name) + + def test_get_ongoing_task_info_dispatches_to_the_cdc_sink_result(self): + command = GetOngoingTaskInfoOperation("orders-cdc", OngoingTaskType.CDC_SINK).get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db")) + command.set_response(json.dumps(self.RESPONSE), False) + + self.assertIn("type=CdcSink", request.url) + self.assertIsInstance(command.result, OngoingTaskCdcSink) + self.assertEqual("orders-cdc", command.result.task_name) + + +class TestCdcSinkInDatabaseRecord(unittest.TestCase): + def test_database_record_carries_cdc_sinks(self): + record = DatabaseRecord("db") + record.cdc_sinks = [_configuration()] + + self.assertEqual("orders-cdc", record.to_json()["CdcSinks"][0]["Name"]) + + def test_database_record_parses_cdc_sinks(self): + record = DatabaseRecord.from_json( + {"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}, "CdcSinks": [_configuration().to_json()]} + ) + + self.assertEqual(1, len(record.cdc_sinks)) + self.assertEqual("Orders", record.cdc_sinks[0].tables[0].collection_name) + + def test_a_record_from_a_server_without_cdc_sinks_gets_an_empty_list(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}}) + + self.assertEqual([], record.cdc_sinks) From 708afa7391437252e257f88a393a1efd30001b69 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:36 +0200 Subject: [PATCH 08/29] Manage server-wide connection strings and report what uses them --- .../get_connection_string_operation.py | 69 ++-- .../operations/connection_strings.py | 70 +++- .../operations/connection_strings.py | 313 ++++++++++++++++++ .../test_server_wide_connection_strings.py | 250 ++++++++++++++ 4 files changed, 658 insertions(+), 44 deletions(-) create mode 100644 ravendb/serverwide/operations/connection_strings.py create mode 100644 ravendb/tests/operations_tests/test_server_wide_connection_strings.py diff --git a/ravendb/documents/operations/connection_string/get_connection_string_operation.py b/ravendb/documents/operations/connection_string/get_connection_string_operation.py index 13dbec34..6000e876 100644 --- a/ravendb/documents/operations/connection_string/get_connection_string_operation.py +++ b/ravendb/documents/operations/connection_string/get_connection_string_operation.py @@ -1,10 +1,11 @@ import json -from typing import Dict, Optional +from typing import Dict, Optional, Type, TypeVar import requests from ravendb import RavenCommand, ServerNode from ravendb.documents.operations.ai.ai_connection_string import AiConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage from ravendb.documents.operations.definitions import MaintenanceOperation from ravendb.documents.operations.etl.configuration import RavenConnectionString from ravendb.documents.operations.etl.elastic_search.connection import ElasticSearchConnectionString @@ -14,6 +15,8 @@ from ravendb.documents.operations.etl.sql import SqlConnectionString from ravendb.serverwide.server_operation_executor import ConnectionStringType +_T = TypeVar("_T", bound=ConnectionString) + class GetConnectionStringsResult: def __init__( @@ -45,52 +48,32 @@ def to_json(self) -> Dict: "SnowflakeConnectionStrings": [x.to_json() for x in self.snowflake_connection_strings.values()], } + @classmethod + def _parse(cls, json_dict: Optional[Dict[str, Dict]], connection_string_type: Type[_T]) -> Optional[Dict[str, _T]]: + if not json_dict: + return None + + result = {} + for key, value in json_dict.items(): + connection_string = connection_string_type.from_json(value) + # UsedBy is computed server-side and only present on reads. + connection_string.used_by = ConnectionStringUsage.list_from_json(value.get("UsedBy")) + result[key] = connection_string + return result + @classmethod def from_json(cls, json_dict: Dict[str, Dict]) -> "GetConnectionStringsResult": return cls( - raven_connection_strings=( - {key: RavenConnectionString.from_json(rcs) for key, rcs in json_dict["RavenConnectionStrings"].items()} - if json_dict["RavenConnectionStrings"] - else None - ), - sql_connection_strings=( - {key: SqlConnectionString.from_json(sqlcs) for key, sqlcs in json_dict["SqlConnectionStrings"].items()} - if json_dict["SqlConnectionStrings"] - else None - ), - olap_connection_strings=( - { - key: OlapConnectionString.from_json(olapcs) - for key, olapcs in json_dict["OlapConnectionStrings"].items() - } - if json_dict["OlapConnectionStrings"] - else None - ), - ai_connection_strings=( - {key: AiConnectionString.from_json(aics) for key, aics in json_dict["AiConnectionStrings"].items()} - if json_dict["AiConnectionStrings"] - else None - ), - elastic_search_connection_strings=( - { - key: ElasticSearchConnectionString.from_json(escs) - for key, escs in json_dict["ElasticSearchConnectionStrings"].items() - } - if json_dict["ElasticSearchConnectionStrings"] - else None - ), - queue_connection_strings=( - {key: QueueConnectionString.from_json(qcs) for key, qcs in json_dict["QueueConnectionStrings"].items()} - if json_dict["QueueConnectionStrings"] - else None + raven_connection_strings=cls._parse(json_dict.get("RavenConnectionStrings"), RavenConnectionString), + sql_connection_strings=cls._parse(json_dict.get("SqlConnectionStrings"), SqlConnectionString), + olap_connection_strings=cls._parse(json_dict.get("OlapConnectionStrings"), OlapConnectionString), + ai_connection_strings=cls._parse(json_dict.get("AiConnectionStrings"), AiConnectionString), + elastic_search_connection_strings=cls._parse( + json_dict.get("ElasticSearchConnectionStrings"), ElasticSearchConnectionString ), - snowflake_connection_strings=( - { - key: SnowflakeConnectionString.from_json(scs) - for key, scs in json_dict["SnowflakeConnectionStrings"].items() - } - if json_dict["SnowflakeConnectionStrings"] - else None + queue_connection_strings=cls._parse(json_dict.get("QueueConnectionStrings"), QueueConnectionString), + snowflake_connection_strings=cls._parse( + json_dict.get("SnowflakeConnectionStrings"), SnowflakeConnectionString ), ) diff --git a/ravendb/documents/operations/connection_strings.py b/ravendb/documents/operations/connection_strings.py index be5c9a44..c3e2aacd 100644 --- a/ravendb/documents/operations/connection_strings.py +++ b/ravendb/documents/operations/connection_strings.py @@ -1,10 +1,78 @@ +import enum from abc import abstractmethod -from typing import Dict, Any +from typing import Any, Dict, List, Optional + + +class ConnectionStringUsageKind(enum.Enum): + """The kind of task or agent that references a connection string.""" + + RAVEN_ETL = "RavenEtl" + SQL_ETL = "SqlEtl" + OLAP_ETL = "OlapEtl" + ELASTIC_SEARCH_ETL = "ElasticSearchEtl" + QUEUE_ETL = "QueueEtl" + SNOWFLAKE_ETL = "SnowflakeEtl" + QUEUE_SINK = "QueueSink" + EXTERNAL_REPLICATION = "ExternalReplication" + PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" + EMBEDDINGS_GENERATION = "EmbeddingsGeneration" + GEN_AI = "GenAi" + AI_AGENT = "AiAgent" + CDC_SINK = "CdcSink" + + def __str__(self): + return self.value + + +class ConnectionStringUsage: + """ + One task or agent that references a connection string. Computed server-side and + returned when reading connection strings; it is never sent back on a write. + """ + + def __init__( + self, + kind: ConnectionStringUsageKind = None, + id_: Optional[int] = None, + identifier: str = None, + name: str = None, + ): + self.kind = kind + # The numeric task id, for ongoing tasks (ETL, replication, sinks). None for AI agents. + self.id_ = id_ + # The string identifier, for AI agents. None for ongoing tasks. + self.identifier = identifier + self.name = name + + def to_json(self) -> Dict[str, Any]: + return { + "Kind": self.kind.value if self.kind else None, + "Id": self.id_, + "Identifier": self.identifier, + "Name": self.name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "ConnectionStringUsage": + kind = json_dict.get("Kind") + return cls( + kind=ConnectionStringUsageKind(kind) if kind else None, + id_=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + ) + + @classmethod + def list_from_json(cls, json_list: Optional[List[Dict[str, Any]]]) -> List["ConnectionStringUsage"]: + return [cls.from_json(usage) for usage in json_list or []] class ConnectionString: def __init__(self, name: str): self.name = name + # Populated when reading connection strings back from the server. Left empty on + # anything the client builds, and never written by to_json. + self.used_by: List[ConnectionStringUsage] = [] @abstractmethod def get_type(self): diff --git a/ravendb/serverwide/operations/connection_strings.py b/ravendb/serverwide/operations/connection_strings.py new file mode 100644 index 00000000..e78e53dd --- /dev/null +++ b/ravendb/serverwide/operations/connection_strings.py @@ -0,0 +1,313 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +import requests + +from ravendb.documents.operations.connection_strings import ( + ConnectionString, + ConnectionStringUsage, + ConnectionStringUsageKind, +) +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.common import ServerOperation +from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.tools.utils import Utils +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class ServerWideConnectionStringUsage(ConnectionStringUsage): + """ + A usage of a server-wide connection string. Adds the database the referencing task + or agent lives in, since server-wide usages are aggregated across every database. + """ + + def __init__( + self, + kind: ConnectionStringUsageKind = None, + id_: Optional[int] = None, + identifier: str = None, + name: str = None, + database_name: str = None, + ): + super().__init__(kind, id_, identifier, name) + self.database_name = database_name + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["DatabaseName"] = self.database_name + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> ServerWideConnectionStringUsage: + kind = json_dict.get("Kind") + return cls( + kind=ConnectionStringUsageKind(kind) if kind else None, + id_=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + database_name=json_dict.get("DatabaseName"), + ) + + +class ServerWideConnectionString: + """ + A connection string that the cluster propagates to every database, except those + listed in excluded_databases. + """ + + NAME_PREFIX = "Server Wide Connection String" + + def __init__( + self, + connection_string: ConnectionString = None, + excluded_databases: List[str] = None, + used_by: List[ServerWideConnectionStringUsage] = None, + ): + self.connection_string = connection_string + # Databases that should not receive this connection string. When None or empty it + # is propagated everywhere. + self.excluded_databases = excluded_databases + # Computed server-side when reading; anything a client sends here is ignored. + self.used_by = used_by or [] + + @property + def name(self) -> Optional[str]: + return self.connection_string.name if self.connection_string else None + + @property + def type(self) -> ConnectionStringType: + if self.connection_string is None: + return ConnectionStringType.NONE + return ConnectionStringType(self.connection_string.get_type) + + @staticmethod + def get_database_record_connection_string_name(name: str) -> str: + """The name a propagated server-wide connection string carries inside a database record.""" + return f"{ServerWideConnectionString.NAME_PREFIX}, {name}" + + def to_json(self) -> Dict[str, Any]: + # The connection string is flattened into the top-level object rather than nested, + # which is what the server-wide endpoint expects. + json_dict = self.connection_string.to_json() if self.connection_string else {} + json_dict["Type"] = self.type.value + json_dict["ExcludedDatabases"] = self.excluded_databases + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> Optional[ServerWideConnectionString]: + if json_dict is None: + return None + + type_ = json_dict.get("Type") + if type_ is None: + return None + + return cls( + connection_string=cls._deserialize_connection_string(json_dict, ConnectionStringType(type_)), + excluded_databases=json_dict.get("ExcludedDatabases"), + used_by=[ServerWideConnectionStringUsage.from_json(usage) for usage in json_dict.get("UsedBy") or []], + ) + + @staticmethod + def _deserialize_connection_string( + json_dict: Dict[str, Any], connection_string_type: ConnectionStringType + ) -> ConnectionString: + from ravendb.documents.operations.ai.ai_connection_string import AiConnectionString + from ravendb.documents.operations.etl.configuration import RavenConnectionString + from ravendb.documents.operations.etl.elastic_search.connection import ElasticSearchConnectionString + from ravendb.documents.operations.etl.olap.connection import OlapConnectionString + from ravendb.documents.operations.etl.queue.connection import QueueConnectionString + from ravendb.documents.operations.etl.snowflake.connection import SnowflakeConnectionString + from ravendb.documents.operations.etl.sql import SqlConnectionString + + types = { + ConnectionStringType.RAVEN: RavenConnectionString, + ConnectionStringType.SQL: SqlConnectionString, + ConnectionStringType.OLAP: OlapConnectionString, + ConnectionStringType.ELASTIC_SEARCH: ElasticSearchConnectionString, + ConnectionStringType.QUEUE: QueueConnectionString, + ConnectionStringType.SNOWFLAKE: SnowflakeConnectionString, + ConnectionStringType.AI: AiConnectionString, + } + + if connection_string_type not in types: + raise NotImplementedError(f"Unknown connection string type: {connection_string_type}") + + return types[connection_string_type].from_json(json_dict) + + +class GetServerWideConnectionStringsResult: + def __init__(self, results: List[ServerWideConnectionString] = None): + self.results = results or [] + + def to_json(self) -> Dict[str, Any]: + return {"Results": [connection_string.to_json() for connection_string in self.results]} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> GetServerWideConnectionStringsResult: + return cls(results=[ServerWideConnectionString.from_json(result) for result in json_dict.get("Results") or []]) + + +class GetServerWideConnectionStringsOperation(ServerOperation[GetServerWideConnectionStringsResult]): + """ + Reads server-wide connection strings from the cluster, either all of them or + filtered by name and type. + """ + + def __init__(self, connection_string_name: str = None, connection_string_type: ConnectionStringType = None): + if connection_string_name is not None and not connection_string_name.strip(): + raise ValueError("Connection string name must not be empty.") + + self._connection_string_name = connection_string_name + self._type = connection_string_type + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[GetServerWideConnectionStringsResult]: + return self._GetServerWideConnectionStringsCommand(self._connection_string_name, self._type) + + class _GetServerWideConnectionStringsCommand(RavenCommand[GetServerWideConnectionStringsResult]): + def __init__(self, connection_string_name: str = None, connection_string_type: ConnectionStringType = None): + super().__init__(GetServerWideConnectionStringsResult) + self._connection_string_name = connection_string_name + self._type = connection_string_type + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + + query_params = [] + if self._connection_string_name is not None: + query_params.append(f"name={Utils.quote_key(self._connection_string_name)}") + if self._type is not None and self._type != ConnectionStringType.NONE: + query_params.append(f"type={self._type.value}") + + if query_params: + url += f"?{'&'.join(query_params)}" + + return requests.Request("GET", url) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = GetServerWideConnectionStringsResult.from_json(json.loads(response)) + + +class PutServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> PutServerWideConnectionStringResult: + return cls(json_dict.get("RaftCommandIndex")) + + +class PutServerWideConnectionStringOperation(ServerOperation[PutServerWideConnectionStringResult]): + """ + Creates or updates a server-wide connection string. The cluster propagates it to + every database that is not listed in excluded_databases. + """ + + def __init__(self, connection_string: ServerWideConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + + if connection_string.connection_string is None: + raise ValueError("ServerWideConnectionString.connection_string must not be None.") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[PutServerWideConnectionStringResult]: + return self._PutServerWideConnectionStringCommand(self._connection_string) + + class _PutServerWideConnectionStringCommand(RavenCommand[PutServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ServerWideConnectionString): + super().__init__(PutServerWideConnectionStringResult) + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + + request = requests.Request("PUT", url) + request.data = self._connection_string.to_json() + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = PutServerWideConnectionStringResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +class RemoveServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> RemoveServerWideConnectionStringResult: + return cls(json_dict.get("RaftCommandIndex")) + + +class RemoveServerWideConnectionStringOperation(ServerOperation[RemoveServerWideConnectionStringResult]): + """ + Removes a server-wide connection string from the cluster and from every database + record that received it. Fails when an ongoing task still uses it. + """ + + def __init__(self, connection_string: ConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + + if not connection_string.name or connection_string.name.isspace(): + raise ValueError("Connection string name must not be empty.") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[RemoveServerWideConnectionStringResult]: + return self._RemoveServerWideConnectionStringCommand(self._connection_string) + + class _RemoveServerWideConnectionStringCommand(RavenCommand[RemoveServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ConnectionString): + super().__init__(RemoveServerWideConnectionStringResult) + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = ( + f"{node.url}/admin/configuration/server-wide/connection-strings" + f"?name={Utils.quote_key(self._connection_string.name)}" + f"&type={self._connection_string.get_type}" + ) + + return requests.Request("DELETE", url) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = RemoveServerWideConnectionStringResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() diff --git a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py new file mode 100644 index 00000000..65314419 --- /dev/null +++ b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py @@ -0,0 +1,250 @@ +""" +Tests for the server-wide connection string operations added in 7.2.5, and for the +UsedBy metadata the server now returns with every connection string. +""" + +import json +import unittest + +from ravendb.documents.operations.connection_string.get_connection_string_operation import ( + GetConnectionStringsResult, +) +from ravendb.documents.operations.connection_strings import ConnectionStringUsage, ConnectionStringUsageKind +from ravendb.documents.operations.etl.configuration import RavenConnectionString +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + PutServerWideConnectionStringOperation, + RemoveServerWideConnectionStringOperation, + ServerWideConnectionString, + ServerWideConnectionStringUsage, +) +from ravendb.serverwide.server_operation_executor import ConnectionStringType + + +class TestConnectionStringUsage(unittest.TestCase): + def test_ongoing_tasks_are_identified_by_a_numeric_id(self): + usage = ConnectionStringUsage.from_json({"Kind": "RavenEtl", "Id": 3, "Identifier": None, "Name": "etl-to-b"}) + + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, usage.kind) + self.assertEqual(3, usage.id_) + self.assertIsNone(usage.identifier) + self.assertEqual("etl-to-b", usage.name) + + def test_ai_agents_are_identified_by_a_string_identifier(self): + usage = ConnectionStringUsage.from_json( + {"Kind": "AiAgent", "Id": None, "Identifier": "agents/support", "Name": "support"} + ) + + self.assertEqual(ConnectionStringUsageKind.AI_AGENT, usage.kind) + self.assertIsNone(usage.id_) + self.assertEqual("agents/support", usage.identifier) + + def test_cdc_sink_is_a_known_usage_kind(self): + self.assertEqual("CdcSink", ConnectionStringUsageKind.CDC_SINK.value) + + def test_read_connection_strings_carry_their_usages(self): + result = GetConnectionStringsResult.from_json( + { + "RavenConnectionStrings": { + "to-b": { + "Name": "to-b", + "Database": "b", + "TopologyDiscoveryUrls": ["http://localhost:8080"], + "UsedBy": [ + {"Kind": "RavenEtl", "Id": 3, "Name": "etl-1"}, + {"Kind": "AiAgent", "Identifier": "agents/1", "Name": "helper"}, + ], + } + } + } + ) + connection_string = result.raven_connection_strings["to-b"] + + self.assertEqual(2, len(connection_string.used_by)) + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, connection_string.used_by[0].kind) + self.assertEqual("agents/1", connection_string.used_by[1].identifier) + + def test_a_connection_string_nothing_uses_has_an_empty_usage_list(self): + result = GetConnectionStringsResult.from_json( + {"SqlConnectionStrings": {"sql1": {"Name": "sql1", "ConnectionString": "x", "FactoryName": "y"}}} + ) + + self.assertEqual([], result.sql_connection_strings["sql1"].used_by) + + def test_usages_are_never_written_back(self): + # UsedBy is computed server-side; sending it back would be meaningless. + connection_string = RavenConnectionString("to-b", "b", ["http://localhost:8080"]) + connection_string.used_by = [ConnectionStringUsage(ConnectionStringUsageKind.RAVEN_ETL, 3, None, "etl-1")] + + self.assertNotIn("UsedBy", connection_string.to_json()) + + def test_an_absent_bucket_reads_as_none(self): + result = GetConnectionStringsResult.from_json({"RavenConnectionStrings": {}}) + + self.assertIsNone(result.raven_connection_strings) + self.assertIsNone(result.queue_connection_strings) + + +class TestServerWideConnectionString(unittest.TestCase): + def _raven(self) -> ServerWideConnectionString: + return ServerWideConnectionString( + connection_string=RavenConnectionString("shared-raven", "orders", ["http://localhost:8080"]), + excluded_databases=["scratch"], + ) + + def test_name_and_type_are_delegated_to_the_wrapped_connection_string(self): + server_wide = self._raven() + + self.assertEqual("shared-raven", server_wide.name) + self.assertEqual(ConnectionStringType.RAVEN, server_wide.type) + + def test_an_empty_wrapper_reports_no_type(self): + server_wide = ServerWideConnectionString() + + self.assertIsNone(server_wide.name) + self.assertEqual(ConnectionStringType.NONE, server_wide.type) + + def test_the_connection_string_is_flattened_rather_than_nested(self): + serialized = self._raven().to_json() + + self.assertEqual("shared-raven", serialized["Name"]) + self.assertEqual("orders", serialized["Database"]) + self.assertEqual("Raven", serialized["Type"]) + self.assertEqual(["scratch"], serialized["ExcludedDatabases"]) + + def test_it_round_trips_through_the_flat_shape(self): + serialized = self._raven().to_json() + parsed = ServerWideConnectionString.from_json(serialized) + + self.assertIsInstance(parsed.connection_string, RavenConnectionString) + self.assertEqual(serialized, parsed.to_json()) + + def test_every_connection_string_type_is_dispatched_by_its_type_field(self): + queue = ServerWideConnectionString( + connection_string=QueueConnectionString("shared-queue", QueueBrokerType.AZURE_SERVICE_BUS) + ) + parsed = ServerWideConnectionString.from_json(queue.to_json()) + + self.assertIsInstance(parsed.connection_string, QueueConnectionString) + self.assertEqual(QueueBrokerType.AZURE_SERVICE_BUS, parsed.connection_string.broker_type) + + def test_a_payload_without_a_type_cannot_be_parsed(self): + self.assertIsNone(ServerWideConnectionString.from_json({"Name": "x"})) + self.assertIsNone(ServerWideConnectionString.from_json(None)) + + def test_usages_carry_the_database_they_come_from(self): + parsed = ServerWideConnectionString.from_json( + { + "Name": "shared-raven", + "Database": "orders", + "TopologyDiscoveryUrls": [], + "Type": "Raven", + "UsedBy": [{"Kind": "RavenEtl", "Id": 3, "Name": "etl-1", "DatabaseName": "shop"}], + } + ) + + self.assertIsInstance(parsed.used_by[0], ServerWideConnectionStringUsage) + self.assertEqual("shop", parsed.used_by[0].database_name) + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, parsed.used_by[0].kind) + + def test_the_propagated_name_inside_a_database_record_is_prefixed(self): + self.assertEqual( + "Server Wide Connection String, shared-raven", + ServerWideConnectionString.get_database_record_connection_string_name("shared-raven"), + ) + + +class TestServerWideConnectionStringOperations(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + self.server_wide = ServerWideConnectionString( + connection_string=RavenConnectionString("shared-raven", "orders", ["http://localhost:8080"]) + ) + + def test_get_without_a_filter_asks_for_everything(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + request = command.create_request(self.node) + + self.assertEqual("GET", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + self.assertTrue(command.is_read_request()) + + def test_get_filters_by_name_and_type(self): + command = GetServerWideConnectionStringsOperation("shared raven", ConnectionStringType.RAVEN).get_command(None) + url = command.create_request(self.node).url + + self.assertIn("?name=shared%20raven", url) + self.assertIn("&type=Raven", url) + + def test_a_blank_name_is_rejected_client_side(self): + with self.assertRaises(ValueError): + GetServerWideConnectionStringsOperation(" ") + + def test_get_parses_the_results_list(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + command.set_response( + json.dumps( + { + "Results": [ + { + "Name": "shared-raven", + "Database": "orders", + "TopologyDiscoveryUrls": [], + "Type": "Raven", + "ExcludedDatabases": ["scratch"], + } + ] + } + ), + False, + ) + + self.assertEqual(1, len(command.result.results)) + self.assertEqual("shared-raven", command.result.results[0].name) + self.assertEqual(["scratch"], command.result.results[0].excluded_databases) + + def test_put_sends_the_flattened_connection_string(self): + command = PutServerWideConnectionStringOperation(self.server_wide).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + self.assertEqual(self.server_wide.to_json(), request.data) + self.assertIsInstance(command, RaftCommand) + + def test_put_reads_back_the_raft_index(self): + command = PutServerWideConnectionStringOperation(self.server_wide).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 11}), False) + + self.assertEqual(11, command.result.raft_command_index) + + def test_put_requires_a_wrapped_connection_string(self): + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(ServerWideConnectionString()) + + def test_remove_addresses_the_connection_string_by_name_and_type(self): + command = RemoveServerWideConnectionStringOperation(SqlConnectionString("shared sql")).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("DELETE", request.method) + self.assertIn("?name=shared%20sql", request.url) + self.assertIn("&type=Sql", request.url) + self.assertIsInstance(command, RaftCommand) + + def test_remove_requires_a_named_connection_string(self): + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(SqlConnectionString(None)) + + def test_remove_reads_back_the_raft_index(self): + command = RemoveServerWideConnectionStringOperation(SqlConnectionString("sql1")).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 12}), False) + + self.assertEqual(12, command.result.raft_command_index) From 16450bd0da6f40320d16890302b2ecb46e51cc15 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:47:36 +0200 Subject: [PATCH 09/29] Export the new 7.2.5 types from the package root --- ravendb/__init__.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 172d57b8..f3df37c8 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -75,7 +75,11 @@ ) from ravendb.documents.operations.configuration.definitions import StudioConfiguration, StudioEnvironment -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ( + ConnectionString, + ConnectionStringUsage, + ConnectionStringUsageKind, +) # AI Operations from ravendb.documents.ai import ( @@ -115,6 +119,13 @@ GetAiAgentsResponse, AddOrUpdateAiAgentOperation, DeleteAiAgentOperation, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, ) from ravendb.documents.operations.ai import ( ChunkingOptions, @@ -188,6 +199,25 @@ from ravendb.documents.operations.ongoing_tasks import ( OngoingTaskPullReplicationAsSink, OngoingTaskPullReplicationAsHub, + OngoingTaskCdcSink, +) +from ravendb.documents.operations.cdc_sink import ( + AddCdcSinkOperation, + AddCdcSinkOperationResult, + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkProcessState, + CdcSinkRelationType, + CdcSinkTableConfig, + CdcSinkTableLoadState, + CdcSinkTaskState, + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, ) from ravendb.documents.operations.revisions import ( RevisionsCollectionConfiguration, @@ -356,6 +386,19 @@ GetCertificatesResponse, PutClientCertificateOperation, SecurityClearance, + CertificateUsage, + SsoIdentifier, + SsoProvider, +) +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + GetServerWideConnectionStringsResult, + PutServerWideConnectionStringOperation, + PutServerWideConnectionStringResult, + RemoveServerWideConnectionStringOperation, + RemoveServerWideConnectionStringResult, + ServerWideConnectionString, + ServerWideConnectionStringUsage, ) from ravendb.serverwide.operations.common import ( BuildNumber, From edc55a77feb11907b0cebaea2504c120c081b906 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 09:51:25 +0200 Subject: [PATCH 10/29] Drop an unused import from the conversation-messages tests --- ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py index 00adeb5d..ddce3c8c 100644 --- a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py +++ b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py @@ -3,7 +3,6 @@ conversation back, and for the cancel-pending-action-tools flag on a run. """ -import json import unittest from datetime import datetime, timedelta, timezone From 5ca953459f3d8a3bde41765d839809540fbdf7da Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 18:27:29 +0200 Subject: [PATCH 11/29] Add the Queue Sink client surface --- ravendb/documents/operations/ongoing_tasks.py | 77 +++++ .../operations/queue_sink/__init__.py | 114 ++++++++ .../azure_service_bus_sink_source.py | 42 +++ .../operations/queue_sink/configuration.py | 145 ++++++++++ .../tests/operations_tests/test_queue_sink.py | 272 ++++++++++++++++++ 5 files changed, 650 insertions(+) create mode 100644 ravendb/documents/operations/queue_sink/__init__.py create mode 100644 ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py create mode 100644 ravendb/documents/operations/queue_sink/configuration.py create mode 100644 ravendb/tests/operations_tests/test_queue_sink.py diff --git a/ravendb/documents/operations/ongoing_tasks.py b/ravendb/documents/operations/ongoing_tasks.py index cbbb28cf..e09648ca 100644 --- a/ravendb/documents/operations/ongoing_tasks.py +++ b/ravendb/documents/operations/ongoing_tasks.py @@ -18,6 +18,8 @@ if TYPE_CHECKING: from ravendb.documents.conventions import DocumentConventions from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration + from ravendb.documents.operations.etl.queue.connection import QueueBrokerType + from ravendb.documents.operations.queue_sink.configuration import QueueSinkConfiguration from ravendb.documents.operations.ai.gen_ai_configuration import GenAiConfiguration from ravendb.documents.operations.ai.embeddings_generation_configuration import EmbeddingsGenerationConfiguration @@ -472,6 +474,79 @@ def from_json(cls, json_dict: dict) -> Optional["OngoingTaskPullReplicationAsSin ) +class OngoingTaskQueueSink(OngoingTask): + """Ongoing task information for a queue sink task.""" + + def __init__( + self, + task_id: Optional[int] = None, + responsible_node: Optional[NodeId] = None, + task_state: Optional[OngoingTaskState] = None, + task_connection_status: Optional[OngoingTaskConnectionStatus] = None, + task_name: Optional[str] = None, + error: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: Optional[bool] = None, + configuration: Optional["QueueSinkConfiguration"] = None, + broker_type: Optional["QueueBrokerType"] = None, + connection_string_name: Optional[str] = None, + url: Optional[str] = None, + ): + super().__init__( + task_id=task_id, + task_type=OngoingTaskType.QUEUE_SINK, + responsible_node=responsible_node, + task_state=task_state, + task_connection_status=task_connection_status, + task_name=task_name, + error=error, + mentor_node=mentor_node, + pin_to_mentor_node=pin_to_mentor_node, + ) + self.configuration = configuration + self.broker_type = broker_type + self.connection_string_name = connection_string_name + # The broker URL the server resolved from the connection string. + self.url = url + + def to_json(self) -> dict: + result = super().to_json() + result["BrokerType"] = self.broker_type.value if self.broker_type else None + result["ConnectionStringName"] = self.connection_string_name + result["Url"] = self.url + result["Configuration"] = self.configuration.to_json() if self.configuration else None + return result + + @classmethod + def from_json(cls, json_dict: dict) -> Optional["OngoingTaskQueueSink"]: + if json_dict is None: + return None + + from ravendb.documents.operations.etl.queue.connection import QueueBrokerType + from ravendb.documents.operations.queue_sink.configuration import QueueSinkConfiguration + + task_state_str = json_dict.get("TaskState") + task_connection_status_str = json_dict.get("TaskConnectionStatus") + broker_type_str = json_dict.get("BrokerType") + configuration = json_dict.get("Configuration") + return cls( + task_id=json_dict.get("TaskId"), + responsible_node=NodeId.from_json(json_dict.get("ResponsibleNode")), + task_state=OngoingTaskState(task_state_str) if task_state_str else None, + task_connection_status=( + OngoingTaskConnectionStatus(task_connection_status_str) if task_connection_status_str else None + ), + task_name=json_dict.get("TaskName"), + error=json_dict.get("Error"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode"), + configuration=QueueSinkConfiguration.from_json(configuration) if configuration else None, + broker_type=QueueBrokerType(broker_type_str) if broker_type_str else None, + connection_string_name=json_dict.get("ConnectionStringName"), + url=json_dict.get("Url"), + ) + + class OngoingTaskCdcSink(OngoingTask): """Ongoing task information for a CDC Sink task.""" @@ -748,6 +823,8 @@ def _deserialize_task( return OngoingTaskPullReplicationAsSink.from_json(json_dict) elif self._task_type == OngoingTaskType.CDC_SINK: return OngoingTaskCdcSink.from_json(json_dict) + elif self._task_type == OngoingTaskType.QUEUE_SINK: + return OngoingTaskQueueSink.from_json(json_dict) else: # todo: handle more types of tasks return OngoingTask.from_json(json_dict) diff --git a/ravendb/documents/operations/queue_sink/__init__.py b/ravendb/documents/operations/queue_sink/__init__.py new file mode 100644 index 00000000..8482a68a --- /dev/null +++ b/ravendb/documents/operations/queue_sink/__init__.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import requests + +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.operations.queue_sink.azure_service_bus_sink_source import AzureServiceBusSinkSource +from ravendb.documents.operations.queue_sink.configuration import ( + AddQueueSinkOperationResult, + QueueSinkConfiguration, + QueueSinkProcessState, + QueueSinkScript, + UpdateQueueSinkOperationResult, +) +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class AddQueueSinkOperation(MaintenanceOperation[AddQueueSinkOperationResult]): + """ + Adds a queue sink task, which has RavenDB consume messages from an external broker + such as Kafka or RabbitMQ and store them in the database. + """ + + def __init__(self, configuration: QueueSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[AddQueueSinkOperationResult]: + return self._AddQueueSinkCommand(self._configuration) + + class _AddQueueSinkCommand(RavenCommand[AddQueueSinkOperationResult], RaftCommand): + def __init__(self, configuration: QueueSinkConfiguration): + super().__init__(AddQueueSinkOperationResult) + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/queue-sink" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = AddQueueSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +class UpdateQueueSinkOperation(MaintenanceOperation[UpdateQueueSinkOperationResult]): + """Updates an existing queue sink task.""" + + def __init__(self, task_id: int, configuration: QueueSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._task_id = task_id + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[UpdateQueueSinkOperationResult]: + return self._UpdateQueueSinkCommand(self._task_id, self._configuration) + + class _UpdateQueueSinkCommand(RavenCommand[UpdateQueueSinkOperationResult], RaftCommand): + def __init__(self, task_id: int, configuration: QueueSinkConfiguration): + super().__init__(UpdateQueueSinkOperationResult) + self._task_id = task_id + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/queue-sink?id={self._task_id}" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = UpdateQueueSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +__all__ = [ + "AddQueueSinkOperation", + "AddQueueSinkOperationResult", + "AzureServiceBusSinkSource", + "QueueSinkConfiguration", + "QueueSinkProcessState", + "QueueSinkScript", + "UpdateQueueSinkOperation", + "UpdateQueueSinkOperationResult", +] diff --git a/ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py b/ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py new file mode 100644 index 00000000..fd9844af --- /dev/null +++ b/ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py @@ -0,0 +1,42 @@ +from __future__ import annotations + + +class AzureServiceBusSinkSource: + """ + Builds the strings that go into QueueSinkScript.queues for an Azure Service Bus sink. + + A queue is its own name; a topic subscription is encoded as "topic;subscription". + Service Bus naming rules forbid ';' in queue, topic and subscription names, so the + separator cannot collide with a real name. + """ + + SEPARATOR = ";" + + @staticmethod + def queue(queue_name: str) -> str: + """The entry for a Service Bus queue. A pass-through, kept for symmetry with subscription().""" + if not queue_name or queue_name.isspace(): + raise ValueError("Queue name must be non-empty.") + + if AzureServiceBusSinkSource.SEPARATOR in queue_name: + raise ValueError(f"Queue name must not contain the '{AzureServiceBusSinkSource.SEPARATOR}' character.") + + return queue_name + + @staticmethod + def subscription(topic_name: str, subscription_name: str) -> str: + """The entry for a topic subscription, in the form 'topic;subscription'.""" + if not topic_name or topic_name.isspace(): + raise ValueError("Topic name must be non-empty.") + + if not subscription_name or subscription_name.isspace(): + raise ValueError("Subscription name must be non-empty.") + + separator = AzureServiceBusSinkSource.SEPARATOR + if separator in topic_name: + raise ValueError(f"Topic name must not contain the '{separator}' character.") + + if separator in subscription_name: + raise ValueError(f"Subscription name must not contain the '{separator}' character.") + + return f"{topic_name}{separator}{subscription_name}" diff --git a/ravendb/documents/operations/queue_sink/configuration.py b/ravendb/documents/operations/queue_sink/configuration.py new file mode 100644 index 00000000..eb261fbe --- /dev/null +++ b/ravendb/documents/operations/queue_sink/configuration.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType + + +class QueueSinkScript: + """ + A user-defined script that consumes messages from one or more queues and decides + how they are stored in RavenDB. + """ + + def __init__( + self, + name: str = None, + queues: List[str] = None, + script: str = None, + disabled: bool = False, + ): + self.name = name + # Broker-specific source entries. For Azure Service Bus these are built with + # AzureServiceBusSinkSource.queue() / .subscription(). + self.queues = queues or [] + self.script = script + self.disabled = disabled + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "Script": self.script, + "Queues": self.queues, + "Disabled": self.disabled, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> QueueSinkScript: + return cls( + name=json_dict.get("Name"), + queues=json_dict.get("Queues"), + script=json_dict.get("Script"), + disabled=json_dict.get("Disabled", False), + ) + + +class QueueSinkConfiguration: + """ + A queue sink task: RavenDB consumes messages from an external broker (Kafka, + RabbitMQ, Azure Queue Storage, Amazon SQS, Azure Service Bus) and stores them. + """ + + def __init__( + self, + name: str = None, + broker_type: QueueBrokerType = None, + connection_string_name: str = None, + scripts: List[QueueSinkScript] = None, + task_id: int = 0, + disabled: bool = False, + mentor_node: str = None, + pin_to_mentor_node: bool = False, + ): + self.name = name + self.broker_type = broker_type + self.connection_string_name = connection_string_name + self.scripts = scripts or [] + self.task_id = task_id + self.disabled = disabled + self.mentor_node = mentor_node + self.pin_to_mentor_node = pin_to_mentor_node + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "TaskId": self.task_id, + "Disabled": self.disabled, + "ConnectionStringName": self.connection_string_name, + "MentorNode": self.mentor_node, + "PinToMentorNode": self.pin_to_mentor_node, + "Scripts": [script.to_json() for script in self.scripts], + "BrokerType": self.broker_type.value if self.broker_type else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> QueueSinkConfiguration: + broker_type = json_dict.get("BrokerType") + return cls( + name=json_dict.get("Name"), + broker_type=QueueBrokerType(broker_type) if broker_type else None, + connection_string_name=json_dict.get("ConnectionStringName"), + scripts=[QueueSinkScript.from_json(script) for script in json_dict.get("Scripts") or []], + task_id=json_dict.get("TaskId", 0), + disabled=json_dict.get("Disabled", False), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode", False), + ) + + +class QueueSinkProcessState: + """Which node a queue sink script last ran on.""" + + def __init__(self, node_tag: str = None, configuration_name: str = None, script_name: str = None): + self.node_tag = node_tag + self.configuration_name = configuration_name + self.script_name = script_name + + def to_json(self) -> Dict[str, Any]: + return { + "ConfigurationName": self.configuration_name, + "ScriptName": self.script_name, + "NodeTag": self.node_tag, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> QueueSinkProcessState: + return cls( + node_tag=json_dict.get("NodeTag"), + configuration_name=json_dict.get("ConfigurationName"), + script_name=json_dict.get("ScriptName"), + ) + + @staticmethod + def generate_item_name(database_name: str, configuration_name: str, transformation_name: str) -> str: + return f"values/{database_name}/queuesink/{configuration_name.lower()}/{transformation_name.lower()}" + + +class AddQueueSinkOperationResult: + def __init__(self, raft_command_index: Optional[int] = None, task_id: Optional[int] = None): + self.raft_command_index = raft_command_index + self.task_id = task_id + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index, "TaskId": self.task_id} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AddQueueSinkOperationResult: + return cls( + raft_command_index=json_dict.get("RaftCommandIndex"), + task_id=json_dict.get("TaskId"), + ) + + +class UpdateQueueSinkOperationResult(AddQueueSinkOperationResult): + # Same shape as the add result; kept as its own name so callers read what they got back. + pass diff --git a/ravendb/tests/operations_tests/test_queue_sink.py b/ravendb/tests/operations_tests/test_queue_sink.py new file mode 100644 index 00000000..71f1d63f --- /dev/null +++ b/ravendb/tests/operations_tests/test_queue_sink.py @@ -0,0 +1,272 @@ +""" +Tests for the Queue Sink client surface: the configuration, the Add/Update operations, +the Azure Service Bus source encoding, and the QueueSink ongoing task. +""" + +import json +import unittest + +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType +from ravendb.documents.operations.ongoing_tasks import ( + GetOngoingTaskInfoOperation, + OngoingTaskQueueSink, + OngoingTaskType, +) +from ravendb.documents.operations.queue_sink import ( + AddQueueSinkOperation, + AddQueueSinkOperationResult, + AzureServiceBusSinkSource, + QueueSinkConfiguration, + QueueSinkProcessState, + QueueSinkScript, + UpdateQueueSinkOperation, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.etl.queue.connection import QueueConnectionString +from ravendb.documents.operations.etl.queue.kafka_connection_settings import KafkaConnectionSettings +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.database_record import DatabaseRecord +from ravendb.tests.test_base import TestBase + + +def _configuration() -> QueueSinkConfiguration: + return QueueSinkConfiguration( + name="orders-sink", + broker_type=QueueBrokerType.KAFKA, + connection_string_name="kafka", + scripts=[ + QueueSinkScript( + name="orders", + queues=["orders-topic"], + script="put('orders/', this);", + ) + ], + ) + + +class TestQueueSinkConfiguration(unittest.TestCase): + def test_configuration_survives_a_json_round_trip(self): + serialized = _configuration().to_json() + + self.assertEqual(serialized, QueueSinkConfiguration.from_json(serialized).to_json()) + + def test_configuration_serializes_the_task_level_fields(self): + serialized = _configuration().to_json() + + self.assertEqual("orders-sink", serialized["Name"]) + self.assertEqual("Kafka", serialized["BrokerType"]) + self.assertEqual("kafka", serialized["ConnectionStringName"]) + self.assertEqual(0, serialized["TaskId"]) + self.assertFalse(serialized["Disabled"]) + + def test_a_script_carries_its_queues_and_disabled_flag(self): + # The wire format comes from reflection over the public properties, so Disabled + # ships even though the server-side ToJson leaves it out. + script = QueueSinkScript("s1", ["q1", "q2"], "put('x/', this);", disabled=True) + serialized = script.to_json() + + self.assertEqual(["q1", "q2"], serialized["Queues"]) + self.assertTrue(serialized["Disabled"]) + self.assertEqual(serialized, QueueSinkScript.from_json(serialized).to_json()) + + def test_collections_default_to_empty_rather_than_none(self): + self.assertEqual([], QueueSinkConfiguration().scripts) + self.assertEqual([], QueueSinkScript().queues) + + def test_a_configuration_from_an_older_server_parses(self): + configuration = QueueSinkConfiguration.from_json({"Name": "sink", "BrokerType": "RabbitMq"}) + + self.assertEqual(QueueBrokerType.RABBIT_MQ, configuration.broker_type) + self.assertEqual([], configuration.scripts) + + def test_process_state_item_name_is_lower_cased(self): + # The server stores the cluster value under a lower-cased key. + self.assertEqual( + "values/DB/queuesink/orders-sink/orders", + QueueSinkProcessState.generate_item_name("DB", "Orders-Sink", "Orders"), + ) + + def test_process_state_round_trips(self): + payload = {"ConfigurationName": "orders-sink", "ScriptName": "orders", "NodeTag": "A"} + + self.assertEqual(payload, QueueSinkProcessState.from_json(payload).to_json()) + + +class TestAzureServiceBusSinkSource(unittest.TestCase): + def test_a_queue_entry_is_the_queue_name(self): + self.assertEqual("orders", AzureServiceBusSinkSource.queue("orders")) + + def test_a_subscription_entry_joins_topic_and_subscription(self): + self.assertEqual("events;audit", AzureServiceBusSinkSource.subscription("events", "audit")) + + def test_an_empty_name_is_rejected(self): + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.queue("") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.queue(" ") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("events", "") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("", "audit") + + def test_the_separator_cannot_appear_inside_a_name(self): + # Service Bus forbids ';' in names, so an entry carrying one could not be decoded. + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.queue("a;b") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("a;b", "audit") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("events", "a;b") + + +class TestQueueSinkOperations(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def test_add_sends_the_configuration_to_the_admin_endpoint(self): + configuration = _configuration() + command = AddQueueSinkOperation(configuration).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/queue-sink", request.url) + self.assertEqual(configuration.to_json(), request.data) + self.assertFalse(command.is_read_request()) + + def test_update_addresses_the_task_by_id(self): + command = UpdateQueueSinkOperation(11, _configuration()).get_command(None) + + self.assertEqual( + "http://localhost:8080/databases/db/admin/queue-sink?id=11", + command.create_request(self.node).url, + ) + + def test_both_operations_are_raft_commands(self): + add = AddQueueSinkOperation(_configuration()).get_command(None) + update = UpdateQueueSinkOperation(1, _configuration()).get_command(None) + + self.assertIsInstance(add, RaftCommand) + self.assertIsInstance(update, RaftCommand) + self.assertTrue(add.get_raft_unique_request_id()) + + def test_a_missing_configuration_is_rejected_client_side(self): + with self.assertRaises(ValueError): + AddQueueSinkOperation(None) + with self.assertRaises(ValueError): + UpdateQueueSinkOperation(1, None) + + def test_add_reads_the_task_id_off_the_response(self): + command = AddQueueSinkOperation(_configuration()).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 9, "TaskId": 4}), False) + + self.assertIsInstance(command.result, AddQueueSinkOperationResult) + self.assertEqual(9, command.result.raft_command_index) + self.assertEqual(4, command.result.task_id) + + +class TestQueueSinkOngoingTask(unittest.TestCase): + RESPONSE = { + "TaskId": 4, + "TaskType": "QueueSink", + "TaskName": "orders-sink", + "TaskState": "Enabled", + "ResponsibleNode": {"NodeTag": "A", "NodeUrl": "http://localhost:8080"}, + "BrokerType": "Kafka", + "ConnectionStringName": "kafka", + "Url": "localhost:9092", + "Configuration": {"Name": "orders-sink", "BrokerType": "Kafka", "Scripts": [{"Name": "orders"}]}, + } + + def test_ongoing_task_carries_the_broker_and_url(self): + task = OngoingTaskQueueSink.from_json(self.RESPONSE) + + self.assertEqual(OngoingTaskType.QUEUE_SINK, task.task_type) + self.assertEqual(QueueBrokerType.KAFKA, task.broker_type) + self.assertEqual("localhost:9092", task.url) + self.assertEqual("kafka", task.connection_string_name) + + def test_ongoing_task_parses_the_nested_configuration(self): + task = OngoingTaskQueueSink.from_json(self.RESPONSE) + + self.assertIsInstance(task.configuration, QueueSinkConfiguration) + self.assertEqual("orders", task.configuration.scripts[0].name) + + def test_get_ongoing_task_info_dispatches_to_the_queue_sink_result(self): + command = GetOngoingTaskInfoOperation("orders-sink", OngoingTaskType.QUEUE_SINK).get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db")) + command.set_response(json.dumps(self.RESPONSE), False) + + self.assertIn("type=QueueSink", request.url) + self.assertIsInstance(command.result, OngoingTaskQueueSink) + self.assertEqual("orders-sink", command.result.task_name) + + +class TestQueueSinkInDatabaseRecord(unittest.TestCase): + def test_database_record_carries_queue_sinks(self): + record = DatabaseRecord("db") + record.queue_sinks = [_configuration()] + + self.assertEqual("orders-sink", record.to_json()["QueueSinks"][0]["Name"]) + + def test_database_record_parses_queue_sinks(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "QueueSinks": [_configuration().to_json()]}) + + self.assertEqual(1, len(record.queue_sinks)) + self.assertEqual(QueueBrokerType.KAFKA, record.queue_sinks[0].broker_type) + + def test_a_record_from_a_server_without_queue_sinks_gets_an_empty_list(self): + self.assertEqual([], DatabaseRecord.from_json({"DatabaseName": "db"}).queue_sinks) + + +class TestQueueSinkAgainstServer(TestBase): + def setUp(self): + super().setUp() + self.store.maintenance.send( + PutConnectionStringOperation( + QueueConnectionString( + name="kafka", + broker_type=QueueBrokerType.KAFKA, + kafka_settings=KafkaConnectionSettings(bootstrap_servers="localhost:9092"), + ) + ) + ) + + def test_the_server_stores_a_queue_sink_task_and_reads_it_back(self): + result = self.store.maintenance.send(AddQueueSinkOperation(_configuration())) + self.assertGreater(result.task_id, 0) + self.assertGreater(result.raft_command_index, 0) + + task = self.store.maintenance.send(GetOngoingTaskInfoOperation("orders-sink", OngoingTaskType.QUEUE_SINK)) + self.assertIsInstance(task, OngoingTaskQueueSink) + self.assertEqual(QueueBrokerType.KAFKA, task.broker_type) + # The server resolves the broker URL from the connection string. + self.assertEqual("localhost:9092", task.url) + self.assertEqual(["orders-topic"], task.configuration.scripts[0].queues) + self.assertEqual("put('orders/', this);", task.configuration.scripts[0].script) + + def test_the_task_can_be_updated(self): + task_id = self.store.maintenance.send(AddQueueSinkOperation(_configuration())).task_id + + updated = _configuration() + updated.task_id = task_id + updated.scripts[0].queues = ["orders-topic", "returns-topic"] + self.store.maintenance.send(UpdateQueueSinkOperation(task_id, updated)) + + task = self.store.maintenance.send(GetOngoingTaskInfoOperation("orders-sink", OngoingTaskType.QUEUE_SINK)) + self.assertEqual(["orders-topic", "returns-topic"], task.configuration.scripts[0].queues) + + def test_a_configuration_with_no_scripts_is_refused_by_the_server(self): + empty = QueueSinkConfiguration( + name="empty-sink", broker_type=QueueBrokerType.KAFKA, connection_string_name="kafka" + ) + + self.assertRaisesWithMessageContaining( + self.store.maintenance.send, + RavenException, + "'Scripts' list cannot be empty", + AddQueueSinkOperation(empty), + ) From 82299ad6d984c11cb7c8bd7f4a477639c3990bf1 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 18:27:46 +0200 Subject: [PATCH 12/29] Add the database smuggler, reachable as store.smuggler --- ravendb/documents/smuggler/common.py | 290 ++++++++++++++++ .../documents/smuggler/database_smuggler.py | 285 +++++++++++++++ ravendb/documents/store/definition.py | 14 +- .../tests/documents_tests/test_smuggler.py | 326 ++++++++++++++++++ 4 files changed, 914 insertions(+), 1 deletion(-) create mode 100644 ravendb/documents/smuggler/database_smuggler.py create mode 100644 ravendb/tests/documents_tests/test_smuggler.py diff --git a/ravendb/documents/smuggler/common.py b/ravendb/documents/smuggler/common.py index aa9cf44d..e9189b32 100644 --- a/ravendb/documents/smuggler/common.py +++ b/ravendb/documents/smuggler/common.py @@ -1,7 +1,15 @@ +from __future__ import annotations + import enum +from typing import Any, Dict, Iterable, List, Optional, Set, Type, TypeVar class DatabaseItemType(enum.Enum): + """ + What an export or import operates on. The server takes a combination of these, so + the smuggler options carry them as a set rather than a single value. + """ + NONE = "None" DOCUMENTS = "Documents" REVISION_DOCUMENTS = "RevisionDocuments" @@ -15,6 +23,7 @@ class DatabaseItemType(enum.Enum): LEGACY_ATTACHMENT_DELETIONS = "LegacyAttachmentDeletions" DATABASE_RECORD = "DatabaseRecord" UNKNOWN = "Unknown" + # Kept for callers written against older servers; 7.x replaced it with COUNTER_GROUPS. COUNTERS = "Counters" ATTACHMENTS = "Attachments" COUNTER_GROUPS = "CounterGroups" @@ -22,3 +31,284 @@ class DatabaseItemType(enum.Enum): COMPARE_EXCHANGE_TOMBSTONES = "CompareExchangeTombstones" TIME_SERIES = "TimeSeries" REPLICATION_HUB_CERTIFICATES = "ReplicationHubCertificates" + TIME_SERIES_DELETED_RANGES = "TimeSeriesDeletedRanges" + + def __str__(self) -> str: + return self.value + + +class DatabaseRecordItemType(enum.Enum): + """Which parts of the database record an export or import operates on.""" + + NONE = "None" + CONFLICT_SOLVER_CONFIG = "ConflictSolverConfig" + SETTINGS = "Settings" + REVISIONS = "Revisions" + EXPIRATION = "Expiration" + PERIODIC_BACKUPS = "PeriodicBackups" + EXTERNAL_REPLICATIONS = "ExternalReplications" + RAVEN_CONNECTION_STRINGS = "RavenConnectionStrings" + SQL_CONNECTION_STRINGS = "SqlConnectionStrings" + RAVEN_ETLS = "RavenEtls" + SQL_ETLS = "SqlEtls" + CLIENT = "Client" + SORTERS = "Sorters" + SINK_PULL_REPLICATIONS = "SinkPullReplications" + HUB_PULL_REPLICATIONS = "HubPullReplications" + TIME_SERIES = "TimeSeries" + DOCUMENTS_COMPRESSION = "DocumentsCompression" + ANALYZERS = "Analyzers" + LOCK_MODE = "LockMode" + OLAP_CONNECTION_STRINGS = "OlapConnectionStrings" + OLAP_ETLS = "OlapEtls" + ELASTIC_SEARCH_CONNECTION_STRINGS = "ElasticSearchConnectionStrings" + ELASTIC_SEARCH_ETLS = "ElasticSearchEtls" + POSTGRE_SQL_INTEGRATION = "PostgreSQLIntegration" + QUEUE_CONNECTION_STRINGS = "QueueConnectionStrings" + QUEUE_ETLS = "QueueEtls" + INDEXES_HISTORY = "IndexesHistory" + REFRESH = "Refresh" + QUEUE_SINKS = "QueueSinks" + DATA_ARCHIVAL = "DataArchival" + SNOWFLAKE_CONNECTION_STRINGS = "SnowflakeConnectionStrings" + SNOWFLAKE_ETLS = "SnowflakeEtls" + EMBEDDINGS_GENERATIONS = "EmbeddingsGenerations" + AI_CONNECTION_STRINGS = "AiConnectionStrings" + GEN_AI_ETLS = "GenAiEtls" + AI_AGENTS = "AiAgents" + REMOTE_ATTACHMENTS = "RemoteAttachments" + SCHEMA_VALIDATION = "SchemaValidation" + CDC_SINKS = "CdcSinks" + + def __str__(self) -> str: + return self.value + + +class ExportCompressionAlgorithm(enum.Enum): + ZSTD = "Zstd" + GZIP = "Gzip" + + def __str__(self) -> str: + return self.value + + +_TFlag = TypeVar("_TFlag", DatabaseItemType, DatabaseRecordItemType) + + +def flags_to_string(members: Optional[Iterable[_TFlag]], enum_type: Type[_TFlag]) -> str: + """ + Renders a set of members the way the server reads a [Flags] enum: a comma-separated + list of names, or "None" when nothing is selected. + """ + selected = set(members or ()) + selected.discard(enum_type.NONE) + if not selected: + return enum_type.NONE.value + + # Declaration order keeps the output stable and matches the bit order the server uses. + return ", ".join(member.value for member in enum_type if member in selected) + + +def flags_from_string(value: Optional[str], enum_type: Type[_TFlag]) -> Set[_TFlag]: + """The inverse of flags_to_string.""" + if not value: + return set() + + members = {enum_type(part.strip()) for part in value.split(",") if part.strip()} + members.discard(enum_type.NONE) + return members + + +DEFAULT_OPERATE_ON_TYPES: Set[DatabaseItemType] = { + DatabaseItemType.INDEXES, + DatabaseItemType.DOCUMENTS, + DatabaseItemType.REVISION_DOCUMENTS, + DatabaseItemType.CONFLICTS, + DatabaseItemType.DATABASE_RECORD, + DatabaseItemType.REPLICATION_HUB_CERTIFICATES, + DatabaseItemType.IDENTITIES, + DatabaseItemType.COMPARE_EXCHANGE, + DatabaseItemType.ATTACHMENTS, + DatabaseItemType.COUNTER_GROUPS, + DatabaseItemType.SUBSCRIPTIONS, + DatabaseItemType.TIME_SERIES, + DatabaseItemType.TIME_SERIES_DELETED_RANGES, +} + +DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES: Set[DatabaseRecordItemType] = { + DatabaseRecordItemType.CLIENT, + DatabaseRecordItemType.CONFLICT_SOLVER_CONFIG, + DatabaseRecordItemType.EXPIRATION, + DatabaseRecordItemType.EXTERNAL_REPLICATIONS, + DatabaseRecordItemType.PERIODIC_BACKUPS, + DatabaseRecordItemType.RAVEN_CONNECTION_STRINGS, + DatabaseRecordItemType.RAVEN_ETLS, + DatabaseRecordItemType.REVISIONS, + DatabaseRecordItemType.SETTINGS, + DatabaseRecordItemType.SQL_CONNECTION_STRINGS, + DatabaseRecordItemType.SORTERS, + DatabaseRecordItemType.SQL_ETLS, + DatabaseRecordItemType.HUB_PULL_REPLICATIONS, + DatabaseRecordItemType.SINK_PULL_REPLICATIONS, + DatabaseRecordItemType.TIME_SERIES, + DatabaseRecordItemType.DOCUMENTS_COMPRESSION, + DatabaseRecordItemType.ANALYZERS, + DatabaseRecordItemType.LOCK_MODE, + DatabaseRecordItemType.OLAP_CONNECTION_STRINGS, + DatabaseRecordItemType.OLAP_ETLS, + DatabaseRecordItemType.ELASTIC_SEARCH_CONNECTION_STRINGS, + DatabaseRecordItemType.ELASTIC_SEARCH_ETLS, + DatabaseRecordItemType.POSTGRE_SQL_INTEGRATION, + DatabaseRecordItemType.QUEUE_CONNECTION_STRINGS, + DatabaseRecordItemType.QUEUE_ETLS, + DatabaseRecordItemType.INDEXES_HISTORY, + DatabaseRecordItemType.REFRESH, + DatabaseRecordItemType.DATA_ARCHIVAL, + DatabaseRecordItemType.QUEUE_SINKS, + DatabaseRecordItemType.SNOWFLAKE_ETLS, + DatabaseRecordItemType.SNOWFLAKE_CONNECTION_STRINGS, + DatabaseRecordItemType.EMBEDDINGS_GENERATIONS, + DatabaseRecordItemType.AI_CONNECTION_STRINGS, + DatabaseRecordItemType.GEN_AI_ETLS, + DatabaseRecordItemType.AI_AGENTS, + DatabaseRecordItemType.REMOTE_ATTACHMENTS, + DatabaseRecordItemType.SCHEMA_VALIDATION, + DatabaseRecordItemType.CDC_SINKS, +} + +DEFAULT_MAX_STEPS_FOR_TRANSFORM_SCRIPT = 10 * 1000 + + +class DatabaseSmugglerOptions: + """What an export or import covers, and how it transforms what it moves.""" + + def __init__( + self, + operate_on_types: Set[DatabaseItemType] = None, + operate_on_database_record_types: Set[DatabaseRecordItemType] = None, + include_expired: bool = True, + include_artificial: bool = False, + include_archived: bool = True, + remove_analyzers: bool = False, + transform_script: str = None, + max_steps_for_transform_script: int = DEFAULT_MAX_STEPS_FOR_TRANSFORM_SCRIPT, + encryption_key: str = None, + collections: List[str] = None, + max_read_ops_per_second: Optional[int] = None, + skip_corrupted_data: bool = False, + ): + self.operate_on_types = set(operate_on_types if operate_on_types is not None else DEFAULT_OPERATE_ON_TYPES) + self.operate_on_database_record_types = set( + operate_on_database_record_types + if operate_on_database_record_types is not None + else DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES + ) + self.include_expired = include_expired + self.include_artificial = include_artificial + self.include_archived = include_archived + self.remove_analyzers = remove_analyzers + self.transform_script = transform_script + self.max_steps_for_transform_script = max_steps_for_transform_script + self.encryption_key = encryption_key + # Empty means every collection. + self.collections = collections if collections is not None else [] + self.max_read_ops_per_second = max_read_ops_per_second + # Lets an export continue past corrupted data, reporting it in the result instead + # of stopping. Useful when the database has lost its compression dictionaries. + self.skip_corrupted_data = skip_corrupted_data + + def to_json(self) -> Dict[str, Any]: + return { + "OperateOnTypes": flags_to_string(self.operate_on_types, DatabaseItemType), + "OperateOnDatabaseRecordTypes": flags_to_string( + self.operate_on_database_record_types, DatabaseRecordItemType + ), + "IncludeExpired": self.include_expired, + "IncludeArtificial": self.include_artificial, + "IncludeArchived": self.include_archived, + "RemoveAnalyzers": self.remove_analyzers, + "TransformScript": self.transform_script, + "MaxStepsForTransformScript": self.max_steps_for_transform_script, + "EncryptionKey": self.encryption_key, + "Collections": self.collections, + "MaxReadOpsPerSecond": self.max_read_ops_per_second, + "SkipCorruptedData": self.skip_corrupted_data, + } + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + self.operate_on_types = flags_from_string(json_dict.get("OperateOnTypes"), DatabaseItemType) + self.operate_on_database_record_types = flags_from_string( + json_dict.get("OperateOnDatabaseRecordTypes"), DatabaseRecordItemType + ) + self.include_expired = json_dict.get("IncludeExpired", True) + self.include_artificial = json_dict.get("IncludeArtificial", False) + self.include_archived = json_dict.get("IncludeArchived", True) + self.remove_analyzers = json_dict.get("RemoveAnalyzers", False) + self.transform_script = json_dict.get("TransformScript") + self.max_steps_for_transform_script = json_dict.get( + "MaxStepsForTransformScript", DEFAULT_MAX_STEPS_FOR_TRANSFORM_SCRIPT + ) + self.encryption_key = json_dict.get("EncryptionKey") + self.collections = json_dict.get("Collections") or [] + self.max_read_ops_per_second = json_dict.get("MaxReadOpsPerSecond") + self.skip_corrupted_data = json_dict.get("SkipCorruptedData", False) + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> DatabaseSmugglerOptions: + options = cls() + options._fill_from_json(json_dict) + return options + + +class DatabaseSmugglerExportOptions(DatabaseSmugglerOptions): + def __init__(self, compression_algorithm: ExportCompressionAlgorithm = None, **kwargs): + super().__init__(**kwargs) + # None leaves the choice to the server's own default. + self.compression_algorithm = compression_algorithm + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + if self.compression_algorithm is not None: + json_dict["CompressionAlgorithm"] = self.compression_algorithm.value + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> DatabaseSmugglerExportOptions: + options = cls() + options._fill_from_json(json_dict) + compression_algorithm = json_dict.get("CompressionAlgorithm") + options.compression_algorithm = ( + ExportCompressionAlgorithm(compression_algorithm) if compression_algorithm else None + ) + return options + + +class DatabaseSmugglerImportOptions(DatabaseSmugglerOptions): + def __init__(self, skip_revision_creation: bool = False, **kwargs): + super().__init__(**kwargs) + self.skip_revision_creation = skip_revision_creation + + @classmethod + def from_options(cls, options: DatabaseSmugglerOptions) -> DatabaseSmugglerImportOptions: + """Carries the shared options over from an export, the way export-to-database does.""" + return cls( + operate_on_types=set(options.operate_on_types), + include_expired=options.include_expired, + include_artificial=options.include_artificial, + include_archived=options.include_archived, + max_steps_for_transform_script=options.max_steps_for_transform_script, + remove_analyzers=options.remove_analyzers, + transform_script=options.transform_script, + ) + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["SkipRevisionCreation"] = self.skip_revision_creation + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> DatabaseSmugglerImportOptions: + options = cls() + options._fill_from_json(json_dict) + options.skip_revision_creation = json_dict.get("SkipRevisionCreation", False) + return options diff --git a/ravendb/documents/smuggler/database_smuggler.py b/ravendb/documents/smuggler/database_smuggler.py new file mode 100644 index 00000000..96171465 --- /dev/null +++ b/ravendb/documents/smuggler/database_smuggler.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import json +import os +import tempfile +from typing import IO, TYPE_CHECKING, List, Optional, Union + +import requests + +from ravendb.documents.commands.bulkinsert import GetNextOperationIdCommand +from ravendb.documents.operations.operation import Operation +from ravendb.documents.smuggler.common import ( + DatabaseItemType, + DatabaseSmugglerExportOptions, + DatabaseSmugglerImportOptions, + DatabaseSmugglerOptions, +) +from ravendb.http.misc import ResponseDisposeHandling +from ravendb.http.raven_command import VoidRavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants +from ravendb.util.request_utils import RequestUtils + +if TYPE_CHECKING: + from ravendb.documents.store.definition import DocumentStore + +_IMPORT_OPTIONS_PART = "importOptions" +_COPY_BUFFER_SIZE = 8192 + +_PERIODIC_BACKUP = constants.Documents.PeriodicBackup +_LEGACY_INCREMENTAL_BACKUP_EXTENSION = ".ravendb-incremental-dump" +_LEGACY_FULL_BACKUP_EXTENSION = ".ravendb-full-dump" + +# A full backup or snapshot is restored before the incremental files that build on it. +_FULL_BACKUP_EXTENSIONS = ( + _PERIODIC_BACKUP.FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.SNAPSHOT_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_SNAPSHOT_EXTENSTION, +) + +# Snapshots are deliberately absent: they are restored, not imported. +_BACKUP_EXTENSIONS = ( + _PERIODIC_BACKUP.FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.INCREMENTAL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_INCREMENTAL_BACKUP_EXTENSTION, + _LEGACY_INCREMENTAL_BACKUP_EXTENSION, + _LEGACY_FULL_BACKUP_EXTENSION, +) + + +def _is_backup_file(file_path: str) -> bool: + return os.path.splitext(file_path)[1].lower() in _BACKUP_EXTENSIONS + + +def _backup_sort_key(file_path: str): + name, extension = os.path.splitext(os.path.basename(file_path)) + # A full backup is restored before the incremental files that build on it. + return name, 0 if extension.lower() in _FULL_BACKUP_EXTENSIONS else 1, os.path.getmtime(file_path) + + +class DatabaseSmuggler: + """ + Exports a database to a file or stream and imports it back, through the server's + smuggler endpoints. Reached as ``store.smuggler``. + """ + + def __init__(self, store: "DocumentStore", database_name: str = None): + self._store = store + self._database_name = database_name if database_name is not None else store.database + self._request_executor = None + + @property + def _executor(self): + if self._request_executor is None and self._database_name is not None: + self._request_executor = self._store.get_request_executor(self._database_name) + return self._request_executor + + def for_database(self, database_name: str) -> DatabaseSmuggler: + if self._database_name is not None and database_name is not None: + if database_name.lower() == self._database_name.lower(): + return self + + return DatabaseSmuggler(self._store, database_name) + + def _assert_database_set(self) -> None: + if self._executor is None: + raise RuntimeError("Cannot use Smuggler without a database defined, did you forget to call for_database?") + + def _next_operation_id(self): + command = GetNextOperationIdCommand() + self._executor.execute_command(command) + return command.result, command.node_tag + + def _operation_for(self, operation_id: int, node_tag: str) -> Operation: + return Operation( + self._executor, + lambda: None, + self._executor.conventions, + operation_id, + node_tag, + ) + + def export( + self, + options: DatabaseSmugglerExportOptions, + to_file_or_stream: Union[str, IO[bytes]], + ) -> Operation: + """ + Exports the database. ``to_file_or_stream`` is either a path, in which case the + file (and any missing parent directory) is created, or a writable binary stream, + which is left open. + + The download runs to completion before this returns, so the operation handed back + is only there to read the server-side result and progress. + """ + if options is None: + raise ValueError("options cannot be None") + if to_file_or_stream is None: + raise ValueError("to_file_or_stream cannot be None") + + self._assert_database_set() + + if isinstance(to_file_or_stream, str): + directory = os.path.dirname(os.path.abspath(to_file_or_stream)) + if directory and not os.path.isdir(directory): + os.makedirs(directory, exist_ok=True) + + with open(to_file_or_stream, "wb") as destination: + return self._export_to_stream(options, destination) + + return self._export_to_stream(options, to_file_or_stream) + + def _export_to_stream(self, options: DatabaseSmugglerExportOptions, destination: IO[bytes]) -> Operation: + operation_id, node_tag = self._next_operation_id() + self._executor.execute_command(self._ExportCommand(options, destination, operation_id, node_tag)) + return self._operation_for(operation_id, node_tag) + + def export_to_database( + self, + options: DatabaseSmugglerExportOptions, + to_smuggler: DatabaseSmuggler, + ) -> Operation: + """ + Streams an export straight into another database. Returns the import operation on + the receiving side, which is the one worth waiting on. + """ + if options is None: + raise ValueError("options cannot be None") + if to_smuggler is None: + raise ValueError("to_smuggler cannot be None") + + import_options = DatabaseSmugglerImportOptions.from_options(options) + + # The sync client cannot hand one side's response stream to the other side's + # request body, so the export is staged on disk first. + with tempfile.TemporaryDirectory() as directory: + staged = os.path.join(directory, "export.ravendbdump") + self.export(options, staged).wait_for_completion() + return to_smuggler.import_data(import_options, staged) + + def import_data( + self, + options: DatabaseSmugglerImportOptions, + from_file_or_stream: Union[str, IO[bytes]], + ) -> Operation: + """ + Imports a dump. ``from_file_or_stream`` is either a path, which is opened and + closed here, or a readable binary stream, which is left open. + """ + if options is None: + raise ValueError("options cannot be None") + if from_file_or_stream is None: + raise ValueError("from_file_or_stream cannot be None") + + self._assert_database_set() + + if isinstance(from_file_or_stream, str): + with open(from_file_or_stream, "rb") as source: + return self._import_from_stream(options, source) + + return self._import_from_stream(options, from_file_or_stream) + + def _import_from_stream(self, options: DatabaseSmugglerImportOptions, source: IO[bytes]) -> Operation: + operation_id, node_tag = self._next_operation_id() + self._executor.execute_command(self._ImportCommand(options, source, operation_id, node_tag)) + return self._operation_for(operation_id, node_tag) + + def import_incremental(self, options: DatabaseSmugglerImportOptions, from_directory: str) -> None: + """ + Imports a backup directory in order, waiting for each file. Indexes and + subscriptions come from the last file only, so a later incremental file cannot + resurrect an index the backup dropped. + """ + if options is None: + raise ValueError("options cannot be None") + + files = sorted( + ( + os.path.join(from_directory, name) + for name in os.listdir(from_directory) + if _is_backup_file(os.path.join(from_directory, name)) + ), + key=_backup_sort_key, + ) + + if not files: + return + + original_operate_on_types = self._configure_options_for_incremental_import(options) + for file_path in files[:-1]: + self.import_data(options, file_path).wait_for_completion() + + options.operate_on_types = original_operate_on_types + self.import_data(options, files[-1]).wait_for_completion() + + @staticmethod + def _configure_options_for_incremental_import(options: DatabaseSmugglerOptions) -> set: + options.operate_on_types.add(DatabaseItemType.TOMBSTONES) + options.operate_on_types.add(DatabaseItemType.COMPARE_EXCHANGE_TOMBSTONES) + + original = set(options.operate_on_types) + options.operate_on_types.discard(DatabaseItemType.INDEXES) + options.operate_on_types.discard(DatabaseItemType.SUBSCRIPTIONS) + return original + + class _ExportCommand(VoidRavenCommand): + def __init__( + self, + options: DatabaseSmugglerExportOptions, + destination: IO[bytes], + operation_id: int, + node_tag: str = None, + ): + super().__init__() + self._options = options + self._destination = destination + self._operation_id = operation_id + self._selected_node_tag = node_tag + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/smuggler/export?operationId={self._operation_id}" + + request = requests.Request("POST", url) + request.data = self._options.to_json() + return request + + def send(self, session: requests.Session, request: requests.Request) -> requests.Response: + prepared_request = session.prepare_request(request) + RequestUtils.remove_zstd_encoding(prepared_request) + return session.send(prepared_request, cert=session.cert, stream=True) + + def process_response(self, cache, response: requests.Response, url) -> ResponseDisposeHandling: + for chunk in response.iter_content(chunk_size=_COPY_BUFFER_SIZE): + if chunk: + self._destination.write(chunk) + + return ResponseDisposeHandling.AUTOMATIC + + class _ImportCommand(VoidRavenCommand): + def __init__( + self, + options: DatabaseSmugglerImportOptions, + source: IO[bytes], + operation_id: int, + node_tag: str = None, + ): + super().__init__() + self._options = options + self._source = source + self._operation_id = operation_id + self._selected_node_tag = node_tag + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/smuggler/import?operationId={self._operation_id}" + + request = requests.Request("POST", url) + # The server reads the options section first and the dump second, so the order + # of these parts matters. + request.files = { + _IMPORT_OPTIONS_PART: (None, json.dumps(self._options.to_json()), "application/json"), + "file": ("name", self._source, "application/octet-stream"), + } + return request diff --git a/ravendb/documents/store/definition.py b/ravendb/documents/store/definition.py index 0f3ed0be..e7fcab26 100644 --- a/ravendb/documents/store/definition.py +++ b/ravendb/documents/store/definition.py @@ -44,6 +44,7 @@ if TYPE_CHECKING: from ravendb.documents.indexes.abstract_index_creation_tasks import AbstractIndexCreationTask + from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler from ravendb.http.misc import AggressiveCacheMode, AggressiveCacheOptions @@ -341,7 +342,7 @@ def __init__(self, urls: Union[str, List[str]] = None, database: Optional[str] = self.__aggressive_cache_changes: Dict[str, "DocumentStore._AggressiveCacheInvalidator"] = {} self.__maintenance_operation_executor: Optional[MaintenanceOperationExecutor] = None self.__operation_executor: Optional[OperationExecutor] = None - # todo: database smuggler + self.__smuggler: Optional["DatabaseSmuggler"] = None self.__multi_db_hilo: Optional[MultiDatabaseHiLoGenerator] = None self.__identifier: Optional[str] = None self.__add_change_lock = threading.Lock() @@ -714,6 +715,17 @@ def operations(self) -> OperationExecutor: return self.__operation_executor + @property + def smuggler(self) -> "DatabaseSmuggler": + self.assert_initialized() + + if self.__smuggler is None: + from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler + + self.__smuggler = DatabaseSmuggler(self) + + return self.__smuggler + @property def time_series(self) -> TimeSeriesOperations: if self.__time_series_operation is None: diff --git a/ravendb/tests/documents_tests/test_smuggler.py b/ravendb/tests/documents_tests/test_smuggler.py new file mode 100644 index 00000000..5047cf0c --- /dev/null +++ b/ravendb/tests/documents_tests/test_smuggler.py @@ -0,0 +1,326 @@ +""" +Tests for the database smuggler: the options that say what an export or import covers, +the request shapes, backup-file ordering for incremental imports, and a full export / +import round trip against a real server. +""" + +import io +import json +import os +import tempfile +import unittest + +from ravendb.documents.smuggler.common import ( + DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES, + DEFAULT_OPERATE_ON_TYPES, + DatabaseItemType, + DatabaseRecordItemType, + DatabaseSmugglerExportOptions, + DatabaseSmugglerImportOptions, + DatabaseSmugglerOptions, + ExportCompressionAlgorithm, + flags_from_string, + flags_to_string, +) +from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler, _backup_sort_key, _is_backup_file +from ravendb.http.server_node import ServerNode +from ravendb.tests.test_base import TestBase, User + + +class TestItemTypeFlags(unittest.TestCase): + def test_nothing_selected_renders_as_none(self): + self.assertEqual("None", flags_to_string(set(), DatabaseItemType)) + self.assertEqual("None", flags_to_string(None, DatabaseItemType)) + self.assertEqual("None", flags_to_string({DatabaseItemType.NONE}, DatabaseItemType)) + + def test_members_render_as_a_comma_separated_list(self): + # This is the shape the server's Enum.Parse reads a [Flags] value back from. + self.assertEqual( + "Documents, Indexes", + flags_to_string({DatabaseItemType.INDEXES, DatabaseItemType.DOCUMENTS}, DatabaseItemType), + ) + + def test_rendering_is_stable_regardless_of_set_order(self): + one = flags_to_string({DatabaseItemType.DOCUMENTS, DatabaseItemType.ATTACHMENTS}, DatabaseItemType) + other = flags_to_string({DatabaseItemType.ATTACHMENTS, DatabaseItemType.DOCUMENTS}, DatabaseItemType) + + self.assertEqual(one, other) + + def test_flags_round_trip(self): + members = {DatabaseItemType.DOCUMENTS, DatabaseItemType.TIME_SERIES_DELETED_RANGES} + + self.assertEqual(members, flags_from_string(flags_to_string(members, DatabaseItemType), DatabaseItemType)) + + def test_parsing_tolerates_spacing_and_drops_none(self): + self.assertEqual( + {DatabaseItemType.DOCUMENTS, DatabaseItemType.INDEXES}, + flags_from_string("None,Documents , Indexes", DatabaseItemType), + ) + self.assertEqual(set(), flags_from_string("", DatabaseItemType)) + + def test_the_database_record_types_the_7_2_5_server_knows(self): + for name in ("QueueSinks", "CdcSinks", "SchemaValidation", "RemoteAttachments", "AiAgents"): + self.assertEqual(name, DatabaseRecordItemType(name).value) + + +class TestSmugglerOptions(unittest.TestCase): + def test_defaults_match_the_server_side_defaults(self): + options = DatabaseSmugglerOptions() + + self.assertEqual(DEFAULT_OPERATE_ON_TYPES, options.operate_on_types) + self.assertEqual(DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES, options.operate_on_database_record_types) + self.assertTrue(options.include_expired) + self.assertFalse(options.include_artificial) + self.assertTrue(options.include_archived) + self.assertEqual(10000, options.max_steps_for_transform_script) + self.assertEqual([], options.collections) + + def test_each_instance_gets_its_own_collections(self): + # A shared default would leak one export's narrowing into the next. + first = DatabaseSmugglerOptions() + first.operate_on_types.add(DatabaseItemType.TOMBSTONES) + first.collections.append("Orders") + + second = DatabaseSmugglerOptions() + self.assertNotIn(DatabaseItemType.TOMBSTONES, second.operate_on_types) + self.assertEqual([], second.collections) + + def test_body_carries_the_selections_as_flag_strings(self): + options = DatabaseSmugglerOptions( + operate_on_types={DatabaseItemType.DOCUMENTS}, + operate_on_database_record_types={DatabaseRecordItemType.SETTINGS}, + ) + body = options.to_json() + + self.assertEqual("Documents", body["OperateOnTypes"]) + self.assertEqual("Settings", body["OperateOnDatabaseRecordTypes"]) + + def test_options_round_trip(self): + options = DatabaseSmugglerOptions( + operate_on_types={DatabaseItemType.DOCUMENTS, DatabaseItemType.ATTACHMENTS}, + collections=["Orders"], + transform_script="this.Foo = 1;", + max_read_ops_per_second=200, + skip_corrupted_data=True, + ) + + self.assertEqual(options.to_json(), DatabaseSmugglerOptions.from_json(options.to_json()).to_json()) + + def test_export_options_leave_the_compression_choice_to_the_server(self): + self.assertNotIn("CompressionAlgorithm", DatabaseSmugglerExportOptions().to_json()) + + def test_export_options_send_a_chosen_compression_algorithm(self): + options = DatabaseSmugglerExportOptions(compression_algorithm=ExportCompressionAlgorithm.GZIP) + + self.assertEqual("Gzip", options.to_json()["CompressionAlgorithm"]) + self.assertEqual( + ExportCompressionAlgorithm.GZIP, + DatabaseSmugglerExportOptions.from_json(options.to_json()).compression_algorithm, + ) + + def test_import_options_always_state_the_revision_choice(self): + self.assertFalse(DatabaseSmugglerImportOptions().to_json()["SkipRevisionCreation"]) + self.assertTrue(DatabaseSmugglerImportOptions(skip_revision_creation=True).to_json()["SkipRevisionCreation"]) + + def test_import_options_copy_only_the_shared_settings_of_an_export(self): + # Collections and the database-record selection stay at the import defaults, the + # same subset the C# copy constructor carries over. + source = DatabaseSmugglerOptions( + operate_on_types={DatabaseItemType.DOCUMENTS}, + operate_on_database_record_types={DatabaseRecordItemType.SETTINGS}, + collections=["Orders"], + transform_script="this.Foo = 1;", + include_expired=False, + ) + copied = DatabaseSmugglerImportOptions.from_options(source) + + self.assertEqual({DatabaseItemType.DOCUMENTS}, copied.operate_on_types) + self.assertEqual("this.Foo = 1;", copied.transform_script) + self.assertFalse(copied.include_expired) + self.assertEqual([], copied.collections) + self.assertEqual(DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES, copied.operate_on_database_record_types) + + +class TestSmugglerRequests(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def test_export_posts_the_options_with_the_operation_id(self): + options = DatabaseSmugglerExportOptions() + command = DatabaseSmuggler._ExportCommand(options, io.BytesIO(), 42, "A") + request = command.create_request(self.node) + + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/smuggler/export?operationId=42", request.url) + self.assertEqual(options.to_json(), request.data) + self.assertEqual("A", command.selected_node_tag) + + def test_import_posts_the_options_and_the_dump_as_two_parts(self): + options = DatabaseSmugglerImportOptions(skip_revision_creation=True) + command = DatabaseSmuggler._ImportCommand(options, io.BytesIO(b"dump"), 43, "B") + request = command.create_request(self.node) + + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/smuggler/import?operationId=43", request.url) + # The server reads the options section before the dump, so the order matters. + self.assertEqual(["importOptions", "file"], list(request.files)) + self.assertTrue(json.loads(request.files["importOptions"][1])["SkipRevisionCreation"]) + self.assertEqual("B", command.selected_node_tag) + + def test_a_smuggler_without_a_database_refuses_to_run(self): + class _StoreWithoutDatabase: + database = None + + def get_request_executor(self, database=None): + return None + + smuggler = DatabaseSmuggler(_StoreWithoutDatabase()) + with self.assertRaises(RuntimeError): + smuggler.export(DatabaseSmugglerExportOptions(), io.BytesIO()) + with self.assertRaises(RuntimeError): + smuggler.import_data(DatabaseSmugglerImportOptions(), io.BytesIO()) + + def test_missing_arguments_are_rejected_client_side(self): + smuggler = DatabaseSmuggler(type("_Store", (), {"database": "db"})()) + + with self.assertRaises(ValueError): + smuggler.export(None, io.BytesIO()) + with self.assertRaises(ValueError): + smuggler.export(DatabaseSmugglerExportOptions(), None) + with self.assertRaises(ValueError): + smuggler.import_data(None, io.BytesIO()) + with self.assertRaises(ValueError): + smuggler.import_data(DatabaseSmugglerImportOptions(), None) + + +class TestBackupFileOrdering(unittest.TestCase): + def test_backup_extensions_are_recognized(self): + for name in ( + "2026-06-16.ravendb-full-backup", + "2026-06-16.ravendb-incremental-backup", + "2026-06-16.ravendb-encrypted-full-backup", + "2026-06-16.ravendb-encrypted-incremental-backup", + "old.ravendb-full-dump", + "old.ravendb-incremental-dump", + ): + self.assertTrue(_is_backup_file(name), name) + + def test_other_files_are_ignored(self): + for name in ("notes.txt", "export.ravendbdump", "2026-06-16.ravendb-snapshot"): + self.assertFalse(_is_backup_file(name), name) + + def test_a_full_backup_sorts_before_its_incrementals(self): + with tempfile.TemporaryDirectory() as directory: + names = [ + "2026-06-16-10-00.ravendb-incremental-backup", + "2026-06-16-10-00.ravendb-full-backup", + "2026-06-15-10-00.ravendb-full-backup", + ] + for name in names: + open(os.path.join(directory, name), "wb").close() + + ordered = sorted((os.path.join(directory, name) for name in names), key=_backup_sort_key) + + self.assertEqual( + [ + "2026-06-15-10-00.ravendb-full-backup", + "2026-06-16-10-00.ravendb-full-backup", + "2026-06-16-10-00.ravendb-incremental-backup", + ], + [os.path.basename(path) for path in ordered], + ) + + def test_incremental_import_narrows_and_restores_the_selection(self): + # Indexes and subscriptions come from the last file only, so an earlier + # incremental cannot bring back an index a later backup dropped. + options = DatabaseSmugglerImportOptions() + original = DatabaseSmuggler._configure_options_for_incremental_import(options) + + self.assertIn(DatabaseItemType.TOMBSTONES, options.operate_on_types) + self.assertIn(DatabaseItemType.COMPARE_EXCHANGE_TOMBSTONES, options.operate_on_types) + self.assertNotIn(DatabaseItemType.INDEXES, options.operate_on_types) + self.assertNotIn(DatabaseItemType.SUBSCRIPTIONS, options.operate_on_types) + self.assertIn(DatabaseItemType.INDEXES, original) + self.assertIn(DatabaseItemType.SUBSCRIPTIONS, original) + + +class TestSmugglerAgainstServer(TestBase): + def setUp(self): + super().setUp() + + def _store_users(self, store, count: int): + with store.open_session() as session: + for i in range(count): + session.store(User(name=f"user-{i}"), f"users/{i}") + session.save_changes() + + def test_exports_a_database_to_a_file_and_imports_it_into_another(self): + self._store_users(self.store, 5) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "export.ravendbdump") + self.store.smuggler.export(DatabaseSmugglerExportOptions(), dump).wait_for_completion() + + self.assertTrue(os.path.isfile(dump)) + self.assertGreater(os.path.getsize(dump), 0) + + with self.get_document_store() as target: + target.smuggler.import_data(DatabaseSmugglerImportOptions(), dump).wait_for_completion() + + with target.open_session() as session: + self.assertEqual("user-3", session.load("users/3", User).name) + self.assertEqual(5, len(session.load(["users/0", "users/1", "users/2", "users/3", "users/4"]))) + + def test_exports_into_a_stream(self): + self._store_users(self.store, 2) + + destination = io.BytesIO() + self.store.smuggler.export(DatabaseSmugglerExportOptions(), destination).wait_for_completion() + + self.assertGreater(len(destination.getvalue()), 0) + + def test_an_export_narrowed_to_one_collection_leaves_the_rest_behind(self): + self._store_users(self.store, 3) + with self.store.open_session() as session: + session.store({"Name": "an order", "@metadata": {"@collection": "Orders"}}, "orders/1") + session.save_changes() + + options = DatabaseSmugglerExportOptions(operate_on_types={DatabaseItemType.DOCUMENTS}, collections=["Users"]) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "users-only.ravendbdump") + self.store.smuggler.export(options, dump).wait_for_completion() + + with self.get_document_store() as target: + target.smuggler.import_data(DatabaseSmugglerImportOptions(), dump).wait_for_completion() + + with target.open_session() as session: + self.assertIsNotNone(session.load("users/0", User)) + self.assertIsNone(session.load("orders/1")) + + def test_export_creates_the_target_directory(self): + self._store_users(self.store, 1) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "nested", "deeper", "export.ravendbdump") + self.store.smuggler.export(DatabaseSmugglerExportOptions(), dump).wait_for_completion() + + self.assertTrue(os.path.isfile(dump)) + + def test_for_database_targets_another_database_on_the_same_store(self): + self._store_users(self.store, 2) + + with self.get_document_store() as target: + smuggler = self.store.smuggler + self.assertIs(smuggler, smuggler.for_database(self.store.database)) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "export.ravendbdump") + smuggler.export(DatabaseSmugglerExportOptions(), dump).wait_for_completion() + + self.store.smuggler.for_database(target.database).import_data( + DatabaseSmugglerImportOptions(), dump + ).wait_for_completion() + + with target.open_session() as session: + self.assertEqual("user-1", session.load("users/1", User).name) From ab2c14a9495c48dcf6e2c5cbf149792390313100 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 18:27:50 +0200 Subject: [PATCH 13/29] Carry queue sinks on the database record and harden its deserialization from_json required an AutoIndexes key and a LockMode, so any payload without them raised. Both C# fields have no initializer: a missing AutoIndexes leaves null, and a missing LockMode lands on Unlock. to_json also iterated the auto-index dict without .items(), so it never serialized one. --- ravendb/serverwide/database_record.py | 24 +++++-- .../operations_tests/test_database_record.py | 70 +++++++++++++++++++ 2 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 ravendb/tests/operations_tests/test_database_record.py diff --git a/ravendb/serverwide/database_record.py b/ravendb/serverwide/database_record.py index 580a0544..a449e599 100644 --- a/ravendb/serverwide/database_record.py +++ b/ravendb/serverwide/database_record.py @@ -12,6 +12,7 @@ ) from ravendb.documents.operations.backups.settings import PeriodicBackupConfiguration from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration +from ravendb.documents.operations.queue_sink.configuration import QueueSinkConfiguration from ravendb.documents.operations.etl.configuration import RavenConnectionString, RavenEtlConfiguration from ravendb.documents.operations.etl.olap.connection import OlapConnectionString, OlapEtlConfiguration from ravendb.documents.operations.etl.sql import SqlConnectionString, SqlEtlConfiguration @@ -71,6 +72,7 @@ def __init__(self, database_name: Optional[str] = None): self.sql_etls: List[SqlEtlConfiguration] = [] self.olap_etls: List[OlapEtlConfiguration] = [] self.cdc_sinks: List[CdcSinkConfiguration] = [] + self.queue_sinks: List[QueueSinkConfiguration] = [] self.embeddings_generations: List = [] self.client: Optional[ClientConfiguration] = None self.studio: Optional[StudioConfiguration] = None @@ -101,7 +103,7 @@ def to_json(self): "Indexes": self.indexes, "IndexesHistory": self.indexes_history_story, "AutoIndexes": ( - {key: AutoIndexDefinition.to_json(auto_index) for key, auto_index in self.auto_indexes} + {key: auto_index.to_json() for key, auto_index in self.auto_indexes.items()} if self.auto_indexes else None ), @@ -122,6 +124,7 @@ def to_json(self): "SqlEtls": self.sql_etls, "OlapEtls": self.olap_etls, "CdcSinks": [cdc_sink.to_json() for cdc_sink in self.cdc_sinks or []], + "QueueSinks": [queue_sink.to_json() for queue_sink in self.queue_sinks or []], "Client": self.client, "Studio": self.studio, "TruncatedClusterTransactionCommand": self.truncated_cluster_transaction_commands_count, @@ -136,7 +139,12 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.deletion_in_progress = json_dict.get("DeletionInProgress", None) record.rolling_indexes = json_dict.get("RollingIndexes", None) record.database_state = json_dict.get("DatabaseState", None) - record.lock_mode = DatabaseRecord.DatabaseLockMode(json_dict.get("LockMode", None)) + lock_mode = json_dict.get("LockMode") + # A record without a lock mode is unlocked - the C# field has no initializer, so it + # lands on the enum's zero value. + record.lock_mode = ( + DatabaseRecord.DatabaseLockMode(lock_mode) if lock_mode else DatabaseRecord.DatabaseLockMode.UNLOCK + ) record.topology = json_dict.get("Topology", None) record.conflict_solver_config = json_dict.get("ConflictSolverConfig", None) record.documents_compression = ( @@ -148,9 +156,12 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.analyzers = json_dict.get("Analyzers", None) record.indexes = json_dict.get("Indexes", None) record.indexes_history_story = json_dict.get("IndexesHistory", None) - record.auto_indexes = { - key: AutoIndexDefinition.from_json(auto_index) for key, auto_index in json_dict.get("AutoIndexes").items() - } + auto_indexes = json_dict.get("AutoIndexes") + record.auto_indexes = ( + {key: AutoIndexDefinition.from_json(auto_index) for key, auto_index in auto_indexes.items()} + if auto_indexes + else None + ) record.settings = json_dict.get("Settings", None) record.revisions = json_dict.get("Revisions", None) if json_dict.get("TimeSeries", None): @@ -171,6 +182,9 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.sql_etls = json_dict.get("SqlEtls", None) record.olap_etls = json_dict.get("OlapEtls", None) record.cdc_sinks = [CdcSinkConfiguration.from_json(cdc_sink) for cdc_sink in json_dict.get("CdcSinks") or []] + record.queue_sinks = [ + QueueSinkConfiguration.from_json(queue_sink) for queue_sink in json_dict.get("QueueSinks") or [] + ] embeddings_generations_data = json_dict.get("EmbeddingsGenerations", []) if embeddings_generations_data: from ravendb.documents.operations.ai.embeddings_generation_configuration import ( diff --git a/ravendb/tests/operations_tests/test_database_record.py b/ravendb/tests/operations_tests/test_database_record.py new file mode 100644 index 00000000..c6cc4940 --- /dev/null +++ b/ravendb/tests/operations_tests/test_database_record.py @@ -0,0 +1,70 @@ +""" +Tests for DatabaseRecord serialization, in particular the fields whose C# counterparts +have no initializer and so are simply absent from a server payload. +""" + +import unittest + +from ravendb.documents.indexes.definitions import AutoIndexDefinition +from ravendb.serverwide.database_record import DatabaseRecord + +AUTO_INDEX = { + "Type": "AutoMap", + "Name": "Auto/Orders/ByCompany", + "Priority": "Normal", + "State": "Normal", + "Collection": "Orders", + "MapFields": {}, + "GroupByFields": {}, +} + + +class TestDatabaseRecordFromJson(unittest.TestCase): + def test_a_payload_without_optional_sections_parses(self): + # The C# fields for these have no initializer, so a server is free to omit them. + record = DatabaseRecord.from_json({"DatabaseName": "db"}) + + self.assertEqual("db", record.database_name) + self.assertIsNone(record.auto_indexes) + self.assertIsNone(record.indexes) + + def test_a_record_without_a_lock_mode_is_unlocked(self): + record = DatabaseRecord.from_json({"DatabaseName": "db"}) + + self.assertEqual(DatabaseRecord.DatabaseLockMode.UNLOCK, record.lock_mode) + + def test_a_lock_mode_is_read_when_the_server_sends_one(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "LockMode": "PreventDeletesError"}) + + self.assertEqual(DatabaseRecord.DatabaseLockMode.PREVENT_DELETES_ERROR, record.lock_mode) + + def test_auto_indexes_are_parsed_into_definitions(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {"Auto/Orders/ByCompany": AUTO_INDEX}}) + + self.assertIsInstance(record.auto_indexes["Auto/Orders/ByCompany"], AutoIndexDefinition) + self.assertEqual("Orders", record.auto_indexes["Auto/Orders/ByCompany"].collection) + + def test_an_empty_auto_index_map_reads_as_no_auto_indexes(self): + self.assertIsNone(DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {}}).auto_indexes) + + +class TestDatabaseRecordToJson(unittest.TestCase): + def test_auto_indexes_are_serialized_by_name(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {"Auto/Orders/ByCompany": AUTO_INDEX}}) + serialized = record.to_json()["AutoIndexes"] + + self.assertEqual(["Auto/Orders/ByCompany"], list(serialized)) + self.assertEqual("Orders", serialized["Auto/Orders/ByCompany"]["Collection"]) + self.assertEqual("AutoMap", serialized["Auto/Orders/ByCompany"]["Type"]) + + def test_a_record_with_no_auto_indexes_writes_none(self): + self.assertIsNone(DatabaseRecord("db").to_json()["AutoIndexes"]) + + def test_a_parsed_record_survives_a_serialization_round_trip(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {"Auto/Orders/ByCompany": AUTO_INDEX}}) + round_tripped = DatabaseRecord.from_json(record.to_json()) + + self.assertEqual( + record.auto_indexes["Auto/Orders/ByCompany"].collection, + round_tripped.auto_indexes["Auto/Orders/ByCompany"].collection, + ) From fc0de6534188b844b1af032720327a513eaac438 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 18:27:50 +0200 Subject: [PATCH 14/29] Export the queue sink and smuggler types from the package root --- ravendb/__init__.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/ravendb/__init__.py b/ravendb/__init__.py index f3df37c8..1732cc2c 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -200,7 +200,27 @@ OngoingTaskPullReplicationAsSink, OngoingTaskPullReplicationAsHub, OngoingTaskCdcSink, -) + OngoingTaskQueueSink, +) +from ravendb.documents.operations.queue_sink import ( + AddQueueSinkOperation, + AddQueueSinkOperationResult, + AzureServiceBusSinkSource, + QueueSinkConfiguration, + QueueSinkProcessState, + QueueSinkScript, + UpdateQueueSinkOperation, + UpdateQueueSinkOperationResult, +) +from ravendb.documents.smuggler.common import ( + DatabaseItemType, + DatabaseRecordItemType, + DatabaseSmugglerExportOptions, + DatabaseSmugglerImportOptions, + DatabaseSmugglerOptions, + ExportCompressionAlgorithm, +) +from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler from ravendb.documents.operations.cdc_sink import ( AddCdcSinkOperation, AddCdcSinkOperationResult, @@ -732,15 +752,6 @@ # IChangesConnectionState # todo: Smuggler -# DatabaseItemType -# DatabaseRecordItemType -# DatabaseSmuggler -# DatabaseSmugglerExportOptions -# IDatabaseSmugglerExportOptions -# DatabaseSmugglerImportOptions -# IDatabaseSmugglerImportOptions -# DatabaseSmugglerOptions -# IDatabaseSmugglerOptions # todo: Certificates # AddDatabaseNodeOperation From 25e137a028543029c0de8901914ef8874245dd80 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Tue, 8 Sep 2026 19:02:21 +0200 Subject: [PATCH 15/29] Gate the licensed queue sink tests behind RAVENDB_LICENSE --- ravendb/tests/operations_tests/test_queue_sink.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ravendb/tests/operations_tests/test_queue_sink.py b/ravendb/tests/operations_tests/test_queue_sink.py index 71f1d63f..e9a25e7e 100644 --- a/ravendb/tests/operations_tests/test_queue_sink.py +++ b/ravendb/tests/operations_tests/test_queue_sink.py @@ -4,6 +4,7 @@ """ import json +import os import unittest from ravendb.documents.operations.etl.queue.connection import QueueBrokerType @@ -223,6 +224,10 @@ def test_a_record_from_a_server_without_queue_sinks_gets_an_empty_list(self): class TestQueueSinkAgainstServer(TestBase): + # Creating a queue sink task is licensed: an unlicensed server accepts it on a fresh + # start but rejects it once the suite has built up databases and tasks, so the two + # tests that create one are gated the way the rest of this repo gates licensed tests. + def setUp(self): super().setUp() self.store.maintenance.send( @@ -235,6 +240,7 @@ def setUp(self): ) ) + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") def test_the_server_stores_a_queue_sink_task_and_reads_it_back(self): result = self.store.maintenance.send(AddQueueSinkOperation(_configuration())) self.assertGreater(result.task_id, 0) @@ -248,6 +254,7 @@ def test_the_server_stores_a_queue_sink_task_and_reads_it_back(self): self.assertEqual(["orders-topic"], task.configuration.scripts[0].queues) self.assertEqual("put('orders/', this);", task.configuration.scripts[0].script) + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") def test_the_task_can_be_updated(self): task_id = self.store.maintenance.send(AddQueueSinkOperation(_configuration())).task_id From 3e444fedc7d5186987a5569aa52f7a69018555ea Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:46:53 +0200 Subject: [PATCH 16/29] Type license limits with LimitType and LicenseLimitException --- ravendb/exceptions/commercial.py | 89 +++++++++++++++++++ ravendb/exceptions/exception_dispatcher.py | 3 + .../test_commercial_limits.py | 61 +++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 ravendb/exceptions/commercial.py create mode 100644 ravendb/tests/operations_tests/test_commercial_limits.py diff --git a/ravendb/exceptions/commercial.py b/ravendb/exceptions/commercial.py new file mode 100644 index 00000000..8e7cfcc8 --- /dev/null +++ b/ravendb/exceptions/commercial.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import enum +from typing import Optional + +from ravendb.exceptions.raven_exceptions import RavenException + + +class LimitType(enum.Enum): + """ + Which licensed capability a server refused. Carried by LicenseLimitException when the + server names one. + """ + + INVALID_LICENSE = "InvalidLicense" # Invalid License + FORBIDDEN_HOST = "ForbiddenHost" # Forbidden Host + DYNAMIC_NODE_DISTRIBUTION = "DynamicNodeDistribution" # Dynamic Nodes Distribution + CLUSTER_SIZE = "ClusterSize" # Cluster Size + SNAPSHOT_BACKUP = "SnapshotBackup" # Snapshot Backup + CLOUD_BACKUP = "CloudBackup" # Cloud Backup + ENCRYPTION = "Encryption" + DOCUMENTS_COMPRESSION = "DocumentsCompression" # Documents Compression + ROLLING_INDEXES = "RollingIndexes" # Rolling Indexes + EXTERNAL_REPLICATION = "ExternalReplication" # External Replication + RAVEN_ETL = "RavenEtl" # Raven ETL + SQL_ETL = "SqlEtl" # SQL ETL + OLAP_ETL = "OlapEtl" # OLAP ETL + ELASTIC_SEARCH_ETL = "ElasticSearchEtl" # ElasticSearch ETL + QUEUE_ETL = "QueueEtl" # Queue ETL + SNOWFLAKE_ETL = "SnowflakeEtl" # Snowflake ETL + EMBEDDINGS_GENERATION = "EmbeddingsGeneration" # Embeddings Generation + GEN_AI = "GenAi" # Gen AI + AI_AGENT = "AiAgent" # AI Agent + AI_ASSISTANT = "AiAssistant" # AI Assistant + QUILL = "Quill" + CORES = "Cores" # Cores Limit + SNMP = "Snmp" # SNMP + POSTGRE_SQL_INTEGRATION = "PostgreSqlIntegration" # PostgreSql Integration + POWER_B_I = "PowerBI" # Power BI + DELAYED_EXTERNAL_REPLICATION = "DelayedExternalReplication" # Delayed External Replication + HIGHLY_AVAILABLE_TASKS = "HighlyAvailableTasks" # Highly Available Tasks + PULL_REPLICATION_AS_HUB = "PullReplicationAsHub" # Pull Replication As Hub + PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" # Pull Replication As Sink + TIME_SERIES_ROLLUPS_AND_RETENTION = "TimeSeriesRollupsAndRetention" # Time Series Rollups and Retention + ENCRYPTED_BACKUP = "EncryptedBackup" # Encrypted Backup + ADDITIONAL_ASSEMBLIES_FROM_NU_GET = "AdditionalAssembliesFromNuGet" # Additional Assemblies from NuGet + MONITORING_ENDPOINTS = "MonitoringEndpoints" # Monitoring Endpoints + READ_ONLY_CERTIFICATES = "ReadOnlyCertificates" # Read-only Certificates + CONCURRENT_SUBSCRIPTIONS = "ConcurrentSubscriptions" # Concurrent Subscriptions + TCP_DATA_COMPRESSION = "TcpDataCompression" # TCP Data Compression + SERVER_WIDE_BACKUPS = "ServerWideBackups" # Server Wide Backups + SERVER_WIDE_EXTERNAL_REPLICATIONS = "ServerWideExternalReplications" # Server Wide External Replications + SERVER_WIDE_CUSTOM_SORTERS = "ServerWideCustomSorters" # Server Wide Custom Sorters + SERVER_WIDE_ANALYZERS = "ServerWideAnalyzers" # Server Wide Analyzers + SERVER_WIDE_CONNECTION_STRINGS = "ServerWideConnectionStrings" # Server Wide Connection Strings + INDEX_CLEANUP = "IndexCleanup" # Index Cleanup + PERIODIC_BACKUP = "PeriodicBackup" # Periodic Backup + CLIENT_CONFIGURATION = "ClientConfiguration" # Client Configuration + STUDIO_CONFIGURATION = "StudioConfiguration" # Studio Configuration + QUEUE_SINK = "QueueSink" # Queue Sink + CDC_SINK = "CdcSink" # CDC Sink + DATA_ARCHIVAL = "DataArchival" # Data Archival + SHARDING = "Sharding" + SUBSCRIPTIONS = "Subscriptions" + REVISIONS_CONFIGURATION = "RevisionsConfiguration" # Revisions Configuration + EXPIRATION = "Expiration" + REFRESH = "Refresh" + INDEXES = "Indexes" + CUSTOM_SORTERS = "CustomSorters" # Custom Sorters + CUSTOM_ANALYZERS = "CustomAnalyzers" # Custom Analyzers + REMOTE_ATTACHMENTS = "RemoteAttachments" # Remote Attachments + SCHEMA_VALIDATION = "SchemaValidation" # Schema Validation + SSO = "Sso" # SSO + + def __str__(self) -> str: + return self.value + + +class LicenseLimitException(RavenException): + """ + Raised when the server refuses an operation because the license does not cover it. + + The server answers HTTP 402 and names the feature in the message; when it also sends a + machine-readable limit, it lands in :attr:`limit_type`. + """ + + def __init__(self, message: str = None, limit_type: Optional[LimitType] = None): + super().__init__(message) + self.limit_type = limit_type diff --git a/ravendb/exceptions/exception_dispatcher.py b/ravendb/exceptions/exception_dispatcher.py index b06ca2a2..2ac3e600 100644 --- a/ravendb/exceptions/exception_dispatcher.py +++ b/ravendb/exceptions/exception_dispatcher.py @@ -5,6 +5,7 @@ from datetime import timedelta from ravendb.exceptions.cluster import NodeIsPassiveException, NoLoaderException +from ravendb.exceptions.commercial import LicenseLimitException from ravendb.exceptions.documents import DocumentConflictException, DocumentDoesNotExistException from ravendb.exceptions.documents.bulkinsert import BulkInsertAbortedException, BulkInsertProtocolViolationException from ravendb.exceptions.documents.indexes import IndexDoesNotExistException @@ -60,6 +61,8 @@ # cluster "NodeIsPassiveException": NodeIsPassiveException, "NoLoaderException": NoLoaderException, + # commercial + "LicenseLimitException": LicenseLimitException, } diff --git a/ravendb/tests/operations_tests/test_commercial_limits.py b/ravendb/tests/operations_tests/test_commercial_limits.py new file mode 100644 index 00000000..ea5279f8 --- /dev/null +++ b/ravendb/tests/operations_tests/test_commercial_limits.py @@ -0,0 +1,61 @@ +""" +Tests for the commercial limit surface: the LimitType enum the 7.2.5 patch extended, and +LicenseLimitException coming back typed from a 402 instead of a bare RavenException. +""" + +import unittest + +from ravendb.exceptions.commercial import LicenseLimitException, LimitType +from ravendb.exceptions.exception_dispatcher import ExceptionDispatcher +from ravendb.exceptions.raven_exceptions import RavenException + + +class TestLimitType(unittest.TestCase): + def test_the_limits_7_2_5_added(self): + self.assertEqual("ServerWideConnectionStrings", LimitType.SERVER_WIDE_CONNECTION_STRINGS.value) + self.assertEqual("CdcSink", LimitType.CDC_SINK.value) + self.assertEqual("Sso", LimitType.SSO.value) + + def test_a_limit_parses_from_the_name_the_server_uses(self): + self.assertEqual(LimitType.QUEUE_SINK, LimitType("QueueSink")) + self.assertEqual(LimitType.SCHEMA_VALIDATION, LimitType("SchemaValidation")) + + def test_every_member_stringifies_to_its_wire_name(self): + for member in LimitType: + self.assertEqual(member.value, str(member)) + + +class TestLicenseLimitException(unittest.TestCase): + def test_it_is_a_raven_exception(self): + self.assertIsInstance(LicenseLimitException("nope"), RavenException) + + def test_it_can_carry_the_limit_it_hit(self): + exception = LicenseLimitException("nope", LimitType.QUEUE_SINK) + + self.assertEqual(LimitType.QUEUE_SINK, exception.limit_type) + + def test_the_dispatcher_returns_it_for_a_402(self): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", + object_type="Raven.Client.Exceptions.Commercial.LicenseLimitException", + message="no", + error="Your license doesn't support using the queue sink feature.", + ) + + exception = ExceptionDispatcher.get(schema, 402) + + self.assertIsInstance(exception, LicenseLimitException) + self.assertIn("queue sink", str(exception)) + # The server does not put the limit on the wire, so it stays unset here, exactly + # as it does in the C# client. + self.assertIsNone(exception.limit_type) + + def test_a_402_from_another_exception_type_is_left_alone(self): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", + object_type="Raven.Client.Exceptions.RavenException", + message="no", + error="something else", + ) + + self.assertNotIsInstance(ExceptionDispatcher.get(schema, 402), LicenseLimitException) From c4fe8493effde916b230d9a0aa03581c7b6a46d3 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:46:54 +0200 Subject: [PATCH 17/29] Stop AutoIndexDefinition.from_json requiring optional sections --- ravendb/documents/indexes/definitions.py | 31 ++++++++++---- .../operations_tests/test_database_record.py | 41 ++++++++++++++++++- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/ravendb/documents/indexes/definitions.py b/ravendb/documents/indexes/definitions.py index 27d04376..b6b05000 100644 --- a/ravendb/documents/indexes/definitions.py +++ b/ravendb/documents/indexes/definitions.py @@ -393,25 +393,38 @@ def __init__( @classmethod def from_json(cls, json_dict: Dict) -> AutoIndexDefinition: + index_type = json_dict.get("Type") + priority = json_dict.get("Priority") + state = json_dict.get("State") + # The C# constructor starts both field maps empty, and the rest are nullable, so a + # payload that leaves any of them out is well-formed. return cls( - IndexType(json_dict.get("Type")), + IndexType(index_type) if index_type else None, json_dict.get("Name"), - IndexPriority(json_dict.get("Priority")), - IndexState(json_dict.get("State")) if json_dict.get("State", None) else None, + IndexPriority(priority) if priority else None, + IndexState(state) if state else None, json_dict.get("Collection"), - {name: AutoIndexFieldOptions.from_json(value) for name, value in json_dict.get("MapFields").items()}, - {name: AutoIndexFieldOptions.from_json(value) for name, value in json_dict.get("GroupByFields").items()}, + { + name: AutoIndexFieldOptions.from_json(value) + for name, value in (json_dict.get("MapFields") or {}).items() + }, + { + name: AutoIndexFieldOptions.from_json(value) + for name, value in (json_dict.get("GroupByFields") or {}).items() + }, ) def to_json(self) -> Dict: return { - "Type": self.index_type.value, + "Type": self.index_type.value if self.index_type is not None else None, "Name": self.name, - "Priority": self.priority.value, + "Priority": self.priority.value if self.priority is not None else None, "State": self.state.value if self.state is not None else None, "Collection": self.collection, - "MapFields": {key: map_field.to_json() for key, map_field in self.map_fields.items()}, - "GroupByFields": {key: group_by_field.to_json() for key, group_by_field in self.group_by_fields.items()}, + "MapFields": {key: map_field.to_json() for key, map_field in (self.map_fields or {}).items()}, + "GroupByFields": { + key: group_by_field.to_json() for key, group_by_field in (self.group_by_fields or {}).items() + }, } diff --git a/ravendb/tests/operations_tests/test_database_record.py b/ravendb/tests/operations_tests/test_database_record.py index c6cc4940..24f76e76 100644 --- a/ravendb/tests/operations_tests/test_database_record.py +++ b/ravendb/tests/operations_tests/test_database_record.py @@ -5,7 +5,7 @@ import unittest -from ravendb.documents.indexes.definitions import AutoIndexDefinition +from ravendb.documents.indexes.definitions import AutoIndexDefinition, IndexPriority, IndexType from ravendb.serverwide.database_record import DatabaseRecord AUTO_INDEX = { @@ -68,3 +68,42 @@ def test_a_parsed_record_survives_a_serialization_round_trip(self): record.auto_indexes["Auto/Orders/ByCompany"].collection, round_tripped.auto_indexes["Auto/Orders/ByCompany"].collection, ) + + +class TestAutoIndexDefinitionFromJson(unittest.TestCase): + """ + The C# constructor starts MapFields and GroupByFields empty and leaves the rest + nullable, so a payload that omits any of them is well-formed. + """ + + def test_a_minimal_payload_parses(self): + definition = AutoIndexDefinition.from_json({"Name": "Auto/Orders", "Collection": "Orders"}) + + self.assertEqual("Auto/Orders", definition.name) + self.assertEqual("Orders", definition.collection) + self.assertEqual({}, definition.map_fields) + self.assertEqual({}, definition.group_by_fields) + self.assertIsNone(definition.index_type) + self.assertIsNone(definition.priority) + + def test_an_empty_payload_parses(self): + self.assertIsNone(AutoIndexDefinition.from_json({}).name) + + def test_a_full_payload_still_parses(self): + definition = AutoIndexDefinition.from_json(AUTO_INDEX) + + self.assertEqual(IndexType.AUTO_MAP, definition.index_type) + self.assertEqual(IndexPriority.NORMAL, definition.priority) + self.assertEqual("Orders", definition.collection) + + def test_to_json_survives_the_optional_fields_being_unset(self): + serialized = AutoIndexDefinition.from_json({"Name": "Auto/Orders"}).to_json() + + self.assertIsNone(serialized["Type"]) + self.assertIsNone(serialized["Priority"]) + self.assertEqual({}, serialized["MapFields"]) + + def test_definition_round_trips(self): + definition = AutoIndexDefinition.from_json(AUTO_INDEX) + + self.assertEqual(definition.to_json(), AutoIndexDefinition.from_json(definition.to_json()).to_json()) From bd9b6e4be7f6e01c39e7f09db61d141b57ba55e6 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:46:54 +0200 Subject: [PATCH 18/29] Convert between backup and remote-attachment cloud settings Ports RemoteAttachmentExtensions. Backup S3Settings gains the StorageClass it was missing, and the S3 and Azure from_json / to_json paths stop requiring keys that C# treats as optional, since a converted instance carries no backup script. --- .../operations/attachments/__init__.py | 43 ++++ .../documents/operations/backups/settings.py | 93 ++++++-- .../operations_tests/test_s3_settings.py | 207 ++++++++++++++---- 3 files changed, 275 insertions(+), 68 deletions(-) diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index b4924446..53c0efa8 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -449,6 +449,30 @@ def to_json(self) -> dict: result["StorageClass"] = self.storage_class.value return result + def to_s3_settings(self) -> Optional["S3Settings"]: + """ + The same bucket as periodic-backup settings, enabled for direct upload. + Returns None when the bucket is not set, which is the minimum the server needs. + """ + from ravendb.documents.operations.backups.settings import S3Settings + + if not self.bucket_name or self.bucket_name.isspace(): + return None + + return S3Settings( + disabled=False, + aws_access_key=self.aws_access_key, + aws_secret_key=self.aws_secret_key, + aws_session_token=self.aws_session_token, + aws_region_name=self.aws_region_name, + remote_folder_name=self.remote_folder_name, + bucket_name=self.bucket_name, + custom_server_url=self.custom_server_url, + force_path_style=self.force_path_style, + disable_checksum_validation=self.disable_checksum_validation, + storage_class=self.storage_class, + ) + class RemoteAttachmentsAzureSettings: def __init__( @@ -484,6 +508,25 @@ def to_json(self) -> dict: "SasToken": self.sas_token, } + def to_azure_settings(self) -> Optional["AzureSettings"]: + """ + The same container as periodic-backup settings, enabled for direct upload. + Returns None when the container is not set, which is the minimum the server needs. + """ + from ravendb.documents.operations.backups.settings import AzureSettings + + if not self.storage_container or self.storage_container.isspace(): + return None + + return AzureSettings( + disabled=False, + storage_container=self.storage_container, + remote_folder_name=self.remote_folder_name, + account_name=self.account_name, + account_key=self.account_key, + sas_token=self.sas_token, + ) + class RemoteAttachmentsDestinationConfiguration: def __init__( diff --git a/ravendb/documents/operations/backups/settings.py b/ravendb/documents/operations/backups/settings.py index 0a242863..99f01014 100644 --- a/ravendb/documents/operations/backups/settings.py +++ b/ravendb/documents/operations/backups/settings.py @@ -62,7 +62,9 @@ def __init__(self, disabled: bool = None, get_backup_configuration_script: GetBa def to_json(self) -> Dict[str, Any]: return { "Disabled": self.disabled, - "GetBackupConfigurationScript": self.get_backup_configuration_script.to_json(), + "GetBackupConfigurationScript": ( + self.get_backup_configuration_script.to_json() if self.get_backup_configuration_script else None + ), } @@ -131,6 +133,7 @@ def __init__( custom_server_url: str = None, force_path_style: bool = None, disable_checksum_validation: bool = None, + storage_class: "S3StorageClass" = None, ): super().__init__( disabled, @@ -145,27 +148,37 @@ def __init__( self.custom_server_url = custom_server_url self.force_path_style = force_path_style self.disable_checksum_validation = disable_checksum_validation + self.storage_class = storage_class @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: return cls( - json_dict["Disabled"], - GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]), - json_dict["AwsAccessKey"], - json_dict["AwsSecretKey"], - json_dict["AwsSessionToken"], - json_dict["AwsRegionName"], - json_dict["RemoteFolderName"], - json_dict["BucketName"], - json_dict["CustomServerUrl"], - json_dict["ForcePathStyle"], + json_dict.get("Disabled"), + ( + GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]) + if json_dict.get("GetBackupConfigurationScript") + else None + ), + # C# deserializes by reflection, so a key the server left out just keeps the + # field's default rather than failing. + json_dict.get("AwsAccessKey"), + json_dict.get("AwsSecretKey"), + json_dict.get("AwsSessionToken"), + json_dict.get("AwsRegionName"), + json_dict.get("RemoteFolderName"), + json_dict.get("BucketName"), + json_dict.get("CustomServerUrl"), + json_dict.get("ForcePathStyle"), json_dict.get("DisableChecksumValidation"), + S3StorageClass(json_dict["StorageClass"]) if json_dict.get("StorageClass") else None, ) def to_json(self) -> Dict[str, Any]: return { "Disabled": self.disabled, - "GetBackupConfigurationScript": self.get_backup_configuration_script.to_json(), + "GetBackupConfigurationScript": ( + self.get_backup_configuration_script.to_json() if self.get_backup_configuration_script else None + ), "AwsAccessKey": self.aws_access_key, "AwsSecretKey": self.aws_secret_key, "AwsSessionToken": self.aws_session_token, @@ -175,8 +188,28 @@ def to_json(self) -> Dict[str, Any]: "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, "DisableChecksumValidation": self.disable_checksum_validation, + # The server treats an absent storage class as its own default, so only send one + # when the caller picked it. + **({"StorageClass": self.storage_class.value} if self.storage_class is not None else {}), } + def to_remote_attachments_s3_settings(self) -> "RemoteAttachmentsS3Settings": + """The same bucket as remote-attachment settings, dropping the backup-only fields.""" + from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings + + return RemoteAttachmentsS3Settings( + aws_access_key=self.aws_access_key, + aws_secret_key=self.aws_secret_key, + aws_session_token=self.aws_session_token, + aws_region_name=self.aws_region_name, + remote_folder_name=self.remote_folder_name, + bucket_name=self.bucket_name, + custom_server_url=self.custom_server_url, + force_path_style=self.force_path_style, + disable_checksum_validation=self.disable_checksum_validation, + storage_class=self.storage_class, + ) + class GlacierSettings(AmazonSettings): def __init__( @@ -205,7 +238,11 @@ def __init__( def from_json(cls, json_dict: Dict[str, Any]) -> GlacierSettings: return cls( json_dict["Disabled"], - GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]), + ( + GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]) + if json_dict.get("GetBackupConfigurationScript") + else None + ), json_dict["AwsAccessKey"], json_dict["AwsSecretKey"], json_dict["AwsSessionToken"], @@ -217,7 +254,9 @@ def from_json(cls, json_dict: Dict[str, Any]) -> GlacierSettings: def to_json(self) -> Dict[str, Any]: return { "Disabled": self.disabled, - "GetBackupConfigurationScript": self.get_backup_configuration_script.to_json(), + "GetBackupConfigurationScript": ( + self.get_backup_configuration_script.to_json() if self.get_backup_configuration_script else None + ), "AwsAccessKey": self.aws_access_key, "AwsSecretKey": self.aws_secret_key, "AwsSessionToken": self.aws_session_token, @@ -248,17 +287,17 @@ def __init__( @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> AzureSettings: return cls( - json_dict["Disabled"], + json_dict.get("Disabled"), ( GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]) - if json_dict["GetBackupConfigurationScript"] + if json_dict.get("GetBackupConfigurationScript") else None ), - json_dict["StorageContainer"], - json_dict["RemoteFolderName"], - json_dict["AccountName"], - json_dict["AccountKey"], - json_dict["SasToken"], + json_dict.get("StorageContainer"), + json_dict.get("RemoteFolderName"), + json_dict.get("AccountName"), + json_dict.get("AccountKey"), + json_dict.get("SasToken"), ) def to_json(self) -> Dict[str, Any]: @@ -274,6 +313,18 @@ def to_json(self) -> Dict[str, Any]: "SasToken": self.sas_token, } + def to_remote_attachments_azure_settings(self) -> "RemoteAttachmentsAzureSettings": + """The same container as remote-attachment settings, dropping the backup-only fields.""" + from ravendb.documents.operations.attachments import RemoteAttachmentsAzureSettings + + return RemoteAttachmentsAzureSettings( + storage_container=self.storage_container, + remote_folder_name=self.remote_folder_name, + account_name=self.account_name, + account_key=self.account_key, + sas_token=self.sas_token, + ) + class FtpSettings(BackupSettings): def __init__( diff --git a/ravendb/tests/operations_tests/test_s3_settings.py b/ravendb/tests/operations_tests/test_s3_settings.py index f4a93300..7d04c2f1 100644 --- a/ravendb/tests/operations_tests/test_s3_settings.py +++ b/ravendb/tests/operations_tests/test_s3_settings.py @@ -1,69 +1,182 @@ """ -Tests for DisableChecksumValidation, added in 7.2.5 to both S3 settings classes for -S3-compatible storage that does not support modern object integrity checks. +Tests for the S3 settings the 7.2.5 patch touched: the DisableChecksumValidation flag on +both settings classes, and the conversions between periodic-backup and remote-attachment +settings that the patch kept in step. """ import unittest -from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings -from ravendb.documents.operations.backups.settings import GetBackupConfigurationScript, S3Settings +from ravendb.documents.operations.attachments import RemoteAttachmentsAzureSettings, RemoteAttachmentsS3Settings +from ravendb.documents.operations.backups.settings import ( + AzureSettings, + GetBackupConfigurationScript, + S3Settings, + S3StorageClass, +) -class TestBackupS3Settings(unittest.TestCase): - def _settings(self, **kwargs) -> S3Settings: - return S3Settings( - disabled=False, - get_backup_configuration_script=GetBackupConfigurationScript(), - aws_access_key="key", - aws_secret_key="secret", - aws_session_token=None, - aws_region_name="eu-central-1", - remote_folder_name="backups", - bucket_name="rvn", - custom_server_url="https://minio.local", - force_path_style=True, - **kwargs, - ) +def _remote_settings() -> RemoteAttachmentsS3Settings: + return RemoteAttachmentsS3Settings( + aws_access_key="key", + aws_secret_key="secret", + aws_session_token="token", + aws_region_name="eu-central-1", + remote_folder_name="attachments", + bucket_name="bucket", + custom_server_url="https://minio.local", + force_path_style=True, + disable_checksum_validation=True, + storage_class=S3StorageClass.GLACIER, + ) + + +class TestDisableChecksumValidation(unittest.TestCase): + def test_remote_attachment_settings_carry_the_flag(self): + serialized = _remote_settings().to_json() + + self.assertTrue(serialized["DisableChecksumValidation"]) + self.assertTrue(RemoteAttachmentsS3Settings.from_json(serialized).disable_checksum_validation) + + def test_backup_settings_carry_the_flag(self): + settings = S3Settings(bucket_name="bucket", disable_checksum_validation=True) + + self.assertTrue(settings.to_json()["DisableChecksumValidation"]) + self.assertTrue(S3Settings.from_json(settings.to_json()).disable_checksum_validation) + + def test_the_flag_is_absent_rather_than_true_by_default(self): + # Checksum validation protects data integrity, so opting out has to be deliberate. + self.assertIsNone(S3Settings(bucket_name="bucket").to_json()["DisableChecksumValidation"]) + self.assertIsNone(RemoteAttachmentsS3Settings(bucket_name="bucket").to_json()["DisableChecksumValidation"]) + + +class TestBackupSettingsStorageClass(unittest.TestCase): + def test_a_chosen_storage_class_is_sent(self): + settings = S3Settings(bucket_name="bucket", storage_class=S3StorageClass.GLACIER) + + self.assertEqual("Glacier", settings.to_json()["StorageClass"]) + + def test_an_unset_storage_class_is_left_to_the_server(self): + self.assertNotIn("StorageClass", S3Settings(bucket_name="bucket").to_json()) + + def test_storage_class_round_trips(self): + settings = S3Settings(bucket_name="bucket", storage_class=S3StorageClass.STANDARD) + + self.assertEqual(S3StorageClass.STANDARD, S3Settings.from_json(settings.to_json()).storage_class) + + +class TestBackupSettingsWithoutAScript(unittest.TestCase): + def test_settings_without_a_configuration_script_serialize(self): + # C# writes GetBackupConfigurationScript?.ToJson(), so an absent script is normal. + self.assertIsNone(S3Settings(bucket_name="bucket").to_json()["GetBackupConfigurationScript"]) + + def test_settings_without_a_configuration_script_parse(self): + settings = S3Settings.from_json({"Disabled": False, "BucketName": "bucket"}) - def test_checksum_validation_is_left_alone_by_default(self): - self.assertIsNone(self._settings().disable_checksum_validation) + self.assertEqual("bucket", settings.bucket_name) + self.assertIsNone(settings.get_backup_configuration_script) - def test_the_flag_is_written(self): - self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + def test_a_script_still_round_trips(self): + settings = S3Settings(bucket_name="bucket", get_backup_configuration_script=GetBackupConfigurationScript("run")) - def test_the_flag_round_trips(self): - serialized = self._settings(disable_checksum_validation=True).to_json() + self.assertEqual("run", S3Settings.from_json(settings.to_json()).get_backup_configuration_script.exec) - self.assertEqual(serialized, S3Settings.from_json(serialized).to_json()) - def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): - serialized = self._settings().to_json() - del serialized["DisableChecksumValidation"] +class TestS3SettingsConversions(unittest.TestCase): + def test_remote_settings_become_backup_settings(self): + backup = _remote_settings().to_s3_settings() - self.assertIsNone(S3Settings.from_json(serialized).disable_checksum_validation) + self.assertIsInstance(backup, S3Settings) + self.assertEqual("bucket", backup.bucket_name) + self.assertEqual("eu-central-1", backup.aws_region_name) + self.assertTrue(backup.force_path_style) + self.assertTrue(backup.disable_checksum_validation) + self.assertEqual(S3StorageClass.GLACIER, backup.storage_class) + def test_the_conversion_enables_the_settings_for_direct_upload(self): + self.assertFalse(_remote_settings().to_s3_settings().disabled) -class TestRemoteAttachmentsS3Settings(unittest.TestCase): - def _settings(self, **kwargs) -> RemoteAttachmentsS3Settings: - return RemoteAttachmentsS3Settings( + def test_a_bucketless_remote_setting_converts_to_nothing(self): + # The bucket is the minimum the server needs, so there is nothing to convert. + self.assertIsNone(RemoteAttachmentsS3Settings(aws_access_key="key").to_s3_settings()) + self.assertIsNone(RemoteAttachmentsS3Settings(bucket_name=" ").to_s3_settings()) + + def test_backup_settings_become_remote_settings(self): + remote = S3Settings( + bucket_name="bucket", aws_access_key="key", - aws_secret_key="secret", - aws_region_name="eu-central-1", - bucket_name="rvn", - force_path_style=True, - **kwargs, + disable_checksum_validation=True, + storage_class=S3StorageClass.GLACIER, + ).to_remote_attachments_s3_settings() + + self.assertIsInstance(remote, RemoteAttachmentsS3Settings) + self.assertEqual("bucket", remote.bucket_name) + self.assertTrue(remote.disable_checksum_validation) + self.assertEqual(S3StorageClass.GLACIER, remote.storage_class) + + def test_a_conversion_round_trip_keeps_every_shared_field(self): + original = _remote_settings() + + self.assertEqual(original.to_json(), original.to_s3_settings().to_remote_attachments_s3_settings().to_json()) + + def test_backup_only_fields_are_dropped_on_the_way_out(self): + backup = S3Settings( + bucket_name="bucket", + disabled=True, + get_backup_configuration_script=GetBackupConfigurationScript("run"), + ) + remote = backup.to_remote_attachments_s3_settings() + + self.assertNotIn("Disabled", remote.to_json()) + self.assertNotIn("GetBackupConfigurationScript", remote.to_json()) + + +class TestAzureSettingsConversions(unittest.TestCase): + def _remote(self) -> RemoteAttachmentsAzureSettings: + return RemoteAttachmentsAzureSettings( + storage_container="container", + remote_folder_name="attachments", + account_name="account", + account_key="key", + sas_token="token", ) - def test_checksum_validation_is_left_alone_by_default(self): - self.assertIsNone(self._settings().disable_checksum_validation) + def test_remote_settings_become_backup_settings(self): + backup = self._remote().to_azure_settings() + + self.assertIsInstance(backup, AzureSettings) + self.assertEqual("container", backup.storage_container) + self.assertEqual("account", backup.account_name) + self.assertFalse(backup.disabled) + + def test_a_containerless_remote_setting_converts_to_nothing(self): + self.assertIsNone(RemoteAttachmentsAzureSettings(account_name="account").to_azure_settings()) + self.assertIsNone(RemoteAttachmentsAzureSettings(storage_container=" ").to_azure_settings()) + + def test_backup_settings_become_remote_settings(self): + remote = AzureSettings( + storage_container="container", account_name="account" + ).to_remote_attachments_azure_settings() - def test_the_flag_is_written(self): - self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + self.assertIsInstance(remote, RemoteAttachmentsAzureSettings) + self.assertEqual("container", remote.storage_container) - def test_the_flag_round_trips(self): - serialized = self._settings(disable_checksum_validation=True).to_json() + def test_a_conversion_round_trip_keeps_every_shared_field(self): + original = self._remote() + + self.assertEqual( + original.to_json(), original.to_azure_settings().to_remote_attachments_azure_settings().to_json() + ) + + def test_backup_only_fields_are_dropped_on_the_way_out(self): + backup = AzureSettings( + storage_container="container", + disabled=True, + get_backup_configuration_script=GetBackupConfigurationScript("run"), + ) + remote = backup.to_remote_attachments_azure_settings() - self.assertEqual(serialized, RemoteAttachmentsS3Settings.from_json(serialized).to_json()) + self.assertNotIn("Disabled", remote.to_json()) + self.assertNotIn("GetBackupConfigurationScript", remote.to_json()) - def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): - self.assertIsNone(RemoteAttachmentsS3Settings.from_json({"BucketName": "rvn"}).disable_checksum_validation) + def test_partial_backup_settings_parse(self): + self.assertEqual("container", AzureSettings.from_json({"StorageContainer": "container"}).storage_container) From ca3c52045798968af8435f80a6d93003d4be272a Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:46:54 +0200 Subject: [PATCH 19/29] Give smuggler operations a typed result Operation.wait_for_completion now hands back the result the server carried instead of dropping it, and the smuggler narrows that to a SmugglerResult with per-item-type counts. No existing caller used the return value. --- ravendb/documents/operations/operation.py | 5 +- .../documents/smuggler/database_smuggler.py | 22 +- ravendb/documents/smuggler/result.py | 421 ++++++++++++++++++ .../tests/documents_tests/test_smuggler.py | 102 ++++- 4 files changed, 539 insertions(+), 11 deletions(-) create mode 100644 ravendb/documents/smuggler/result.py diff --git a/ravendb/documents/operations/operation.py b/ravendb/documents/operations/operation.py index 868198d9..18273ceb 100644 --- a/ravendb/documents/operations/operation.py +++ b/ravendb/documents/operations/operation.py @@ -40,13 +40,14 @@ def _get_operation_state_command( ) -> RavenCommand[dict]: return GetOperationStateOperation.GetOperationStateCommand(self.__key, node_tag) - def wait_for_completion(self) -> None: + def wait_for_completion(self) -> Optional[dict]: + """Blocks until the operation finishes, then hands back whatever result it carried.""" while True: status = self.fetch_operations_status() operation_status = status.get("Status") if operation_status == "Completed": - return + return status.get("Result") elif operation_status == "Canceled": raise OperationCancelledException() elif operation_status == "Faulted": diff --git a/ravendb/documents/smuggler/database_smuggler.py b/ravendb/documents/smuggler/database_smuggler.py index 96171465..7e37f061 100644 --- a/ravendb/documents/smuggler/database_smuggler.py +++ b/ravendb/documents/smuggler/database_smuggler.py @@ -15,6 +15,7 @@ DatabaseSmugglerImportOptions, DatabaseSmugglerOptions, ) +from ravendb.documents.smuggler.result import SmugglerResult from ravendb.http.misc import ResponseDisposeHandling from ravendb.http.raven_command import VoidRavenCommand from ravendb.http.server_node import ServerNode @@ -50,6 +51,13 @@ ) +class SmugglerOperation(Operation): + """An export or import operation, whose result is a typed SmugglerResult.""" + + def wait_for_completion(self) -> SmugglerResult: + return SmugglerResult.from_json(super().wait_for_completion()) + + def _is_backup_file(file_path: str) -> bool: return os.path.splitext(file_path)[1].lower() in _BACKUP_EXTENSIONS @@ -93,8 +101,8 @@ def _next_operation_id(self): self._executor.execute_command(command) return command.result, command.node_tag - def _operation_for(self, operation_id: int, node_tag: str) -> Operation: - return Operation( + def _operation_for(self, operation_id: int, node_tag: str) -> SmugglerOperation: + return SmugglerOperation( self._executor, lambda: None, self._executor.conventions, @@ -106,7 +114,7 @@ def export( self, options: DatabaseSmugglerExportOptions, to_file_or_stream: Union[str, IO[bytes]], - ) -> Operation: + ) -> SmugglerOperation: """ Exports the database. ``to_file_or_stream`` is either a path, in which case the file (and any missing parent directory) is created, or a writable binary stream, @@ -132,7 +140,7 @@ def export( return self._export_to_stream(options, to_file_or_stream) - def _export_to_stream(self, options: DatabaseSmugglerExportOptions, destination: IO[bytes]) -> Operation: + def _export_to_stream(self, options: DatabaseSmugglerExportOptions, destination: IO[bytes]) -> SmugglerOperation: operation_id, node_tag = self._next_operation_id() self._executor.execute_command(self._ExportCommand(options, destination, operation_id, node_tag)) return self._operation_for(operation_id, node_tag) @@ -141,7 +149,7 @@ def export_to_database( self, options: DatabaseSmugglerExportOptions, to_smuggler: DatabaseSmuggler, - ) -> Operation: + ) -> SmugglerOperation: """ Streams an export straight into another database. Returns the import operation on the receiving side, which is the one worth waiting on. @@ -164,7 +172,7 @@ def import_data( self, options: DatabaseSmugglerImportOptions, from_file_or_stream: Union[str, IO[bytes]], - ) -> Operation: + ) -> SmugglerOperation: """ Imports a dump. ``from_file_or_stream`` is either a path, which is opened and closed here, or a readable binary stream, which is left open. @@ -182,7 +190,7 @@ def import_data( return self._import_from_stream(options, from_file_or_stream) - def _import_from_stream(self, options: DatabaseSmugglerImportOptions, source: IO[bytes]) -> Operation: + def _import_from_stream(self, options: DatabaseSmugglerImportOptions, source: IO[bytes]) -> SmugglerOperation: operation_id, node_tag = self._next_operation_id() self._executor.execute_command(self._ImportCommand(options, source, operation_id, node_tag)) return self._operation_for(operation_id, node_tag) diff --git a/ravendb/documents/smuggler/result.py b/ravendb/documents/smuggler/result.py new file mode 100644 index 00000000..359efe4e --- /dev/null +++ b/ravendb/documents/smuggler/result.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from ravendb.tools.utils import Utils + + +class Counts: + """How much of one item type an export or import got through.""" + + def __init__( + self, + start_time: Optional[datetime] = None, + processed: bool = False, + read_count: int = 0, + skipped: bool = False, + errored_count: int = 0, + size_in_bytes: int = 0, + ): + self.start_time = start_time + self.processed = processed + self.read_count = read_count + self.skipped = skipped + self.errored_count = errored_count + self.size_in_bytes = size_in_bytes + + def to_json(self) -> Dict[str, Any]: + return { + "StartTime": Utils.datetime_to_string(self.start_time) if self.start_time else None, + "Processed": self.processed, + "ReadCount": self.read_count, + "Skipped": self.skipped, + "ErroredCount": self.errored_count, + "SizeInBytes": self.size_in_bytes, + } + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + start_time = json_dict.get("StartTime") + self.start_time = Utils.string_to_datetime(start_time) if start_time else None + self.processed = json_dict.get("Processed", False) + self.read_count = json_dict.get("ReadCount", 0) + self.skipped = json_dict.get("Skipped", False) + self.errored_count = json_dict.get("ErroredCount", 0) + self.size_in_bytes = json_dict.get("SizeInBytes", 0) + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]): + counts = cls() + if json_dict: + counts._fill_from_json(json_dict) + return counts + + def __repr__(self) -> str: + text = f"Read: {self.read_count:,}." + if self.errored_count: + text += f" Errored: {self.errored_count:,}." + if self.size_in_bytes: + text += f" Size: {self.size_in_bytes:,} bytes." + return text + + +class CountsWithLastEtag(Counts): + def __init__(self, last_etag: int = 0, **kwargs): + super().__init__(**kwargs) + self.last_etag = last_etag + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["LastEtag"] = self.last_etag + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.last_etag = json_dict.get("LastEtag", 0) + + +class CountsWithLastEtagAndAttachments(CountsWithLastEtag): + def __init__(self, attachments: Optional[Counts] = None, **kwargs): + super().__init__(**kwargs) + self.attachments = attachments if attachments is not None else Counts() + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["Attachments"] = self.attachments.to_json() + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.attachments = Counts.from_json(json_dict.get("Attachments")) + + def __repr__(self) -> str: + if self.attachments and self.attachments.read_count: + return f"{super().__repr__()} Attachments: {self.attachments!r}" + return super().__repr__() + + +class CountsWithSkippedCountAndLastEtag(CountsWithLastEtag): + def __init__(self, skipped_count: int = 0, **kwargs): + super().__init__(**kwargs) + self.skipped_count = skipped_count + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["SkippedCount"] = self.skipped_count + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.skipped_count = json_dict.get("SkippedCount", 0) + + def __repr__(self) -> str: + return f"Skipped: {self.skipped_count:,}. {super().__repr__()}" + + +class CountsWithSkippedCountAndLastEtagAndAttachments(CountsWithLastEtagAndAttachments): + def __init__(self, skipped_count: int = 0, **kwargs): + super().__init__(**kwargs) + self.skipped_count = skipped_count + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["SkippedCount"] = self.skipped_count + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.skipped_count = json_dict.get("SkippedCount", 0) + + def __repr__(self) -> str: + if self.skipped_count: + return f"Skipped: {self.skipped_count:,}. {super().__repr__()}" + return super().__repr__() + + +class DatabaseRecordProgress(Counts): + """ + Which parts of the database record an import wrote. The server sends only the flags it + actually set, so anything it leaves out reads as False. + """ + + def __init__( + self, + sorters_updated: bool = False, + analyzers_updated: bool = False, + sink_pull_replications_updated: bool = False, + hub_pull_replications_updated: bool = False, + raven_etls_updated: bool = False, + sql_etls_updated: bool = False, + snowflake_etls_updated: bool = False, + embeddings_generations_updated: bool = False, + gen_ai_tasks_updated: bool = False, + external_replications_updated: bool = False, + periodic_backups_updated: bool = False, + conflict_solver_config_updated: bool = False, + schema_validation_config_updated: bool = False, + time_series_configuration_updated: bool = False, + documents_compression_configuration_updated: bool = False, + revisions_configuration_updated: bool = False, + expiration_configuration_updated: bool = False, + refresh_configuration_updated: bool = False, + data_archival_configuration_updated: bool = False, + remote_attachments_configuration_updated: bool = False, + raven_connection_strings_updated: bool = False, + sql_connection_strings_updated: bool = False, + snowflake_connection_strings_updated: bool = False, + ai_connection_strings_updated: bool = False, + ai_agents_updated: bool = False, + client_configuration_updated: bool = False, + unused_database_ids_updated: bool = False, + lock_mode_updated: bool = False, + olap_etls_updated: bool = False, + olap_connection_strings_updated: bool = False, + elastic_search_etls_updated: bool = False, + elastic_search_connection_strings_updated: bool = False, + postre_s_q_l_configuration_updated: bool = False, + queue_etls_updated: bool = False, + queue_connection_strings_updated: bool = False, + queue_sinks_updated: bool = False, + cdc_sinks_updated: bool = False, + indexes_history_updated: bool = False, + **kwargs, + ): + super().__init__(**kwargs) + self.sorters_updated = sorters_updated + self.analyzers_updated = analyzers_updated + self.sink_pull_replications_updated = sink_pull_replications_updated + self.hub_pull_replications_updated = hub_pull_replications_updated + self.raven_etls_updated = raven_etls_updated + self.sql_etls_updated = sql_etls_updated + self.snowflake_etls_updated = snowflake_etls_updated + self.embeddings_generations_updated = embeddings_generations_updated + self.gen_ai_tasks_updated = gen_ai_tasks_updated + self.external_replications_updated = external_replications_updated + self.periodic_backups_updated = periodic_backups_updated + self.conflict_solver_config_updated = conflict_solver_config_updated + self.schema_validation_config_updated = schema_validation_config_updated + self.time_series_configuration_updated = time_series_configuration_updated + self.documents_compression_configuration_updated = documents_compression_configuration_updated + self.revisions_configuration_updated = revisions_configuration_updated + self.expiration_configuration_updated = expiration_configuration_updated + self.refresh_configuration_updated = refresh_configuration_updated + self.data_archival_configuration_updated = data_archival_configuration_updated + self.remote_attachments_configuration_updated = remote_attachments_configuration_updated + self.raven_connection_strings_updated = raven_connection_strings_updated + self.sql_connection_strings_updated = sql_connection_strings_updated + self.snowflake_connection_strings_updated = snowflake_connection_strings_updated + self.ai_connection_strings_updated = ai_connection_strings_updated + self.ai_agents_updated = ai_agents_updated + self.client_configuration_updated = client_configuration_updated + self.unused_database_ids_updated = unused_database_ids_updated + self.lock_mode_updated = lock_mode_updated + self.olap_etls_updated = olap_etls_updated + self.olap_connection_strings_updated = olap_connection_strings_updated + self.elastic_search_etls_updated = elastic_search_etls_updated + self.elastic_search_connection_strings_updated = elastic_search_connection_strings_updated + self.postre_s_q_l_configuration_updated = postre_s_q_l_configuration_updated + self.queue_etls_updated = queue_etls_updated + self.queue_connection_strings_updated = queue_connection_strings_updated + self.queue_sinks_updated = queue_sinks_updated + self.cdc_sinks_updated = cdc_sinks_updated + self.indexes_history_updated = indexes_history_updated + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + if self.sorters_updated: + json_dict["SortersUpdated"] = True + if self.analyzers_updated: + json_dict["AnalyzersUpdated"] = True + if self.sink_pull_replications_updated: + json_dict["SinkPullReplicationsUpdated"] = True + if self.hub_pull_replications_updated: + json_dict["HubPullReplicationsUpdated"] = True + if self.raven_etls_updated: + json_dict["RavenEtlsUpdated"] = True + if self.sql_etls_updated: + json_dict["SqlEtlsUpdated"] = True + if self.snowflake_etls_updated: + json_dict["SnowflakeEtlsUpdated"] = True + if self.embeddings_generations_updated: + json_dict["EmbeddingsGenerationsUpdated"] = True + if self.gen_ai_tasks_updated: + json_dict["GenAiTasksUpdated"] = True + if self.external_replications_updated: + json_dict["ExternalReplicationsUpdated"] = True + if self.periodic_backups_updated: + json_dict["PeriodicBackupsUpdated"] = True + if self.conflict_solver_config_updated: + json_dict["ConflictSolverConfigUpdated"] = True + if self.schema_validation_config_updated: + json_dict["SchemaValidationConfigUpdated"] = True + if self.time_series_configuration_updated: + json_dict["TimeSeriesConfigurationUpdated"] = True + if self.documents_compression_configuration_updated: + json_dict["DocumentsCompressionConfigurationUpdated"] = True + if self.revisions_configuration_updated: + json_dict["RevisionsConfigurationUpdated"] = True + if self.expiration_configuration_updated: + json_dict["ExpirationConfigurationUpdated"] = True + if self.refresh_configuration_updated: + json_dict["RefreshConfigurationUpdated"] = True + if self.data_archival_configuration_updated: + json_dict["DataArchivalConfigurationUpdated"] = True + if self.remote_attachments_configuration_updated: + json_dict["RemoteAttachmentsConfigurationUpdated"] = True + if self.raven_connection_strings_updated: + json_dict["RavenConnectionStringsUpdated"] = True + if self.sql_connection_strings_updated: + json_dict["SqlConnectionStringsUpdated"] = True + if self.snowflake_connection_strings_updated: + json_dict["SnowflakeConnectionStringsUpdated"] = True + if self.ai_connection_strings_updated: + json_dict["AiConnectionStringsUpdated"] = True + if self.ai_agents_updated: + json_dict["AiAgentsUpdated"] = True + if self.client_configuration_updated: + json_dict["ClientConfigurationUpdated"] = True + if self.unused_database_ids_updated: + json_dict["UnusedDatabaseIdsUpdated"] = True + if self.lock_mode_updated: + json_dict["LockModeUpdated"] = True + if self.olap_etls_updated: + json_dict["OlapEtlsUpdated"] = True + if self.olap_connection_strings_updated: + json_dict["OlapConnectionStringsUpdated"] = True + if self.elastic_search_etls_updated: + json_dict["ElasticSearchEtlsUpdated"] = True + if self.elastic_search_connection_strings_updated: + json_dict["ElasticSearchConnectionStringsUpdated"] = True + if self.postre_s_q_l_configuration_updated: + json_dict["PostreSQLConfigurationUpdated"] = True + if self.queue_etls_updated: + json_dict["QueueEtlsUpdated"] = True + if self.queue_connection_strings_updated: + json_dict["QueueConnectionStringsUpdated"] = True + if self.queue_sinks_updated: + json_dict["QueueSinksUpdated"] = True + if self.cdc_sinks_updated: + json_dict["CdcSinksUpdated"] = True + if self.indexes_history_updated: + json_dict["IndexesHistoryUpdated"] = True + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.sorters_updated = json_dict.get("SortersUpdated", False) + self.analyzers_updated = json_dict.get("AnalyzersUpdated", False) + self.sink_pull_replications_updated = json_dict.get("SinkPullReplicationsUpdated", False) + self.hub_pull_replications_updated = json_dict.get("HubPullReplicationsUpdated", False) + self.raven_etls_updated = json_dict.get("RavenEtlsUpdated", False) + self.sql_etls_updated = json_dict.get("SqlEtlsUpdated", False) + self.snowflake_etls_updated = json_dict.get("SnowflakeEtlsUpdated", False) + self.embeddings_generations_updated = json_dict.get("EmbeddingsGenerationsUpdated", False) + self.gen_ai_tasks_updated = json_dict.get("GenAiTasksUpdated", False) + self.external_replications_updated = json_dict.get("ExternalReplicationsUpdated", False) + self.periodic_backups_updated = json_dict.get("PeriodicBackupsUpdated", False) + self.conflict_solver_config_updated = json_dict.get("ConflictSolverConfigUpdated", False) + self.schema_validation_config_updated = json_dict.get("SchemaValidationConfigUpdated", False) + self.time_series_configuration_updated = json_dict.get("TimeSeriesConfigurationUpdated", False) + self.documents_compression_configuration_updated = json_dict.get( + "DocumentsCompressionConfigurationUpdated", False + ) + self.revisions_configuration_updated = json_dict.get("RevisionsConfigurationUpdated", False) + self.expiration_configuration_updated = json_dict.get("ExpirationConfigurationUpdated", False) + self.refresh_configuration_updated = json_dict.get("RefreshConfigurationUpdated", False) + self.data_archival_configuration_updated = json_dict.get("DataArchivalConfigurationUpdated", False) + self.remote_attachments_configuration_updated = json_dict.get("RemoteAttachmentsConfigurationUpdated", False) + self.raven_connection_strings_updated = json_dict.get("RavenConnectionStringsUpdated", False) + self.sql_connection_strings_updated = json_dict.get("SqlConnectionStringsUpdated", False) + self.snowflake_connection_strings_updated = json_dict.get("SnowflakeConnectionStringsUpdated", False) + self.ai_connection_strings_updated = json_dict.get("AiConnectionStringsUpdated", False) + self.ai_agents_updated = json_dict.get("AiAgentsUpdated", False) + self.client_configuration_updated = json_dict.get("ClientConfigurationUpdated", False) + self.unused_database_ids_updated = json_dict.get("UnusedDatabaseIdsUpdated", False) + self.lock_mode_updated = json_dict.get("LockModeUpdated", False) + self.olap_etls_updated = json_dict.get("OlapEtlsUpdated", False) + self.olap_connection_strings_updated = json_dict.get("OlapConnectionStringsUpdated", False) + self.elastic_search_etls_updated = json_dict.get("ElasticSearchEtlsUpdated", False) + self.elastic_search_connection_strings_updated = json_dict.get("ElasticSearchConnectionStringsUpdated", False) + self.postre_s_q_l_configuration_updated = json_dict.get("PostreSQLConfigurationUpdated", False) + self.queue_etls_updated = json_dict.get("QueueEtlsUpdated", False) + self.queue_connection_strings_updated = json_dict.get("QueueConnectionStringsUpdated", False) + self.queue_sinks_updated = json_dict.get("QueueSinksUpdated", False) + self.cdc_sinks_updated = json_dict.get("CdcSinksUpdated", False) + self.indexes_history_updated = json_dict.get("IndexesHistoryUpdated", False) + + @property + def updated(self) -> List[str]: + """The parts that were written, for when you just want to see the list.""" + return sorted(name for name, value in vars(self).items() if name.endswith("_updated") and value is True) + + +class SmugglerProgressBase: + """Per-item-type counts, shared by a smuggler operation's progress and its result.""" + + _SECTIONS = ( + ("database_record", "DatabaseRecord", DatabaseRecordProgress), + ("documents", "Documents", CountsWithSkippedCountAndLastEtagAndAttachments), + ("revision_documents", "RevisionDocuments", CountsWithSkippedCountAndLastEtagAndAttachments), + ("tombstones", "Tombstones", CountsWithLastEtag), + ("conflicts", "Conflicts", CountsWithLastEtag), + ("identities", "Identities", CountsWithLastEtag), + ("indexes", "Indexes", Counts), + ("compare_exchange", "CompareExchange", CountsWithLastEtag), + ("subscriptions", "Subscriptions", Counts), + ("counters", "Counters", CountsWithSkippedCountAndLastEtag), + ("compare_exchange_tombstones", "CompareExchangeTombstones", Counts), + ("time_series", "TimeSeries", CountsWithSkippedCountAndLastEtag), + ("replication_hub_certificates", "ReplicationHubCertificates", Counts), + ("time_series_deleted_ranges", "TimeSeriesDeletedRanges", CountsWithSkippedCountAndLastEtag), + ) + + def __init__(self): + for attribute, _, counts_type in self._SECTIONS: + setattr(self, attribute, counts_type()) + + def to_json(self) -> Dict[str, Any]: + return {key: getattr(self, attribute).to_json() for attribute, key, _ in self._SECTIONS} + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + for attribute, key, counts_type in self._SECTIONS: + setattr(self, attribute, counts_type.from_json(json_dict.get(key))) + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]): + progress = cls() + if json_dict: + progress._fill_from_json(json_dict) + return progress + + +class SmugglerResult(SmugglerProgressBase): + """ + What a smuggler export or import actually moved. Read it off a finished operation: + + operation = store.smuggler.import_data(options, "dump.ravendbdump") + result = operation.wait_for_completion() + print(result.documents.read_count) + """ + + def __init__(self): + super().__init__() + self.messages: List[str] = [] + self.elapsed: Optional[str] = None + self.message: Optional[str] = None + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["Messages"] = self.messages + json_dict["Elapsed"] = self.elapsed + json_dict["Message"] = self.message + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.messages = json_dict.get("Messages") or [] + self.elapsed = json_dict.get("Elapsed") + self.message = json_dict.get("Message") + + def __repr__(self) -> str: + return f"Documents: {self.documents!r} Revisions: {self.revision_documents!r} Indexes: {self.indexes!r}" diff --git a/ravendb/tests/documents_tests/test_smuggler.py b/ravendb/tests/documents_tests/test_smuggler.py index 5047cf0c..9748ddba 100644 --- a/ravendb/tests/documents_tests/test_smuggler.py +++ b/ravendb/tests/documents_tests/test_smuggler.py @@ -23,6 +23,7 @@ flags_to_string, ) from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler, _backup_sort_key, _is_backup_file +from ravendb.documents.smuggler.result import SmugglerResult from ravendb.http.server_node import ServerNode from ravendb.tests.test_base import TestBase, User @@ -265,7 +266,10 @@ def test_exports_a_database_to_a_file_and_imports_it_into_another(self): self.assertGreater(os.path.getsize(dump), 0) with self.get_document_store() as target: - target.smuggler.import_data(DatabaseSmugglerImportOptions(), dump).wait_for_completion() + imported = target.smuggler.import_data(DatabaseSmugglerImportOptions(), dump).wait_for_completion() + self.assertIsInstance(imported, SmugglerResult) + self.assertEqual(5, imported.documents.read_count) + self.assertTrue(imported.messages) with target.open_session() as session: self.assertEqual("user-3", session.load("users/3", User).name) @@ -275,9 +279,11 @@ def test_exports_into_a_stream(self): self._store_users(self.store, 2) destination = io.BytesIO() - self.store.smuggler.export(DatabaseSmugglerExportOptions(), destination).wait_for_completion() + exported = self.store.smuggler.export(DatabaseSmugglerExportOptions(), destination).wait_for_completion() self.assertGreater(len(destination.getvalue()), 0) + self.assertEqual(2, exported.documents.read_count) + self.assertIsNotNone(exported.elapsed) def test_an_export_narrowed_to_one_collection_leaves_the_rest_behind(self): self._store_users(self.store, 3) @@ -324,3 +330,95 @@ def test_for_database_targets_another_database_on_the_same_store(self): with target.open_session() as session: self.assertEqual("user-1", session.load("users/1", User).name) + + +class TestSmugglerResult(unittest.TestCase): + RESPONSE = { + "Documents": { + "ReadCount": 5, + "SkippedCount": 1, + "ErroredCount": 0, + "SizeInBytes": 120, + "LastEtag": 9, + "Attachments": {"ReadCount": 2, "SizeInBytes": 40}, + }, + "RevisionDocuments": {"ReadCount": 3}, + "Indexes": {"ReadCount": 1}, + "TimeSeriesDeletedRanges": {"ReadCount": 4}, + "DatabaseRecord": {"ReadCount": 1, "QueueSinksUpdated": True, "CdcSinksUpdated": True}, + "Messages": ["Processed 5 documents."], + "Elapsed": "00:00:01.2340000", + } + + def test_counts_are_parsed_per_item_type(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(5, result.documents.read_count) + self.assertEqual(1, result.documents.skipped_count) + self.assertEqual(9, result.documents.last_etag) + self.assertEqual(3, result.revision_documents.read_count) + self.assertEqual(1, result.indexes.read_count) + self.assertEqual(4, result.time_series_deleted_ranges.read_count) + + def test_attachments_hang_off_the_documents_count(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(2, result.documents.attachments.read_count) + self.assertEqual(40, result.documents.attachments.size_in_bytes) + + def test_messages_and_elapsed_are_carried(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(["Processed 5 documents."], result.messages) + self.assertEqual("00:00:01.2340000", result.elapsed) + + def test_the_database_record_reports_only_what_was_written(self): + record = SmugglerResult.from_json(self.RESPONSE).database_record + + self.assertTrue(record.queue_sinks_updated) + self.assertTrue(record.cdc_sinks_updated) + self.assertFalse(record.sorters_updated) + self.assertEqual(["cdc_sinks_updated", "queue_sinks_updated"], record.updated) + + def test_the_database_record_writes_back_only_the_true_flags(self): + # This is how the server sends them, and how C# writes them out. + serialized = SmugglerResult.from_json(self.RESPONSE).database_record.to_json() + + self.assertEqual(["QueueSinksUpdated", "CdcSinksUpdated"], [k for k in serialized if k.endswith("Updated")]) + + def test_a_missing_section_reads_as_zero_rather_than_raising(self): + result = SmugglerResult.from_json({}) + + self.assertEqual(0, result.documents.read_count) + self.assertEqual(0, result.identities.read_count) + self.assertEqual([], result.messages) + self.assertEqual([], result.database_record.updated) + + def test_a_null_result_reads_as_an_empty_one(self): + self.assertEqual(0, SmugglerResult.from_json(None).documents.read_count) + + def test_result_round_trips(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(result.to_json(), SmugglerResult.from_json(result.to_json()).to_json()) + + def test_every_section_the_server_sends_has_a_home(self): + keys = set(SmugglerResult().to_json()) + + for name in ( + "DatabaseRecord", + "Documents", + "RevisionDocuments", + "Tombstones", + "Conflicts", + "Identities", + "Indexes", + "CompareExchange", + "Subscriptions", + "Counters", + "CompareExchangeTombstones", + "TimeSeries", + "ReplicationHubCertificates", + "TimeSeriesDeletedRanges", + ): + self.assertIn(name, keys) From d7b8d11078e6106560592ab31f9f99b333233b8f Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:46:54 +0200 Subject: [PATCH 20/29] Add CDC Sink schema discovery and mapping preview --- .../documents/operations/cdc_sink/__init__.py | 16 ++ .../documents/operations/cdc_sink/schema.py | 271 ++++++++++++++++++ .../documents/operations/cdc_sink/testing.py | 213 ++++++++++++++ .../tests/operations_tests/test_cdc_sink.py | 268 +++++++++++++++++ 4 files changed, 768 insertions(+) create mode 100644 ravendb/documents/operations/cdc_sink/schema.py create mode 100644 ravendb/documents/operations/cdc_sink/testing.py diff --git a/ravendb/documents/operations/cdc_sink/__init__.py b/ravendb/documents/operations/cdc_sink/__init__.py index e76c8a87..4eb5100f 100644 --- a/ravendb/documents/operations/cdc_sink/__init__.py +++ b/ravendb/documents/operations/cdc_sink/__init__.py @@ -21,6 +21,22 @@ CdcSinkTaskState, UpdateCdcSinkOperationResult, ) +from ravendb.documents.operations.cdc_sink.schema import ( + CdcSinkSchemaRequest, + CdcSinkSourceColumn, + CdcSinkSourceForeignKey, + CdcSinkSourceSchema, + CdcSinkSourceTable, + GetCdcSinkSchemaOperation, +) +from ravendb.documents.operations.cdc_sink.testing import ( + TestCdcSinkMappingOperation, + TestCdcSinkMappingRequest, + TestCdcSinkMappingResult, + TestCdcSinkOperation, + TestCdcSinkRowResult, + TestCdcSinkRowSelector, +) from ravendb.documents.operations.definitions import MaintenanceOperation from ravendb.http.raven_command import RavenCommand from ravendb.http.server_node import ServerNode diff --git a/ravendb/documents/operations/cdc_sink/schema.py b/ravendb/documents/operations/cdc_sink/schema.py new file mode 100644 index 00000000..798a47a8 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/schema.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import requests + +from ravendb.documents.operations.cdc_sink.configuration import CdcColumnType +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class CdcSinkSourceColumn: + """ + One source column as CDC schema discovery sees it. The field names mirror + CdcColumnMapping so a discovered column can be turned into a mapping directly. + """ + + def __init__( + self, + name: str = None, + native_type: str = None, + suggested_type: CdcColumnType = CdcColumnType.DEFAULT, + is_primary_key: bool = False, + is_cdc_capturable: bool = False, + unsupported_reason: str = None, + ): + self.name = name + # The source-side type as the database reports it, e.g. "varchar", "jsonb". + self.native_type = native_type + self.suggested_type = suggested_type + self.is_primary_key = is_primary_key + # False when the source type has no CDC mapping, or the column is not in + # SQL Server's capture list. The reason is then in unsupported_reason. + self.is_cdc_capturable = is_cdc_capturable + self.unsupported_reason = unsupported_reason + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "NativeType": self.native_type, + "SuggestedType": self.suggested_type.value if self.suggested_type else None, + "IsPrimaryKey": self.is_primary_key, + "IsCdcCapturable": self.is_cdc_capturable, + "UnsupportedReason": self.unsupported_reason, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceColumn: + suggested_type = json_dict.get("SuggestedType") + return cls( + name=json_dict.get("Name"), + native_type=json_dict.get("NativeType"), + suggested_type=CdcColumnType(suggested_type) if suggested_type else CdcColumnType.DEFAULT, + is_primary_key=json_dict.get("IsPrimaryKey", False), + is_cdc_capturable=json_dict.get("IsCdcCapturable", False), + unsupported_reason=json_dict.get("UnsupportedReason"), + ) + + +class CdcSinkSourceForeignKey: + """A foreign key leaving a source table, which is what a linked table is built from.""" + + def __init__( + self, + columns: List[str] = None, + referenced_schema: str = None, + referenced_table: str = None, + referenced_columns: List[str] = None, + ): + self.columns = columns or [] + self.referenced_schema = referenced_schema + self.referenced_table = referenced_table + self.referenced_columns = referenced_columns or [] + + def to_json(self) -> Dict[str, Any]: + return { + "Columns": self.columns, + "ReferencedSchema": self.referenced_schema, + "ReferencedTable": self.referenced_table, + "ReferencedColumns": self.referenced_columns, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceForeignKey: + return cls( + columns=json_dict.get("Columns") or [], + referenced_schema=json_dict.get("ReferencedSchema"), + referenced_table=json_dict.get("ReferencedTable"), + referenced_columns=json_dict.get("ReferencedColumns") or [], + ) + + +class CdcSinkSourceTable: + """One source table as CDC schema discovery sees it, annotated with capturability.""" + + def __init__( + self, + source_table_schema: str = None, + source_table_name: str = None, + columns: List[CdcSinkSourceColumn] = None, + primary_key_columns: List[str] = None, + foreign_keys: List[CdcSinkSourceForeignKey] = None, + is_cdc_enabled: bool = False, + unsupported_reason: str = None, + warnings: List[str] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.columns = columns or [] + self.primary_key_columns = primary_key_columns or [] + self.foreign_keys = foreign_keys or [] + # Whether CDC tracking is already active at the source. Always true for + # PostgreSQL and MySQL, where membership is a database-level concern. + self.is_cdc_enabled = is_cdc_enabled + # Set when the whole table cannot be captured. + self.unsupported_reason = unsupported_reason + # Table-scoped findings that do not make it unusable, such as a REPLICA + # IDENTITY that will not carry row-identifying columns on DELETE. + self.warnings = warnings or [] + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "ForeignKeys": [foreign_key.to_json() for foreign_key in self.foreign_keys], + "IsCdcEnabled": self.is_cdc_enabled, + "UnsupportedReason": self.unsupported_reason, + "Warnings": self.warnings, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceTable: + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + columns=[CdcSinkSourceColumn.from_json(column) for column in json_dict.get("Columns") or []], + primary_key_columns=json_dict.get("PrimaryKeyColumns") or [], + foreign_keys=[ + CdcSinkSourceForeignKey.from_json(foreign_key) for foreign_key in json_dict.get("ForeignKeys") or [] + ], + is_cdc_enabled=json_dict.get("IsCdcEnabled", False), + unsupported_reason=json_dict.get("UnsupportedReason"), + warnings=json_dict.get("Warnings") or [], + ) + + +class CdcSinkSourceSchema: + """The source database's tables, columns, keys and CDC readiness.""" + + def __init__( + self, + catalog_name: str = None, + tables: List[CdcSinkSourceTable] = None, + errors: List[str] = None, + has_permission_to_setup: bool = False, + warnings: List[str] = None, + ): + self.catalog_name = catalog_name + self.tables = tables or [] + # Whole-request failures: validation, an unreachable source, or a connection-level + # blocker such as PostgreSQL wal_level not being logical. + self.errors = errors or [] + # Whether the connecting user can provision CDC itself. Distinct from a table's + # is_cdc_enabled, which says whether CDC is already running. + self.has_permission_to_setup = has_permission_to_setup + self.warnings = warnings or [] + + @property + def success(self) -> bool: + """True when nothing blocks setting CDC up. Warnings do not count against it.""" + return len(self.errors) == 0 + + def to_json(self) -> Dict[str, Any]: + return { + "CatalogName": self.catalog_name, + "Tables": [table.to_json() for table in self.tables], + "Errors": self.errors, + "HasPermissionToSetup": self.has_permission_to_setup, + "Warnings": self.warnings, + "Success": self.success, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceSchema: + return cls( + catalog_name=json_dict.get("CatalogName"), + tables=[CdcSinkSourceTable.from_json(table) for table in json_dict.get("Tables") or []], + errors=json_dict.get("Errors") or [], + has_permission_to_setup=json_dict.get("HasPermissionToSetup", False), + warnings=json_dict.get("Warnings") or [], + ) + + +class CdcSinkSchemaRequest: + """The body of a schema-discovery call. Give it either a connection or a name.""" + + def __init__( + self, + connection: SqlConnectionString = None, + connection_string_name: str = None, + schemas: List[str] = None, + ): + # Inline credentials, for when the connection has not been saved to the database + # record yet. When set, connection_string_name is ignored. + self.connection = connection + self.connection_string_name = connection_string_name + # PostgreSQL only, defaults to ["public"] server-side. Each entry is validated + # against ^[A-Za-z_][A-Za-z0-9_]*$, so quoted names with hyphens are rejected. + self.schemas = schemas + + def to_json(self) -> Dict[str, Any]: + return { + "Connection": self.connection.to_json() if self.connection else None, + "ConnectionStringName": self.connection_string_name, + "Schemas": self.schemas, + } + + +class GetCdcSinkSchemaOperation(MaintenanceOperation[CdcSinkSourceSchema]): + """ + Browses the source database a CDC Sink would read: tables, columns, primary and + foreign keys, each annotated with what CDC can capture. Requires DatabaseAdmin. + """ + + def __init__( + self, + connection_or_name: Any = None, + schemas: List[str] = None, + request: CdcSinkSchemaRequest = None, + ): + if request is not None: + self._request = request + elif isinstance(connection_or_name, SqlConnectionString): + self._request = CdcSinkSchemaRequest(connection=connection_or_name, schemas=schemas) + elif isinstance(connection_or_name, str): + self._request = CdcSinkSchemaRequest(connection_string_name=connection_or_name, schemas=schemas) + else: + raise ValueError("Pass either a SqlConnectionString, a connection string name, or a request") + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[CdcSinkSourceSchema]: + return self._GetCdcSinkSchemaCommand(self._request) + + class _GetCdcSinkSchemaCommand(RavenCommand[CdcSinkSourceSchema]): + def __init__(self, request: CdcSinkSchemaRequest): + super().__init__(CdcSinkSourceSchema) + self._request = request + + def is_read_request(self) -> bool: + # A POST that changes nothing server-side, so failover to the fastest node is fine. + return True + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink/schema" + + request = requests.Request("POST", url) + request.data = self._request.to_json() + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = CdcSinkSourceSchema.from_json(json.loads(response)) diff --git a/ravendb/documents/operations/cdc_sink/testing.py b/ravendb/documents/operations/cdc_sink/testing.py new file mode 100644 index 00000000..40ddce5c --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/testing.py @@ -0,0 +1,213 @@ +""" +The CDC Sink mapping preview: run rows from the source table through the configured +mapping and patches without saving anything, to see what documents would come out. + +Named ``testing`` rather than ``test`` so ``unittest discover`` does not pick the module +up as a test file. +""" + +from __future__ import annotations + +import enum +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import requests + +from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class TestCdcSinkRowSelector(enum.Enum): + """Sample the first N rows by primary key, or fetch one row by its key values.""" + + FIRST = "First" + BY_PRIMARY_KEY = "ByPrimaryKey" + + def __str__(self) -> str: + return self.value + + +class TestCdcSinkOperation(enum.Enum): + """Whether to drive each row through the table's Patch or its OnDelete.Patch.""" + + UPSERT = "Upsert" + DELETE = "Delete" + + def __str__(self) -> str: + return self.value + + +class TestCdcSinkRowResult: + """ + One row's worth of preview output. ``document`` and ``source_row`` come back as JSON + text rather than parsed objects, so parse them yourself when you need the values. + """ + + def __init__( + self, + document_id: str = None, + document: str = None, + source_row: str = None, + would_delete: bool = False, + ignore_deletes: bool = False, + debug_output: List[str] = None, + error: str = None, + ): + self.document_id = document_id + # The mapped and patched document, as JSON text. Stays at the pre-patch mapping + # if the script called del() or put(). + self.document = document + self.source_row = source_row + # True for a Delete run when OnDelete.IgnoreDeletes is false. + self.would_delete = would_delete + self.ignore_deletes = ignore_deletes + # Whatever the patch script passed to output(). + self.debug_output = debug_output + # Set when this row failed on its own, for example the patch threw. + self.error = error + + def to_json(self) -> Dict[str, Any]: + return { + "DocumentId": self.document_id, + "Document": self.document, + "SourceRow": self.source_row, + "WouldDelete": self.would_delete, + "IgnoreDeletes": self.ignore_deletes, + "DebugOutput": self.debug_output, + "Error": self.error, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> TestCdcSinkRowResult: + return cls( + document_id=json_dict.get("DocumentId"), + document=json_dict.get("Document"), + source_row=json_dict.get("SourceRow"), + would_delete=json_dict.get("WouldDelete", False), + ignore_deletes=json_dict.get("IgnoreDeletes", False), + debug_output=json_dict.get("DebugOutput"), + error=json_dict.get("Error"), + ) + + +class TestCdcSinkMappingResult: + """ + Always an array shape, so a single-row test and a multi-row sample read the same. + A row that failed on its own carries its error; a failure that stopped the whole + request lands in ``errors`` and leaves ``results`` empty. + """ + + def __init__( + self, + results: List[TestCdcSinkRowResult] = None, + errors: List[str] = None, + warnings: List[str] = None, + ): + self.results = results or [] + self.errors = errors or [] + # Advisory notes that do not invalidate the results, for example that linked and + # embedded tables are not exercised in test mode. + self.warnings = warnings or [] + + def to_json(self) -> Dict[str, Any]: + return { + "Results": [result.to_json() for result in self.results], + "Errors": self.errors, + "Warnings": self.warnings, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> TestCdcSinkMappingResult: + return cls( + results=[TestCdcSinkRowResult.from_json(result) for result in json_dict.get("Results") or []], + errors=json_dict.get("Errors") or [], + warnings=json_dict.get("Warnings") or [], + ) + + +class TestCdcSinkMappingRequest: + """The body of a mapping preview: the task configuration plus which rows to run.""" + + def __init__( + self, + configuration: CdcSinkConfiguration = None, + connection: SqlConnectionString = None, + source_table_schema: str = None, + source_table_name: str = None, + row_selector: TestCdcSinkRowSelector = TestCdcSinkRowSelector.FIRST, + primary_key_values: List[str] = None, + operation: TestCdcSinkOperation = TestCdcSinkOperation.UPSERT, + max_rows: int = 1, + ): + # The same configuration used to create the task: driver, table lookup, column + # mapping and patch scripts all come from here. + self.configuration = configuration + # Inline credentials, for when the connection has not been saved yet. When null, + # the configuration's connection_string_name is used. + self.connection = connection + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.row_selector = row_selector + # In the target table's primary key order. Required with BY_PRIMARY_KEY. + self.primary_key_values = primary_key_values + self.operation = operation + # Only meaningful with FIRST, and must be 1 with BY_PRIMARY_KEY. Capped at 5,000 + # server-side: this endpoint is a preview, not a bulk fetch. + self.max_rows = max_rows + + def to_json(self) -> Dict[str, Any]: + return { + "Configuration": self.configuration.to_json() if self.configuration else None, + "Connection": self.connection.to_json() if self.connection else None, + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "RowSelector": self.row_selector.value if self.row_selector else None, + "PrimaryKeyValues": self.primary_key_values, + "Operation": self.operation.value if self.operation else None, + "MaxRows": self.max_rows, + } + + +class TestCdcSinkMappingOperation(MaintenanceOperation[TestCdcSinkMappingResult]): + """ + Previews how source rows would become documents, before saving a CDC Sink task. + Requires DatabaseAdmin. + """ + + def __init__(self, request: TestCdcSinkMappingRequest): + if request is None: + raise ValueError("request cannot be None") + + self._request = request + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[TestCdcSinkMappingResult]: + return self._TestCdcSinkMappingCommand(self._request) + + class _TestCdcSinkMappingCommand(RavenCommand[TestCdcSinkMappingResult]): + def __init__(self, request: TestCdcSinkMappingRequest): + super().__init__(TestCdcSinkMappingResult) + self._request = request + + def is_read_request(self) -> bool: + # A POST that changes nothing server-side, so failover to the fastest node is fine. + return True + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink/test" + + request = requests.Request("POST", url) + request.data = self._request.to_json() + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = TestCdcSinkMappingResult.from_json(json.loads(response)) diff --git a/ravendb/tests/operations_tests/test_cdc_sink.py b/ravendb/tests/operations_tests/test_cdc_sink.py index 9bc02d2e..8dcb0a15 100644 --- a/ravendb/tests/operations_tests/test_cdc_sink.py +++ b/ravendb/tests/operations_tests/test_cdc_sink.py @@ -4,6 +4,7 @@ """ import json +import os import unittest from ravendb.documents.operations.cdc_sink import ( @@ -27,7 +28,23 @@ OngoingTaskCdcSink, OngoingTaskType, ) +from ravendb.documents.operations.cdc_sink.schema import ( + CdcSinkSourceSchema, + GetCdcSinkSchemaOperation, +) +from ravendb.documents.operations.cdc_sink.testing import ( + TestCdcSinkMappingOperation, + TestCdcSinkMappingRequest, + TestCdcSinkMappingResult, + TestCdcSinkOperation, + TestCdcSinkRowSelector, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.etl.sql import SqlConnectionString from ravendb.http.server_node import ServerNode +from ravendb.tests.test_base import TestBase from ravendb.http.topology import RaftCommand from ravendb.serverwide.database_record import DatabaseRecord @@ -282,3 +299,254 @@ def test_a_record_from_a_server_without_cdc_sinks_gets_an_empty_list(self): record = DatabaseRecord.from_json({"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}}) self.assertEqual([], record.cdc_sinks) + + +class TestCdcSinkSchemaDiscovery(unittest.TestCase): + RESPONSE = { + "CatalogName": "northwind", + "HasPermissionToSetup": True, + "Warnings": ["SQL Server Agent is not running."], + "Errors": [], + "Tables": [ + { + "SourceTableSchema": "public", + "SourceTableName": "orders", + "IsCdcEnabled": True, + "PrimaryKeyColumns": ["id"], + "Warnings": ["REPLICA IDENTITY will not carry row-identifying columns on DELETE."], + "Columns": [ + { + "Name": "id", + "NativeType": "bigint", + "SuggestedType": "Default", + "IsPrimaryKey": True, + "IsCdcCapturable": True, + }, + {"Name": "payload", "NativeType": "jsonb", "SuggestedType": "Json", "IsCdcCapturable": True}, + {"Name": "blob", "NativeType": "bytea", "SuggestedType": "Attachment", "IsCdcCapturable": True}, + { + "Name": "weird", + "NativeType": "cube", + "IsCdcCapturable": False, + "UnsupportedReason": "No CDC mapping for this type.", + }, + ], + "ForeignKeys": [ + { + "Columns": ["customer_id"], + "ReferencedSchema": "public", + "ReferencedTable": "customers", + "ReferencedColumns": ["id"], + } + ], + } + ], + } + + def test_schema_response_is_parsed(self): + schema = CdcSinkSourceSchema.from_json(self.RESPONSE) + + self.assertEqual("northwind", schema.catalog_name) + self.assertTrue(schema.has_permission_to_setup) + self.assertEqual(1, len(schema.tables)) + self.assertEqual(["SQL Server Agent is not running."], schema.warnings) + + def test_success_follows_errors_not_warnings(self): + self.assertTrue(CdcSinkSourceSchema.from_json(self.RESPONSE).success) + self.assertFalse(CdcSinkSourceSchema.from_json({"Errors": ["boom"]}).success) + # A warning is advisory, so it must not flip success. + self.assertTrue(CdcSinkSourceSchema.from_json({"Warnings": ["heads up"]}).success) + + def test_columns_carry_their_suggested_mapping(self): + columns = CdcSinkSourceSchema.from_json(self.RESPONSE).tables[0].columns + + self.assertEqual(CdcColumnType.DEFAULT, columns[0].suggested_type) + self.assertTrue(columns[0].is_primary_key) + self.assertEqual(CdcColumnType.JSON, columns[1].suggested_type) + self.assertEqual(CdcColumnType.ATTACHMENT, columns[2].suggested_type) + + def test_an_uncapturable_column_says_why(self): + column = CdcSinkSourceSchema.from_json(self.RESPONSE).tables[0].columns[3] + + self.assertFalse(column.is_cdc_capturable) + self.assertEqual("No CDC mapping for this type.", column.unsupported_reason) + + def test_foreign_keys_are_parsed(self): + foreign_key = CdcSinkSourceSchema.from_json(self.RESPONSE).tables[0].foreign_keys[0] + + self.assertEqual(["customer_id"], foreign_key.columns) + self.assertEqual("customers", foreign_key.referenced_table) + self.assertEqual(["id"], foreign_key.referenced_columns) + + def test_schema_round_trips(self): + schema = CdcSinkSourceSchema.from_json(self.RESPONSE) + + self.assertEqual(schema.to_json(), CdcSinkSourceSchema.from_json(schema.to_json()).to_json()) + + def test_an_empty_response_parses(self): + schema = CdcSinkSourceSchema.from_json({}) + + self.assertEqual([], schema.tables) + self.assertTrue(schema.success) + + def test_the_operation_takes_a_connection_or_a_name(self): + node = ServerNode("http://localhost:8080", "db") + connection = SqlConnectionString("pg", "Host=localhost", "Npgsql") + + by_connection = GetCdcSinkSchemaOperation(connection, ["public"]).get_command(None) + request = by_connection.create_request(node) + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink/schema", request.url) + self.assertEqual("pg", request.data["Connection"]["Name"]) + self.assertEqual(["public"], request.data["Schemas"]) + self.assertIsNone(request.data["ConnectionStringName"]) + + by_name = GetCdcSinkSchemaOperation("pg").get_command(None) + self.assertEqual("pg", by_name.create_request(node).data["ConnectionStringName"]) + + def test_the_operation_allows_fastest_node_failover(self): + # A POST, but nothing changes server-side. + self.assertTrue(GetCdcSinkSchemaOperation("pg").get_command(None).is_read_request()) + + def test_the_operation_needs_a_connection_or_a_name(self): + with self.assertRaises(ValueError): + GetCdcSinkSchemaOperation() + with self.assertRaises(ValueError): + GetCdcSinkSchemaOperation(42) + + +class TestCdcSinkMappingPreview(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + self.request = TestCdcSinkMappingRequest( + configuration=_configuration(), + connection=SqlConnectionString("pg", "Host=localhost", "Npgsql"), + source_table_schema="public", + source_table_name="orders", + max_rows=3, + ) + + def test_the_request_carries_the_configuration_and_the_row_choice(self): + body = self.request.to_json() + + self.assertEqual("orders", body["SourceTableName"]) + self.assertEqual("First", body["RowSelector"]) + self.assertEqual("Upsert", body["Operation"]) + self.assertEqual(3, body["MaxRows"]) + self.assertEqual("orders-cdc", body["Configuration"]["Name"]) + + def test_a_by_primary_key_request_names_the_key_values(self): + request = TestCdcSinkMappingRequest( + configuration=_configuration(), + row_selector=TestCdcSinkRowSelector.BY_PRIMARY_KEY, + primary_key_values=["42"], + operation=TestCdcSinkOperation.DELETE, + ) + body = request.to_json() + + self.assertEqual("ByPrimaryKey", body["RowSelector"]) + self.assertEqual(["42"], body["PrimaryKeyValues"]) + self.assertEqual("Delete", body["Operation"]) + + def test_the_operation_posts_to_the_test_endpoint(self): + command = TestCdcSinkMappingOperation(self.request).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink/test", request.url) + self.assertTrue(command.is_read_request()) + + def test_a_missing_request_is_rejected_client_side(self): + with self.assertRaises(ValueError): + TestCdcSinkMappingOperation(None) + + def test_row_results_are_parsed(self): + command = TestCdcSinkMappingOperation(self.request).get_command(None) + command.set_response( + json.dumps( + { + "Results": [ + { + "DocumentId": "orders/42", + "Document": '{"Id":42}', + "SourceRow": '{"id":42}', + "WouldDelete": False, + "IgnoreDeletes": False, + "DebugOutput": ["mapped"], + }, + {"DocumentId": "orders/43", "Error": "the patch threw"}, + ], + "Errors": [], + "Warnings": ["Linked tables are not exercised in test mode."], + } + ), + False, + ) + result = command.result + + self.assertIsInstance(result, TestCdcSinkMappingResult) + self.assertEqual(2, len(result.results)) + self.assertEqual("orders/42", result.results[0].document_id) + self.assertEqual(["mapped"], result.results[0].debug_output) + # A row that failed on its own carries the error; the request still succeeded. + self.assertEqual("the patch threw", result.results[1].error) + self.assertEqual([], result.errors) + self.assertEqual(1, len(result.warnings)) + + def test_a_whole_request_failure_leaves_no_rows(self): + result = TestCdcSinkMappingResult.from_json({"Errors": ["Cannot open connection"]}) + + self.assertEqual([], result.results) + self.assertEqual(["Cannot open connection"], result.errors) + + def test_result_round_trips(self): + result = TestCdcSinkMappingResult.from_json( + {"Results": [{"DocumentId": "orders/1", "WouldDelete": True}], "Errors": [], "Warnings": []} + ) + + self.assertEqual(result.to_json(), TestCdcSinkMappingResult.from_json(result.to_json()).to_json()) + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestCdcSinkAgainstServer(TestBase): + # CDC Sink is licensed, and each of these needs the server to accept the payload + # before it ever reaches a source database. + + def test_the_server_answers_schema_discovery_with_a_structured_result(self): + # There is no PostgreSQL to reach, so what matters is that the server parsed the + # request and answered in the shape the client expects instead of failing. + schema = self.store.maintenance.send(GetCdcSinkSchemaOperation("no-such-connection-string")) + + self.assertIsInstance(schema, CdcSinkSourceSchema) + self.assertFalse(schema.success) + self.assertTrue(any("no-such-connection-string" in error for error in schema.errors)) + + def test_the_server_reads_a_full_configuration_out_of_a_mapping_preview(self): + request = TestCdcSinkMappingRequest( + configuration=_configuration(), + connection=SqlConnectionString("pg", "Host=localhost;Database=nope", "Npgsql"), + source_table_schema="public", + source_table_name="orders", + ) + + result = self.store.maintenance.send(TestCdcSinkMappingOperation(request)) + + self.assertIsInstance(result, TestCdcSinkMappingResult) + # It reached the driver, which means the whole configuration tree deserialized. + self.assertTrue(any("source database" in error for error in result.errors)) + + def test_a_cdc_sink_task_is_stored_and_read_back(self): + self.store.maintenance.send( + PutConnectionStringOperation(SqlConnectionString("pg", "Host=localhost;Database=nope", "Npgsql")) + ) + + result = self.store.maintenance.send(AddCdcSinkOperation(_configuration())) + self.assertGreater(result.task_id, 0) + + task = self.store.maintenance.send(GetOngoingTaskInfoOperation("orders-cdc", OngoingTaskType.CDC_SINK)) + self.assertIsInstance(task, OngoingTaskCdcSink) + self.assertEqual("pg", task.connection_string_name) + self.assertEqual("Orders", task.configuration.tables[0].collection_name) + self.assertEqual( + ["Lines"], [embedded.property_name for embedded in task.configuration.tables[0].embedded_tables] + ) From 7ddb0ec03ecfe59568dfc8c7b655ba55a9a47833 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:46:55 +0200 Subject: [PATCH 21/29] Exercise server-wide connection strings against a licensed server --- .../test_server_wide_connection_strings.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py index 65314419..871832b2 100644 --- a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py +++ b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py @@ -4,6 +4,7 @@ """ import json +import os import unittest from ravendb.documents.operations.connection_string.get_connection_string_operation import ( @@ -23,6 +24,7 @@ ServerWideConnectionStringUsage, ) from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.tests.test_base import TestBase class TestConnectionStringUsage(unittest.TestCase): @@ -248,3 +250,63 @@ def test_remove_reads_back_the_raft_index(self): command.set_response(json.dumps({"RaftCommandIndex": 12}), False) self.assertEqual(12, command.result.raft_command_index) + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestServerWideConnectionStringsAgainstServer(TestBase): + # Server-wide connection strings are licensed: writing one hits the license gate on an + # unlicensed server, so these only run when a license is configured. + + def _connection_string(self, name: str) -> ServerWideConnectionString: + return ServerWideConnectionString( + RavenConnectionString(name, database="db1", topology_discovery_urls=[self.store.urls[0]]) + ) + + def test_a_connection_string_is_stored_listed_and_removed(self): + put = self.store.maintenance.server.send( + PutServerWideConnectionStringOperation(self._connection_string("sw-raven")) + ) + self.assertGreater(put.raft_command_index, 0) + + listed = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + self.assertIn("sw-raven", [result.name for result in listed.results]) + + removed = self.store.maintenance.server.send( + RemoveServerWideConnectionStringOperation(RavenConnectionString("sw-raven")) + ) + self.assertGreater(removed.raft_command_index, 0) + + listed = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + self.assertNotIn("sw-raven", [result.name for result in listed.results]) + + def test_a_stored_connection_string_reads_back_with_its_type_and_urls(self): + self.store.maintenance.server.send(PutServerWideConnectionStringOperation(self._connection_string("sw-typed"))) + try: + listed = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + stored = next(result for result in listed.results if result.name == "sw-typed") + + self.assertEqual(ConnectionStringType.RAVEN, stored.type) + self.assertIsInstance(stored.connection_string, RavenConnectionString) + self.assertEqual("db1", stored.connection_string.database) + self.assertEqual([self.store.urls[0]], stored.connection_string.topology_discovery_urls) + # Nothing references it yet. + self.assertEqual([], stored.used_by) + finally: + self.store.maintenance.server.send( + RemoveServerWideConnectionStringOperation(RavenConnectionString("sw-typed")) + ) + + def test_filtering_by_name_and_type_narrows_the_listing(self): + for name in ("sw-one", "sw-two"): + self.store.maintenance.server.send(PutServerWideConnectionStringOperation(self._connection_string(name))) + try: + listed = self.store.maintenance.server.send( + GetServerWideConnectionStringsOperation("sw-one", ConnectionStringType.RAVEN) + ) + + self.assertEqual(["sw-one"], [result.name for result in listed.results]) + finally: + for name in ("sw-one", "sw-two"): + self.store.maintenance.server.send( + RemoveServerWideConnectionStringOperation(RavenConnectionString(name)) + ) From 58afe65f4c89ee803e6827623ef2ee89430aeeae Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:46:55 +0200 Subject: [PATCH 22/29] Export the follow-up types from the package root --- ravendb/__init__.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 1732cc2c..0199060a 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -221,6 +221,34 @@ ExportCompressionAlgorithm, ) from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler +from ravendb.documents.operations.cdc_sink.schema import ( + CdcSinkSchemaRequest, + CdcSinkSourceColumn, + CdcSinkSourceForeignKey, + CdcSinkSourceSchema, + CdcSinkSourceTable, + GetCdcSinkSchemaOperation, +) +from ravendb.documents.operations.cdc_sink.testing import ( + TestCdcSinkMappingOperation, + TestCdcSinkMappingRequest, + TestCdcSinkMappingResult, + TestCdcSinkOperation, + TestCdcSinkRowResult, + TestCdcSinkRowSelector, +) +from ravendb.documents.smuggler.result import ( + Counts, + CountsWithLastEtag, + CountsWithLastEtagAndAttachments, + CountsWithSkippedCountAndLastEtag, + CountsWithSkippedCountAndLastEtagAndAttachments, + DatabaseRecordProgress, + SmugglerProgressBase, + SmugglerResult, +) +from ravendb.documents.smuggler.database_smuggler import SmugglerOperation +from ravendb.exceptions.commercial import LicenseLimitException, LimitType from ravendb.documents.operations.cdc_sink import ( AddCdcSinkOperation, AddCdcSinkOperationResult, From 58ba64aae866c3b6d1a0a3d84151b861b71dfa2e Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 12:56:47 +0200 Subject: [PATCH 23/29] Assert what a completed operation reports, not that it reports nothing This test pinned wait_for_completion returning None, which is the behaviour the previous commit deliberately changed. It now checks the result the server sent. --- .../tests/raven_commands_tests/test_by_index_actions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ravendb/tests/raven_commands_tests/test_by_index_actions.py b/ravendb/tests/raven_commands_tests/test_by_index_actions.py index a0457c3c..e5b5e3a6 100644 --- a/ravendb/tests/raven_commands_tests/test_by_index_actions.py +++ b/ravendb/tests/raven_commands_tests/test_by_index_actions.py @@ -112,8 +112,12 @@ def test_delete_by_index_success(self): response.operation_id, response.operation_node_tag, ) - # wait_for_completion doesnt return anything (None) when operation state is 'Completed' - self.assertIsNone(x.wait_for_completion()) + # wait_for_completion hands back the result the server reported for the operation. + # 45, not 50: the range is quoted, so the server compares DocNumber as a string + # and "5".."9" fall outside "0".."49" lexicographically. + result = x.wait_for_completion() + self.assertEqual(45, result["Total"]) + self.assertEqual("Processed 45 items.", result["Message"]) if __name__ == "__main__": From cfa81f1f04adde3fd5deee8480dee2f14d02c526 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 13:06:32 +0200 Subject: [PATCH 24/29] Track the 7.2.6 client version and its CDC slot-naming note --- ravendb/documents/operations/cdc_sink/configuration.py | 4 ++-- ravendb/http/request_executor.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ravendb/documents/operations/cdc_sink/configuration.py b/ravendb/documents/operations/cdc_sink/configuration.py index 291757f7..752dd0c9 100644 --- a/ravendb/documents/operations/cdc_sink/configuration.py +++ b/ravendb/documents/operations/cdc_sink/configuration.py @@ -104,9 +104,9 @@ class CdcSinkPostgresSettings: """ def __init__(self, publication_name: str = None, slot_name: str = None): - # Publication used for logical replication, auto-filled as rvn_cdc_p_{guid}. + # Publication used for logical replication, auto-filled as rvn_cdc_p_{taskId}. self.publication_name = publication_name - # Logical replication slot, auto-filled as rvn_cdc_s_{guid}. + # Logical replication slot, auto-filled as rvn_cdc_s_{taskId}. self.slot_name = slot_name def to_json(self) -> Dict[str, Any]: diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index 360da4dc..242e1c79 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -56,7 +56,7 @@ class RequestExecutor: __INITIAL_TOPOLOGY_ETAG = -2 __GLOBAL_APPLICATION_IDENTIFIER = uuid.uuid4() - CLIENT_VERSION = "7.2.5" + CLIENT_VERSION = "7.2.6" logger = logging.getLogger("request_executor") # todo: initializer should take also cryptography certificates From d0c6d12fe4645c31c1bb2be70fc208dfa28e577b Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 13:14:47 +0200 Subject: [PATCH 25/29] Let a conversation turn override the agent's output schema AiOutputOptions carries a sample object, an explicit JSON schema, or no schema at all, and run_with_schema / stream_with_schema apply it to a single turn. Nothing is sent when no options are given, so the agent's own schema stays in charge. --- ravendb/__init__.py | 1 + ravendb/documents/ai/__init__.py | 1 + ravendb/documents/ai/ai_conversation.py | 42 ++++++ ravendb/documents/ai/ai_output_options.py | 72 +++++++++ .../ai/agents/run_conversation_operation.py | 17 ++- .../ai_agent_tests/test_ai_output_options.py | 139 ++++++++++++++++++ 6 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 ravendb/documents/ai/ai_output_options.py create mode 100644 ravendb/tests/ai_agent_tests/test_ai_output_options.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 0199060a..8499ff81 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -249,6 +249,7 @@ ) from ravendb.documents.smuggler.database_smuggler import SmugglerOperation from ravendb.exceptions.commercial import LicenseLimitException, LimitType +from ravendb.documents.ai.ai_output_options import AiOutputOptions from ravendb.documents.operations.cdc_sink import ( AddCdcSinkOperation, AddCdcSinkOperationResult, diff --git a/ravendb/documents/ai/__init__.py b/ravendb/documents/ai/__init__.py index 9c2c750a..6860b6b2 100644 --- a/ravendb/documents/ai/__init__.py +++ b/ravendb/documents/ai/__init__.py @@ -15,3 +15,4 @@ "AiMessagePromptFields", "AiMessagePromptTypes", ] +from ravendb.documents.ai.ai_output_options import AiOutputOptions diff --git a/ravendb/documents/ai/ai_conversation.py b/ravendb/documents/ai/ai_conversation.py index ea625616..50216309 100644 --- a/ravendb/documents/ai/ai_conversation.py +++ b/ravendb/documents/ai/ai_conversation.py @@ -6,6 +6,7 @@ from datetime import timedelta from ravendb.documents.ai.ai_answer import AiAnswer, AiConversationStatus +from ravendb.documents.ai.ai_output_options import AiOutputOptions from ravendb.documents.ai.content_part import ContentPart, TextPart from ravendb.documents.operations.ai.agents import ( AiAgentActionRequest, @@ -149,16 +150,56 @@ def run(self) -> AiAnswer: if self._handle_server_reply(r): return r + def run_with_schema(self, output_options: "AiOutputOptions") -> AiAnswer: + """ + Runs one turn with the output format overridden for that turn only, leaving the + agent's own schema in place for later turns. Pass + ``AiOutputOptions(no_schema=True)`` to get free-form text back instead of JSON. + """ + if output_options is None: + raise ValueError("output_options cannot be None") + + self._dispatched_tool_ids.clear() + + while True: + r = self._run_internal(output_options=output_options) + if self._handle_server_reply(r): + return r + def stream(self, stream_property_path: str = None, on_chunk: Optional[Callable[[str], None]] = None) -> AiAnswer: while True: r = self._run_internal(stream_property_path=stream_property_path, streamed_chunks_callback=on_chunk) if self._handle_server_reply(r): return r + def stream_with_schema( + self, + stream_property_path: str = None, + on_chunk: Optional[Callable[[str], None]] = None, + output_options: "AiOutputOptions" = None, + ) -> AiAnswer: + """ + Streams one turn with the output format overridden for that turn only. + ``stream_property_path`` is ignored when the options ask for no schema, since + free-form text has no property to stream from. + """ + if output_options is None: + raise ValueError("output_options cannot be None") + + while True: + r = self._run_internal( + stream_property_path=stream_property_path, + streamed_chunks_callback=on_chunk, + output_options=output_options, + ) + if self._handle_server_reply(r): + return r + def _run_internal( self, stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, + output_options: Optional["AiOutputOptions"] = None, ) -> AiAnswer: from ravendb.documents.operations.ai.agents import RunConversationOperation import time @@ -197,6 +238,7 @@ def _run_internal( stream_property_path=stream_property_path, streamed_chunks_callback=streamed_chunks_callback, attachments_commands=self._attachments_commands, + output_options=output_options, debug=self._debug, cancel_pending_action_tools=self._cancel_pending_action_tools, ) diff --git a/ravendb/documents/ai/ai_output_options.py b/ravendb/documents/ai/ai_output_options.py new file mode 100644 index 00000000..d97a5f9d --- /dev/null +++ b/ravendb/documents/ai/ai_output_options.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, Optional + + +class AiOutputOptions: + """ + Overrides the output format for a single conversation turn. + + The agent still needs a default output schema when it is created; these options take + precedence over it, and only for the turn they are passed to. + + Give it exactly one of: + + - ``sample_object`` - an object the server turns into a JSON schema at request time + - ``output_schema`` - an explicit JSON schema string + - ``no_schema=True`` - no structured output at all, so the model answers in free text + """ + + def __init__( + self, + sample_object: Any = None, + output_schema: str = None, + no_schema: bool = False, + ): + if no_schema and (sample_object is not None or output_schema is not None): + raise ValueError( + "no_schema asks the model for free-form text, so it cannot be combined with " + "an output schema or a sample object. Drop one of them." + ) + + if output_schema is not None and (not output_schema or output_schema.isspace()): + raise ValueError("output_schema cannot be empty or whitespace") + + self.sample_object = sample_object + # Takes precedence over sample_object when both are set. + self.output_schema = output_schema + self.no_schema = no_schema + + def to_json(self) -> Dict[str, Any]: + json_dict = {} + + if self.sample_object is not None: + sample = self.sample_object + if callable(getattr(sample, "to_json", None)): + sample = sample.to_json() + # The server reads this as a JSON string, not as a nested object. + json_dict["SampleObject"] = json.dumps(sample) + + if self.output_schema is not None: + json_dict["OutputSchema"] = self.output_schema + + if self.no_schema: + json_dict["NoSchema"] = True + + return json_dict + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> Optional[AiOutputOptions]: + if not json_dict: + return None + + options = cls.__new__(cls) + sample = json_dict.get("SampleObject") + if sample is not None: + options.sample_object = json.loads(sample) if isinstance(sample, str) else sample + else: + options.sample_object = None + options.output_schema = json_dict.get("OutputSchema") + options.no_schema = json_dict.get("NoSchema", False) + return options diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index b73add25..49a6d7ee 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -10,6 +10,7 @@ from ravendb.http.server_node import ServerNode import requests from ravendb.http.misc import ResponseDisposeHandling +from ravendb.documents.ai.ai_output_options import AiOutputOptions from ravendb.documents.ai.content_part import ContentPart TSchema = TypeVar("TSchema") @@ -256,15 +257,17 @@ def __init__( user_prompt: Optional[List[ContentPart]] = None, creation_options: Optional[AiConversationCreationOptions] = None, attachment_commands: Optional[List[Any]] = None, + output_options: Optional[AiOutputOptions] = None, ): self.action_responses: Optional[List[AiAgentActionResponse]] = action_responses self.artificial_actions: Optional[List[AiAgentArtificialActionResponse]] = artificial_actions self.user_prompt: Optional[List[ContentPart]] = user_prompt self.creation_options: Optional[AiConversationCreationOptions] = creation_options self.attachment_commands: Optional[List[Any]] = attachment_commands + self.output_options: Optional[AiOutputOptions] = output_options def to_json(self) -> Dict[str, Any]: - return { + body = { "ActionResponses": ( [resp.to_json() for resp in self.action_responses] if self.action_responses is not None else None ), @@ -280,6 +283,12 @@ def to_json(self) -> Dict[str, Any]: ), } + # Only sent when the caller overrides the agent's own schema for this turn. + if self.output_options is not None: + body["OutputOptions"] = self.output_options.to_json() + + return body + class RunConversationOperation(MaintenanceOperation[ConversationResult[TSchema]]): def __init__( @@ -294,6 +303,7 @@ def __init__( stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, + output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, ): @@ -314,6 +324,7 @@ def __init__( self._stream_property_path = stream_property_path self._streamed_chunks_callback = streamed_chunks_callback self._attachments_commands = attachments_commands or [] + self._output_options = output_options self._debug = debug self._cancel_pending_action_tools = cancel_pending_action_tools @@ -330,6 +341,7 @@ def get_command(self, conventions: DocumentConventions) -> RavenCommand[Conversa streamed_chunks_callback=self._streamed_chunks_callback, conventions=conventions, attachments_commands=self._attachments_commands, + output_options=self._output_options, debug=self._debug, cancel_pending_action_tools=self._cancel_pending_action_tools, ) @@ -349,6 +361,7 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, + output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, ): @@ -366,6 +379,7 @@ def __init__( self._stream_property_path = stream_property_path self._streamed_chunks_callback = streamed_chunks_callback self._conventions = conventions + self._output_options = output_options self._debug = debug self._cancel_pending_action_tools = cancel_pending_action_tools self._attachments_commands = attachments_commands or [] @@ -424,6 +438,7 @@ def create_request(self, node: ServerNode) -> requests.Request: user_prompt=self._prompt_parts, creation_options=self._options, attachment_commands=self._attachments_commands if self._attachments_commands else None, + output_options=self._output_options, ) body = json.dumps(request_body.to_json()) request = requests.Request("POST", url) diff --git a/ravendb/tests/ai_agent_tests/test_ai_output_options.py b/ravendb/tests/ai_agent_tests/test_ai_output_options.py new file mode 100644 index 00000000..6add1585 --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_ai_output_options.py @@ -0,0 +1,139 @@ +""" +Tests for the per-turn output schema override added in 7.2.6: AiOutputOptions, the +run_with_schema / stream_with_schema entry points, and how the options reach the wire. +""" + +import json +import unittest + +from ravendb.documents.ai.ai_conversation import AiConversation +from ravendb.documents.ai.ai_output_options import AiOutputOptions +from ravendb.documents.operations.ai.agents.run_conversation_operation import ( + ConversationRequestBody, + ConversationResult, + RunConversationOperation, +) +from ravendb.http.server_node import ServerNode + + +def _request_body(operation: RunConversationOperation) -> dict: + request = operation.get_command(None).create_request(ServerNode("http://localhost:8080", "db")) + return json.loads(request.data) if isinstance(request.data, str) else request.data + + +class TestAiOutputOptions(unittest.TestCase): + def test_a_sample_object_is_sent_as_json_text(self): + # The server reads SampleObject as a string, not as a nested object. + options = AiOutputOptions(sample_object={"Name": "sample", "Age": 1}) + + self.assertEqual({"Name": "sample", "Age": 1}, json.loads(options.to_json()["SampleObject"])) + + def test_a_sample_object_may_be_an_entity_with_to_json(self): + class _Answer: + def to_json(self): + return {"Name": "sample"} + + self.assertEqual( + {"Name": "sample"}, json.loads(AiOutputOptions(sample_object=_Answer()).to_json()["SampleObject"]) + ) + + def test_an_explicit_schema_is_sent_verbatim(self): + schema = '{"type":"object","properties":{"Name":{"type":"string"}}}' + + self.assertEqual(schema, AiOutputOptions(output_schema=schema).to_json()["OutputSchema"]) + + def test_no_schema_asks_for_free_form_text(self): + self.assertEqual({"NoSchema": True}, AiOutputOptions(no_schema=True).to_json()) + + def test_empty_options_send_nothing(self): + # Nothing set means the agent's own schema stays in charge. + self.assertEqual({}, AiOutputOptions().to_json()) + + def test_no_schema_cannot_be_combined_with_a_schema(self): + with self.assertRaises(ValueError): + AiOutputOptions(no_schema=True, output_schema="{}") + with self.assertRaises(ValueError): + AiOutputOptions(no_schema=True, sample_object={"Name": "x"}) + + def test_an_empty_schema_string_is_rejected(self): + with self.assertRaises(ValueError): + AiOutputOptions(output_schema="") + with self.assertRaises(ValueError): + AiOutputOptions(output_schema=" ") + + def test_options_round_trip(self): + for options in ( + AiOutputOptions(sample_object={"Name": "x"}), + AiOutputOptions(output_schema='{"type":"object"}'), + AiOutputOptions(no_schema=True), + ): + self.assertEqual(options.to_json(), AiOutputOptions.from_json(options.to_json()).to_json()) + + def test_nothing_parses_to_nothing(self): + self.assertIsNone(AiOutputOptions.from_json(None)) + self.assertIsNone(AiOutputOptions.from_json({})) + + +class TestOutputOptionsOnTheWire(unittest.TestCase): + def test_the_request_body_carries_the_options(self): + body = _request_body( + RunConversationOperation( + agent_id="agent", + conversation_id="chats/1", + prompt_parts=[], + output_options=AiOutputOptions(no_schema=True), + ) + ) + + self.assertEqual({"NoSchema": True}, body["OutputOptions"]) + + def test_a_turn_without_options_does_not_mention_them(self): + # Leaving the key out is what keeps the agent's configured schema in charge. + body = _request_body(RunConversationOperation(agent_id="agent", conversation_id="chats/1", prompt_parts=[])) + + self.assertNotIn("OutputOptions", body) + + def test_the_request_body_serializes_a_schema_override(self): + body = ConversationRequestBody(output_options=AiOutputOptions(output_schema='{"type":"string"}')).to_json() + + self.assertEqual('{"type":"string"}', body["OutputOptions"]["OutputSchema"]) + + def test_the_other_body_fields_are_untouched(self): + body = _request_body( + RunConversationOperation( + agent_id="agent", + conversation_id="chats/1", + prompt_parts=[], + output_options=AiOutputOptions(no_schema=True), + ) + ) + + for key in ("ActionResponses", "ArtificialActions", "CreationOptions", "UserPrompt"): + self.assertIn(key, body) + + +class TestConversationRunEntryPoints(unittest.TestCase): + def test_the_schema_overriding_entry_points_exist(self): + for name in ("run", "run_with_schema", "stream", "stream_with_schema"): + self.assertTrue(hasattr(AiConversation, name), name) + + def test_they_refuse_to_run_without_options(self): + conversation = AiConversation.__new__(AiConversation) + + with self.assertRaises(ValueError): + conversation.run_with_schema(None) + with self.assertRaises(ValueError): + conversation.stream_with_schema("Answer", lambda chunk: None, None) + + +class TestFreeFormAnswers(unittest.TestCase): + def test_a_raw_text_answer_is_read_as_a_string(self): + # With no schema the server answers with a bare string rather than an object. + result = ConversationResult.from_json({"ConversationId": "chats/1", "Response": "just some prose"}) + + self.assertEqual("just some prose", result.response) + + def test_a_structured_answer_is_still_read_as_an_object(self): + result = ConversationResult.from_json({"ConversationId": "chats/1", "Response": {"Name": "sample"}}) + + self.assertEqual({"Name": "sample"}, result.response) From 519a9a6b277eeff477e58ec56e062c9dd1bef196 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 13:26:51 +0200 Subject: [PATCH 26/29] Let embeddings generation store each chunk's text --- .../ai/embeddings_generation_configuration.py | 6 +++ .../test_chunking_options.py | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/ravendb/documents/operations/ai/embeddings_generation_configuration.py b/ravendb/documents/operations/ai/embeddings_generation_configuration.py index b20ef25b..9022cd9d 100644 --- a/ravendb/documents/operations/ai/embeddings_generation_configuration.py +++ b/ravendb/documents/operations/ai/embeddings_generation_configuration.py @@ -37,6 +37,7 @@ def __init__( chunking_options_for_querying: ChunkingOptions = None, embeddings_cache_expiration: timedelta = None, embeddings_cache_for_querying_expiration: timedelta = None, + store_chunk_text: bool = False, disabled: bool = False, mentor_node: str = None, pin_to_mentor_node: bool = False, @@ -59,6 +60,9 @@ def __init__( self.embeddings_transformation = embeddings_transformation self.quantization = quantization self.chunking_options_for_querying = chunking_options_for_querying + # Keeps each chunk's text next to its embedding, which helps when debugging or + # highlighting but costs storage. Off by default. + self.store_chunk_text = store_chunk_text self.embeddings_cache_expiration = ( embeddings_cache_expiration if embeddings_cache_expiration is not None @@ -220,6 +224,7 @@ def to_json(self) -> Dict[str, Any]: if self.embeddings_cache_for_querying_expiration else None ), + "StoreChunkText": self.store_chunk_text, } ) return result @@ -255,4 +260,5 @@ def from_json(cls, json_dict: Dict[str, Any]) -> "EmbeddingsGenerationConfigurat mentor_node=json_dict.get("MentorNode", None), pin_to_mentor_node=json_dict.get("PinToMentorNode", False), allow_etl_on_non_encrypted_channel=json_dict.get("AllowEtlOnNonEncryptedChannel", False), + store_chunk_text=json_dict.get("StoreChunkText", False), ) diff --git a/ravendb/tests/embeddings_generation_tests/test_chunking_options.py b/ravendb/tests/embeddings_generation_tests/test_chunking_options.py index 4c930a65..4a957035 100644 --- a/ravendb/tests/embeddings_generation_tests/test_chunking_options.py +++ b/ravendb/tests/embeddings_generation_tests/test_chunking_options.py @@ -1,6 +1,9 @@ import unittest from ravendb.documents.operations.ai.chunking_options import ChunkingOptions, ChunkingMethod +from ravendb.documents.operations.ai.embeddings_generation_configuration import ( + EmbeddingsGenerationConfiguration, +) class TestChunkingOptionsContextPrefix(unittest.TestCase): @@ -78,3 +81,40 @@ def test_from_json_missing_int_keys_use_csharp_defaults(self): if __name__ == "__main__": unittest.main() + + +class TestStoreChunkText(unittest.TestCase): + """StoreChunkText, added in 7.2.6, keeps each chunk's text next to its embedding.""" + + @staticmethod + def _configuration(**kwargs) -> EmbeddingsGenerationConfiguration: + return EmbeddingsGenerationConfiguration( + name="embeddings", + identifier="embeddings", + collection="Orders", + connection_string_name="ai", + chunking_options_for_querying=ChunkingOptions( + chunking_method=ChunkingMethod.PLAIN_TEXT_SPLIT, max_tokens_per_chunk=512 + ), + **kwargs, + ) + + def test_it_is_off_unless_asked_for(self): + # It costs storage, so opting in has to be deliberate. + self.assertFalse(self._configuration().store_chunk_text) + self.assertFalse(self._configuration().to_json()["StoreChunkText"]) + + def test_it_reaches_the_wire_when_set(self): + self.assertTrue(self._configuration(store_chunk_text=True).to_json()["StoreChunkText"]) + + def test_it_round_trips(self): + serialized = self._configuration(store_chunk_text=True).to_json() + + self.assertTrue(EmbeddingsGenerationConfiguration.from_json(serialized).store_chunk_text) + self.assertEqual(serialized, EmbeddingsGenerationConfiguration.from_json(serialized).to_json()) + + def test_a_configuration_from_an_older_server_reads_as_off(self): + serialized = self._configuration().to_json() + del serialized["StoreChunkText"] + + self.assertFalse(EmbeddingsGenerationConfiguration.from_json(serialized).store_chunk_text) From a5ede50d7ac449e41304a657f56f8cb23ec1d796 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 13:26:52 +0200 Subject: [PATCH 27/29] Type a failed agent query tool as QueryToolFailedException --- ravendb/exceptions/exception_dispatcher.py | 2 ++ ravendb/exceptions/raven_exceptions.py | 6 +++++ .../test_commercial_limits.py | 25 ++++++++++++++++--- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ravendb/exceptions/exception_dispatcher.py b/ravendb/exceptions/exception_dispatcher.py index 2ac3e600..5b5dbf74 100644 --- a/ravendb/exceptions/exception_dispatcher.py +++ b/ravendb/exceptions/exception_dispatcher.py @@ -17,6 +17,7 @@ IndexCompactionInProgressException, InsufficientQuotaException, MissingAiAgentParameterException, + QueryToolFailedException, PortInUseException, RateLimitException, RavenException, @@ -48,6 +49,7 @@ "InsufficientQuotaException": InsufficientQuotaException, "TooManyTokensException": TooManyTokensException, "MissingAiAgentParameterException": MissingAiAgentParameterException, + "QueryToolFailedException": QueryToolFailedException, # documents "DocumentConflictException": DocumentConflictException, "DocumentDoesNotExistException": DocumentDoesNotExistException, diff --git a/ravendb/exceptions/raven_exceptions.py b/ravendb/exceptions/raven_exceptions.py index 6b6efee9..12f48cad 100644 --- a/ravendb/exceptions/raven_exceptions.py +++ b/ravendb/exceptions/raven_exceptions.py @@ -82,6 +82,12 @@ class TooManyTokensException(TooManyRequestsException): pass +class QueryToolFailedException(AiException): + """Raised when an agent's query tool could not run the query it was asked for.""" + + pass + + class MissingAiAgentParameterException(RavenException): pass diff --git a/ravendb/tests/operations_tests/test_commercial_limits.py b/ravendb/tests/operations_tests/test_commercial_limits.py index ea5279f8..92edff90 100644 --- a/ravendb/tests/operations_tests/test_commercial_limits.py +++ b/ravendb/tests/operations_tests/test_commercial_limits.py @@ -1,13 +1,14 @@ """ -Tests for the commercial limit surface: the LimitType enum the 7.2.5 patch extended, and -LicenseLimitException coming back typed from a 402 instead of a bare RavenException. +Tests for the typed exceptions the 7.2.5 and 7.2.6 patches added: the LimitType enum, +LicenseLimitException coming back from a 402 instead of a bare RavenException, and +QueryToolFailedException for a failed agent query tool. """ import unittest from ravendb.exceptions.commercial import LicenseLimitException, LimitType from ravendb.exceptions.exception_dispatcher import ExceptionDispatcher -from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.exceptions.raven_exceptions import AiException, QueryToolFailedException, RavenException class TestLimitType(unittest.TestCase): @@ -59,3 +60,21 @@ def test_a_402_from_another_exception_type_is_left_alone(self): ) self.assertNotIsInstance(ExceptionDispatcher.get(schema, 402), LicenseLimitException) + + +class TestQueryToolFailedException(unittest.TestCase): + def test_it_is_an_ai_exception(self): + self.assertIsInstance(QueryToolFailedException("nope"), AiException) + + def test_the_dispatcher_returns_it(self): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", + object_type="Raven.Client.Exceptions.QueryToolFailedException", + message="no", + error="The agent's query tool could not run the query.", + ) + + exception = ExceptionDispatcher.get(schema, 500) + + self.assertIsInstance(exception, QueryToolFailedException) + self.assertIn("query tool", str(exception)) From f89590f09d5a9a332c4da9bfa236c59e2f47bb62 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Thu, 10 Sep 2026 13:26:53 +0200 Subject: [PATCH 28/29] Keep a vector search intact when a query alias is applied VectorSearchToken inherited the base add_alias, which builds a plain WhereToken and so dropped the similarity threshold, candidate count and exactness. It now rebuilds itself with the field qualified. The caller also discarded what add_alias returned, so no alias was ever applied. C# assigns the new token back into the list; this does the same. --- ravendb/documents/session/query.py | 6 +- .../tokens/query_tokens/definitions.py | 21 +++++ .../test_query_token_aliasing.py | 94 +++++++++++++++++++ 3 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 ravendb/tests/session_tests/test_query_token_aliasing.py diff --git a/ravendb/documents/session/query.py b/ravendb/documents/session/query.py index 5d3542bd..9d0b1084 100644 --- a/ravendb/documents/session/query.py +++ b/ravendb/documents/session/query.py @@ -1317,9 +1317,11 @@ def add_from_alias_to_where_tokens(self, from_alias: str) -> None: raise RuntimeError("Alias cannot be None or empty") tokens = self.__get_current_where_tokens() - for token in tokens: + for index, token in enumerate(tokens): if isinstance(token, WhereToken): - token.add_alias(from_alias) + # add_alias returns a new token rather than mutating this one, so the + # qualified field name only takes effect once it replaces the original. + tokens[index] = token.add_alias(from_alias) def add_alias_to_includes_tokens(self, from_alias: str) -> str: if self._includes_alias is None: diff --git a/ravendb/documents/session/tokens/query_tokens/definitions.py b/ravendb/documents/session/tokens/query_tokens/definitions.py index 23f261fb..8d8b63a4 100644 --- a/ravendb/documents/session/tokens/query_tokens/definitions.py +++ b/ravendb/documents/session/tokens/query_tokens/definitions.py @@ -1076,6 +1076,27 @@ def __init__( self._task_name = task_name self._document_id = document_id + def add_alias(self, alias: str) -> WhereToken: + # The base implementation builds a plain WhereToken, which would drop every + # vector-search setting. Rebuild this token instead, with the field qualified. + if self.field_name == "id()": + return self + + token = VectorSearchToken( + wrapped_field_name=f"{alias}.{self.field_name}", + parameter_name=self.parameter_name, + source_quantization_type=self._source_quantization_type, + target_quantization_type=self._target_quantization_type, + similarity_threshold=self._similarity_threshold, + number_of_candidates_for_querying=self._number_of_candidates_for_querying, + is_exact=self._is_exact, + task_name=self._task_name, + document_id=self._document_id, + ) + token.options = self.options + token.where_operator = self.where_operator + return token + def write_to(self, writer: List[str]) -> None: """ Builds the vector search query string components and appends them to the writer list. diff --git a/ravendb/tests/session_tests/test_query_token_aliasing.py b/ravendb/tests/session_tests/test_query_token_aliasing.py new file mode 100644 index 00000000..8fd73165 --- /dev/null +++ b/ravendb/tests/session_tests/test_query_token_aliasing.py @@ -0,0 +1,94 @@ +""" +Tests for qualifying where-token field names with a query alias, which 7.2.6 made +overridable so a vector search does not lose its settings on the way through. +""" + +import unittest + +from ravendb.documents.indexes.vector.options import VectorEmbeddingType +from ravendb.documents.session.tokens.query_tokens.definitions import ( + MoreLikeThisToken, + VectorSearchToken, + WhereOperator, + WhereToken, +) + + +def _vector_token(field_name: str = "Vector") -> VectorSearchToken: + return VectorSearchToken( + wrapped_field_name=field_name, + parameter_name="p0", + source_quantization_type=VectorEmbeddingType.SINGLE, + target_quantization_type=VectorEmbeddingType.SINGLE, + similarity_threshold=0.8, + number_of_candidates_for_querying=32, + is_exact=True, + ) + + +def _render(token) -> str: + writer = [] + token.write_to(writer) + return "".join(writer) + + +class TestWhereTokenAliasing(unittest.TestCase): + def test_a_plain_where_token_gains_the_alias(self): + token = WhereToken.create(WhereOperator.EQUALS, "Name", "p0") + + self.assertEqual("x.Name", token.add_alias("x").field_name) + + def test_the_document_id_field_is_left_alone(self): + token = WhereToken.create(WhereOperator.EQUALS, "id()", "p0") + + self.assertIs(token, token.add_alias("x")) + + def test_add_alias_does_not_mutate_the_original(self): + # It returns a new token, which is why callers have to use the return value. + token = WhereToken.create(WhereOperator.EQUALS, "Name", "p0") + aliased = token.add_alias("x") + + self.assertIsNot(token, aliased) + self.assertEqual("Name", token.field_name) + + +class TestVectorSearchTokenAliasing(unittest.TestCase): + def test_aliasing_keeps_the_token_type(self): + # The base implementation would hand back a plain WhereToken. + self.assertIsInstance(_vector_token().add_alias("x"), VectorSearchToken) + + def test_aliasing_qualifies_the_field(self): + self.assertEqual("x.Vector", _vector_token().add_alias("x").field_name) + + def test_aliasing_keeps_every_vector_setting(self): + aliased = _vector_token().add_alias("x") + + self.assertEqual(0.8, aliased._similarity_threshold) + self.assertEqual(32, aliased._number_of_candidates_for_querying) + self.assertTrue(aliased._is_exact) + + def test_the_aliased_token_still_renders_as_a_vector_search(self): + self.assertEqual("exact(vector.search(x.Vector, $p0, 0.8, 32))", _render(_vector_token().add_alias("x"))) + + def test_the_document_id_field_is_left_alone(self): + token = _vector_token("id()") + + self.assertIs(token, token.add_alias("x")) + + def test_a_task_backed_token_keeps_its_task(self): + token = VectorSearchToken( + wrapped_field_name="Vector", + parameter_name="p0", + source_quantization_type=VectorEmbeddingType.SINGLE, + target_quantization_type=VectorEmbeddingType.SINGLE, + task_name="embeddings-task", + ) + + self.assertEqual("embeddings-task", token.add_alias("x")._task_name) + + +class TestMoreLikeThisToken(unittest.TestCase): + def test_it_is_not_a_where_token_here(self): + # C# had to override AddAlias on it because MoreLikeThisToken derives from + # WhereToken there. In this client it does not, so alias rewriting skips it. + self.assertFalse(issubclass(MoreLikeThisToken, WhereToken)) From 6b8b3848eabda9ca56436295d6bee561e0f20d91 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Fri, 11 Sep 2026 11:12:03 +0200 Subject: [PATCH 29/29] Keep every new parameter additive and leave wait_for_completion alone Four constructors took a new parameter in the middle of their existing signature, which breaks anyone passing positionally: output_options on RunConversationOperation and its command, store_chunk_text on EmbeddingsGenerationConfiguration, and disable_checksum_validation on RemoteAttachmentsS3Settings. All four move to the end. Operation.wait_for_completion also started returning the server's result instead of None, which is a change every caller can see. It returns None again; the result now comes from _wait_for_completion_result, and the smuggler is its only caller. Also covers the six symbols the coverage audit found untested: import_incremental, export_to_database, get_conversation_messages, CdcSinkTableLoadState, ConnectionStringUsage.list_from_json and UpdateQueueSinkOperationResult. --- .../ai/agents/run_conversation_operation.py | 4 +- .../ai/embeddings_generation_configuration.py | 2 +- .../operations/attachments/__init__.py | 22 ++-- ravendb/documents/operations/operation.py | 11 +- .../documents/smuggler/database_smuggler.py | 2 +- .../test_ai_conversation_messages.py | 40 +++++++ .../tests/documents_tests/test_smuggler.py | 102 ++++++++++++++++++ .../tests/operations_tests/test_cdc_sink.py | 23 ++++ .../tests/operations_tests/test_queue_sink.py | 9 ++ .../test_server_wide_connection_strings.py | 21 ++++ .../test_by_index_actions.py | 10 +- 11 files changed, 223 insertions(+), 23 deletions(-) diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index 49a6d7ee..5dae0647 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -303,9 +303,9 @@ def __init__( stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, - output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, + output_options: Optional[AiOutputOptions] = None, ): if not agent_id or (isinstance(agent_id, str) and agent_id.isspace()): raise ValueError("agent_id cannot be None or empty") @@ -361,9 +361,9 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, - output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, + output_options: Optional[AiOutputOptions] = None, ): from ravendb.util.util import RaftIdGenerator from ravendb.documents.commands.batches import PutAttachmentCommandData diff --git a/ravendb/documents/operations/ai/embeddings_generation_configuration.py b/ravendb/documents/operations/ai/embeddings_generation_configuration.py index 9022cd9d..90b2937d 100644 --- a/ravendb/documents/operations/ai/embeddings_generation_configuration.py +++ b/ravendb/documents/operations/ai/embeddings_generation_configuration.py @@ -37,12 +37,12 @@ def __init__( chunking_options_for_querying: ChunkingOptions = None, embeddings_cache_expiration: timedelta = None, embeddings_cache_for_querying_expiration: timedelta = None, - store_chunk_text: bool = False, disabled: bool = False, mentor_node: str = None, pin_to_mentor_node: bool = False, task_id: int = 0, allow_etl_on_non_encrypted_channel: bool = False, + store_chunk_text: bool = False, ): super().__init__( name=name, diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index 53c0efa8..fdc8718e 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -403,8 +403,8 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, - disable_checksum_validation: bool = None, storage_class: Optional[S3StorageClass] = None, + disable_checksum_validation: bool = None, ): self.aws_access_key = aws_access_key self.aws_secret_key = aws_secret_key @@ -421,16 +421,16 @@ def __init__( def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: storage_class_raw = json_dict.get("StorageClass") return cls( - json_dict.get("AwsAccessKey"), - json_dict.get("AwsSecretKey"), - json_dict.get("AwsSessionToken"), - json_dict.get("AwsRegionName"), - json_dict.get("RemoteFolderName"), - json_dict.get("BucketName"), - json_dict.get("CustomServerUrl"), - json_dict.get("ForcePathStyle"), - json_dict.get("DisableChecksumValidation"), - S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, + aws_access_key=json_dict.get("AwsAccessKey"), + aws_secret_key=json_dict.get("AwsSecretKey"), + aws_session_token=json_dict.get("AwsSessionToken"), + aws_region_name=json_dict.get("AwsRegionName"), + remote_folder_name=json_dict.get("RemoteFolderName"), + bucket_name=json_dict.get("BucketName"), + custom_server_url=json_dict.get("CustomServerUrl"), + force_path_style=json_dict.get("ForcePathStyle"), + storage_class=S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, + disable_checksum_validation=json_dict.get("DisableChecksumValidation"), ) def to_json(self) -> dict: diff --git a/ravendb/documents/operations/operation.py b/ravendb/documents/operations/operation.py index 18273ceb..0ea5b667 100644 --- a/ravendb/documents/operations/operation.py +++ b/ravendb/documents/operations/operation.py @@ -40,8 +40,15 @@ def _get_operation_state_command( ) -> RavenCommand[dict]: return GetOperationStateOperation.GetOperationStateCommand(self.__key, node_tag) - def wait_for_completion(self) -> Optional[dict]: - """Blocks until the operation finishes, then hands back whatever result it carried.""" + def wait_for_completion(self) -> None: + """Blocks until the operation finishes.""" + self._wait_for_completion_result() + + def _wait_for_completion_result(self) -> Optional[dict]: + """ + The same wait, handing back the result the server reported. Kept separate so + wait_for_completion goes on returning nothing, which is what callers expect. + """ while True: status = self.fetch_operations_status() operation_status = status.get("Status") diff --git a/ravendb/documents/smuggler/database_smuggler.py b/ravendb/documents/smuggler/database_smuggler.py index 7e37f061..7fa9ad40 100644 --- a/ravendb/documents/smuggler/database_smuggler.py +++ b/ravendb/documents/smuggler/database_smuggler.py @@ -55,7 +55,7 @@ class SmugglerOperation(Operation): """An export or import operation, whose result is a typed SmugglerResult.""" def wait_for_completion(self) -> SmugglerResult: - return SmugglerResult.from_json(super().wait_for_completion()) + return SmugglerResult.from_json(self._wait_for_completion_result()) def _is_backup_file(file_path: str) -> bool: diff --git a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py index ddce3c8c..9284c1d3 100644 --- a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py +++ b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py @@ -17,6 +17,7 @@ GetConversationMessagesOptions, RunConversationOperation, ) +from ravendb.documents.ai.ai_operations import AiOperations from ravendb.http.server_node import ServerNode from ravendb.primitives import constants @@ -215,3 +216,42 @@ def test_the_flag_is_sent_on_every_run(self): def test_the_flag_is_sent_when_it_is_set(self): self.assertIn("&cancelPendingActionTools=True", self._url(cancel_pending_action_tools=True)) + + +class TestGetConversationMessagesEntryPoint(unittest.TestCase): + """AiOperations.get_conversation_messages takes either an id or full options.""" + + class _RecordingMaintenance: + def __init__(self): + self.sent = None + + def send(self, operation): + self.sent = operation + return "result" + + class _Store: + def __init__(self, maintenance): + self.maintenance = maintenance + + def _operations(self): + maintenance = self._RecordingMaintenance() + return AiOperations(self._Store(maintenance)), maintenance + + def test_a_bare_id_becomes_an_operation(self): + operations, maintenance = self._operations() + + self.assertEqual("result", operations.get_conversation_messages("chats/1")) + self.assertIsInstance(maintenance.sent, GetConversationMessagesOperation) + + def test_options_are_passed_through(self): + operations, maintenance = self._operations() + options = GetConversationMessagesOptions( + conversation_id="chats/1", page_size=10, detail_level=AiConversationDetailLevel.FULL + ) + + operations.get_conversation_messages(options) + url = maintenance.sent.get_command(None).create_request(ServerNode("http://localhost:8080", "db")).url + + self.assertIn("conversationId=chats%2F1", url) + self.assertIn("pageSize=10", url) + self.assertIn("detailLevel=Full", url) diff --git a/ravendb/tests/documents_tests/test_smuggler.py b/ravendb/tests/documents_tests/test_smuggler.py index 9748ddba..97e0ca85 100644 --- a/ravendb/tests/documents_tests/test_smuggler.py +++ b/ravendb/tests/documents_tests/test_smuggler.py @@ -422,3 +422,105 @@ def test_every_section_the_server_sends_has_a_home(self): "TimeSeriesDeletedRanges", ): self.assertIn(name, keys) + + +class TestSmugglerIncrementalImport(TestBase): + """ + import_incremental walks a backup directory in order. Indexes and subscriptions come + from the last file only, so an earlier incremental cannot resurrect what a later + backup dropped. + """ + + def _dump_of(self, store, options: DatabaseSmugglerExportOptions, path: str) -> None: + store.smuggler.export(options, path).wait_for_completion() + + def test_it_imports_every_file_in_order(self): + with self.store.open_session() as session: + session.store(User(name="first"), "users/1") + session.save_changes() + + with tempfile.TemporaryDirectory() as directory: + full = os.path.join(directory, "2026-06-16-10-00.ravendb-full-backup") + self._dump_of(self.store, DatabaseSmugglerExportOptions(), full) + + with self.store.open_session() as session: + session.store(User(name="second"), "users/2") + session.save_changes() + + incremental = os.path.join(directory, "2026-06-16-11-00.ravendb-incremental-backup") + self._dump_of(self.store, DatabaseSmugglerExportOptions(), incremental) + + with self.get_document_store() as target: + target.smuggler.import_incremental(DatabaseSmugglerImportOptions(), directory) + + with target.open_session() as session: + self.assertEqual("first", session.load("users/1", User).name) + self.assertEqual("second", session.load("users/2", User).name) + + def test_an_empty_directory_is_a_no_op(self): + with tempfile.TemporaryDirectory() as directory: + # Nothing that looks like a backup file, so there is nothing to import. + open(os.path.join(directory, "notes.txt"), "wb").close() + + self.assertIsNone(self.store.smuggler.import_incremental(DatabaseSmugglerImportOptions(), directory)) + + def test_it_restores_the_selection_it_narrowed(self): + options = DatabaseSmugglerImportOptions() + before = set(options.operate_on_types) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "2026-06-16-10-00.ravendb-full-backup") + self._dump_of(self.store, DatabaseSmugglerExportOptions(), dump) + self.store.smuggler.import_incremental(options, directory) + + # Tombstones are added on the way in and kept; indexes and subscriptions come back. + self.assertIn(DatabaseItemType.INDEXES, options.operate_on_types) + self.assertIn(DatabaseItemType.SUBSCRIPTIONS, options.operate_on_types) + self.assertTrue(before.issubset(options.operate_on_types)) + + def test_missing_options_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + self.store.smuggler.import_incremental(None, directory) + + +class TestSmugglerExportToDatabase(TestBase): + def test_it_moves_documents_into_another_database(self): + with self.store.open_session() as session: + for i in range(4): + session.store(User(name=f"user-{i}"), f"users/{i}") + session.save_changes() + + with self.get_document_store() as target: + result = self.store.smuggler.export_to_database( + DatabaseSmugglerExportOptions(), target.smuggler + ).wait_for_completion() + + # The import side is the one worth reporting, so that is what comes back. + self.assertIsInstance(result, SmugglerResult) + self.assertEqual(4, result.documents.read_count) + + with target.open_session() as session: + self.assertEqual("user-2", session.load("users/2", User).name) + + def test_it_carries_the_export_selection_over(self): + with self.store.open_session() as session: + session.store(User(name="a user"), "users/1") + session.store({"Name": "an order", "@metadata": {"@collection": "Orders"}}, "orders/1") + session.save_changes() + + with self.get_document_store() as target: + self.store.smuggler.export_to_database( + DatabaseSmugglerExportOptions(operate_on_types={DatabaseItemType.DOCUMENTS}, collections=["Users"]), + target.smuggler, + ).wait_for_completion() + + with target.open_session() as session: + self.assertIsNotNone(session.load("users/1", User)) + self.assertIsNone(session.load("orders/1")) + + def test_missing_arguments_are_rejected(self): + with self.assertRaises(ValueError): + self.store.smuggler.export_to_database(None, self.store.smuggler) + with self.assertRaises(ValueError): + self.store.smuggler.export_to_database(DatabaseSmugglerExportOptions(), None) diff --git a/ravendb/tests/operations_tests/test_cdc_sink.py b/ravendb/tests/operations_tests/test_cdc_sink.py index 8dcb0a15..97de1bab 100644 --- a/ravendb/tests/operations_tests/test_cdc_sink.py +++ b/ravendb/tests/operations_tests/test_cdc_sink.py @@ -20,6 +20,7 @@ CdcSinkProcessState, CdcSinkRelationType, CdcSinkTableConfig, + CdcSinkTableLoadState, CdcSinkTaskState, UpdateCdcSinkOperation, ) @@ -550,3 +551,25 @@ def test_a_cdc_sink_task_is_stored_and_read_back(self): self.assertEqual( ["Lines"], [embedded.property_name for embedded in task.configuration.tables[0].embedded_tables] ) + + +class TestCdcSinkTableLoadState(unittest.TestCase): + """Per-table initial-load progress, so an interrupted load can pick up where it stopped.""" + + def test_it_round_trips(self): + payload = {"InitialLoadCompleted": True, "LastKeyValues": ["42"], "KeyColumns": ["id"]} + + self.assertEqual(payload, CdcSinkTableLoadState.from_json(payload).to_json()) + + def test_it_carries_the_resume_position(self): + state = CdcSinkTableLoadState.from_json({"LastKeyValues": ["42", "b"], "KeyColumns": ["id", "code"]}) + + self.assertEqual(["42", "b"], state.last_key_values) + self.assertEqual(["id", "code"], state.key_columns) + self.assertFalse(state.initial_load_completed) + + def test_an_empty_payload_reads_as_not_started(self): + state = CdcSinkTableLoadState.from_json({}) + + self.assertFalse(state.initial_load_completed) + self.assertIsNone(state.last_key_values) diff --git a/ravendb/tests/operations_tests/test_queue_sink.py b/ravendb/tests/operations_tests/test_queue_sink.py index e9a25e7e..e618b42b 100644 --- a/ravendb/tests/operations_tests/test_queue_sink.py +++ b/ravendb/tests/operations_tests/test_queue_sink.py @@ -21,6 +21,7 @@ QueueSinkProcessState, QueueSinkScript, UpdateQueueSinkOperation, + UpdateQueueSinkOperationResult, ) from ravendb.documents.operations.connection_string.put_connection_string_operation import ( PutConnectionStringOperation, @@ -160,6 +161,14 @@ def test_a_missing_configuration_is_rejected_client_side(self): with self.assertRaises(ValueError): UpdateQueueSinkOperation(1, None) + def test_update_reads_the_task_id_off_the_response(self): + command = UpdateQueueSinkOperation(4, _configuration()).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 11, "TaskId": 4}), False) + + self.assertIsInstance(command.result, UpdateQueueSinkOperationResult) + self.assertEqual(11, command.result.raft_command_index) + self.assertEqual(4, command.result.task_id) + def test_add_reads_the_task_id_off_the_response(self): command = AddQueueSinkOperation(_configuration()).get_command(None) command.set_response(json.dumps({"RaftCommandIndex": 9, "TaskId": 4}), False) diff --git a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py index 871832b2..d2478aec 100644 --- a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py +++ b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py @@ -310,3 +310,24 @@ def test_filtering_by_name_and_type_narrows_the_listing(self): self.store.maintenance.server.send( RemoveServerWideConnectionStringOperation(RavenConnectionString(name)) ) + + +class TestConnectionStringUsageList(unittest.TestCase): + def test_a_list_of_usages_is_parsed(self): + usages = ConnectionStringUsage.list_from_json( + [ + {"Kind": "RavenEtl", "Id": 1, "Name": "etl"}, + {"Kind": "AiAgent", "Identifier": "agents/1", "Name": "agent"}, + ] + ) + + self.assertEqual(2, len(usages)) + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, usages[0].kind) + self.assertEqual(1, usages[0].id_) + # An AI agent is identified by a string rather than a numeric task id. + self.assertEqual("agents/1", usages[1].identifier) + self.assertIsNone(usages[1].id_) + + def test_nothing_parses_to_an_empty_list(self): + self.assertEqual([], ConnectionStringUsage.list_from_json(None)) + self.assertEqual([], ConnectionStringUsage.list_from_json([])) diff --git a/ravendb/tests/raven_commands_tests/test_by_index_actions.py b/ravendb/tests/raven_commands_tests/test_by_index_actions.py index e5b5e3a6..a51fbe3d 100644 --- a/ravendb/tests/raven_commands_tests/test_by_index_actions.py +++ b/ravendb/tests/raven_commands_tests/test_by_index_actions.py @@ -112,12 +112,10 @@ def test_delete_by_index_success(self): response.operation_id, response.operation_node_tag, ) - # wait_for_completion hands back the result the server reported for the operation. - # 45, not 50: the range is quoted, so the server compares DocNumber as a string - # and "5".."9" fall outside "0".."49" lexicographically. - result = x.wait_for_completion() - self.assertEqual(45, result["Total"]) - self.assertEqual("Processed 45 items.", result["Message"]) + # wait_for_completion returns nothing; _wait_for_completion_result is what + # carries the server's report, and the smuggler is its only caller. + self.assertIsNone(x.wait_for_completion()) + self.assertEqual(45, x._wait_for_completion_result()["Total"]) if __name__ == "__main__":