diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/_helpers.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/_helpers.py index 2d45140bd96f..4793a7d2696e 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/_helpers.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/_helpers.py @@ -16,6 +16,7 @@ from __future__ import annotations +import collections.abc import datetime import json import re @@ -48,8 +49,12 @@ from google.cloud.firestore_v1.bson import ( BSONBinary, BSONDecimal128, + BSONInt32, + BSONMaxKey, + BSONMinKey, BSONObjectID, BSONRegex, + BSONTimestamp, BSONType, ) from google.cloud.firestore_v1.field_path import FieldPath, parse_field_path @@ -225,6 +230,18 @@ def encode_value(value: Any) -> document.Value: options_str = _extract_regex_options(flags_attr) return encode_value(BSONRegex(pattern_attr, options_str)) + # Duck-typing input bridge for external PyMongo / bson package objects (zero dependency) + if hasattr(value, "binary") and not isinstance( + value, (bytes, bytearray, BSONBinary) + ): + return encode_value(BSONObjectID(getattr(value, "binary"))) + elif hasattr(value, "to_decimal") and not isinstance(value, BSONDecimal128): + return encode_value(BSONDecimal128(getattr(value, "to_decimal")())) + elif hasattr(value, "pattern") and not isinstance(value, (str, BSONRegex)): + return encode_value( + BSONRegex(getattr(value, "pattern"), str(getattr(value, "flags", ""))) + ) + # Must come before int since ``bool`` is an integer subtype. if isinstance(value, bool): return document.Value(boolean_value=value) @@ -441,7 +458,100 @@ def decode_value( raise ValueError("Unknown ``value_type``", value_type) -def decode_dict(value_fields, client) -> Union[dict, Vector]: +def _parse_oid(val: Any) -> BSONObjectID: + if not isinstance(val, str): + raise ValueError(f"Invalid BSONObjectID map value, expected str: {val!r}") + return BSONObjectID(val) + + +def _parse_decimal128(val: Any) -> BSONDecimal128: + if not isinstance(val, str): + raise ValueError(f"Invalid BSONDecimal128 map value, expected str: {val!r}") + return BSONDecimal128(val) + + +def _parse_int32(val: Any) -> BSONInt32: + if type(val) is not int or isinstance(val, bool): + raise ValueError(f"Invalid BSONInt32 map value, expected int: {val!r}") + return BSONInt32(val) + + +def _parse_minkey(val: Any) -> BSONMinKey: + if type(val) is not int or isinstance(val, bool): + raise ValueError(f"Invalid BSONMinKey map value, expected int: {val!r}") + return BSONMinKey() + + +def _parse_maxkey(val: Any) -> BSONMaxKey: + if type(val) is not int or isinstance(val, bool): + raise ValueError(f"Invalid BSONMaxKey map value, expected int: {val!r}") + return BSONMaxKey() + + +def _parse_timestamp(val: Any) -> BSONTimestamp: + if not isinstance(val, collections.abc.Mapping): + raise ValueError(f"Invalid BSONTimestamp map value, expected mapping: {val!r}") + sec = val.get("seconds") + inc = val.get("increment") + if ( + type(sec) is not int + or type(inc) is not int + or isinstance(sec, bool) + or isinstance(inc, bool) + or len(val) != 2 + ): + raise ValueError(f"Invalid BSONTimestamp fields: {val!r}") + return BSONTimestamp(sec, inc) + + +def _parse_regex(val: Any) -> BSONRegex: + if not isinstance(val, collections.abc.Mapping): + raise ValueError(f"Invalid BSONRegex map value, expected mapping: {val!r}") + pat = val.get("pattern") + opt = val.get("options", "") + if not isinstance(pat, str) or not isinstance(opt, str) or len(val) not in (1, 2): + raise ValueError(f"Invalid BSONRegex fields: {val!r}") + return BSONRegex(pat, opt) + + +def _parse_binary(val: Any) -> BSONBinary: + if not isinstance(val, collections.abc.Mapping): + raise ValueError(f"Invalid BSONBinary map value, expected mapping: {val!r}") + sub = val.get("sub_type") + bdata = val.get("bytes") + if ( + type(sub) is not int + or isinstance(sub, bool) + or not isinstance(bdata, (bytes, bytearray, memoryview)) + or len(val) != 2 + ): + raise ValueError(f"Invalid BSONBinary fields: {val!r}") + return BSONBinary(bdata, subtype=sub) + + +_BSON_MAP_PARSERS = { + "__oid__": _parse_oid, + "__decimal128__": _parse_decimal128, + "__int__": _parse_int32, + "__minkey__": _parse_minkey, + "__maxkey__": _parse_maxkey, + "__timestamp__": _parse_timestamp, + "__regex__": _parse_regex, + "__binary__": _parse_binary, +} + + +def _parse_bson_mapping(key: str, val: Any) -> Optional[Any]: + """Converts legacy BSON map value representations to native BSON instances.""" + parser = _BSON_MAP_PARSERS.get(key) + if parser is not None: + return parser(val) + return None + + +def decode_dict( + value_fields, client, decode_bson: Optional[bool] = None +) -> Union[dict, Vector]: """Converts a protobuf map of Firestore ``Value``-s. Args: @@ -449,21 +559,29 @@ def decode_dict(value_fields, client) -> Union[dict, Vector]: protobuf map of Firestore ``Value``-s. client (:class:`~google.cloud.firestore_v1.client.Client`): A client that has a document factory. + decode_bson (Optional[bool]): Flag indicating whether to decode BSON map representations. Returns: - Dict[str, Union[NoneType, bool, int, float, datetime.datetime, \ - str, bytes, dict, ~google.cloud.Firestore.GeoPoint]]: A dictionary - of native Python values converted from the ``value_fields``. + Dict[str, Any]: A dictionary converted from ``value_fields``. """ + effective_decode = ( + decode_bson + if decode_bson is not None + else getattr(client, "decode_bson", False) + ) value_fields_pb = getattr(value_fields, "_pb", value_fields) res = {key: decode_value(value, client) for key, value in value_fields_pb.items()} if res.get("__type__", None) == "__vector__": - # Vector data type is represented as mapping. - # {"__type__":"__vector__", "value": [1.0, 2.0, 3.0]}. values = cast(Sequence[float], res["value"]) return Vector(values) + if effective_decode and len(res) == 1: + single_key = next(iter(res)) + parsed = _parse_bson_mapping(single_key, res[single_key]) + if parsed is not None: + return parsed + return res diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/async_client.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/async_client.py index 3167335e0385..6463d8a163f9 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/async_client.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/async_client.py @@ -105,6 +105,7 @@ def __init__( database=None, client_info=_CLIENT_INFO, client_options=None, + decode_bson: bool = False, ) -> None: super(AsyncClient, self).__init__( project=project, @@ -112,6 +113,7 @@ def __init__( database=database, client_info=client_info, client_options=client_options, + decode_bson=decode_bson, ) def _to_sync_copy(self): @@ -124,6 +126,7 @@ def _to_sync_copy(self): database=self._database, client_info=self._client_info, client_options=self._client_options, + decode_bson=self.decode_bson, ) return self._sync_copy diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_client.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_client.py index 95166266bef2..9f43548841a9 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_client.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_client.py @@ -132,6 +132,7 @@ def __init__( database=None, client_info=_CLIENT_INFO, client_options=None, + decode_bson: bool = False, ) -> None: database = database or DEFAULT_DATABASE # NOTE: This API has no use for the _http argument, but sending it @@ -165,6 +166,7 @@ def __init__( self._client_options = client_options self._database = database + self.decode_bson = decode_bson def _firestore_api_helper(self, transport, client_class, client_module) -> Any: """Lazy-loading getter GAPIC Firestore API. @@ -610,7 +612,8 @@ def _parse_batch_get( result_type = get_doc_response._pb.WhichOneof("result") if result_type == "found": reference = _get_reference(get_doc_response.found.name, reference_map) - data = _helpers.decode_dict(get_doc_response.found.fields, client) + fields = get_doc_response.found.fields + data = _helpers.decode_dict(fields, client) snapshot = DocumentSnapshot( reference, data, @@ -618,6 +621,7 @@ def _parse_batch_get( read_time=get_doc_response.read_time, create_time=get_doc_response.found.create_time, update_time=get_doc_response.found.update_time, + raw_fields=fields, ) elif result_type == "missing": reference = _get_reference(get_doc_response.missing, reference_map) diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_document.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_document.py index 92d8daa21fd6..2b972ba82f4b 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_document.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_document.py @@ -388,21 +388,92 @@ class DocumentSnapshot(object): """ def __init__( - self, reference, data, exists, read_time, create_time, update_time + self, + reference, + data, + exists, + read_time, + create_time, + update_time, + raw_fields=None, + decode_bson: Optional[bool] = None, ) -> None: self._reference = reference - # We want immutable data, so callers can't modify this value - # out from under us. - self._data = copy.deepcopy(data) self._exists = exists self.read_time = read_time self.create_time = create_time self.update_time = update_time + self._raw_fields = raw_fields + + client = getattr(reference, "_client", None) if reference else None + self._decode_bson = ( + decode_bson + if decode_bson is not None + else (getattr(client, "decode_bson", False) if client else False) + ) + self._data_raw = None + self._data_bson = None + + if raw_fields is not None: + if self._decode_bson: + self._data_bson = copy.deepcopy(data) + else: + self._data_raw = copy.deepcopy(data) + else: + self._data_raw = copy.deepcopy(data) if data is not None else None + + def _get_data(self, decode_bson: Optional[bool] = None) -> Optional[Dict[str, Any]]: + effective_decode = ( + decode_bson + if decode_bson is not None + else ( + self._decode_bson + if hasattr(self, "_decode_bson") and self._decode_bson is not None + else ( + self._reference._client.decode_bson + if ( + self._reference + and hasattr(self._reference, "_client") + and self._reference._client + ) + else False + ) + ) + ) + + if effective_decode: + if self._data_bson is None: + if self._raw_fields is not None: + client = self._reference._client if self._reference else None + self._data_bson = _helpers.decode_dict( + self._raw_fields, client, decode_bson=True + ) + elif self._data_raw is not None: + self._data_bson = self._data_raw + return self._data_bson + else: + if self._data_raw is None: + if self._raw_fields is not None: + client = self._reference._client if self._reference else None + self._data_raw = _helpers.decode_dict( + self._raw_fields, client, decode_bson=False + ) + elif self._data_bson is not None: + self._data_raw = self._data_bson + return self._data_raw + + @property + def _data(self) -> Optional[Dict[str, Any]]: + return self._get_data() def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented - return self._reference == other._reference and self._data == other._data + return ( + self._reference == other._reference + and self.read_time == other.read_time + and self._get_data(decode_bson=False) == other._get_data(decode_bson=False) + ) def __hash__(self): return hash(self._reference) + hash(self.update_time) @@ -448,84 +519,22 @@ def reference(self) -> BaseDocumentReference: """ return self._reference - def get(self, field_path: str) -> Any: - """Get a value from the snapshot data. - - If the data is nested, for example: - - .. code-block:: python - - >>> snapshot.to_dict() - { - 'top1': { - 'middle2': { - 'bottom3': 20, - 'bottom4': 22, - }, - 'middle5': True, - }, - 'top6': b'\x00\x01 foo', - } - - a **field path** can be used to access the nested data. For - example: - - .. code-block:: python - - >>> snapshot.get('top1') - { - 'middle2': { - 'bottom3': 20, - 'bottom4': 22, - }, - 'middle5': True, - } - >>> snapshot.get('top1.middle2') - { - 'bottom3': 20, - 'bottom4': 22, - } - >>> snapshot.get('top1.middle2.bottom3') - 20 - - See :meth:`~google.cloud.firestore_v1.client.Client.field_path` for - more information on **field paths**. - - A copy is returned since the data may contain mutable values, - but the data stored in the snapshot must remain immutable. - - Args: - field_path (str): A field path (``.``-delimited list of - field names). - - Returns: - Any or None: - (A copy of) the value stored for the ``field_path`` or - None if snapshot document does not exist. - - Raises: - KeyError: If the ``field_path`` does not match nested data - in the snapshot. - """ + def get(self, field_path: str, decode_bson: Optional[bool] = None) -> Any: + """Get a value from the snapshot data.""" if not self._exists: return None - nested_data = field_path_module.get_nested_value(field_path, self._data) + data = self._get_data(decode_bson=decode_bson) + nested_data = field_path_module.get_nested_value(field_path, data) return copy.deepcopy(nested_data) - def to_dict(self) -> Union[Dict[str, Any], None]: - """Retrieve the data contained in this snapshot. - - A copy is returned since the data may contain mutable values, - but the data stored in the snapshot must remain immutable. - - Returns: - Dict[str, Any] or None: - The data in the snapshot. Returns None if reference - does not exist. - """ + def to_dict( + self, decode_bson: Optional[bool] = None + ) -> Union[Dict[str, Any], None]: + """Retrieve the data contained in this snapshot.""" if not self._exists: return None - return copy.deepcopy(self._data) + data = self._get_data(decode_bson=decode_bson) + return copy.deepcopy(data) def _to_protobuf(self) -> Optional[Document]: return _helpers.document_snapshot_to_protobuf(self) diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/client.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/client.py index e29d07cb09ac..7b97f3ac13a5 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/client.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/client.py @@ -94,6 +94,7 @@ def __init__( database=None, client_info=_CLIENT_INFO, client_options=None, + decode_bson: bool = False, ) -> None: super(Client, self).__init__( project=project, @@ -101,6 +102,7 @@ def __init__( database=database, client_info=client_info, client_options=client_options, + decode_bson=decode_bson, ) @property diff --git a/packages/google-cloud-firestore/tests/system/test_bson.py b/packages/google-cloud-firestore/tests/system/test_bson.py index 747786580b5a..f88bf874f051 100644 --- a/packages/google-cloud-firestore/tests/system/test_bson.py +++ b/packages/google-cloud-firestore/tests/system/test_bson.py @@ -103,3 +103,53 @@ def __init__(self, pat: str, flags: str): snapshot = doc_ref.get() assert snapshot.exists + + +@pytest.mark.parametrize("database", [FIRESTORE_ENTERPRISE_DB], indirect=True) +def test_bson_document_writes_and_reads(client, cleanup, database): + """Test standard write and read operations for BSON types on Enterprise DB.""" + collection_id = "bson_docs_" + UNIQUE_RESOURCE_ID + doc_ref = client.collection(collection_id).document("bson_doc") + cleanup(doc_ref.delete) + + bson_payload = { + "_id": BSONObjectID("507f191e810c19729de860ea"), + "price": BSONDecimal128("199.99"), + "qty": BSONInt32(50), + "pattern": BSONRegex(pattern="^prod.*", flags="i"), + "ts": BSONTimestamp(seconds=1710000000, increment=2), + "binary_data": BSONBinary(sub_type=1, data=b"binary_payload"), + "min_key": BSONMinKey(), + "max_key": BSONMaxKey(), + } + + doc_ref.set(bson_payload) + + snapshot = doc_ref.get(decode_bson=True) + assert snapshot.exists + assert snapshot.to_dict(decode_bson=True) == bson_payload + + +@pytest.mark.parametrize("database", [FIRESTORE_ENTERPRISE_DB], indirect=True) +async def test_bson_document_writes_and_reads_async(client, cleanup, database): + """Test async write and read operations for BSON types on Enterprise DB.""" + collection_id = "bson_docs_async_" + UNIQUE_RESOURCE_ID + doc_ref = client.collection(collection_id).document("bson_doc") + cleanup(doc_ref.delete) + + bson_payload = { + "_id": BSONObjectID("507f191e810c19729de860ea"), + "price": BSONDecimal128("299.99"), + "qty": BSONInt32(100), + "pattern": BSONRegex(pattern="^async.*", flags="m"), + "ts": BSONTimestamp(seconds=1720000000, increment=1), + "binary_data": BSONBinary(sub_type=1, data=b"async_binary_payload"), + "min_key": BSONMinKey(), + "max_key": BSONMaxKey(), + } + + await doc_ref.set(bson_payload) + + snapshot = await doc_ref.get(decode_bson=True) + assert snapshot.exists + assert snapshot.to_dict(decode_bson=True) == bson_payload diff --git a/packages/google-cloud-firestore/tests/unit/v1/test__helpers.py b/packages/google-cloud-firestore/tests/unit/v1/test__helpers.py index 8322089664d6..b7412425e58d 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test__helpers.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test__helpers.py @@ -393,6 +393,20 @@ def __init__(self, pat: str, flags: str): ) +def test_decode_dict_malformed_bson_raises(): + import pytest + + from google.cloud.firestore_v1._helpers import decode_dict + from google.cloud.firestore_v1.types import document + + # Invalid payload schema raises ValueError when decode_bson=True + malformed = { + "__oid__": document.Value(integer_value=12345) + } # should be string_value + with pytest.raises(ValueError, match="Invalid BSONObjectID map value"): + decode_dict(malformed, client=None, decode_bson=True) + + def test_reference_value_to_document_w_bad_format(): from google.cloud.firestore_v1._helpers import ( BAD_REFERENCE_ERROR, diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_async_bson.py b/packages/google-cloud-firestore/tests/unit/v1/test_async_bson.py new file mode 100644 index 000000000000..9e97d85037f2 --- /dev/null +++ b/packages/google-cloud-firestore/tests/unit/v1/test_async_bson.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud.firestore_v1._helpers import decode_dict, encode_dict +from google.cloud.firestore_v1.async_client import AsyncClient +from google.cloud.firestore_v1.bson import ( + BSONBinary, + BSONDecimal128, + BSONInt32, + BSONMaxKey, + BSONMinKey, + BSONObjectID, + BSONRegex, + BSONTimestamp, +) + + +def test_async_client_decode_bson_flag(): + client_default = AsyncClient(project="project", decode_bson=False) + assert client_default.decode_bson is False + assert client_default._to_sync_copy().decode_bson is False + + client_enabled = AsyncClient(project="project", decode_bson=True) + assert client_enabled.decode_bson is True + assert client_enabled._to_sync_copy().decode_bson is True + + +def test_async_bson_encode_decode_roundtrip(): + data = { + "oid": BSONObjectID("507f1f77bcf86cd799439011"), + "dec": BSONDecimal128("123.456"), + "ts": BSONTimestamp(100, 200), + "reg": BSONRegex("foo", "i"), + "bin": BSONBinary(b"bar", subtype=1), + "int32": BSONInt32(42), + "min": BSONMinKey(), + "max": BSONMaxKey(), + } + + encoded = encode_dict(data) + + client_disabled = AsyncClient(project="proj", decode_bson=False) + decoded_raw = decode_dict(encoded, client=client_disabled) + assert isinstance(decoded_raw["oid"], dict) + assert decoded_raw["oid"]["__oid__"] == "507f1f77bcf86cd799439011" + + client_enabled = AsyncClient(project="proj", decode_bson=True) + decoded_bson = decode_dict(encoded, client=client_enabled) + assert isinstance(decoded_bson["oid"], BSONObjectID) + assert decoded_bson["oid"].value == "507f1f77bcf86cd799439011" + assert isinstance(decoded_bson["dec"], BSONDecimal128) + assert isinstance(decoded_bson["ts"], BSONTimestamp) + assert isinstance(decoded_bson["reg"], BSONRegex) + assert isinstance(decoded_bson["bin"], BSONBinary) + assert isinstance(decoded_bson["int32"], BSONInt32) + assert isinstance(decoded_bson["min"], BSONMinKey) + assert isinstance(decoded_bson["max"], BSONMaxKey)