From a6d32434b848ada0a635dae38406811dbf4a7c67 Mon Sep 17 00:00:00 2001 From: Google Team Member Date: Mon, 21 Sep 2026 19:39:26 -0700 Subject: [PATCH] feat: expose SpeechMetadata, VoiceConfig.voice, and SpeechAnnotation in public GenAI SDKs PiperOrigin-RevId: 985665934 --- .../_gaos/resources/interactions/__init__.py | 2 + .../interactions/annotation/__init__.py | 22 +++++ .../_gaos/types/interactions/__init__.py | 5 + .../_gaos/types/interactions/annotation.py | 19 +++- .../types/interactions/speechannotation.py | 93 +++++++++++++++++++ google/genai/_live_converters.py | 9 ++ google/genai/_tokens_converters.py | 3 + google/genai/batches.py | 3 + google/genai/caches.py | 6 ++ google/genai/models.py | 9 ++ google/genai/tunings.py | 6 ++ google/genai/types.py | 45 +++++++++ 12 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 google/genai/_gaos/resources/interactions/annotation/__init__.py create mode 100644 google/genai/_gaos/types/interactions/speechannotation.py diff --git a/google/genai/_gaos/resources/interactions/__init__.py b/google/genai/_gaos/resources/interactions/__init__.py index 2b1c2cfb9..ccb5b4813 100644 --- a/google/genai/_gaos/resources/interactions/__init__.py +++ b/google/genai/_gaos/resources/interactions/__init__.py @@ -114,6 +114,7 @@ from ...types.interactions.videoresponseformat import VideoResponseFormat from ...types.interactions.webhookconfig import WebhookConfig from ...types.interactions.wordinfo import WordInfo +from . import annotation from . import codeexecutioncallstep from . import codemenderagentconfig from . import environment @@ -224,6 +225,7 @@ "VideoResponseFormat", "WebhookConfig", "WordInfo", + "annotation", "codeexecutioncallstep", "codemenderagentconfig", "environment", diff --git a/google/genai/_gaos/resources/interactions/annotation/__init__.py b/google/genai/_gaos/resources/interactions/annotation/__init__.py new file mode 100644 index 000000000..43491fe81 --- /dev/null +++ b/google/genai/_gaos/resources/interactions/annotation/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pyformat: disable +# pylint: skip-file + +"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.""" + +from ....types.interactions.speechannotation import SpeechAnnotation as SpeechMetadata + +__all__ = ["SpeechMetadata"] diff --git a/google/genai/_gaos/types/interactions/__init__.py b/google/genai/_gaos/types/interactions/__init__.py index 552ec8f3c..4ea484662 100644 --- a/google/genai/_gaos/types/interactions/__init__.py +++ b/google/genai/_gaos/types/interactions/__init__.py @@ -350,6 +350,7 @@ ) from .source import Source, SourceParam, SourceType from .speakerconfig import SpeakerConfig, SpeakerConfigParam + from .speechannotation import SpeechAnnotation, SpeechAnnotationParam from .speechconfig import SpeechConfig, SpeechConfigParam from .staticmediaprocessing import StaticMediaProcessing, StaticMediaProcessingParam from .status import Status, StatusParam @@ -740,6 +741,8 @@ "SourceType", "SpeakerConfig", "SpeakerConfigParam", + "SpeechAnnotation", + "SpeechAnnotationParam", "SpeechConfig", "SpeechConfigParam", "SpeechConfigUnion", @@ -1156,6 +1159,8 @@ "SourceType": ".source", "SpeakerConfig": ".speakerconfig", "SpeakerConfigParam": ".speakerconfig", + "SpeechAnnotation": ".speechannotation", + "SpeechAnnotationParam": ".speechannotation", "SpeechConfig": ".speechconfig", "SpeechConfigParam": ".speechconfig", "StaticMediaProcessing": ".staticmediaprocessing", diff --git a/google/genai/_gaos/types/interactions/annotation.py b/google/genai/_gaos/types/interactions/annotation.py index 755876cef..45b6baa4c 100644 --- a/google/genai/_gaos/types/interactions/annotation.py +++ b/google/genai/_gaos/types/interactions/annotation.py @@ -20,6 +20,7 @@ from __future__ import annotations from .filecitation import FileCitation, FileCitationParam from .placecitation import PlaceCitation, PlaceCitationParam +from .speechannotation import SpeechAnnotation, SpeechAnnotationParam from .urlcitation import URLCitation, URLCitationParam from .wordinfo import WordInfo, WordInfoParam from functools import partial @@ -33,7 +34,13 @@ AnnotationParam = TypeAliasType( "AnnotationParam", - Union[URLCitationParam, PlaceCitationParam, WordInfoParam, FileCitationParam], + Union[ + SpeechAnnotationParam, + URLCitationParam, + PlaceCitationParam, + WordInfoParam, + FileCitationParam, + ], ) r"""Citation information for model-generated content.""" @@ -51,13 +58,21 @@ class UnknownAnnotation(BaseModel): _ANNOTATION_VARIANTS: dict[str, Any] = { "file_citation": FileCitation, "place_citation": PlaceCitation, + "speech_metadata": SpeechAnnotation, "url_citation": URLCitation, "word_info": WordInfo, } Annotation = Annotated[ - Union[FileCitation, PlaceCitation, URLCitation, WordInfo, UnknownAnnotation], + Union[ + FileCitation, + PlaceCitation, + SpeechAnnotation, + URLCitation, + WordInfo, + UnknownAnnotation, + ], BeforeValidator( partial( parse_open_union, diff --git a/google/genai/_gaos/types/interactions/speechannotation.py b/google/genai/_gaos/types/interactions/speechannotation.py new file mode 100644 index 000000000..69080ddc6 --- /dev/null +++ b/google/genai/_gaos/types/interactions/speechannotation.py @@ -0,0 +1,93 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pyformat: disable +# pylint: skip-file + +"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.""" + +from __future__ import annotations +from .. import BaseModel, UNSET_SENTINEL +from ...utils import validate_const +import pydantic +from pydantic import model_serializer +from pydantic.functional_validators import AfterValidator +from typing import Literal, Optional +from typing_extensions import Annotated, NotRequired, TypedDict + + +class SpeechAnnotationParam(TypedDict): + r"""Speech annotation for text content.""" + + end_index: NotRequired[int] + r"""End of the attributed segment, exclusive.""" + speaker: NotRequired[str] + r"""The speaker to associate with this turn.""" + start_index: NotRequired[int] + r"""Start of segment of the response that is attributed to this source. + + Index indicates the start of the segment, measured in bytes. + """ + style: NotRequired[str] + r"""Style instruction for the speech synthesis.""" + type: Literal["speech_metadata"] + + +class SpeechAnnotation(BaseModel): + r"""Speech annotation for text content.""" + + end_index: Optional[int] = None + r"""End of the attributed segment, exclusive.""" + + speaker: Optional[str] = None + r"""The speaker to associate with this turn.""" + + start_index: Optional[int] = None + r"""Start of segment of the response that is attributed to this source. + + Index indicates the start of the segment, measured in bytes. + """ + + style: Optional[str] = None + r"""Style instruction for the speech synthesis.""" + + type: Annotated[ + Annotated[ + Literal["speech_metadata"], + AfterValidator(validate_const("speech_metadata")), + ], + pydantic.Field(alias="type"), + ] = "speech_metadata" + + @model_serializer(mode="wrap") + def serialize_model(self, handler): + optional_fields = set(["end_index", "speaker", "start_index", "style"]) + serialized = handler(self) + m = {} + + for n, f in type(self).model_fields.items(): + k = f.alias or n + val = serialized.get(k, serialized.get(n)) + + if val != UNSET_SENTINEL: + if val is not None or k not in optional_fields: + m[k] = val + + return m + + +try: + SpeechAnnotation.model_rebuild() +except NameError: + pass diff --git a/google/genai/_live_converters.py b/google/genai/_live_converters.py index 20465ac87..130b2face 100644 --- a/google/genai/_live_converters.py +++ b/google/genai/_live_converters.py @@ -1538,6 +1538,9 @@ def _Part_to_mldev( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object @@ -1623,6 +1626,9 @@ def _Part_to_vertex( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object @@ -2066,4 +2072,7 @@ def _VoiceConfig_to_vertex( getv(from_object, ['prebuilt_voice_config']), ) + if getv(from_object, ['voice']) is not None: + setv(to_object, ['voice'], getv(from_object, ['voice'])) + return to_object diff --git a/google/genai/_tokens_converters.py b/google/genai/_tokens_converters.py index f72b65ad4..3cbe5d1c6 100644 --- a/google/genai/_tokens_converters.py +++ b/google/genai/_tokens_converters.py @@ -536,6 +536,9 @@ def _Part_to_mldev( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object diff --git a/google/genai/batches.py b/google/genai/batches.py index 30e3890ad..a83371ede 100644 --- a/google/genai/batches.py +++ b/google/genai/batches.py @@ -1583,6 +1583,9 @@ def _Part_to_mldev( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object diff --git a/google/genai/caches.py b/google/genai/caches.py index dbf3509e2..c8b21f423 100644 --- a/google/genai/caches.py +++ b/google/genai/caches.py @@ -719,6 +719,9 @@ def _Part_to_mldev( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object @@ -804,6 +807,9 @@ def _Part_to_vertex( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object diff --git a/google/genai/models.py b/google/genai/models.py index fdea72ddd..0e965a839 100644 --- a/google/genai/models.py +++ b/google/genai/models.py @@ -3363,6 +3363,9 @@ def _Part_to_mldev( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object @@ -3449,6 +3452,9 @@ def _Part_to_vertex( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object @@ -4622,6 +4628,9 @@ def _VoiceConfig_to_vertex( getv(from_object, ['prebuilt_voice_config']), ) + if getv(from_object, ['voice']) is not None: + setv(to_object, ['voice'], getv(from_object, ['voice'])) + return to_object diff --git a/google/genai/tunings.py b/google/genai/tunings.py index a270085d6..3d2ae2d44 100644 --- a/google/genai/tunings.py +++ b/google/genai/tunings.py @@ -1608,6 +1608,9 @@ def _Part_to_vertex( to_object, ['mediaProcessing'], getv(from_object, ['media_processing']) ) + if getv(from_object, ['speech_metadata']) is not None: + setv(to_object, ['speechMetadata'], getv(from_object, ['speech_metadata'])) + return to_object @@ -2603,6 +2606,9 @@ def _VoiceConfig_to_vertex( getv(from_object, ['prebuilt_voice_config']), ) + if getv(from_object, ['voice']) is not None: + setv(to_object, ['voice'], getv(from_object, ['voice'])) + return to_object diff --git a/google/genai/types.py b/google/genai/types.py index 5cfa1381c..d4bae8c99 100644 --- a/google/genai/types.py +++ b/google/genai/types.py @@ -1616,6 +1616,36 @@ class ToolResponseDict(TypedDict, total=False): ToolResponseOrDict = Union[ToolResponse, ToolResponseDict] +class SpeechMetadata(_common.BaseModel): + """Extra metadata associated with the part for speech synthesis.""" + + speaker: Optional[str] = Field( + default=None, + description="""The speaker for this part, which must match a `speaker` name in + `MultiSpeakerVoiceConfig.speaker_voice_configs`.""", + ) + style: Optional[str] = Field( + default=None, + description="""The style instruction for how the voice should be synthesized + (e.g. "excited, fast-paced").""", + ) + + +class SpeechMetadataDict(TypedDict, total=False): + """Extra metadata associated with the part for speech synthesis.""" + + speaker: Optional[str] + """The speaker for this part, which must match a `speaker` name in + `MultiSpeakerVoiceConfig.speaker_voice_configs`.""" + + style: Optional[str] + """The style instruction for how the voice should be synthesized + (e.g. "excited, fast-paced").""" + + +SpeechMetadataOrDict = Union[SpeechMetadata, SpeechMetadataDict] + + class CodeExecutionResult(_common.BaseModel): """Result of executing the ExecutableCode. @@ -2323,6 +2353,11 @@ class Part(_common.BaseModel): default=None, description="""How the model processes this part's media for understanding.""", ) + speech_metadata: Optional[SpeechMetadata] = Field( + default=None, + description="""Extra metadata associated with the part for speech synthesis, such + as speaker and style. Only valid when `Part.data` is set to `text`.""", + ) def __init__( self, @@ -2560,6 +2595,10 @@ class PartDict(TypedDict, total=False): media_processing: Optional[MediaProcessing] """How the model processes this part's media for understanding.""" + speech_metadata: Optional[SpeechMetadataDict] + """Extra metadata associated with the part for speech synthesis, such + as speaker and style. Only valid when `Part.data` is set to `text`.""" + PartOrDict = Union[Part, PartDict] @@ -5566,6 +5605,9 @@ class VoiceConfig(_common.BaseModel): prebuilt_voice_config: Optional[PrebuiltVoiceConfig] = Field( default=None, description="""The configuration for a prebuilt voice.""" ) + voice: Optional[str] = Field( + default=None, description="""The speaker identifier for synthesis.""" + ) class VoiceConfigDict(TypedDict, total=False): @@ -5579,6 +5621,9 @@ class VoiceConfigDict(TypedDict, total=False): prebuilt_voice_config: Optional[PrebuiltVoiceConfigDict] """The configuration for a prebuilt voice.""" + voice: Optional[str] + """The speaker identifier for synthesis.""" + VoiceConfigOrDict = Union[VoiceConfig, VoiceConfigDict]