From 88ee2aa74db915c5ab236684d4d1d16a50c2c63b Mon Sep 17 00:00:00 2001 From: Dhruv Arya Date: Wed, 12 Aug 2026 00:58:57 +0000 Subject: [PATCH 1/2] OpenAPI: REST catalog support for table format version 4 Applies the format-version-4 changes to the table schemas in place, so the semantic diff is easy to see, and moves the affected table endpoints to /v2. A follow-up commit repackages this to retain the /v1 endpoints unchanged and expose the v4 behavior through V2-suffixed variants; this commit is the readable "what actually changed" view. - Snapshot: add `content-root` and make `manifest-list` optional, with a oneOf requiring exactly one of the two. Format versions 1-3 use `manifest-list` and version 4 uses `content-root`. - TableMetadata: raise the format-version maximum to 4; document that `location` is optional in v4. - LoadTableResult / UnregisterTableResult: add `table-location` for resolving relative paths in v4 metadata. - CommitTableResponse: add `table-location`; `metadata-location` is no longer required (absent for catalog-managed tables). - Move the table, register, and unregister endpoints to /v2. make lint and make generate both pass. --- open-api/rest-catalog-open-api.py | 135 +++++++++++++++++++++++++--- open-api/rest-catalog-open-api.yaml | 106 ++++++++++++++++++---- 2 files changed, 214 insertions(+), 27 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5e3580ae3825..d49b67808f5d 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -56,14 +56,16 @@ class CatalogConfig(BaseModel): ) endpoints: list[str] | None = Field( None, - description='A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character.', + description='A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character. Table endpoints are available under `/v1` and `/v2`. Clients that understand format version 4 should prefer the `/v2` APIs when available.', examples=[ [ 'GET /v1/{prefix}/namespaces/{namespace}', 'GET /v1/{prefix}/namespaces', 'POST /v1/{prefix}/namespaces', - 'GET /v1/{prefix}/namespaces/{namespace}/tables/{table}', 'GET /v1/{prefix}/namespaces/{namespace}/views/{view}', + 'GET /v2/{prefix}/namespaces/{namespace}/tables/{table}', + 'POST /v2/{prefix}/namespaces/{namespace}/tables/{table}', + 'POST /v2/{prefix}/namespaces/{namespace}/register', ] ], ) @@ -330,7 +332,15 @@ class Summary(BaseModel): operation: Literal['append', 'replace', 'overwrite', 'delete'] -class Snapshot(BaseModel): +class Snapshot1(BaseModel): + """ + A snapshot of the table's contents at a point in time. + + + Format versions 1-3 use `manifest-list` and format version 4 uses `content-root`. + + """ + snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') sequence_number: int | None = Field(None, alias='sequence-number') @@ -338,7 +348,49 @@ class Snapshot(BaseModel): manifest_list: str = Field( ..., alias='manifest-list', - description="Location of the snapshot's manifest list file", + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead.", + ) + content_root: str | None = Field( + None, + alias='content-root', + description="Location of the snapshot's content root. Required for format version 4 and must be absent for format versions 1-3.", + ) + first_row_id: int | None = Field( + None, + alias='first-row-id', + description='The first _row_id assigned to the first row in the first data file in the first manifest', + ) + added_rows: int | None = Field( + None, + alias='added-rows', + description='The upper bound of the number of rows with assigned row IDs', + ) + summary: Summary + schema_id: int | None = Field(None, alias='schema-id') + + +class Snapshot2(BaseModel): + """ + A snapshot of the table's contents at a point in time. + + + Format versions 1-3 use `manifest-list` and format version 4 uses `content-root`. + + """ + + snapshot_id: int = Field(..., alias='snapshot-id') + parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') + sequence_number: int | None = Field(None, alias='sequence-number') + timestamp_ms: int = Field(..., alias='timestamp-ms') + manifest_list: str | None = Field( + None, + alias='manifest-list', + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead.", + ) + content_root: str = Field( + ..., + alias='content-root', + description="Location of the snapshot's content root. Required for format version 4 and must be absent for format versions 1-3.", ) first_row_id: int | None = Field( None, @@ -354,6 +406,13 @@ class Snapshot(BaseModel): schema_id: int | None = Field(None, alias='schema-id') +class Snapshot(RootModel[Snapshot1 | Snapshot2]): + root: Snapshot1 | Snapshot2 = Field( + ..., + description="A snapshot of the table's contents at a point in time.\n\n\nFormat versions 1-3 use `manifest-list` and format version 4 uses `content-root`.\n", + ) + + class SnapshotReference(BaseModel): type: Literal['tag', 'branch'] snapshot_id: int = Field(..., alias='snapshot-id') @@ -1650,9 +1709,12 @@ class Apply(BaseModel): class TableMetadata(BaseModel): - format_version: int = Field(..., alias='format-version', ge=1, le=3) + format_version: int = Field(..., alias='format-version', ge=1, le=4) table_uuid: str = Field(..., alias='table-uuid') - location: str | None = None + location: str | None = Field( + None, + description="The table's base location. Required through format version 3, where it may be a path without a URI scheme; readers prepend a scheme for consistency with v4 absolute paths. Optional for format version 4, where the location may be managed externally and supplied by the catalog when the table is loaded, and where it must be an absolute path when present. See the `table-location` field of `LoadTableResult`.", + ) last_updated_ms: int | None = Field(None, alias='last-updated-ms') next_row_id: int | None = Field( None, @@ -1705,11 +1767,35 @@ class AddSchemaUpdate(BaseUpdate): class LoadTableResult(BaseModel): """ - Result used when a table is successfully loaded. + Result used when a table is successfully loaded, for tables at any format version. + + + The table metadata JSON is returned in the `metadata` field. + The location of the table metadata file is returned in the `metadata-location` field when the table has one, and the table's base location in the `table-location` field. + + + ## Metadata location + + + The `metadata-location` field is optional. + It is absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state and no metadata pointer need exist. + + + Clients must not require this field to be present, and must not use it to bypass the catalog for reads or commits. + To obtain a metadata location for a catalog-managed table, use the `unregisterTable` endpoint, which returns the table's last metadata location at the point the table leaves catalog control and further commits are rejected. + + + ## Table location + + The `table-location` field carries the table's base location. + Format version 4 allows location fields in metadata to be relative, and such paths must be resolved against the table location. + Format version 4 also makes `metadata.location` optional, so a table may have metadata that contains relative paths and omits `location`. + Servers must populate `table-location` for any such table, because it cannot be read otherwise. - The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. - Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + + When both `metadata.location` and `table-location` are present, `table-location` takes precedence: + the catalog is authoritative for table state. The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. @@ -1756,7 +1842,12 @@ class LoadTableResult(BaseModel): metadata_location: str | None = Field( None, alias='metadata-location', - description='May be null if the table is staged as part of a transaction', + description='Location of the table metadata file. Absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.', + ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", ) metadata: TableMetadata config: dict[str, str] | None = None @@ -1841,6 +1932,11 @@ class UnregisterTableResult(BaseModel): alias='metadata-location', description='The last metadata location for the table at the time it was unregistered.', ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", + ) metadata: TableMetadata @@ -2026,7 +2122,24 @@ class FunctionStructField(BaseModel): class CommitTableResponse(BaseModel): - metadata_location: str = Field(..., alias='metadata-location') + """ + Result used when a table is successfully updated, for tables at any format version. + + + The `table-location` field carries the table's base location, so that a client can resolve relative paths in the returned metadata. + + """ + + metadata_location: str | None = Field( + None, + alias='metadata-location', + description='Location of the committed table metadata file. Absent when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.', + ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", + ) metadata: TableMetadata diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e9a530a12804..124945d78ce7 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -522,7 +522,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/tables: + /v2/{prefix}/namespaces/{namespace}/tables: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -968,7 +968,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/register: + /v2/{prefix}/namespaces/{namespace}/register: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1024,7 +1024,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/tables/{table}: + /v2/{prefix}/namespaces/{namespace}/tables/{table}: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1299,7 +1299,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/tables/{table}/unregister: + /v2/{prefix}/namespaces/{namespace}/tables/{table}/unregister: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -2305,12 +2305,16 @@ components: type: string description: A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character. + Table endpoints are available under `/v1` and `/v2`. + Clients that understand format version 4 should prefer the `/v2` APIs when available. example: [ "GET /v1/{prefix}/namespaces/{namespace}", "GET /v1/{prefix}/namespaces", "POST /v1/{prefix}/namespaces", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "GET /v1/{prefix}/namespaces/{namespace}/views/{view}" + "GET /v1/{prefix}/namespaces/{namespace}/views/{view}", + "GET /v2/{prefix}/namespaces/{namespace}/tables/{table}", + "POST /v2/{prefix}/namespaces/{namespace}/tables/{table}", + "POST /v2/{prefix}/namespaces/{namespace}/register" ] idempotency-key-lifetime: type: string @@ -2948,12 +2952,21 @@ components: type: string Snapshot: + description: | + A snapshot of the table's contents at a point in time. + + + Format versions 1-3 use `manifest-list` and format version 4 uses `content-root`. type: object required: - snapshot-id - timestamp-ms - - manifest-list - summary + oneOf: + - required: + - manifest-list + - required: + - content-root properties: snapshot-id: type: integer @@ -2969,7 +2982,14 @@ components: format: int64 manifest-list: type: string - description: Location of the snapshot's manifest list file + description: + Location of the snapshot's manifest list file. + Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead. + content-root: + type: string + description: + Location of the snapshot's content root. + Required for format version 4 and must be absent for format versions 1-3. first-row-id: type: integer format: int64 @@ -3055,11 +3075,16 @@ components: format-version: type: integer minimum: 1 - maximum: 3 + maximum: 4 table-uuid: type: string location: type: string + description: + The table's base location. + Required through format version 3, where it may be a path without a URI scheme; readers prepend a scheme for consistency with v4 absolute paths. + Optional for format version 4, where the location may be managed externally and supplied by the catalog when the table is loaded, and where it must be an absolute path when present. + See the `table-location` field of `LoadTableResult`. last-updated-ms: type: integer format: int64 @@ -3851,11 +3876,35 @@ components: LoadTableResult: description: | - Result used when a table is successfully loaded. + Result used when a table is successfully loaded, for tables at any format version. + + + The table metadata JSON is returned in the `metadata` field. + The location of the table metadata file is returned in the `metadata-location` field when the table has one, and the table's base location in the `table-location` field. + + + ## Metadata location + + + The `metadata-location` field is optional. + It is absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state and no metadata pointer need exist. + + + Clients must not require this field to be present, and must not use it to bypass the catalog for reads or commits. + To obtain a metadata location for a catalog-managed table, use the `unregisterTable` endpoint, which returns the table's last metadata location at the point the table leaves catalog control and further commits are rejected. + + ## Table location - The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. - Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + + The `table-location` field carries the table's base location. + Format version 4 allows location fields in metadata to be relative, and such paths must be resolved against the table location. + Format version 4 also makes `metadata.location` optional, so a table may have metadata that contains relative paths and omits `location`. + Servers must populate `table-location` for any such table, because it cannot be read otherwise. + + + When both `metadata.location` and `table-location` are present, `table-location` takes precedence: + the catalog is authoritative for table state. The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. @@ -3896,15 +3945,21 @@ components: - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact. If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI. - type: object required: - metadata properties: metadata-location: type: string - description: May be null if the table is staged as part of a transaction - nullable: true + description: + Location of the table metadata file. + Absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. + Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. + Takes precedence over `metadata.location`. metadata: $ref: '#/components/schemas/TableMetadata' config: @@ -4163,6 +4218,12 @@ components: type: string description: The last metadata location for the table at the time it was unregistered. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. + Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. + Takes precedence over `metadata.location`. metadata: $ref: '#/components/schemas/TableMetadata' @@ -4929,13 +4990,26 @@ components: nullable: true CommitTableResponse: + description: | + Result used when a table is successfully updated, for tables at any format version. + + + The `table-location` field carries the table's base location, so that a client can resolve relative paths in the returned metadata. type: object required: - - metadata-location - metadata properties: metadata-location: type: string + description: + Location of the committed table metadata file. + Absent when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. + Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. + Takes precedence over `metadata.location`. metadata: $ref: '#/components/schemas/TableMetadata' From dc4daf77541661a4d4c2e1f2cba79d3f6fd75ee9 Mon Sep 17 00:00:00 2001 From: Dhruv Arya Date: Wed, 12 Aug 2026 00:58:57 +0000 Subject: [PATCH 2/2] OpenAPI: Retain v1 endpoints and expose v4 through V2-suffixed variants Repackages the previous commit so existing clients are unaffected: the /v1 endpoints and the schemas they use are restored to their original, unchanged definitions, and the format-version-4 behavior is exposed through parallel V2-suffixed schemas, response wrappers, and /v2 endpoints. Net effect against master is additive. Every schema and path without a V2 suffix is byte-identical to master; reviewers can focus on the V2 additions and two small edits. Restore the /v1 table, register, and unregister endpoints unchanged. Restore Snapshot, TableMetadata, LoadTableResult, CommitTableResponse, and UnregisterTableResult to their original definitions. Add SnapshotV2, TableMetadataV2, LoadTableResultV2, CommitTableResponseV2, and UnregisterTableResultV2 carrying the v4 changes from the previous commit, plus the matching V2 response wrappers, wired to the /v2 endpoints. AddSnapshotUpdate.snapshot references SnapshotV2 so a commit can add a v4 snapshot; this shared request type is otherwise unchanged. CatalogConfig endpoints advertises the /v2 endpoints. make lint and make generate both pass. --- open-api/rest-catalog-open-api.py | 158 +++++- open-api/rest-catalog-open-api.yaml | 810 +++++++++++++++++++++++++++- 2 files changed, 931 insertions(+), 37 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index d49b67808f5d..7f35b207b3e7 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -332,7 +332,7 @@ class Summary(BaseModel): operation: Literal['append', 'replace', 'overwrite', 'delete'] -class Snapshot1(BaseModel): +class SnapshotV21(BaseModel): """ A snapshot of the table's contents at a point in time. @@ -369,7 +369,7 @@ class Snapshot1(BaseModel): schema_id: int | None = Field(None, alias='schema-id') -class Snapshot2(BaseModel): +class SnapshotV22(BaseModel): """ A snapshot of the table's contents at a point in time. @@ -406,13 +406,37 @@ class Snapshot2(BaseModel): schema_id: int | None = Field(None, alias='schema-id') -class Snapshot(RootModel[Snapshot1 | Snapshot2]): - root: Snapshot1 | Snapshot2 = Field( +class SnapshotV2(RootModel[SnapshotV21 | SnapshotV22]): + root: SnapshotV21 | SnapshotV22 = Field( ..., description="A snapshot of the table's contents at a point in time.\n\n\nFormat versions 1-3 use `manifest-list` and format version 4 uses `content-root`.\n", ) +class Snapshot(BaseModel): + snapshot_id: int = Field(..., alias='snapshot-id') + parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') + sequence_number: int | None = Field(None, alias='sequence-number') + timestamp_ms: int = Field(..., alias='timestamp-ms') + manifest_list: str = Field( + ..., + alias='manifest-list', + description="Location of the snapshot's manifest list file", + ) + first_row_id: int | None = Field( + None, + alias='first-row-id', + description='The first _row_id assigned to the first row in the first data file in the first manifest', + ) + added_rows: int | None = Field( + None, + alias='added-rows', + description='The upper bound of the number of rows with assigned row IDs', + ) + summary: Summary + schema_id: int | None = Field(None, alias='schema-id') + + class SnapshotReference(BaseModel): type: Literal['tag', 'branch'] snapshot_id: int = Field(..., alias='snapshot-id') @@ -529,7 +553,7 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): action: Literal['add-snapshot'] - snapshot: Snapshot + snapshot: SnapshotV2 class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): @@ -1708,7 +1732,7 @@ class Apply(BaseModel): arguments: list[FunctionArgument] -class TableMetadata(BaseModel): +class TableMetadataV2(BaseModel): format_version: int = Field(..., alias='format-version', ge=1, le=4) table_uuid: str = Field(..., alias='table-uuid') location: str | None = Field( @@ -1731,6 +1755,38 @@ class TableMetadata(BaseModel): sort_orders: list[SortOrder] | None = Field(None, alias='sort-orders') default_sort_order_id: int | None = Field(None, alias='default-sort-order-id') encryption_keys: list[EncryptedKey] | None = Field(None, alias='encryption-keys') + snapshots: list[SnapshotV2] | None = None + refs: SnapshotReferences | None = None + current_snapshot_id: int | None = Field(None, alias='current-snapshot-id') + last_sequence_number: int | None = Field(None, alias='last-sequence-number') + snapshot_log: SnapshotLog | None = Field(None, alias='snapshot-log') + metadata_log: MetadataLog | None = Field(None, alias='metadata-log') + statistics: list[StatisticsFile] | None = None + partition_statistics: list[PartitionStatisticsFile] | None = Field( + None, alias='partition-statistics' + ) + + +class TableMetadata(BaseModel): + format_version: int = Field(..., alias='format-version', ge=1, le=3) + table_uuid: str = Field(..., alias='table-uuid') + location: str | None = None + last_updated_ms: int | None = Field(None, alias='last-updated-ms') + next_row_id: int | None = Field( + None, + alias='next-row-id', + description="A long higher than all assigned row IDs; the next snapshot's first-row-id.", + ) + properties: dict[str, str] | None = None + schemas: list[Schema] | None = None + current_schema_id: int | None = Field(None, alias='current-schema-id') + last_column_id: int | None = Field(None, alias='last-column-id') + partition_specs: list[PartitionSpec] | None = Field(None, alias='partition-specs') + default_spec_id: int | None = Field(None, alias='default-spec-id') + last_partition_id: int | None = Field(None, alias='last-partition-id') + sort_orders: list[SortOrder] | None = Field(None, alias='sort-orders') + default_sort_order_id: int | None = Field(None, alias='default-sort-order-id') + encryption_keys: list[EncryptedKey] | None = Field(None, alias='encryption-keys') snapshots: list[Snapshot] | None = None refs: SnapshotReferences | None = None current_snapshot_id: int | None = Field(None, alias='current-snapshot-id') @@ -1765,7 +1821,7 @@ class AddSchemaUpdate(BaseUpdate): ) -class LoadTableResult(BaseModel): +class LoadTableResultV2(BaseModel): """ Result used when a table is successfully loaded, for tables at any format version. @@ -1849,6 +1905,71 @@ class LoadTableResult(BaseModel): alias='table-location', description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", ) + metadata: TableMetadataV2 + config: dict[str, str] | None = None + storage_credentials: list[StorageCredential] | None = Field( + None, alias='storage-credentials' + ) + remote_signing_config: RemoteSigningConfig | None = Field( + None, alias='remote-signing-config' + ) + + +class LoadTableResult(BaseModel): + """ + Result used when a table is successfully loaded. + + + The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. + Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + + + The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: + - `client`: Clients MUST use client-side scan planning + - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint + + ## AWS Configurations + + The following configurations should be respected when working with tables stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document. + - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + + ## Storage Credentials + + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + + ## Remote Signing + + If remote signing for a specific storage provider is enabled, the server SHOULD use the `remote-signing-config` + field to communicate all signer client settings. When the `remote-signing-config` field is present, clients + SHOULD respect the provided configuration. + + For backward compatibility, the following `config` properties are still supported but **DEPRECATED** and SHOULD NOT be used by clients able to consume the remote signing configuration: + - `signer.endpoint` **DEPRECATED**.: the remote signer endpoint. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI. + - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. + If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact. + If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI. + + """ + + metadata_location: str | None = Field( + None, + alias='metadata-location', + description='May be null if the table is staged as part of a transaction', + ) metadata: TableMetadata config: dict[str, str] | None = None storage_credentials: list[StorageCredential] | None = Field( @@ -1922,7 +2043,7 @@ class CreateTableRequest(BaseModel): properties: dict[str, str] | None = None -class UnregisterTableResult(BaseModel): +class UnregisterTableResultV2(BaseModel): """ Last metadata location and the corresponding table metadata for the table that was successfully unregistered and is no longer tracked by the catalog. """ @@ -1937,6 +2058,19 @@ class UnregisterTableResult(BaseModel): alias='table-location', description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", ) + metadata: TableMetadataV2 + + +class UnregisterTableResult(BaseModel): + """ + Last metadata location and the corresponding table metadata for the table that was successfully unregistered and is no longer tracked by the catalog. + """ + + metadata_location: str = Field( + ..., + alias='metadata-location', + description='The last metadata location for the table at the time it was unregistered.', + ) metadata: TableMetadata @@ -2121,7 +2255,7 @@ class FunctionStructField(BaseModel): type: FunctionDataType -class CommitTableResponse(BaseModel): +class CommitTableResponseV2(BaseModel): """ Result used when a table is successfully updated, for tables at any format version. @@ -2140,6 +2274,11 @@ class CommitTableResponse(BaseModel): alias='table-location', description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", ) + metadata: TableMetadataV2 + + +class CommitTableResponse(BaseModel): + metadata_location: str = Field(..., alias='metadata-location') metadata: TableMetadata @@ -2407,6 +2546,7 @@ class PlanTableScanResult( ComparisonPredicate.model_rebuild() SetPredicate.model_rebuild() Apply.model_rebuild() +TableMetadataV2.model_rebuild() TableMetadata.model_rebuild() ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 124945d78ce7..ad55d94322c3 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -522,7 +522,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v2/{prefix}/namespaces/{namespace}/tables: + /v1/{prefix}/namespaces/{namespace}/tables: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -622,6 +622,106 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' + /v2/{prefix}/namespaces/{namespace}/tables: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + get: + tags: + - Catalog API + summary: List all table identifiers underneath a given namespace + description: Return all table identifiers under this namespace + operationId: listTablesV2 + parameters: + - $ref: '#/components/parameters/page-token' + - $ref: '#/components/parameters/page-size' + responses: + 200: + $ref: '#/components/responses/ListTablesResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Create a table in the given namespace + description: + Create a table or start a create transaction, like atomic CTAS. + + + If `stage-create` is false, the table is created immediately. + + + If `stage-create` is true, the table is not created, but table metadata is initialized and returned. + The service should prepare as needed for a commit to the table commit endpoint to complete the create + transaction. The client uses the returned metadata to begin a transaction. To commit the transaction, + the client sends all create and subsequent changes to the table commit route. Changes from the table + create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial + table state. + operationId: createTableV2 + parameters: + - $ref: '#/components/parameters/data-access' + - $ref: '#/components/parameters/idempotency-key' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateTableRequest' + responses: + 200: + $ref: '#/components/responses/CreateTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The identifier already exists as a table or view + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableAlreadyExists: + $ref: '#/components/examples/TableAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/namespaces/{namespace}/functions: parameters: - $ref: '#/components/parameters/prefix' @@ -968,7 +1068,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v2/{prefix}/namespaces/{namespace}/register: + /v1/{prefix}/namespaces/{namespace}/register: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1024,7 +1124,63 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v2/{prefix}/namespaces/{namespace}/tables/{table}: + /v2/{prefix}/namespaces/{namespace}/register: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + post: + tags: + - Catalog API + summary: Register a table in the given namespace using given metadata file location + parameters: + - $ref: '#/components/parameters/data-access' + - $ref: '#/components/parameters/idempotency-key' + description: + Register a table using given metadata file location. + + operationId: registerTableV2 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterTableRequest' + responses: + 200: + $ref: '#/components/responses/LoadTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The identifier already exists as a table or view + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableAlreadyExists: + $ref: '#/components/examples/TableAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables/{table}: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1299,7 +1455,282 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v2/{prefix}/namespaces/{namespace}/tables/{table}/unregister: + /v2/{prefix}/namespaces/{namespace}/tables/{table}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + get: + tags: + - Catalog API + summary: Load a table from the catalog + operationId: loadTableV2 + description: + Load a table from the catalog. + + + The response contains both configuration and table metadata. The configuration, if non-empty is used + as additional configuration for the table that overrides catalog configuration. For example, this + configuration may change the FileIO implementation to be used for the table. + + + The response also contains the table's full metadata, matching the table metadata JSON file. + + + The catalog configuration may contain credentials that should be used for subsequent requests for the + table. The configuration key "token" is used to pass an access token to be used as a bearer token + for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration + key. For example, "urn:ietf:params:oauth:token-type:jwt=". + parameters: + - $ref: '#/components/parameters/data-access' + - name: If-None-Match + in: header + description: + An optional header that allows the server to return 304 (Not Modified) if the metadata + is current. The content is the value of the ETag received in a CreateTableResponse, + LoadTableResponse or CommitTableResponse. + required: false + schema: + type: string + - in: query + name: snapshots + description: + The snapshots to return in the body of the metadata via the `snapshots` field. Setting + the value to `all` would return the full set of snapshots currently valid for the table. + Setting the value to `refs` would load all snapshots referenced by branches or tags. + + Default if no param is provided is `all`. + required: false + schema: + type: string + enum: [ all, refs ] + - $ref: '#/components/parameters/referenced-by' + responses: + 200: + $ref: '#/components/responses/LoadTableResponseV2' + 304: + description: + Not Modified - Based on the content of the 'If-None-Match' header the table metadata has + not changed since. + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Commit updates to a table + operationId: updateTableV2 + parameters: + - $ref: '#/components/parameters/idempotency-key' + description: + Commit updates to a table. + + + Commits have two parts, requirements and updates. Requirements are assertions that will be validated + before attempting to make and commit changes. For example, `assert-ref-snapshot-id` will check that a + named ref's snapshot ID has a certain value. + Server implementations are required to fail with a 400 status code + if any unknown updates or requirements are received. + + + Updates are changes to make to table metadata. For example, after asserting that the current main ref + is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new + snapshot id. + + + Create table transactions that are started by createTable with `stage-create` set to true are + committed using this route. Transactions should include all changes to the table, including table + initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is + used to ensure that the table was not created concurrently. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CommitTableRequest' + responses: + 200: + $ref: '#/components/responses/CommitTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToUpdateDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 409: + description: + Conflict - CommitFailedException, one or more requirements failed. The client may retry. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 500: + description: + An unknown server-side problem occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Internal Server Error", + "type": "CommitStateUnknownException", + "code": 500 + } + } + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 502: + description: + A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Invalid response from the upstream server", + "type": "CommitStateUnknownException", + "code": 502 + } + } + 504: + description: + A server-side gateway timeout occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Gateway timed out during commit", + "type": "CommitStateUnknownException", + "code": 504 + } + } + 5XX: + description: + A server-side problem that might not be addressable on the client. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Bad Gateway", + "type": "InternalServerError", + "code": 502 + } + } + + delete: + tags: + - Catalog API + summary: Drop a table from the catalog + operationId: dropTableV2 + description: Remove a table from the catalog + parameters: + - $ref: '#/components/parameters/idempotency-key' + - name: purgeRequested + in: query + required: false + description: Whether the user requested to purge the underlying table's data and metadata + schema: + type: boolean + default: false + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, Table to drop does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToDeleteDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + head: + tags: + - Catalog API + summary: Check if a table exists + operationId: tableExistsV2 + description: + Check if a table exists within a given namespace. The response does not contain a body. + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, Table not found + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables/{table}/unregister: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1349,6 +1780,56 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' + /v2/{prefix}/namespaces/{namespace}/tables/{table}/unregister: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + post: + tags: + - Catalog API + summary: Unregister a table without removing its data or metadata files + operationId: unregisterTableV2 + parameters: + - $ref: '#/components/parameters/idempotency-key' + description: + Unregister a table from the catalog. This is the opposite of + `registerTable`. The table no longer exists in the catalog, but the + underlying data and metadata files are left in place so that the table + can be registered in another catalog. + + + On success, this returns the table's last metadata location and the + corresponding table metadata. This table metadata must include all + commits that happened before the unregister operation. All attempted + commits after the unregister operation in this catalog must fail. + responses: + 200: + $ref: '#/components/responses/UnregisterTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to unregister does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToUnregisterDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials: parameters: - $ref: '#/components/parameters/prefix' @@ -2946,27 +3427,79 @@ components: contentEncoding: base64 encrypted-by-id: type: string - properties: + properties: + type: object + additionalProperties: + type: string + + SnapshotV2: + description: | + A snapshot of the table's contents at a point in time. + + + Format versions 1-3 use `manifest-list` and format version 4 uses `content-root`. + type: object + required: + - snapshot-id + - timestamp-ms + - summary + oneOf: + - required: + - manifest-list + - required: + - content-root + properties: + snapshot-id: + type: integer + format: int64 + parent-snapshot-id: + type: integer + format: int64 + sequence-number: + type: integer + format: int64 + timestamp-ms: + type: integer + format: int64 + manifest-list: + type: string + description: + Location of the snapshot's manifest list file. + Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead. + content-root: + type: string + description: + Location of the snapshot's content root. + Required for format version 4 and must be absent for format versions 1-3. + first-row-id: + type: integer + format: int64 + description: The first _row_id assigned to the first row in the first data file in the first manifest + added-rows: + type: integer + format: int64 + description: The upper bound of the number of rows with assigned row IDs + summary: type: object + required: + - operation + properties: + operation: + type: string + enum: ["append", "replace", "overwrite", "delete"] additionalProperties: type: string - - Snapshot: - description: | - A snapshot of the table's contents at a point in time. + schema-id: + type: integer - Format versions 1-3 use `manifest-list` and format version 4 uses `content-root`. + Snapshot: type: object required: - snapshot-id - timestamp-ms + - manifest-list - summary - oneOf: - - required: - - manifest-list - - required: - - content-root properties: snapshot-id: type: integer @@ -2982,14 +3515,7 @@ components: format: int64 manifest-list: type: string - description: - Location of the snapshot's manifest list file. - Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead. - content-root: - type: string - description: - Location of the snapshot's content root. - Required for format version 4 and must be absent for format versions 1-3. + description: Location of the snapshot's manifest list file first-row-id: type: integer format: int64 @@ -3066,7 +3592,7 @@ components: type: integer format: int64 - TableMetadata: + TableMetadataV2: type: object required: - format-version @@ -3127,6 +3653,90 @@ components: items: $ref: '#/components/schemas/EncryptedKey' # snapshot tracking + snapshots: + type: array + items: + $ref: '#/components/schemas/SnapshotV2' + refs: + $ref: '#/components/schemas/SnapshotReferences' + current-snapshot-id: + type: integer + format: int64 + last-sequence-number: + type: integer + format: int64 + # logs + snapshot-log: + $ref: '#/components/schemas/SnapshotLog' + metadata-log: + $ref: '#/components/schemas/MetadataLog' + # statistics + statistics: + type: array + items: + $ref: '#/components/schemas/StatisticsFile' + partition-statistics: + type: array + items: + $ref: '#/components/schemas/PartitionStatisticsFile' + + + TableMetadata: + type: object + required: + - format-version + - table-uuid + properties: + format-version: + type: integer + minimum: 1 + maximum: 3 + table-uuid: + type: string + location: + type: string + last-updated-ms: + type: integer + format: int64 + next-row-id: + type: integer + format: int64 + description: A long higher than all assigned row IDs; the next snapshot's first-row-id. + properties: + type: object + additionalProperties: + type: string + # schema tracking + schemas: + type: array + items: + $ref: '#/components/schemas/Schema' + current-schema-id: + type: integer + last-column-id: + type: integer + # partition spec tracking + partition-specs: + type: array + items: + $ref: '#/components/schemas/PartitionSpec' + default-spec-id: + type: integer + last-partition-id: + type: integer + # sort order tracking + sort-orders: + type: array + items: + $ref: '#/components/schemas/SortOrder' + default-sort-order-id: + type: integer + # encryption + encryption-keys: + type: array + items: + $ref: '#/components/schemas/EncryptedKey' + # snapshot tracking snapshots: type: array items: @@ -3417,7 +4027,7 @@ components: type: string const: "add-snapshot" snapshot: - $ref: '#/components/schemas/Snapshot' + $ref: '#/components/schemas/SnapshotV2' SetSnapshotRefUpdate: allOf: @@ -3874,7 +4484,7 @@ components: items: $ref: '#/components/schemas/StorageCredential' - LoadTableResult: + LoadTableResultV2: description: | Result used when a table is successfully loaded, for tables at any format version. @@ -3960,6 +4570,76 @@ components: The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`. + metadata: + $ref: '#/components/schemas/TableMetadataV2' + config: + type: object + additionalProperties: + type: string + storage-credentials: + type: array + items: + $ref: '#/components/schemas/StorageCredential' + remote-signing-config: + $ref: '#/components/schemas/RemoteSigningConfig' + + + LoadTableResult: + description: | + Result used when a table is successfully loaded. + + + The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. + Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + + + The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: + - `client`: Clients MUST use client-side scan planning + - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint + + ## AWS Configurations + + The following configurations should be respected when working with tables stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document. + - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + + ## Storage Credentials + + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + + ## Remote Signing + + If remote signing for a specific storage provider is enabled, the server SHOULD use the `remote-signing-config` + field to communicate all signer client settings. When the `remote-signing-config` field is present, clients + SHOULD respect the provided configuration. + + For backward compatibility, the following `config` properties are still supported but **DEPRECATED** and SHOULD NOT be used by clients able to consume the remote signing configuration: + - `signer.endpoint` **DEPRECATED**.: the remote signer endpoint. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI. + - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. + If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact. + If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI. + + type: object + required: + - metadata + properties: + metadata-location: + type: string + description: May be null if the table is staged as part of a transaction + nullable: true metadata: $ref: '#/components/schemas/TableMetadata' config: @@ -4204,7 +4884,7 @@ components: type: boolean default: false - UnregisterTableResult: + UnregisterTableResultV2: description: Last metadata location and the corresponding table metadata for the table that was successfully unregistered and is no longer tracked by @@ -4224,6 +4904,24 @@ components: The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`. + metadata: + $ref: '#/components/schemas/TableMetadataV2' + + + UnregisterTableResult: + description: + Last metadata location and the corresponding table metadata for the + table that was successfully unregistered and is no longer tracked by + the catalog. + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string + description: + The last metadata location for the table at the time it was unregistered. metadata: $ref: '#/components/schemas/TableMetadata' @@ -4989,7 +5687,7 @@ components: Server's do not need to implement this. nullable: true - CommitTableResponse: + CommitTableResponseV2: description: | Result used when a table is successfully updated, for tables at any format version. @@ -5010,6 +5708,18 @@ components: The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`. + metadata: + $ref: '#/components/schemas/TableMetadataV2' + + + CommitTableResponse: + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string metadata: $ref: '#/components/schemas/TableMetadata' @@ -5783,6 +6493,17 @@ components: "missing": [ "bar" ] } + CreateTableResponseV2: + description: Table metadata result after creating a table + content: + application/json: + schema: + $ref: '#/components/schemas/LoadTableResultV2' + headers: + etag: + $ref: '#/components/parameters/etag' + + CreateTableResponse: description: Table metadata result after creating a table content: @@ -5814,6 +6535,17 @@ components: schema: $ref: '#/components/schemas/FetchScanTasksResult' + LoadTableResponseV2: + description: Table metadata result when loading a table + content: + application/json: + schema: + $ref: '#/components/schemas/LoadTableResultV2' + headers: + etag: + $ref: '#/components/parameters/etag' + + LoadTableResponse: description: Table metadata result when loading a table content: @@ -5824,6 +6556,14 @@ components: etag: $ref: '#/components/parameters/etag' + UnregisterTableResponseV2: + description: Response when a table is successfully unregistered. + content: + application/json: + schema: + $ref: '#/components/schemas/UnregisterTableResultV2' + + UnregisterTableResponse: description: Response when a table is successfully unregistered. content: @@ -5845,6 +6585,20 @@ components: schema: $ref: '#/components/schemas/LoadViewResult' + CommitTableResponseV2: + description: + Response used when a table is successfully updated. + + The table metadata JSON is returned in the metadata field. The corresponding file location of table metadata must be returned in the metadata-location field. Clients can check whether metadata has changed by comparing metadata locations. + content: + application/json: + schema: + $ref: '#/components/schemas/CommitTableResponseV2' + headers: + etag: + $ref: '#/components/parameters/etag' + + CommitTableResponse: description: Response used when a table is successfully updated.