diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5e3580ae3825..9a18e75e2a72 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -56,14 +56,16 @@ class CatalogConfig(BaseModel): ) endpoints: list[str] | None = Field( None, - description='A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character.', + description='A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character. Table endpoints are available under `/v1` and `/v2`. Clients that understand format version 4 should prefer the `/v2` APIs when available.', examples=[ [ 'GET /v1/{prefix}/namespaces/{namespace}', 'GET /v1/{prefix}/namespaces', 'POST /v1/{prefix}/namespaces', - 'GET /v1/{prefix}/namespaces/{namespace}/tables/{table}', 'GET /v1/{prefix}/namespaces/{namespace}/views/{view}', + 'GET /v2/{prefix}/namespaces/{namespace}/tables/{table}', + 'POST /v2/{prefix}/namespaces/{namespace}/tables/{table}', + 'POST /v2/{prefix}/namespaces/{namespace}/register', ] ], ) @@ -330,6 +332,87 @@ class Summary(BaseModel): operation: Literal['append', 'replace', 'overwrite', 'delete'] +class SnapshotV21(BaseModel): + """ + A snapshot of the table's contents at a point in time. + + + Format versions 1-3 use `manifest-list` and format version 4 uses `root-manifest`. + + """ + + snapshot_id: int = Field(..., alias='snapshot-id') + parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') + sequence_number: int | None = Field(None, alias='sequence-number') + timestamp_ms: int = Field(..., alias='timestamp-ms') + manifest_list: str = Field( + ..., + alias='manifest-list', + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `root-manifest` instead.", + ) + root_manifest: str | None = Field( + None, + alias='root-manifest', + description="Location of the snapshot's root manifest. Required for format version 4 and must be absent for format versions 1-3.", + ) + first_row_id: int | None = Field( + None, + alias='first-row-id', + description='The first _row_id assigned to the first row in the first data file in the first manifest', + ) + added_rows: int | None = Field( + None, + alias='added-rows', + description='The upper bound of the number of rows with assigned row IDs', + ) + summary: Summary + schema_id: int | None = Field(None, alias='schema-id') + + +class SnapshotV22(BaseModel): + """ + A snapshot of the table's contents at a point in time. + + + Format versions 1-3 use `manifest-list` and format version 4 uses `root-manifest`. + + """ + + snapshot_id: int = Field(..., alias='snapshot-id') + parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') + sequence_number: int | None = Field(None, alias='sequence-number') + timestamp_ms: int = Field(..., alias='timestamp-ms') + manifest_list: str | None = Field( + None, + alias='manifest-list', + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `root-manifest` instead.", + ) + root_manifest: str = Field( + ..., + alias='root-manifest', + description="Location of the snapshot's root manifest. Required for format version 4 and must be absent for format versions 1-3.", + ) + first_row_id: int | None = Field( + None, + alias='first-row-id', + description='The first _row_id assigned to the first row in the first data file in the first manifest', + ) + added_rows: int | None = Field( + None, + alias='added-rows', + description='The upper bound of the number of rows with assigned row IDs', + ) + summary: Summary + schema_id: int | None = Field(None, alias='schema-id') + + +class SnapshotV2(RootModel[SnapshotV21 | SnapshotV22]): + root: SnapshotV21 | SnapshotV22 = Field( + ..., + description="A snapshot of the table's contents at a point in time.\n\n\nFormat versions 1-3 use `manifest-list` and format version 4 uses `root-manifest`.\n", + ) + + class Snapshot(BaseModel): snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') @@ -470,7 +553,7 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): action: Literal['add-snapshot'] - snapshot: Snapshot + snapshot: SnapshotV2 class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): @@ -1649,6 +1732,41 @@ class Apply(BaseModel): arguments: list[FunctionArgument] +class TableMetadataV2(BaseModel): + format_version: int = Field(..., alias='format-version', ge=1, le=4) + table_uuid: str = Field(..., alias='table-uuid') + location: str | None = Field( + None, + description="The table's base location. Required through format version 3, where it may be a path without a URI scheme; readers prepend a scheme for consistency with v4 absolute paths. Optional for format version 4, where the location may be managed externally and supplied by the catalog when the table is loaded, and where it must be an absolute path when present. See the `table-location` field of `LoadTableResult`.", + ) + last_updated_ms: int | None = Field(None, alias='last-updated-ms') + next_row_id: int | None = Field( + None, + alias='next-row-id', + description="A long higher than all assigned row IDs; the next snapshot's first-row-id.", + ) + properties: dict[str, str] | None = None + schemas: list[Schema] | None = None + current_schema_id: int | None = Field(None, alias='current-schema-id') + last_column_id: int | None = Field(None, alias='last-column-id') + partition_specs: list[PartitionSpec] | None = Field(None, alias='partition-specs') + default_spec_id: int | None = Field(None, alias='default-spec-id') + last_partition_id: int | None = Field(None, alias='last-partition-id') + sort_orders: list[SortOrder] | None = Field(None, alias='sort-orders') + default_sort_order_id: int | None = Field(None, alias='default-sort-order-id') + encryption_keys: list[EncryptedKey] | None = Field(None, alias='encryption-keys') + snapshots: list[SnapshotV2] | None = None + refs: SnapshotReferences | None = None + current_snapshot_id: int | None = Field(None, alias='current-snapshot-id') + last_sequence_number: int | None = Field(None, alias='last-sequence-number') + snapshot_log: SnapshotLog | None = Field(None, alias='snapshot-log') + metadata_log: MetadataLog | None = Field(None, alias='metadata-log') + statistics: list[StatisticsFile] | None = None + partition_statistics: list[PartitionStatisticsFile] | None = Field( + None, alias='partition-statistics' + ) + + class TableMetadata(BaseModel): format_version: int = Field(..., alias='format-version', ge=1, le=3) table_uuid: str = Field(..., alias='table-uuid') @@ -1703,6 +1821,100 @@ class AddSchemaUpdate(BaseUpdate): ) +class LoadTableResultV2(BaseModel): + """ + Result used when a table is successfully loaded, for tables at any format version. + + + The table metadata JSON is returned in the `metadata` field. + The location of the table metadata file is returned in the `metadata-location` field when the table has one, and the table's base location in the `table-location` field. + + + ## Metadata location + + + The `metadata-location` field is optional. + It is absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state and no metadata pointer need exist. + + + Clients must not require this field to be present, and must not use it to bypass the catalog for reads or commits. + To obtain a metadata location for a catalog-managed table, use the `unregisterTable` endpoint, which returns the table's last metadata location at the point the table leaves catalog control and further commits are rejected. + + + ## Table location + + + The `table-location` field carries the table's base location. + Format version 4 allows location fields in metadata to be relative, and such paths must be resolved against the table location. + Format version 4 also makes `metadata.location` optional, so a table may have metadata that contains relative paths and omits `location`. + Servers must populate `table-location` for any such table, because it cannot be read otherwise. + + + When both `metadata.location` and `table-location` are present, `table-location` takes precedence: + the catalog is authoritative for table state. + + + The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: + - `client`: Clients MUST use client-side scan planning + - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint + + ## AWS Configurations + + The following configurations should be respected when working with tables stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document. + - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + + ## Storage Credentials + + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + + ## Remote Signing + + If remote signing for a specific storage provider is enabled, the server SHOULD use the `remote-signing-config` + field to communicate all signer client settings. When the `remote-signing-config` field is present, clients + SHOULD respect the provided configuration. + + For backward compatibility, the following `config` properties are still supported but **DEPRECATED** and SHOULD NOT be used by clients able to consume the remote signing configuration: + - `signer.endpoint` **DEPRECATED**.: the remote signer endpoint. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI. + - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. + If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact. + If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI. + + """ + + metadata_location: str | None = Field( + None, + alias='metadata-location', + description='Location of the table metadata file. Absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.', + ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", + ) + metadata: TableMetadataV2 + config: dict[str, str] | None = None + storage_credentials: list[StorageCredential] | None = Field( + None, alias='storage-credentials' + ) + remote_signing_config: RemoteSigningConfig | None = Field( + None, alias='remote-signing-config' + ) + + class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -1831,6 +2043,24 @@ class CreateTableRequest(BaseModel): properties: dict[str, str] | None = None +class UnregisterTableResultV2(BaseModel): + """ + Last metadata location and the corresponding table metadata for the table that was successfully unregistered and is no longer tracked by the catalog. + """ + + metadata_location: str = Field( + ..., + alias='metadata-location', + description='The last metadata location for the table at the time it was unregistered.', + ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", + ) + metadata: TableMetadataV2 + + class UnregisterTableResult(BaseModel): """ Last metadata location and the corresponding table metadata for the table that was successfully unregistered and is no longer tracked by the catalog. @@ -2025,6 +2255,28 @@ class FunctionStructField(BaseModel): type: FunctionDataType +class CommitTableResponseV2(BaseModel): + """ + Result used when a table is successfully updated, for tables at any format version. + + + The `table-location` field carries the table's base location, so that a client can resolve relative paths in the returned metadata. + + """ + + metadata_location: str | None = Field( + None, + alias='metadata-location', + description='Location of the committed table metadata file. Absent when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.', + ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", + ) + metadata: TableMetadataV2 + + class CommitTableResponse(BaseModel): metadata_location: str = Field(..., alias='metadata-location') metadata: TableMetadata @@ -2294,6 +2546,7 @@ class PlanTableScanResult( ComparisonPredicate.model_rebuild() SetPredicate.model_rebuild() Apply.model_rebuild() +TableMetadataV2.model_rebuild() TableMetadata.model_rebuild() ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e9a530a12804..1164e669d311 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -622,6 +622,106 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' + /v2/{prefix}/namespaces/{namespace}/tables: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + get: + tags: + - Catalog API + summary: List all table identifiers underneath a given namespace + description: Return all table identifiers under this namespace + operationId: listTablesV2 + parameters: + - $ref: '#/components/parameters/page-token' + - $ref: '#/components/parameters/page-size' + responses: + 200: + $ref: '#/components/responses/ListTablesResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Create a table in the given namespace + description: + Create a table or start a create transaction, like atomic CTAS. + + + If `stage-create` is false, the table is created immediately. + + + If `stage-create` is true, the table is not created, but table metadata is initialized and returned. + The service should prepare as needed for a commit to the table commit endpoint to complete the create + transaction. The client uses the returned metadata to begin a transaction. To commit the transaction, + the client sends all create and subsequent changes to the table commit route. Changes from the table + create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial + table state. + operationId: createTableV2 + parameters: + - $ref: '#/components/parameters/data-access' + - $ref: '#/components/parameters/idempotency-key' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateTableRequest' + responses: + 200: + $ref: '#/components/responses/CreateTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The identifier already exists as a table or view + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableAlreadyExists: + $ref: '#/components/examples/TableAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/namespaces/{namespace}/functions: parameters: - $ref: '#/components/parameters/prefix' @@ -1024,6 +1124,62 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' + /v2/{prefix}/namespaces/{namespace}/register: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + post: + tags: + - Catalog API + summary: Register a table in the given namespace using given metadata file location + parameters: + - $ref: '#/components/parameters/data-access' + - $ref: '#/components/parameters/idempotency-key' + description: + Register a table using given metadata file location. + + operationId: registerTableV2 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterTableRequest' + responses: + 200: + $ref: '#/components/responses/LoadTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The identifier already exists as a table or view + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableAlreadyExists: + $ref: '#/components/examples/TableAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/namespaces/{namespace}/tables/{table}: parameters: - $ref: '#/components/parameters/prefix' @@ -1299,6 +1455,281 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' + /v2/{prefix}/namespaces/{namespace}/tables/{table}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + get: + tags: + - Catalog API + summary: Load a table from the catalog + operationId: loadTableV2 + description: + Load a table from the catalog. + + + The response contains both configuration and table metadata. The configuration, if non-empty is used + as additional configuration for the table that overrides catalog configuration. For example, this + configuration may change the FileIO implementation to be used for the table. + + + The response also contains the table's full metadata, matching the table metadata JSON file. + + + The catalog configuration may contain credentials that should be used for subsequent requests for the + table. The configuration key "token" is used to pass an access token to be used as a bearer token + for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration + key. For example, "urn:ietf:params:oauth:token-type:jwt=". + parameters: + - $ref: '#/components/parameters/data-access' + - name: If-None-Match + in: header + description: + An optional header that allows the server to return 304 (Not Modified) if the metadata + is current. The content is the value of the ETag received in a CreateTableResponse, + LoadTableResponse or CommitTableResponse. + required: false + schema: + type: string + - in: query + name: snapshots + description: + The snapshots to return in the body of the metadata via the `snapshots` field. Setting + the value to `all` would return the full set of snapshots currently valid for the table. + Setting the value to `refs` would load all snapshots referenced by branches or tags. + + Default if no param is provided is `all`. + required: false + schema: + type: string + enum: [ all, refs ] + - $ref: '#/components/parameters/referenced-by' + responses: + 200: + $ref: '#/components/responses/LoadTableResponseV2' + 304: + description: + Not Modified - Based on the content of the 'If-None-Match' header the table metadata has + not changed since. + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Commit updates to a table + operationId: updateTableV2 + parameters: + - $ref: '#/components/parameters/idempotency-key' + description: + Commit updates to a table. + + + Commits have two parts, requirements and updates. Requirements are assertions that will be validated + before attempting to make and commit changes. For example, `assert-ref-snapshot-id` will check that a + named ref's snapshot ID has a certain value. + Server implementations are required to fail with a 400 status code + if any unknown updates or requirements are received. + + + Updates are changes to make to table metadata. For example, after asserting that the current main ref + is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new + snapshot id. + + + Create table transactions that are started by createTable with `stage-create` set to true are + committed using this route. Transactions should include all changes to the table, including table + initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is + used to ensure that the table was not created concurrently. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CommitTableRequest' + responses: + 200: + $ref: '#/components/responses/CommitTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToUpdateDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 409: + description: + Conflict - CommitFailedException, one or more requirements failed. The client may retry. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 500: + description: + An unknown server-side problem occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Internal Server Error", + "type": "CommitStateUnknownException", + "code": 500 + } + } + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 502: + description: + A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Invalid response from the upstream server", + "type": "CommitStateUnknownException", + "code": 502 + } + } + 504: + description: + A server-side gateway timeout occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Gateway timed out during commit", + "type": "CommitStateUnknownException", + "code": 504 + } + } + 5XX: + description: + A server-side problem that might not be addressable on the client. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Bad Gateway", + "type": "InternalServerError", + "code": 502 + } + } + + delete: + tags: + - Catalog API + summary: Drop a table from the catalog + operationId: dropTableV2 + description: Remove a table from the catalog + parameters: + - $ref: '#/components/parameters/idempotency-key' + - name: purgeRequested + in: query + required: false + description: Whether the user requested to purge the underlying table's data and metadata + schema: + type: boolean + default: false + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, Table to drop does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToDeleteDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + head: + tags: + - Catalog API + summary: Check if a table exists + operationId: tableExistsV2 + description: + Check if a table exists within a given namespace. The response does not contain a body. + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, Table not found + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/namespaces/{namespace}/tables/{table}/unregister: parameters: - $ref: '#/components/parameters/prefix' @@ -1349,6 +1780,56 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' + /v2/{prefix}/namespaces/{namespace}/tables/{table}/unregister: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + post: + tags: + - Catalog API + summary: Unregister a table without removing its data or metadata files + operationId: unregisterTableV2 + parameters: + - $ref: '#/components/parameters/idempotency-key' + description: + Unregister a table from the catalog. This is the opposite of + `registerTable`. The table no longer exists in the catalog, but the + underlying data and metadata files are left in place so that the table + can be registered in another catalog. + + + On success, this returns the table's last metadata location and the + corresponding table metadata. This table metadata must include all + commits that happened before the unregister operation. All attempted + commits after the unregister operation in this catalog must fail. + responses: + 200: + $ref: '#/components/responses/UnregisterTableResponseV2' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to unregister does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToUnregisterDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials: parameters: - $ref: '#/components/parameters/prefix' @@ -2305,12 +2786,16 @@ components: type: string description: A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character. + Table endpoints are available under `/v1` and `/v2`. + Clients that understand format version 4 should prefer the `/v2` APIs when available. example: [ "GET /v1/{prefix}/namespaces/{namespace}", "GET /v1/{prefix}/namespaces", "POST /v1/{prefix}/namespaces", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "GET /v1/{prefix}/namespaces/{namespace}/views/{view}" + "GET /v1/{prefix}/namespaces/{namespace}/views/{view}", + "GET /v2/{prefix}/namespaces/{namespace}/tables/{table}", + "POST /v2/{prefix}/namespaces/{namespace}/tables/{table}", + "POST /v2/{prefix}/namespaces/{namespace}/register" ] idempotency-key-lifetime: type: string @@ -2938,14 +3423,75 @@ components: type: string encrypted-key-metadata: type: string - format: byte # for compatibility - contentEncoding: base64 - encrypted-by-id: + format: byte # for compatibility + contentEncoding: base64 + encrypted-by-id: + type: string + properties: + type: object + additionalProperties: + type: string + + SnapshotV2: + description: | + A snapshot of the table's contents at a point in time. + + + Format versions 1-3 use `manifest-list` and format version 4 uses `root-manifest`. + type: object + required: + - snapshot-id + - timestamp-ms + - summary + oneOf: + - required: + - manifest-list + - required: + - root-manifest + properties: + snapshot-id: + type: integer + format: int64 + parent-snapshot-id: + type: integer + format: int64 + sequence-number: + type: integer + format: int64 + timestamp-ms: + type: integer + format: int64 + manifest-list: + type: string + description: + Location of the snapshot's manifest list file. + Used for format versions 1-3 and must be absent for format version 4, which uses `root-manifest` instead. + root-manifest: type: string - properties: + description: + Location of the snapshot's root manifest. + Required for format version 4 and must be absent for format versions 1-3. + first-row-id: + type: integer + format: int64 + description: The first _row_id assigned to the first row in the first data file in the first manifest + added-rows: + type: integer + format: int64 + description: The upper bound of the number of rows with assigned row IDs + summary: type: object + required: + - operation + properties: + operation: + type: string + enum: ["append", "replace", "overwrite", "delete"] additionalProperties: type: string + schema-id: + type: integer + Snapshot: type: object @@ -3046,6 +3592,95 @@ components: type: integer format: int64 + TableMetadataV2: + type: object + required: + - format-version + - table-uuid + properties: + format-version: + type: integer + minimum: 1 + maximum: 4 + table-uuid: + type: string + location: + type: string + description: + The table's base location. + Required through format version 3, where it may be a path without a URI scheme; readers prepend a scheme for consistency with v4 absolute paths. + Optional for format version 4, where the location may be managed externally and supplied by the catalog when the table is loaded, and where it must be an absolute path when present. + See the `table-location` field of `LoadTableResult`. + last-updated-ms: + type: integer + format: int64 + next-row-id: + type: integer + format: int64 + description: A long higher than all assigned row IDs; the next snapshot's first-row-id. + properties: + type: object + additionalProperties: + type: string + # schema tracking + schemas: + type: array + items: + $ref: '#/components/schemas/Schema' + current-schema-id: + type: integer + last-column-id: + type: integer + # partition spec tracking + partition-specs: + type: array + items: + $ref: '#/components/schemas/PartitionSpec' + default-spec-id: + type: integer + last-partition-id: + type: integer + # sort order tracking + sort-orders: + type: array + items: + $ref: '#/components/schemas/SortOrder' + default-sort-order-id: + type: integer + # encryption + encryption-keys: + type: array + items: + $ref: '#/components/schemas/EncryptedKey' + # snapshot tracking + snapshots: + type: array + items: + $ref: '#/components/schemas/SnapshotV2' + refs: + $ref: '#/components/schemas/SnapshotReferences' + current-snapshot-id: + type: integer + format: int64 + last-sequence-number: + type: integer + format: int64 + # logs + snapshot-log: + $ref: '#/components/schemas/SnapshotLog' + metadata-log: + $ref: '#/components/schemas/MetadataLog' + # statistics + statistics: + type: array + items: + $ref: '#/components/schemas/StatisticsFile' + partition-statistics: + type: array + items: + $ref: '#/components/schemas/PartitionStatisticsFile' + + TableMetadata: type: object required: @@ -3392,7 +4027,7 @@ components: type: string const: "add-snapshot" snapshot: - $ref: '#/components/schemas/Snapshot' + $ref: '#/components/schemas/SnapshotV2' SetSnapshotRefUpdate: allOf: @@ -3849,6 +4484,106 @@ components: items: $ref: '#/components/schemas/StorageCredential' + LoadTableResultV2: + description: | + Result used when a table is successfully loaded, for tables at any format version. + + + The table metadata JSON is returned in the `metadata` field. + The location of the table metadata file is returned in the `metadata-location` field when the table has one, and the table's base location in the `table-location` field. + + + ## Metadata location + + + The `metadata-location` field is optional. + It is absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state and no metadata pointer need exist. + + + Clients must not require this field to be present, and must not use it to bypass the catalog for reads or commits. + To obtain a metadata location for a catalog-managed table, use the `unregisterTable` endpoint, which returns the table's last metadata location at the point the table leaves catalog control and further commits are rejected. + + + ## Table location + + + The `table-location` field carries the table's base location. + Format version 4 allows location fields in metadata to be relative, and such paths must be resolved against the table location. + Format version 4 also makes `metadata.location` optional, so a table may have metadata that contains relative paths and omits `location`. + Servers must populate `table-location` for any such table, because it cannot be read otherwise. + + + When both `metadata.location` and `table-location` are present, `table-location` takes precedence: + the catalog is authoritative for table state. + + + The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: + - `client`: Clients MUST use client-side scan planning + - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint + + ## AWS Configurations + + The following configurations should be respected when working with tables stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document. + - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + + ## Storage Credentials + + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + + ## Remote Signing + + If remote signing for a specific storage provider is enabled, the server SHOULD use the `remote-signing-config` + field to communicate all signer client settings. When the `remote-signing-config` field is present, clients + SHOULD respect the provided configuration. + + For backward compatibility, the following `config` properties are still supported but **DEPRECATED** and SHOULD NOT be used by clients able to consume the remote signing configuration: + - `signer.endpoint` **DEPRECATED**.: the remote signer endpoint. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI. + - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. + If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact. + If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI. + type: object + required: + - metadata + properties: + metadata-location: + type: string + description: + Location of the table metadata file. + Absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. + Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. + Takes precedence over `metadata.location`. + metadata: + $ref: '#/components/schemas/TableMetadataV2' + config: + type: object + additionalProperties: + type: string + storage-credentials: + type: array + items: + $ref: '#/components/schemas/StorageCredential' + remote-signing-config: + $ref: '#/components/schemas/RemoteSigningConfig' + + LoadTableResult: description: | Result used when a table is successfully loaded. @@ -4149,6 +4884,30 @@ components: type: boolean default: false + UnregisterTableResultV2: + description: + Last metadata location and the corresponding table metadata for the + table that was successfully unregistered and is no longer tracked by + the catalog. + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string + description: + The last metadata location for the table at the time it was unregistered. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. + Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. + Takes precedence over `metadata.location`. + metadata: + $ref: '#/components/schemas/TableMetadataV2' + + UnregisterTableResult: description: Last metadata location and the corresponding table metadata for the @@ -4928,6 +5687,31 @@ components: Server's do not need to implement this. nullable: true + CommitTableResponseV2: + description: | + Result used when a table is successfully updated, for tables at any format version. + + + The `table-location` field carries the table's base location, so that a client can resolve relative paths in the returned metadata. + type: object + required: + - metadata + properties: + metadata-location: + type: string + description: + Location of the committed table metadata file. + Absent when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. + Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. + Takes precedence over `metadata.location`. + metadata: + $ref: '#/components/schemas/TableMetadataV2' + + CommitTableResponse: type: object required: @@ -5709,6 +6493,17 @@ components: "missing": [ "bar" ] } + CreateTableResponseV2: + description: Table metadata result after creating a table + content: + application/json: + schema: + $ref: '#/components/schemas/LoadTableResultV2' + headers: + etag: + $ref: '#/components/parameters/etag' + + CreateTableResponse: description: Table metadata result after creating a table content: @@ -5740,6 +6535,17 @@ components: schema: $ref: '#/components/schemas/FetchScanTasksResult' + LoadTableResponseV2: + description: Table metadata result when loading a table + content: + application/json: + schema: + $ref: '#/components/schemas/LoadTableResultV2' + headers: + etag: + $ref: '#/components/parameters/etag' + + LoadTableResponse: description: Table metadata result when loading a table content: @@ -5750,6 +6556,14 @@ components: etag: $ref: '#/components/parameters/etag' + UnregisterTableResponseV2: + description: Response when a table is successfully unregistered. + content: + application/json: + schema: + $ref: '#/components/schemas/UnregisterTableResultV2' + + UnregisterTableResponse: description: Response when a table is successfully unregistered. content: @@ -5771,6 +6585,20 @@ components: schema: $ref: '#/components/schemas/LoadViewResult' + CommitTableResponseV2: + description: + Response used when a table is successfully updated. + + The table metadata JSON is returned in the metadata field. The corresponding file location of table metadata must be returned in the metadata-location field. Clients can check whether metadata has changed by comparing metadata locations. + content: + application/json: + schema: + $ref: '#/components/schemas/CommitTableResponseV2' + headers: + etag: + $ref: '#/components/parameters/etag' + + CommitTableResponse: description: Response used when a table is successfully updated.