diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml deleted file mode 100644 index 61f9fa17..00000000 --- a/.github/workflows/beta-release.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Public Beta Release - -on: - push: - tags: '*.*.*b*' - paths-ignore: - - "*/setup.py" - - "*.yml" - - "*.md" - - "*/skyflow/utils/_version.py" - -jobs: - build-and-deploy: - strategy: - matrix: - include: - - variant: v2 - package-name: skyflow - tag-prefix: '' - - variant: flowvault - package-name: skyflow_flowvault - tag-prefix: 'flowvault-' - if: (matrix.variant == 'flowvault' && startsWith(github.ref_name, 'flowvault-')) || (matrix.variant == 'v2' && !startsWith(github.ref_name, 'flowvault-')) - uses: ./.github/workflows/shared-build-and-deploy.yml - with: - ref: ${{ github.ref_name }} - tag: 'beta' - variant: ${{ matrix.variant }} - package-name: ${{ matrix.package-name }} - tag-prefix: ${{ matrix.tag-prefix }} - secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index e82cabd0..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: CI Checks - -on: [pull_request] - -jobs: - check-commit-message: - name: Check Commit Message - runs-on: ubuntu-latest - steps: - - name: Check JIRA ID - uses: gsactions/commit-message-checker@v1 - with: - pattern: '(\[?[A-Z]{1,5}-[1-9][0-9]*)|(\[AUTOMATED\])|(Merge)|(Release).+$' - flags: 'gm' - excludeDescription: 'true' - checkAllCommitMessages: 'true' - accessToken: ${{ secrets.PAT_ACTIONS }} - error: 'One of your your commit messages is not matching the format with JIRA ID Ex: ( SDK-123 commit message )' - - test: - strategy: - fail-fast: false - matrix: - include: - - variant: v2 - package-name: skyflow - coverage-omit: "skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/detect/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py" - - variant: flowvault - package-name: skyflow_flowvault - coverage-omit: "skyflow_flowvault/generated/*" - uses: ./.github/workflows/shared-tests.yml - with: - python-version: '3.9' - variant: ${{ matrix.variant }} - package-name: ${{ matrix.package-name }} - coverage-omit: ${{ matrix.coverage-omit }} - secrets: inherit - - test-common: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - run: pip install -e ./common - - run: pip install coverage - - run: python -m coverage run --source=common --omit="common/generated/*,common/tests/*" -m unittest discover -s common/tests -t . - - run: coverage xml -o test-coverage.xml - - name: Codecov - uses: codecov/codecov-action@v2.1.0 - with: - token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} - files: test-coverage.xml - name: codecov-skyflow-python-common - verbose: true diff --git a/.github/workflows/internal-release.yml b/.github/workflows/internal-release.yml index b91e5b41..75f8f580 100644 --- a/.github/workflows/internal-release.yml +++ b/.github/workflows/internal-release.yml @@ -1,37 +1,53 @@ -name: Internal Release +name: Publish module to the JFrog Artifactory on: push: + # '**' not '*.*': Actions glob '*' does not match '/', so '*.*' let slash + # tags (flowvault/v1.0.0) through and fired this branch-only workflow. tags-ignore: - - '*.*' + - '**' paths-ignore: - - "*/setup.py" - - "*.yml" - "*.md" - - "*/skyflow/utils/_version.py" - - "samples/**" - - "flowvault/samples/**" branches: - - release/* - flowvault-release/* + - skyvault-release/* + # Legacy: predates the per-module naming, still maps to skyvault. + - release/* jobs: + resolve-module: + runs-on: ubuntu-latest + # Skip our own bump commit, or this loops: bump -> push -> release -> bump. + # PAT-authenticated pushes DO trigger workflows; GITHUB_TOKEN pushes do not. + # build-and-deploy needs this job, so skipping here skips the run. + if: ${{ !contains(github.event.head_commit.message, '[AUTOMATED]') }} + outputs: + module: ${{ steps.set-module.outputs.module }} + steps: + # Explicit match, no catch-all: defaulting once published the wrong module. + - name: Resolve module from branch name + id: set-module + env: + BRANCH: ${{ github.ref_name }} + run: | + case "$BRANCH" in + flowvault-release/*) MODULE="flowvault" ;; + skyvault-release/*) MODULE="skyvault" ;; + release/*) MODULE="skyvault" ;; + *) + echo "::error::Branch '$BRANCH' does not map to a module." + exit 1 + ;; + esac + echo "Branch '$BRANCH' -> module '$MODULE'" + echo "module=$MODULE" >> "$GITHUB_OUTPUT" + build-and-deploy: - strategy: - matrix: - include: - - variant: v2 - package-name: skyflow - tag-prefix: '' - - variant: flowvault - package-name: skyflow_flowvault - tag-prefix: 'flowvault-' - if: (matrix.variant == 'flowvault' && startsWith(github.ref_name, 'flowvault-')) || (matrix.variant == 'v2' && !startsWith(github.ref_name, 'flowvault-')) + needs: resolve-module uses: ./.github/workflows/shared-build-and-deploy.yml with: ref: ${{ github.ref_name }} tag: 'internal' - variant: ${{ matrix.variant }} - package-name: ${{ matrix.package-name }} - tag-prefix: ${{ matrix.tag-prefix }} + module: ${{ needs.resolve-module.outputs.module }} secrets: inherit + \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de816ac0..01b8c040 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,39 +7,7 @@ on: jobs: test: - strategy: - fail-fast: false - matrix: - include: - - variant: v2 - package-name: skyflow - coverage-omit: "skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/detect/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py" - - variant: flowvault - package-name: skyflow_flowvault - coverage-omit: "skyflow_flowvault/generated/*" uses: ./.github/workflows/shared-tests.yml with: python-version: '3.9' - variant: ${{ matrix.variant }} - package-name: ${{ matrix.package-name }} - coverage-omit: ${{ matrix.coverage-omit }} secrets: inherit - - test-common: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - run: pip install -e ./common - - run: pip install coverage - - run: python -m coverage run --source=common --omit="common/generated/*,common/tests/*" -m unittest discover -s common/tests -t . - - run: coverage xml -o test-coverage.xml - - name: Codecov - uses: codecov/codecov-action@v2.1.0 - with: - token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} - files: test-coverage.xml - name: codecov-skyflow-python-common - verbose: true diff --git a/.github/workflows/pr-flowvault.yml b/.github/workflows/pr-flowvault.yml new file mode 100644 index 00000000..0ff12416 --- /dev/null +++ b/.github/workflows/pr-flowvault.yml @@ -0,0 +1,84 @@ +name: PR CI Checks (flowvault) + +# flowvault is a folder under main, alongside skyvault - not a branch. +# This workflow fires for PRs targeting main or a flowvault-release/* branch +# that actually touch flowvault or its common dependency, and only builds/ +# tests those two modules. skyvault (and the full 3-module suite) is covered +# by pr.yml, not here. + +on: + pull_request: + branches: [ "main", "flowvault-release/**" ] + paths: + - "flowvault/**" + - "common/**" + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + + # flowvault depends on common as a local path dependency, so common + # must be built and installed first or flowvault's own build/install + # will fail to resolve it. + - name: Build and install common + run: | + python -m pip install --upgrade pip setuptools wheel + cd common + python setup.py sdist bdist_wheel + pip install dist/*.whl + + - name: Build flowvault + run: | + cd flowvault + python setup.py sdist bdist_wheel + + test: + name: Unit Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "credentials.json" + json: ${{ secrets.VALID_SKYFLOW_CREDS_TEST }} + + - name: Run flowvault unit tests + run: | + python -m pip install --upgrade pip setuptools wheel coverage + cp credentials.json flowvault/credentials.json + + # flowvault depends on common as a local path dependency. + cd common + python setup.py sdist bdist_wheel + pip install dist/*.whl + cd .. + + cd flowvault + python setup.py sdist bdist_wheel + pip install dist/*.whl + if [ -f requirements.txt ]; then + pip install -r requirements.txt + fi + python -m coverage run --source=. -m unittest discover + coverage xml -o test-coverage.xml + + - name: Codecov + uses: codecov/codecov-action@v2.1.0 + with: + token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} + files: flowvault/test-coverage.xml + flags: flowvault + name: codecov-skyflow-python-flowvault + verbose: true diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 00000000..e3ddc05d --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,24 @@ +name: PR CI Checks + +on: [pull_request] + +jobs: + check-commit-message: + name: Check Commit Message + runs-on: ubuntu-latest + steps: + - name: Check JIRA ID + uses: gsactions/commit-message-checker@v1 + with: + pattern: '(\[?[A-Z]{1,5}-[1-9][0-9]*)|(\[AUTOMATED\])|(Merge)|(Release).+$' + flags: 'gm' + excludeDescription: 'true' + checkAllCommitMessages: 'true' + accessToken: ${{ secrets.PAT_ACTIONS }} + error: 'One of your your commit messages is not matching the format with JIRA ID Ex: ( SDK-123 commit message )' + + test: + uses: ./.github/workflows/shared-tests.yml + with: + python-version: '3.9' + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01152204..454e3302 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,31 +1,71 @@ -name: Public Release +name: Public release + +# Triggered by publishing a GitHub Release, not a raw tag push: the Release +# carries both facts needed here - target_commitish (the branch picked in the +# UI; a tag records only a commit) and tag_name (module prefix + version). +# +# Beta and final share this workflow - 'release' events cannot be filtered by +# tag pattern, and both behave identically downstream. Kind comes from the tag. on: - push: - tags: "*.*.*" - paths-ignore: - - "*/setup.py" - - "*.yml" - - "*.md" - - "*/skyflow/utils/_version.py" + release: + types: [published] jobs: + resolve-release: + runs-on: ubuntu-latest + outputs: + module: ${{ steps.parse.outputs.module }} + version: ${{ steps.parse.outputs.version }} + kind: ${{ steps.parse.outputs.kind }} + steps: + - name: Parse module, version and release kind from the tag + id: parse + env: + TAG: ${{ github.event.release.tag_name }} + BRANCH: ${{ github.event.release.target_commitish }} + run: | + # Expected: /v[-beta.N] e.g. flowvault/v1.0.0, + # skyvault/v2.1.2, flowvault/v1.0.0-beta.1 + if [[ ! "$TAG" =~ ^[a-z]+/v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]]; then + echo "::error::Tag '$TAG' is not /v[-beta.N]." \ + "Examples: flowvault/v1.0.0, skyvault/v2.1.2, flowvault/v1.0.0-beta.1" + exit 1 + fi + + PREFIX="${TAG%%/*}" # flowvault/v1.0.0 -> flowvault + VERSION="${TAG#*/}" # flowvault/v1.0.0 -> v1.0.0 + VERSION="${VERSION#v}" # v1.0.0 -> 1.0.0 + + # Tag prefix -> module directory (both match the directory name). + case "$PREFIX" in + flowvault) MODULE="flowvault" ;; + skyvault) MODULE="skyvault" ;; + *) + echo "::error::Unknown module prefix '$PREFIX' in tag '$TAG'" + exit 1 + ;; + esac + + if [[ "$VERSION" == *-beta.* ]]; then KIND="beta"; else KIND="public"; fi + + if [ -z "$BRANCH" ]; then + echo "::error::Release has no target_commitish - cannot determine the release branch." + exit 1 + fi + + echo "Tag '$TAG' -> module='$MODULE' version='$VERSION' kind='$KIND' branch='$BRANCH'" + echo "module=$MODULE" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "kind=$KIND" >> "$GITHUB_OUTPUT" + build-and-deploy: - strategy: - matrix: - include: - - variant: v2 - package-name: skyflow - tag-prefix: '' - - variant: flowvault - package-name: skyflow_flowvault - tag-prefix: 'flowvault-' - if: (matrix.variant == 'flowvault' && startsWith(github.ref_name, 'flowvault-')) || (matrix.variant == 'v2' && !startsWith(github.ref_name, 'flowvault-')) + needs: resolve-release uses: ./.github/workflows/shared-build-and-deploy.yml with: - ref: main - tag: 'public' - variant: ${{ matrix.variant }} - package-name: ${{ matrix.package-name }} - tag-prefix: ${{ matrix.tag-prefix }} + ref: ${{ github.event.release.tag_name }} + tag: ${{ needs.resolve-release.outputs.kind }} + module: ${{ needs.resolve-release.outputs.module }} + version: ${{ needs.resolve-release.outputs.version }} + release-branch: ${{ github.event.release.target_commitish }} secrets: inherit diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index c286b921..bce5fc8e 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -20,7 +20,7 @@ jobs: - name: Run Semgrep run: | - semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --exclude "**/generated/**" --sarif . > results.sarif + semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --sarif . > results.sarif - name: Upload SARIF file uses: github/codeql-action/upload-sarif@v3 diff --git a/.github/workflows/shared-build-and-deploy.yml b/.github/workflows/shared-build-and-deploy.yml index adffc3fe..7367c750 100644 --- a/.github/workflows/shared-build-and-deploy.yml +++ b/.github/workflows/shared-build-and-deploy.yml @@ -13,30 +13,57 @@ on: required: true type: string - variant: - description: 'Build variant directory to release (v2 or flowvault)' + module: + description: 'Module to build and publish (skyvault or flowvault)' required: true type: string - package-name: - description: 'Importable package name for this variant (e.g. skyflow or skyflow_flowvault)' + version: + description: >- + Explicit version to release. Set by tag-triggered (beta/public) + callers, which parse it out of a /v tag. When + empty, the version is derived from the module's own setup.py + (internal releases). required: false type: string - default: 'skyflow' + default: '' - tag-prefix: - description: 'Prefix distinguishing this variant''s git tags from other variants'' (e.g. "flowvault-" for flowvault, empty for v2)' + release-branch: + description: >- + Branch that receives the version-bump commit, for beta/public + releases. Supplied by the caller from the GitHub Release's + target_commitish. A tag records only a commit, never a branch, so + the branch has to be supplied rather than inferred. required: false type: string default: '' + dry-run: + description: >- + Validate the release pipeline WITHOUT publishing anything. + Everything still runs - version resolution, the setup.py bump, the + full build - but 'twine upload' is skipped and the version-bump + commit is not pushed. Publishing to PyPI is immutable, so this is + the only safe way to exercise the public path. + required: false + type: boolean + default: false + jobs: - build-and-deploy: + publish: runs-on: ubuntu-latest + env: + MODULE: ${{ inputs.module }} + RELEASE_BRANCH: ${{ inputs.release-branch }} steps: - uses: actions/checkout@v2 with: fetch-depth: 0 + ref: ${{ inputs.ref }} + # Persist an admin credential so the automated version-bump push + # below satisfies the branch-protection ruleset's repo-admin + # bypass; the default GITHUB_TOKEN is not a bypass actor. + token: ${{ secrets.PAT_ACTIONS }} - uses: actions/setup-python@v2 - name: Install dependencies @@ -44,96 +71,119 @@ jobs: python -m pip install --upgrade pip pip install setuptools wheel twine - - name: Resolve Branch for the Tagged Commit - id: resolve-branch + - name: Validate release branch input if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }} run: | - TAG_COMMIT=$(git rev-list -n 1 ${{ github.ref_name }}) - - BRANCH_NAME=$(git branch -r --contains $TAG_COMMIT | grep -o 'origin/.*' | sed 's|origin/||' | head -n 1) - - if [ -z "$BRANCH_NAME" ]; then - echo "Error: Could not resolve branch for the tag." + if [ -z "$RELEASE_BRANCH" ]; then + echo "::error::release-branch is required for ${{ inputs.tag }} releases." exit 1 fi - - echo "Resolved Branch Name: $BRANCH_NAME" - echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV - - - name: Get Previous tag - id: previoustag - uses: WyriHaximus/github-action-get-previous-tag@v1 - with: - fallback: ${{ inputs.tag-prefix }}1.0.0 - pattern: ${{ inputs.tag-prefix }}[0-9]*.[0-9]*.[0-9]* - - - name: Resolve version number - id: version - env: - PREVIOUS_TAG: ${{ steps.previoustag.outputs.tag }} - TAG_PREFIX: ${{ inputs.tag-prefix }} - run: | - TAG="$PREVIOUS_TAG" - VERSION="${TAG#$TAG_PREFIX}" - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Bump Version - working-directory: ${{ inputs.variant }} + # The tagged commit must actually be on that branch, otherwise the + # bump would land somewhere the release was never cut from. + if ! git merge-base --is-ancestor HEAD "origin/$RELEASE_BRANCH"; then + echo "::error::Tagged commit is not an ancestor of origin/$RELEASE_BRANCH." + exit 1 + fi + echo "Release branch: $RELEASE_BRANCH" + + # Version priority: inputs.version (beta/public, parsed from the tag) > + # the module's own setup.py (internal). Tags are a flat repo-wide + # namespace with no module awareness, so internal releases read the + # module's setup.py directly rather than any git-tag lookup - a tag + # lookup would risk stamping one module's version onto another's build. + - name: Resolve base version + id: resolve-version run: | - chmod +x ../ci-scripts/bump_version.sh - if ${{ inputs.tag == 'internal' }}; then - ../ci-scripts/bump_version.sh "${{ steps.version.outputs.version }}" "$(git rev-parse --short "$GITHUB_SHA")" "${{ inputs.package-name }}" + chmod +x ./ci-scripts/bump_version.sh ./ci-scripts/current_module_version.sh + if [ -n "${{ inputs.version }}" ]; then + BASE_VERSION="${{ inputs.version }}" else - ../ci-scripts/bump_version.sh "${{ steps.version.outputs.version }}" "" "${{ inputs.package-name }}" + BASE_VERSION=$(./ci-scripts/current_module_version.sh "$MODULE") fi + echo "base_version=$BASE_VERSION" >> "$GITHUB_OUTPUT" - - name: Commit changes - working-directory: ${{ inputs.variant }} + - name: Bump Version run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - if [[ "${{ inputs.tag }}" == "beta" || "${{ inputs.tag }}" == "public" ]]; then - git checkout ${{ env.branch_name }} - fi - - git add setup.py - git add ${{ inputs.package-name }}/utils/_version.py - if [[ "${{ inputs.tag }}" == "internal" ]]; then - VERSION="${{ steps.version.outputs.version }}.dev0+$(git rev-parse --short $GITHUB_SHA)" - COMMIT_MESSAGE="[AUTOMATED] Private Release (${{ inputs.variant }}) $VERSION" - git commit -m "$COMMIT_MESSAGE" - git push origin ${{ github.ref_name }} -f - fi - if [[ "${{ inputs.tag }}" == "beta" || "${{ inputs.tag }}" == "public" ]]; then - COMMIT_MESSAGE="[AUTOMATED] Public Release (${{ inputs.variant }}) - ${{ steps.previoustag.outputs.tag }}" - git commit -m "$COMMIT_MESSAGE" - git push origin ${{ env.branch_name }} + ./ci-scripts/bump_version.sh "${{ steps.resolve-version.outputs.base_version }}" "$(git rev-parse --short "$GITHUB_SHA")" "$MODULE" + else + ./ci-scripts/bump_version.sh "${{ steps.resolve-version.outputs.base_version }}" "" "$MODULE" fi - - name: Build and install package - working-directory: ${{ inputs.variant }} + # Build and publish happen here, BEFORE the version-bump commit lands on + # release-branch below: the working tree at this point is still exactly + # inputs.ref (the tagged commit) plus the in-place version bump, so the + # published artifact's provenance is the tag, never whatever + # release-branch happens to look like today. Landing the commit first + # would let the branch checkout pull in unrelated branch-tip content + # ahead of the build. + - name: Build ${{ inputs.module }} package run: | + cd "$MODULE" python setup.py sdist bdist_wheel - pip install dist/*.whl - - name: Build and Publish Package - if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }} - working-directory: ${{ inputs.variant }} + - name: Publish to PyPI + if: ${{ (inputs.tag == 'beta' || inputs.tag == 'public') && inputs.dry-run != true }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_TOKEN }} run: | - python setup.py sdist bdist_wheel + cd "$MODULE" twine upload dist/* - - name: Build and Publish to JFrog Artifactory - if: ${{ inputs.tag == 'internal' }} - working-directory: ${{ inputs.variant }} + - name: Publish to JFrog Artifactory + if: ${{ inputs.tag == 'internal' && inputs.dry-run != true }} env: TWINE_USERNAME: ${{ secrets.JFROG_USERNAME }} TWINE_PASSWORD: ${{ secrets.JFROG_PASSWORD }} run: | - python setup.py sdist bdist_wheel + cd "$MODULE" twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/* + + - name: Commit changes + run: | + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + + if [[ "${{ inputs.tag }}" == "beta" || "${{ inputs.tag }}" == "public" ]]; then + git checkout "$RELEASE_BRANCH" + fi + + # Stage exactly what bump_version.sh may have touched: setup.py + # always, plus the module's runtime _version.py if it has one. Not + # `git add -A`: the Build step above already created dist/build/ + # *.egg-info inside $MODULE, and those must never land in a + # version-bump commit. + git add "$MODULE/setup.py" + version_file=$(find "$MODULE" -name "_version.py" -print -quit) + if [ -n "$version_file" ]; then + git add "$version_file" + fi + + # Nothing staged = module already at this version. That is + # success; a bare 'git commit' would exit 1 here. + if git diff --cached --quiet; then + echo "::notice::$MODULE already at the target version - nothing to commit" + exit 0 + fi + + if [[ "${{ inputs.tag }}" == "internal" ]]; then + git commit -m "[AUTOMATED] Private Release ${{ steps.resolve-version.outputs.base_version }}.dev0+$(git rev-parse --short $GITHUB_SHA)" + if [[ "${{ inputs.dry-run }}" == "true" ]]; then + echo "::notice::DRY RUN - not pushing the version-bump commit" + else + git push origin ${{ github.ref_name }} -f + fi + fi + if [[ "${{ inputs.tag }}" == "beta" || "${{ inputs.tag }}" == "public" ]]; then + git commit -m "[AUTOMATED] Public Release - ${{ steps.resolve-version.outputs.base_version }}" + if [[ "${{ inputs.dry-run }}" == "true" ]]; then + echo "::notice::DRY RUN - not pushing the version-bump commit" + else + git push origin "$RELEASE_BRANCH" + fi + fi + + - name: Dry run summary + if: ${{ inputs.dry-run == true }} + run: echo "::notice::DRY RUN - build completed, nothing was published." diff --git a/.github/workflows/shared-tests.yml b/.github/workflows/shared-tests.yml index 796eb89f..dc270360 100644 --- a/.github/workflows/shared-tests.yml +++ b/.github/workflows/shared-tests.yml @@ -7,20 +7,6 @@ on: description: 'Python version to use' required: true type: string - variant: - description: 'Build variant directory to test (v2 or flowvault)' - required: true - type: string - package-name: - description: 'Importable package name for this variant (e.g. skyflow or skyflow_flowvault)' - required: false - type: string - default: 'skyflow' - coverage-omit: - description: 'Comma-separated coverage --omit patterns, relative to the variant directory' - required: false - type: string - default: 'skyflow/generated/*' jobs: run-tests: @@ -37,15 +23,11 @@ jobs: with: name: "credentials.json" json: ${{ secrets.VALID_SKYFLOW_CREDS_TEST }} - dir: ${{ inputs.variant }} - - name: Build and install package - working-directory: ${{ inputs.variant }} + - name: Install dev dependencies run: | - pip install --upgrade pip setuptools wheel - python setup.py sdist bdist_wheel - pip install dist/*.whl - pip install ".[dev]" + python -m pip install --upgrade pip + pip install "codespell>=2.4.1" "ruff>=0.9.0" "coverage>=7.8.0" - name: Run Spell Check run: codespell @@ -53,20 +35,57 @@ jobs: - name: Run Linter Ruff run: ruff check . --output-format=github - - name: 'Run Tests' - working-directory: ${{ inputs.variant }} + # Each module (common, skyvault, flowvault) is its own installable + # distribution with its own setup.py and test suite, mirroring Java's + # per-module Maven build. A module directory that doesn't exist yet + # (pre-migration) is skipped with a notice rather than failing the job. + - name: Build, install and test each module run: | - pip install -r requirements.txt - python -m coverage run --source=${{ inputs.package-name }} --omit=${{ inputs.coverage-omit }} -m unittest discover + for module in common skyvault flowvault; do + if [ ! -f "$module/setup.py" ]; then + echo "::notice::$module/setup.py not found yet - skipping (pre-migration)." + continue + fi + cp credentials.json "$module/credentials.json" + ( + cd "$module" + python -m pip install --upgrade pip setuptools wheel + python setup.py sdist bdist_wheel + pip install dist/*.whl + if [ -f requirements.txt ]; then + pip install -r requirements.txt + fi + python -m coverage run --source=. -m unittest discover + coverage xml -o test-coverage.xml + ) + done - - name: coverage - working-directory: ${{ inputs.variant }} - run: coverage xml -o test-coverage.xml + - name: Codecov (common) + if: hashFiles('common/test-coverage.xml') != '' + uses: codecov/codecov-action@v2.1.0 + with: + token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} + files: common/test-coverage.xml + flags: common + name: codecov-skyflow-python-common + verbose: true + + - name: Codecov (skyvault) + if: hashFiles('skyvault/test-coverage.xml') != '' + uses: codecov/codecov-action@v2.1.0 + with: + token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} + files: skyvault/test-coverage.xml + flags: skyvault + name: codecov-skyflow-python-skyvault + verbose: true - - name: Codecov + - name: Codecov (flowvault) + if: hashFiles('flowvault/test-coverage.xml') != '' uses: codecov/codecov-action@v2.1.0 with: token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} - files: ${{ inputs.variant }}/test-coverage.xml - name: codecov-skyflow-python-${{ inputs.variant }} + files: flowvault/test-coverage.xml + flags: flowvault + name: codecov-skyflow-python-flowvault verbose: true diff --git a/README.md b/README.md index 3cd1fed2..e74d3224 100644 --- a/README.md +++ b/README.md @@ -1,1016 +1,38 @@ # Skyflow Python SDK -> **This is the current, recommended version of the Skyflow SDK.** V2.1.0 brings flexible auth, multi-vault support, native data types, and rich error diagnostics. -> -> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 is in maintenance mode and will reach End of Life on October 31, 2026. +This repository hosts the Skyflow Python SDKs. It is a multi-package workspace — pick the package +that matches the vault you're using. -The Skyflow Python SDK is designed to help with integrating Skyflow into a Python backend. +## Which package do I want? -## Table of Contents - -- [Skyflow Python SDK](#skyflow-python-sdk) - - [Table of Contents](#table-of-contents) - - [Overview](#overview) - - [Installation](#installation) - - [Require](#require) - - [Configuration](#configuration) - - [Quickstart](#quickstart) - - [Authenticate](#authenticate) - - [API Key](#api-key) - - [Bearer Token (static)](#bearer-token-static) - - [Initialize the client](#initialize-the-client) - - [Insert data into the vault, get tokens back](#insert-data-into-the-vault-get-tokens-back) - - [Upgrade from v1 to v2](#upgrade-from-v1-to-v2) - - [Vault](#vault) - - [Insert and tokenize data: `.insert(request)`](#insert-and-tokenize-data-insertrequest) - - [Insert example with `continue_on_error` option](#insert-example-with-continue_on_error-option) - - [Upsert request](#upsert-request) - - [Detokenize: `.detokenize(request, options)`](#detokenize-detokenizerequest-options) - - [Construct a detokenize request](#construct-a-detokenize-request) - - [Get Record(s): `.get(request)`](#get-records-getrequest) - - [Construct a get request](#construct-a-get-request) - - [Get by Skyflow IDs](#get-by-skyflow-ids) - - [Get tokens for records](#get-tokens-for-records) - - [Get by column name and column values](#get-by-column-name-and-column-values) - - [Redaction Types](#redaction-types) - - [Update Records](#update-records) - - [Construct an update request](#construct-an-update-request) - - [Delete Records](#delete-records) - - [Query](#query) - - [Upload File](#upload-file) - - [Retrieve Existing Tokens: `.tokenize(request)`](#retrieve-existing-tokens-tokenizerequest) - - [Construct a `.tokenize()` request](#construct-a-tokenize-request) - - [Detect](#detect) - - [De-identify Text: `.deidentify_text(request)`](#de-identify-text-deidentify_textrequest) - - [Re-identify Text: `.reidentify_text(request)`](#re-identify-text-reidentify_textrequest) - - [De-identify File: `.deidentify_file(request)`](#de-identify-file-deidentify_filerequest) - - [Get Run: `.get_detect_run(request)`](#get-run-get_detect_runrequest) - - [Connections](#connections) - - [Invoke a connection](#invoke-a-connection) - - [Construct an invoke connection request](#construct-an-invoke-connection-request) - - [Authentication & authorization](#authentication--authorization) - - [Types of `credentials`](#types-of-credentials) - - [Generate bearer tokens for authentication & authorization](#generate-bearer-tokens-for-authentication--authorization) - - [Generate a bearer token](#generate-a-bearer-token) - - [`generate_bearer_token(filepath)`](#generate_bearer_tokenfilepath) - - [`generate_bearer_token_from_creds(credentials)`](#generate_bearer_token_from_credscredentials) - - [Generate bearer tokens scoped to certain roles](#generate-bearer-tokens-scoped-to-certain-roles) - - [Generate bearer tokens with `ctx` for context-aware authorization](#generate-bearer-tokens-with-ctx-for-context-aware-authorization) - - [Generate signed data tokens: `generate_signed_data_tokens(filepath, options)`](#generate-signed-data-tokens-generate_signed_data_tokensfilepath-options) - - [Logging](#logging) - - [Example: Setting LogLevel to INFO](#example-setting-loglevel-to-info) - - [Error handling](#error-handling) - - [Catching `SkyflowError` instances](#catching-skyflowerror-instances) - - [Bearer token expiration edge cases](#bearer-token-expiration-edge-cases) - - [Security](#security) - - [Reporting a Vulnerability](#reporting-a-vulnerability) - -## Overview - -The Skyflow SDK enables you to connect to your Skyflow Vault(s) to securely handle sensitive data at rest, in-transit, and in-use. - -> [!TIP] -> Looking for the full list of request parameters, response object attributes, enums, client-management methods, and Detect helper classes? See the **[API Reference](docs/api_reference.md)**. - -> [!IMPORTANT] -> This readme documents SDK version 2. -> For version 1 see the [v1.16.0 README](https://github.com/skyflowapi/skyflow-python/tree/v1). -> For more information on how to migrate see [MIGRATE_TO_V2.md](docs/migrate_to_v2.md). - -## Installation - -### Require - -- Python 3.9 and above (tested with Python 3.9) - -### Configuration - -The package can be installed using pip: +| Package (PyPI) | Import | Vault type | Docs | +|---|---|---|---| +| **`skyflow`** | `import skyflow` | Privacy DB (v2.x) — vault CRUD, tokenize/detokenize, query, files, Detect, Connections | [skyvault/README.md](skyvault/README.md) | +| **`skyflow-flowvault`** | `import skyflow_flowvault` | Flow DB (v1.x) — high-throughput bulk + unary vault operations | [flowvault/README.md](flowvault/README.md) | ```bash -pip install skyflow -``` - -## Quickstart - -Get started quickly with the essential steps: authenticate, initialize the client, and perform a basic vault operation. This section shows you a minimal working example. - -### Before you begin - -To run the examples below, you need a Skyflow account and a few values from the Skyflow Studio console. If you don't have an account yet, [request a demo](https://www.skyflow.com/get-demo). - -| Value | Where to find it | -|-------|------------------| -| `vault_id` | Your vault's details page in Skyflow Studio. | -| `cluster_id` | The first segment of your vault URL: `https://{cluster_id}.vault.skyflowapis.com`. | -| `env` | The environment your vault runs in — `Env.PROD`, `Env.SANDBOX`, `Env.DEV`, or `Env.STAGE` (defaults to `PROD`). | -| Credentials | Create a **service account** in Studio. Choose **API key** during creation for the simplest setup, or download the service-account `credentials.json` for token-based auth. See [Authentication & authorization](#authentication--authorization). | - -The quickstart below assumes a table named `table1` with `card_number` and `cardholder_name` columns. Create a matching table (or adjust the table/column names to your schema) in your vault before running it. See the [Skyflow docs](https://docs.skyflow.com/) for creating vaults, tables, and service accounts. - -### Authenticate - -You can use an API key or a personal bearer token to directly authenticate and authorize requests with the SDK. Use API keys for long-term service authentication. Use bearer tokens for optimal security. - -### API Key - -```python -credentials = { - "api_key": "" -} -``` - -### Bearer Token (static) - -```python -credentials = { - "token": "" -} -``` - -For authenticating via generated bearer tokens including support for scoped tokens, context-aware access tokens, and more, refer to the [Authentication & Authorization](#authentication--authorization) section. - -### Initialize the client - -Initialize the Skyflow client first. You can specify different credential types during initialization. - -```python -from skyflow import Skyflow, LogLevel, Env - -# Configure vault -config = { - 'vault_id': '', - 'cluster_id': '', - 'env': Env.PROD, - 'credentials': { - 'api_key': '' - } -} - -# Initialize Skyflow client -skyflow_client = ( - Skyflow.builder() - .add_vault_config(config) - .set_log_level(LogLevel.ERROR) - .build() -) -``` - -See [docs/advanced_initialization.md](docs/advanced_initialization.md) for advanced initialization examples including multiple vaults and different credential types. - -### Insert data into the vault, get tokens back - -Insert data into your vault using the `insert` method. Set `return_tokens=True` in the request to ensure values are tokenized in the response. - -Create an insert request with the [`InsertRequest`](docs/api_reference.md#insertrequest) class, which includes the values to be inserted as a list of records. - -Below is a simple example to get started. See the [Insert and tokenize data](#insert-and-tokenize-data-insertrequest) section for advanced options. - -```python -from skyflow.vault.data import InsertRequest - -# Insert sensitive data into the vault -insert_data = [ - { 'card_number': '4111111111111111', 'cardholder_name': 'John Doe' }, -] - -insert_request = InsertRequest( - table='table1', - values=insert_data, - return_tokens=True -) - -insert_response = skyflow_client.vault('').insert(insert_request) -print('Insert response:', insert_response) -``` - -Returns an [`InsertResponse`](docs/api_reference.md#insertresponse) (`inserted_fields`, `errors`). With `return_tokens=True`, each entry includes the `skyflow_id` and a token per column: - -```text -Insert response: InsertResponse(inserted_fields=[{'skyflow_id': 'a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', 'card_number': '5391-4629-3722-7102', 'cardholder_name': '0f6b8a2c-90ab-4cde-9def-567890abcdef'}], errors=None) -``` - -## Upgrade from v1 to v2 - -Upgrade from `skyflow-python` v1 using the dedicated guide in [docs/migrate_to_v2.md](docs/migrate_to_v2.md). - -## Vault - -The [Vault](https://docs.skyflow.com/docs/vaults) performs operations on the vault, including inserting records, detokenizing tokens, and retrieving tokens associated with a skyflow_id. - -### Insert and tokenize data: `.insert(request)` - -Pass options to the `insert` method to enable additional functionality such as returning tokenized data, upserting records, or allowing bulk operations to continue despite errors. See [Quickstart](#quickstart) for a basic example. - -```python -from skyflow.vault.data import InsertRequest - -insert_request = InsertRequest( - table='table1', - values=[ - { - '': '', - '': '' - }, - { - '': '', - '': '' - } - ], - return_tokens=True -) - -response = skyflow_client.vault('').insert(insert_request) -print('Insert response:', response) -``` - -Returns an [`InsertResponse`](docs/api_reference.md#insertresponse): - -```text -Insert response: InsertResponse(inserted_fields=[{'skyflow_id': 'a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', '': '', '': ''}], errors=None) -``` - -> With `continue_on_error=True`, each entry also carries a `request_index`, and `errors` is a list of `{request_index, request_id, error, http_code}` for the rows that failed. - -#### Insert example with `continue_on_error` option - -Set the `continue_on_error` flag to `True` to allow insert operations to proceed despite encountering partial errors. - -> [!TIP] -> See the full example in the samples directory: [insert_records.py](samples/vault_api/insert_records.py) - -#### Upsert request - -Turn an insert into an 'update-or-insert' operation using the upsert option. The vault checks for an existing record with the same value in the specified column. If a match exists, the record updates; otherwise, a new record inserts. - -```python -# Specify the column to use as the index for the upsert. -# Note: The column must have the `unique` constraint configured in the vault. -insert_request = InsertRequest( - table='table1', - values=insert_data, - upsert='' -) -``` - -### Detokenize: `.detokenize(request, options)` - -Convert tokens back into plaintext values (or masked values) using the `.detokenize()` method. Detokenization accepts tokens and returns values. - -Create a detokenization request with the [`DetokenizeRequest`](docs/api_reference.md#detokenizerequest) class, which requires a list of tokens and column groups as input. - -Provide optional parameters such as the redaction type and the option to continue on error. - -#### Construct a detokenize request - -```python -from skyflow.vault.tokens import DetokenizeRequest -from skyflow.utils.enums import RedactionType - -detokenize_request = DetokenizeRequest( - data=[ - {'token': 'token1', 'redaction_type': RedactionType.PLAIN_TEXT}, - {'token': 'token2', 'redaction_type': RedactionType.PLAIN_TEXT} - ], - continue_on_error=True -) - -response = skyflow_client.vault('').detokenize(detokenize_request) -print('Detokenization response:', response) -``` - -Returns a [`DetokenizeResponse`](docs/api_reference.md#detokenizeresponse) (`detokenized_fields`, `errors`); each field has `token`, `value`, and `type`: - -```text -Detokenization response: DetokenizeResponse(detokenized_fields=[{'token': 'token1', 'value': '4111111111111111', 'type': 'STRING'}, {'token': 'token2', 'value': 'John Doe', 'type': 'STRING'}], errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [detokenize_records.py](samples/vault_api/detokenize_records.py) - -### Get Record(s): `.get(request)` - -Retrieve data using Skyflow IDs or unique column values with the `get` method. Create a get request with the [`GetRequest`](docs/api_reference.md#getrequest) class, specifying parameters such as the table name, redaction type, Skyflow IDs, column names, and column values. - -> [!NOTE] -> You can't use both Skyflow IDs and column name/value pairs in the same request. - -#### Construct a get request - -```python -from skyflow.vault.data import GetRequest -from skyflow.utils.enums import RedactionType - -get_request = GetRequest( - table='table1', - ids=['', ''], - redaction_type=RedactionType.PLAIN_TEXT, - return_tokens=False -) - -response = skyflow_client.vault('').get(get_request) -print('Get response:', response) -``` - -Returns a [`GetResponse`](docs/api_reference.md#getresponse) (`data`, `errors`), where `data` is a list of record dicts: - -```text -Get response: GetResponse(data=[{'skyflow_id': 'a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', 'card_number': '4111111111111111', 'cardholder_name': 'John Doe'}], errors=None) -``` - -#### Get by Skyflow IDs - -Retrieve specific records using Skyflow IDs. Use this method when you know the exact record IDs. - -```python -from skyflow.vault.data import GetRequest -from skyflow.utils.enums import RedactionType - -get_request = GetRequest( - table='table1', - ids=['', ''], - redaction_type=RedactionType.PLAIN_TEXT -) - -response = skyflow_client.vault('').get(get_request) - -print('Data retrieval successful:', response) -``` - -```text -Data retrieval successful: GetResponse(data=[{'skyflow_id': '', 'card_number': '4111111111111111', 'cardholder_name': 'John Doe'}], errors=None) -``` - -#### Get tokens for records - -Return tokens for records to securely process sensitive data while maintaining data privacy. - -```python -get_request = GetRequest( - table='table1', - ids=[''], - return_tokens=True # Set to `True` to get tokens -) -``` - -> [!TIP] -> See the full example in the samples directory: [get_records.py](samples/vault_api/get_records.py) - -#### Get by column name and column values - -Retrieve records by unique column values when you don't know the Skyflow IDs. Use this method to query data with alternate unique identifiers. - -```python -get_request = GetRequest( - table='table1', - column_name='email', - column_values=['user@email.com'], # Column values of the records to return -) -``` - -> [!TIP] -> See the full example in the samples directory: [get_column_values.py](samples/vault_api/get_column_values.py) - -#### Redaction Types - -Use redaction types to control how sensitive data displays when retrieved from the vault. - -**Available Redaction Types** - -- `DEFAULT`: Applies the vault-configured default redaction setting. -- `REDACTED`: Completely removes sensitive data from view. -- `MASKED`: Partially obscures sensitive information. -- `PLAIN_TEXT`: Displays the full, unmasked data. - -**Choosing the Right Redaction Type** - -- Use `REDACTED` for scenarios requiring maximum data protection to prevent exposure of sensitive information. -- Use `MASKED` to provide partial visibility of sensitive data for less critical use cases. -- Use `PLAIN_TEXT` for internal, authorized access where full data visibility is necessary. - -### Update Records - -Update data in your vault using the `update` method. Create an update request with the [`UpdateRequest`](docs/api_reference.md#updaterequest) class, specifying parameters such as the table name and data (as a dictionary). - -You can pass options like `return_tokens` directly to the request. When `True`, Skyflow returns tokens for the updated records. When `False`, it returns IDs. - -#### Construct an update request - -```python -from skyflow.vault.data import UpdateRequest - -update_request = UpdateRequest( - table='table1', - data={ - 'skyflow_id': '', - '': '', - '': '' - } -) - -response = skyflow_client.vault('').update(update_request) -print('Update response:', response) -``` - -Returns an [`UpdateResponse`](docs/api_reference.md#updateresponse) (`updated_field`, `errors`). With the default `return_tokens=False`, only the `skyflow_id` is returned; with `return_tokens=True`, tokens for the updated columns are included: - -```text -Update response: UpdateResponse(updated_field={'skyflow_id': ''}, errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [update_record.py](samples/vault_api/update_record.py) - -### Delete Records - -Delete records using Skyflow IDs with the `delete` method. Create a delete request with the [`DeleteRequest`](docs/api_reference.md#deleterequest) class, which accepts a list of Skyflow IDs: - -```python -from skyflow.vault.data import DeleteRequest - -delete_request = DeleteRequest( - table='', - ids=['', '', ''] -) - -response = skyflow_client.vault('').delete(delete_request) -print('Delete response:', response) -``` - -Returns a [`DeleteResponse`](docs/api_reference.md#deleteresponse) (`deleted_ids`, `errors`): - -```text -Delete response: DeleteResponse(deleted_ids=['', '', ''], errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [delete_records.py](samples/vault_api/delete_records.py) - -### Query - -Retrieve data with SQL queries using the `query` method. Create a query request with the [`QueryRequest`](docs/api_reference.md#queryrequest) class, which takes the `query` parameter as follows: - -```python -from skyflow.vault.data import QueryRequest - -query_request = QueryRequest( - query="SELECT * FROM table1 WHERE column1 = 'value'" -) - -response = skyflow_client.vault('').query(query_request) -print('Query response:', response) -``` - -Returns a [`QueryResponse`](docs/api_reference.md#queryresponse) (`fields`, `errors`), where `fields` is a list of matching record dicts (each also includes a `tokenized_data` map): - -```text -Query response: QueryResponse(fields=[{'card_number': '4111111111111111', 'cardholder_name': 'John Doe', 'tokenized_data': {}}], errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [query_records.py](samples/vault_api/query_records.py) - -Refer to [Query your data](https://docs.skyflow.com/query-data/) and [Execute Query](https://docs.skyflow.com/record/#QueryService_ExecuteQuery) for guidelines and restrictions on supported SQL statements, operators, and keywords. - -### Upload File - -Upload files to a Skyflow vault using the `upload_file` method. Create a file upload request with the [`FileUploadRequest`](docs/api_reference.md#fileuploadrequest) class. - -**Upload a file to an existing record:** - -```python -from skyflow.vault.data import FileUploadRequest - -# Open the file in binary read mode -with open('path/to/file.pdf', 'rb') as file_obj: - upload_request = FileUploadRequest( - table='', - column_name='', - skyflow_id='', - file_object=file_obj - ) - - response = skyflow_client.vault('').upload_file(upload_request) - print('File upload:', response) -``` - -**Upload a file and create a new record (omit `skyflow_id`):** - -```python -with open('path/to/file.pdf', 'rb') as file_obj: - upload_request = FileUploadRequest( - table='documents', - column_name='attachment', - file_object=file_obj - ) - - response = skyflow_client.vault('').upload_file(upload_request) - print('File upload:', response) -``` - -Both forms return a [`FileUploadResponse`](docs/api_reference.md#fileuploadresponse) (`skyflow_id`, `errors`) with the ID of the record the file was attached to (or the newly created record): - -```text -File upload: FileUploadResponse(skyflow_id='a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [upload_file.py](samples/vault_api/upload_file.py) - -### Retrieve Existing Tokens: `.tokenize(request)` - -Retrieve tokens for values that already exist in the vault using the `.tokenize()` method. This method returns existing tokens only and does not generate new tokens. Build the request with the [`TokenizeRequest`](docs/api_reference.md#tokenizerequest) class. - -#### Construct a `.tokenize()` request - -```python -from skyflow.vault.tokens import TokenizeRequest - -tokenize_request = TokenizeRequest( - values=[ - {"value": "", "column_group": ""}, - {"value": "", "column_group": ""} - ] -) - -response = skyflow_client.vault('').tokenize(tokenize_request) -print('Tokenization result:', response) -``` - -Returns a [`TokenizeResponse`](docs/api_reference.md#tokenizeresponse) (`tokenized_fields`, `errors`); each field carries its `token`: - -```text -Tokenization result: TokenizeResponse(tokenized_fields=[{'token': 'a1b2c3d4-...'}, {'token': 'e5f6g7h8-...'}], errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [tokenize_records.py](samples/vault_api/tokenize_records.py) - -## Detect - -De-identify and reidentify sensitive data in text and files using Skyflow Detect, which supports advanced privacy-preserving workflows. - -### De-identify Text: `.deidentify_text(request)` - -De-identify or anonymize text using the `deidentify_text` method. - -Create a de-identify text request with the [`DeidentifyTextRequest`](docs/api_reference.md#deidentifytextrequest) class. - -```python -from skyflow.vault.detect import DeidentifyTextRequest, TokenFormat, Transformations, DateTransformation -from skyflow.utils.enums import DetectEntities, TokenType - -request = DeidentifyTextRequest( - text="", - entities=[DetectEntities.SSN, DetectEntities.CREDIT_CARD], - token_format=TokenFormat(default=TokenType.VAULT_TOKEN), - transformations=Transformations( - shift_dates=DateTransformation( - max_days=30, # Maximum days to shift - min_days=10, # Minimum days to shift - entities=[DetectEntities.DOB] - ) - ) -) - -response = skyflow_client.detect('').deidentify_text(request) -print('De-identify Text Response:', response) -``` - -Returns a [`DeidentifyTextResponse`](docs/api_reference.md#deidentifytextresponse) (`processed_text`, `entities`, `word_count`, `char_count`, `errors`). `entities` is a list of [`EntityInfo`](docs/api_reference.md#entityinfo) describing each detected entity: - -```text -De-identify Text Response: DeidentifyTextResponse(processed_text='My SSN is [SSN_1].', entities=[...], word_count=4, char_count=18, errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [deidentify_text.py](samples/detect_api/deidentify_text.py) - -### Re-identify Text: `.reidentify_text(request)` - -Re-identify text using the `reidentify_text` method. Create a reidentify text request with the [`ReidentifyTextRequest`](docs/api_reference.md#reidentifytextrequest) class, which includes the redacted or de-identified text to be re-identified. - -```python -from skyflow.vault.detect import ReidentifyTextRequest -from skyflow.utils.enums import DetectEntities - -request = ReidentifyTextRequest( - text="", - redacted_entities=[DetectEntities.SSN], # Keep redacted - masked_entities=[DetectEntities.CREDIT_CARD], # Mask - plain_text_entities=[DetectEntities.NAME] # Reveal -) - -response = skyflow_client.detect().reidentify_text(request) -print('Re-identify Text Response:', response) -``` - -Returns a [`ReidentifyTextResponse`](docs/api_reference.md#reidentifytextresponse) (`processed_text`, `errors`): - -```text -Re-identify Text Response: ReidentifyTextResponse(processed_text='John lives in NYC', errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [reidentify_text.py](samples/detect_api/reidentify_text.py) - -### De-identify File: `.deidentify_file(request)` - -De-identify files using the `deidentify_file` method. Create a request with the [`DeidentifyFileRequest`](docs/api_reference.md#deidentifyfilerequest) class, which includes the file to be deidentified. Provide optional parameters to control how entities are detected and deidentified. - -```python -from skyflow.vault.detect import DeidentifyFileRequest, TokenFormat, FileInput -from skyflow.utils.enums import DetectEntities, TokenType - -# Open file in binary mode -with open('path/to/file.pdf', 'rb') as file_obj: - request = DeidentifyFileRequest( - file=FileInput(file_obj), - entities=[DetectEntities.SSN, DetectEntities.CREDIT_CARD], - token_format=TokenFormat(default=TokenType.ENTITY_ONLY), - output_directory='', - wait_time=64 - ) - - response = skyflow_client.detect().deidentify_file(request) - print('De-identify File Response:', response) -``` - -Returns a [`DeidentifyFileResponse`](docs/api_reference.md#deidentifyfileresponse) with the processed file plus metadata (`file`, `type`, `extension`, `word_count`, `char_count`, `size_in_kb`, `entities`, `run_id`, `status`, `errors`, and more — see the [API Reference](docs/api_reference.md#response-objects)). If processing exceeds `wait_time`, only `run_id` and `status` are returned (poll with `get_detect_run`): - -```text -De-identify File Response: DeidentifyFileResponse(file_base64=None, file=, type='application/pdf', extension='pdf', ..., run_id='r-9c1f2a3b', status='SUCCESS', errors=None) -``` - -**Supported file types:** - -- Documents: `doc`, `docx`, `pdf` -- PDFs: `pdf` -- Images: `bmp`, `jpeg`, `jpg`, `png`, `tif`, `tiff` -- Structured text: `json`, `xml` -- Spreadsheets: `csv`, `xls`, `xlsx` -- Presentations: `ppt`, `pptx` -- Audio: `mp3`, `wav` - -**Notes:** - -- Transformations can't be applied to Documents, Images, or PDFs file formats. -- The `wait_time` option must be ≤ 64 seconds; otherwise, an error is thrown. -- If the API takes more than 64 seconds to process the file, it will return only the `run_id` and `status` in the response. - -> [!TIP] -> See the full example in the samples directory: [deidentify_file.py](samples/detect_api/deidentify_file.py) - -### Get Run: `.get_detect_run(request)` - -Retrieve the results of a previously started file de-identification operation using the `get_detect_run` method. Build the request with the [`GetDetectRunRequest`](docs/api_reference.md#getdetectrunrequest) class, initialized with the `run_id` returned from a prior `deidentify_file` call. - -```python -from skyflow.vault.detect import GetDetectRunRequest - -request = GetDetectRunRequest( - run_id='' -) - -response = skyflow_client.detect().get_detect_run(request) -print('Get Detect Run Response:', response) -``` - -Returns a [`DeidentifyFileResponse`](docs/api_reference.md#deidentifyfileresponse) with the current `status` for the run (and the processed file once `status` is complete): - -```text -Get Detect Run Response: DeidentifyFileResponse(file_base64=None, file=None, ..., run_id='r-9c1f2a3b', status='IN_PROGRESS', errors=None) -``` - -> [!TIP] -> See the full example in the samples directory: [get_detect_run.py](samples/detect_api/get_detect_run.py) - -## Connections - -Securely send and receive data between your systems and first- or third-party services using Skyflow Connections. The [connections](https://github.com/skyflowapi/skyflow-python/tree/v2/skyflow/vault/connection) module invokes both inbound and/or outbound connections. - -- **Inbound connections**: Act as intermediaries between your client and server, tokenizing sensitive data before it reaches your backend, ensuring downstream services handle only tokenized data. -- **Outbound connections**: Enable secure extraction of data from the vault and transfer it to third-party services via your backend server, such as processing checkout or card issuance flows. - -### Invoke a connection - -To invoke a connection, use the `invoke` method of the Skyflow client. Build the request with the [`InvokeConnectionRequest`](docs/api_reference.md#invokeconnectionrequest) class. - -#### Construct an invoke connection request - -```python -from skyflow.vault.connection import InvokeConnectionRequest -from skyflow.utils.enums import RequestMethod - -invoke_request = InvokeConnectionRequest( - method=RequestMethod.POST, - body={ '': '' }, - headers={ '': '' }, - path_params={ '': '' }, - query_params={ '': '' } -) - -response = skyflow_client.connection().invoke(invoke_request) -print('Connection response:', response) -``` - -Returns an [`InvokeConnectionResponse`](docs/api_reference.md#invokeconnectionresponse) (`data`, `metadata`, `errors`), where `data` is the connection's response body: - -```text -Connection response: InvokeConnectionResponse(data={'message': 'success'}, metadata={'request_id': 'b7d3...'}, errors=None) -``` - -`method` supports the following methods (see [`RequestMethod`](docs/api_reference.md#requestmethod)): - -- `GET` -- `POST` -- `PUT` -- `DELETE` - -**path_params, query_params, header, body** are the JSON objects represented as dictionaries that will be sent through the connection integration url. - -> [!TIP] -> See the full example in the samples directory: [invoke_connection.py](samples/vault_api/invoke_connection.py) -> See [docs.skyflow.com](https://docs.skyflow.com) for more details on integrations with Connections, Functions, and Pipelines. - -## Authentication & authorization - -### Types of `credentials` - -The SDK accepts one of several types of credentials object. - -1. **API keys** - A unique identifier used to authenticate and authorize requests to an API. Use for long-term service authentication. To create an API key, first create a 'Service Account' in Skyflow and choose the 'API key' option during creation. - - ```python - credentials = { - "api_key": "" - } - ``` - -2. **Bearer tokens** - A temporary access token used to authenticate API requests. Use for optimal security. As a developer with the right access, you can generate a temporary personal bearer token in Skyflow in the user menu. - - ```python - credentials = { - "token": "" - } - ``` - -3. **Service account credentials file path** - The file path pointing to a JSON file containing credentials for a service account. Use when credentials are managed externally or stored in secure file systems. - - ```python - credentials = { - "path": "" - } - ``` - -4. **Service account credentials string** - JSON-formatted string containing service account credentials. Use when integrating with secret management systems or when credentials are passed programmatically. - - ```python - import os - - credentials = { - "credentials_string": os.getenv("SKYFLOW_CREDENTIALS") - } - ``` - -5. **Environment variables** - If no credentials are explicitly provided, the SDK automatically looks for the SKYFLOW_CREDENTIALS environment variable. Use to avoid hardcoding credentials in source code. This variable must return an object like one of the examples above. - -> [!NOTE] -> Only one type of credential can be used at a time. If multiple credentials are provided, the last one added will take precedence. - -### Generate bearer tokens for authentication & authorization - -Generate and manage bearer tokens to authenticate API calls. This section covers options for scoping to certain roles, passing context, and signing data tokens. - -#### Generate a bearer token - -Generate service account tokens using the [Service Account](https://github.com/skyflowapi/skyflow-python/tree/main/skyflow/service_account) Python package with a service account credentials file provided when a service account is created. Tokens generated by this module are valid for 60 minutes and can be used to make API calls to the [Data](https://docs.skyflow.com/record/) and [Management](https://docs.skyflow.com/management/) APIs, depending on the permissions assigned to the service account. - -##### `generate_bearer_token(filepath)` - -The `generate_bearer_token(filepath)` function takes the `credentials.json` file path for token generation. - -```python -from skyflow.service_account import generate_bearer_token - -token, _ = generate_bearer_token('path/to/credentials.json') -print("Bearer Token:", token) -``` - -##### `generate_bearer_token_from_creds(credentials)` - -Alternatively, you can also send the entire credentials as string by using `generate_bearer_token_from_creds(string)`. - -> [!TIP] -> See the full example in the samples directory: [token_generation_example.py](https://github.com/skyflowapi/skyflow-python/blob/main/samples/service_account/token_generation_example.py) - -#### Generate bearer tokens scoped to certain roles - -Generate bearer tokens with access limited to a specific role by specifying the appropriate roleID when using a service account with multiple roles. Use this to limit access for services with multiple responsibilities, such as segregating access for billing and analytics. Generated bearer tokens are valid for 60 minutes and can only execute operations permitted by the permissions associated with the designated role. - -```python -options = { - 'role_ids': ['roleID1', 'roleID2'] -} -``` - -> [!TIP] -> See the full example in the samples directory: [scoped_token_generation_example.py](samples/service_account/scoped_token_generation_example.py) -> See [docs.skyflow.com](https://docs.skyflow.com) for more details on authentication, access control, and governance for Skyflow. - -#### Generate bearer tokens with `ctx` for context-aware authorization - -Embed context values into a bearer token during generation so you can reference those values in your policies. This enables more flexible access controls, such as tracking end-user identity when making API calls using service accounts, and facilitates using signed data tokens during detokenization. - -Generate bearer tokens containing context information using a service account with the `context_id` identifier. Context information is represented as a JWT claim in a Skyflow-generated bearer token. Tokens generated from such service accounts include a `context_identifier` claim, are valid for 60 minutes, and can be used to make API calls to the Data and Management APIs, depending on the service account's permissions. - -The `ctx` parameter accepts either a **string** or a **dict**: - -**String context** — use when your policy references a single context value: - -```python -options = {'ctx': 'user_12345'} -token, _ = generate_bearer_token(filepath, options) -``` - -**Dict context** — use when your policy needs multiple context values for conditional data access. Each key in the dict maps to a Skyflow CEL policy variable under `request.context.*`: - -```python -options = { - 'ctx': { - 'role': 'admin', - 'department': 'finance', - 'user_id': 'user_12345', - } -} -token, _ = generate_bearer_token(filepath, options) -``` - -With the dict above, your Skyflow policies can reference `request.context.role`, `request.context.department`, and `request.context.user_id` to make conditional access decisions. - -Dict keys must contain only alphanumeric characters and underscores (`[a-zA-Z0-9_]`). Invalid keys will raise a `SkyflowError`. - -> [!TIP] -> See the full example in the samples directory: [token_generation_with_context_example.py](samples/service_account/token_generation_with_context_example.py) -> See Skyflow's [context-aware authorization](https://docs.skyflow.com) and [conditional data access](https://docs.skyflow.com) docs for policy variable syntax like `request.context.*`. - -#### Generate signed data tokens: `generate_signed_data_tokens(filepath, options)` - -Digitally sign data tokens with a service account's private key to add an extra layer of protection. Skyflow generates data tokens when sensitive data is inserted into the vault. Detokenize signed tokens only by providing the signed data token along with a bearer token generated from the service account's credentials. The service account must have the necessary permissions and context to successfully detokenize the signed data tokens. - -The `ctx` parameter on signed data tokens also accepts either a **string** or a **dict**, using the same format as bearer tokens: - -```python -# String context -options = { - 'ctx': 'user_12345', - 'data_tokens': ['dataToken1', 'dataToken2'], - 'time_to_live': 90, -} - -# Dict context -options = { - 'ctx': { - 'role': 'analyst', - 'department': 'research', - }, - 'data_tokens': ['dataToken1', 'dataToken2'], - 'time_to_live': 90, -} -``` - -> [!TIP] -> See the full example in the samples directory: [signed_token_generation_example.py](samples/service_account/signed_token_generation_example.py) -> See [docs.skyflow.com](https://docs.skyflow.com) for more details on authentication, access control, and governance for Skyflow. - -## Logging - -The SDK provides logging using Python's inbuilt `logging` library. By default the logging level of the SDK is set to `LogLevel.ERROR`. This can be changed by using `set_log_level(log_level)` as shown below: - -Currently, the following five log levels are supported: - -- `DEBUG`: -When `LogLevel.DEBUG` is passed, logs at all levels will be printed (DEBUG, INFO, WARN, ERROR). -- `INFO`: -When `LogLevel.INFO` is passed, INFO logs for every event that occurs during SDK flow execution will be printed, along with WARN and ERROR logs. -- `WARN`: -When `LogLevel.WARN` is passed, only WARN and ERROR logs will be printed. -- `ERROR`: -When `LogLevel.ERROR` is passed, only ERROR logs will be printed. -- `OFF`: -`LogLevel.OFF` can be used to turn off all logging from the Skyflow Python SDK. - -**Note:** The ranking of logging levels is as follows: `DEBUG` < `INFO` < `WARN` < `ERROR` < `OFF`. - -### Example: Setting LogLevel to INFO - -```python -from skyflow import Skyflow, LogLevel, Env - -# Define vault configuration -vault_config = { - 'vault_id': '', - 'cluster_id': '', - 'env': Env.PROD, - 'credentials': {'api_key': ''} -} - -skyflow_client = ( - Skyflow.builder() - .add_vault_config(vault_config) - .set_log_level(LogLevel.INFO) # Recommended to use LogLevel.ERROR in production - .build() -) -``` - -## Using the client in production - -**Build the client once and reuse it.** `Skyflow.builder()...build()` returns a long-lived client that lazily creates and caches an HTTP client and bearer token per vault. Construct it once at startup (for example, as a module-level singleton or a dependency-injected instance) and reuse it across requests. Rebuilding the client on every request discards these caches and forces unnecessary token regeneration. - -```python -# At application startup -skyflow_client = ( - Skyflow.builder() - .add_vault_config(vault_config) - .set_log_level(LogLevel.ERROR) - .build() -) - -# Reuse `skyflow_client` for the lifetime of the process -``` - -**Bearer token refresh is automatic.** When you authenticate with a service-account credentials file/string (or API key), the SDK caches the generated bearer token and regenerates it automatically once it expires. You don't need to manage token lifecycle yourself for the common case. (For the rare expire-mid-request case, see [Bearer token expiration edge cases](#bearer-token-expiration-edge-cases).) - -**Configuration mutation is not concurrency-safe.** Methods that change client configuration at runtime — `add_vault_config`, `update_vault_config`, `remove_vault_config`, the `*_connection_config` methods, and `update_skyflow_credentials` — mutate shared client state without locking. Perform configuration changes during setup, not concurrently with in-flight requests from other threads. Once configured, reusing the built client to issue operations is the intended usage pattern. - -**Timeouts and retries.** The SDK does not currently expose request timeout or automatic-retry configuration. If you need strict timeout or retry guarantees, wrap your SDK calls with your own timeout/retry logic at the application layer. - -## Error handling - -### Catching `SkyflowError` instances - -Wrap your calls to the Skyflow SDK in try/except blocks as a best practice. Use the `SkyflowError` class to identify errors coming from Skyflow versus general request/response errors. - -```python -from skyflow.error import SkyflowError - -try: - # ...call the Skyflow SDK - pass -except SkyflowError as error: - # Handle Skyflow specific errors - print("Skyflow Specific Error:", { - "code": error.http_code, - "message": error.message, - "details": error.details, - }) -except Exception as error: - # Handle generic errors - print("Unexpected Error:", error) -``` - -### Bearer token expiration edge cases - -When using bearer tokens for authentication and API requests, a token may expire after verification but before the actual API call completes. This causes the request to fail unexpectedly. An error from this edge case looks like this: - -```txt -message: Authentication failed. Bearer token is expired. Use a valid bearer token. See https://docs.skyflow.com/api-authentication/ +pip install skyflow # Privacy DB SDK +pip install skyflow-flowvault # Flow DB SDK ``` -If you encounter this kind of error, retry the request. During the retry the SDK detects that the previous bearer token has expired and generates a new one for the current and subsequent requests. - -> [!TIP] -> See the full example in the samples directory: [bearer_token_expiry_example.py](samples/service_account/bearer_token_expiry_example.py) -> See [docs.skyflow.com](https://docs.skyflow.com) for more details on authentication, access control, and governance for Skyflow. - -## Troubleshooting - -Most first-run problems come from configuration mismatches. Every error raised by the SDK is a `SkyflowError` exposing `http_code`, `message`, and `details` — inspect these first (see [Error handling](#error-handling)). - -| Symptom | Likely cause | Fix | -|---------|--------------|-----| -| `pip install skyflow` fails / `RuntimeError: skyflow requires Python 3.9+` | Python older than 3.9 | Use Python 3.9 or above. | -| Connection/DNS failures, or 404 on every call | Wrong `cluster_id` | `cluster_id` is the first segment of your vault URL: `https://{cluster_id}.vault.skyflowapis.com`. | -| Requests hit the wrong host / unexpected auth failures | Wrong `env` | Match `env` to where your vault runs (`Env.PROD`, `Env.SANDBOX`, `Env.DEV`, `Env.STAGE`). | -| `401 Unauthorized` | Invalid or expired credentials | Verify your API key / service-account credentials. Regenerate if needed. | -| `403 Forbidden` | Service account lacks permission for the operation | Grant the service account a role with the required permissions, or use a [scoped token](#generate-bearer-tokens-scoped-to-certain-roles) with the right role. | -| `404` referencing a table or column | Table/column doesn't exist or name mismatch | Confirm the table and column names match your vault schema exactly (case-sensitive). | -| Vault not found / 404 with a valid `cluster_id` | Wrong `vault_id` | Copy `vault_id` from the vault's details page in Skyflow Studio. | -| `Authentication failed. Bearer token is expired.` | Token expired between verification and the API call | Retry the request; the SDK regenerates the token. See [Bearer token expiration edge cases](#bearer-token-expiration-edge-cases). | -| Unexpected credential is used | Multiple credentials provided | Only one credential type is used at a time; the last one added takes precedence. Provide exactly one. | -| `RequestMethod.PATCH` raises `AttributeError` | `PATCH` is not a supported connection method | Use `GET`, `POST`, `PUT`, or `DELETE` (see [`RequestMethod`](docs/api_reference.md#requestmethod)). | +> The two artifacts have **independent version lines** and cannot be installed into the same Python +> environment at once. A lower `skyflow-flowvault` version number (1.x) does not mean it is behind +> `skyflow` (2.x) — they are separate products. -If you're stuck, set `set_log_level(LogLevel.DEBUG)` during development for detailed SDK logs (see [Logging](#logging)). +## Repository layout -## Security +| Path | What it is | +|---|---| +| `common/` | Shared client, credentials, config, and error code — depended on by both SDKs, never published on its own. | +| `skyvault/` | The `skyflow` (Privacy DB / v2) SDK. | +| `flowvault/` | The `skyflow-flowvault` (Flow DB / v1) SDK. | +| `docs/` | Reference docs and the [v1 → v2 migration guide](docs/migrate_to_v2.md). | +| `CHANGELOG.md` | Release history. | -### Reporting a Vulnerability +Each SDK ships runnable examples under its own `samples/` directory +([flowvault/samples/](flowvault/samples/), [skyvault/samples/](skyvault/samples/)). -If you discover a potential security issue in this project, reach out to us at [security@skyflow.com](mailto:security@skyflow.com). +## Resources -Don't create public GitHub issues or Pull Requests, as malicious actors could potentially view them. +- [Skyflow docs](https://docs.skyflow.com/) +- [GitHub](https://github.com/skyflowapi/skyflow-python/) diff --git a/ci-scripts/bump_version.sh b/ci-scripts/bump_version.sh index 0fc6e782..507d5707 100755 --- a/ci-scripts/bump_version.sh +++ b/ci-scripts/bump_version.sh @@ -1,27 +1,51 @@ -Version=$1 -SEMVER=$Version -PackageName=${3:-skyflow} +#!/usr/bin/env bash +# Bumps 's version. +# +# Usage: bump_version.sh [] +# +# Always patches /setup.py's `current_version = '...'` line. +# Also patches a runtime version constant, if this module ships one, found +# at /**/_version.py (matching today's skyflow/utils/_version.py +# convention) - skipped with a notice if no such file exists yet, so this +# script stays correct both before and after modules gain their own runtime +# version file. +set -euo pipefail -if [ -z "$2" ] -then - echo "Bumping package version to $1" +Version="${1:?"Usage: bump_version.sh [] "}" +CommitHash="${2:-}" +Module="${3:?"Usage: bump_version.sh [] "}" - sed -E "s/current_version = .+/current_version = '$SEMVER'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile - sed -E "s/SDK_VERSION = .+/SDK_VERSION = '$SEMVER'/g" $PackageName/utils/_version.py > tempfile && cat tempfile > $PackageName/utils/_version.py && rm -f tempfile - sed -E "s/__version__ = .+/__version__ = '$SEMVER'/g" $PackageName/generated/rest/version.py > tempfile && cat tempfile > $PackageName/generated/rest/version.py && rm -f tempfile +SetupFile="$Module/setup.py" - echo -------------------------- - echo "Done, Package now at $1" +if [ ! -f "$SetupFile" ]; then + echo "Error: $SetupFile not found." >&2 + exit 1 +fi + +if [ -z "$CommitHash" ]; then + SEMVER="$Version" else - # Use dev version with commit SHA - DEV_VERSION="${SEMVER}.dev0+$(echo $2 | tr -dc '0-9a-f')" + SEMVER="${Version}.dev0+$(echo "$CommitHash" | tr -dc '0-9a-f')" +fi - echo "Bumping package version to $DEV_VERSION" +echo "Bumping $Module version to $SEMVER" - sed -E "s/current_version = .+/current_version = '$DEV_VERSION'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile - sed -E "s/SDK_VERSION = .+/SDK_VERSION = '$DEV_VERSION'/g" $PackageName/utils/_version.py > tempfile && cat tempfile > $PackageName/utils/_version.py && rm -f tempfile - sed -E "s/__version__ = .+/__version__ = '$DEV_VERSION'/g" $PackageName/generated/rest/version.py > tempfile && cat tempfile > $PackageName/generated/rest/version.py && rm -f tempfile +sed -E "s/current_version = .+/current_version = '$SEMVER'/g" "$SetupFile" > tempfile && cat tempfile > "$SetupFile" && rm -f tempfile - echo -------------------------- - echo "Done, Package now at $DEV_VERSION" +version_files=$(find "$Module" -name "_version.py") +version_file_count=$(echo "$version_files" | grep -c . || true) + +if [ "$version_file_count" -gt 1 ]; then + echo "Error: multiple _version.py files found under $Module - ambiguous, refusing to guess which to bump:" >&2 + echo "$version_files" >&2 + exit 1 +elif [ "$version_file_count" -eq 1 ]; then + version_file="$version_files" + sed -E "s/SDK_VERSION = .+/SDK_VERSION = '$SEMVER'/g" "$version_file" > tempfile && cat tempfile > "$version_file" && rm -f tempfile + echo "Also bumped $version_file" +else + echo "::notice::No _version.py found under $Module yet - skipping runtime version bump" fi + +echo -------------------------- +echo "Done, $Module now at $SEMVER" diff --git a/ci-scripts/current_module_version.sh b/ci-scripts/current_module_version.sh new file mode 100755 index 00000000..9205ccc4 --- /dev/null +++ b/ci-scripts/current_module_version.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Prints 's own current version from its setup.py, with any existing +# .devN+ suffix stripped. Read-only - never modifies setup.py. +# +# Used by internal releases to get a module's base version without touching +# git tags at all: tags are a flat, repo-wide namespace with no module +# awareness, so a tag-based lookup would risk stamping one module's version +# onto another module's release. +set -euo pipefail + +Module="${1:?"Usage: current_module_version.sh "}" +SetupFile="$Module/setup.py" + +if [ ! -f "$SetupFile" ]; then + echo "Error: $SetupFile not found." >&2 + exit 1 +fi + +version=$(grep -E "current_version = " "$SetupFile" | head -n 1 | sed -E "s/.*current_version = '([^']+)'.*/\1/") + +if [ -z "$version" ]; then + echo "Error: could not find a current_version line in $SetupFile" >&2 + exit 1 +fi + +# Strip a trailing .devN+ suffix (internal-release versions), if present. +version=$(echo "$version" | sed -E 's/\.dev[0-9]+\+[0-9a-f]+$//') + +# Sanity check: if the sed extraction above silently failed to match (e.g. +# setup.py used double quotes instead of single), $version would be the +# entire matched grep line rather than a bare version - catch that here +# instead of letting a garbage string flow into a version-bump commit. +if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "Error: could not parse a valid version from $SetupFile (got: '$version')" >&2 + exit 1 +fi + +echo "$version" diff --git a/common/tests/utils/validations/test__validations.py b/common/tests/utils/validations/test__validations.py index b10fc008..7684841b 100644 --- a/common/tests/utils/validations/test__validations.py +++ b/common/tests/utils/validations/test__validations.py @@ -7,6 +7,7 @@ validate_update_vault_config, validate_credentials, validate_log_level, + validate_non_empty_string_list, ) VALID_VAULT_CONFIG = { @@ -165,5 +166,31 @@ def test_uses_injected_messages(self): self.assertIn("FAKE", ctx.exception.message) +class TestValidateNonEmptyStringList(unittest.TestCase): + def test_valid_list_passes(self): + validate_non_empty_string_list(None, ["a", "b"], "boom") # should not raise + + def test_non_list_raises_with_given_error(self): + with self.assertRaises(SkyflowError) as ctx: + validate_non_empty_string_list(None, "not-a-list", "boom") + self.assertEqual(ctx.exception.message, "boom") + + def test_empty_list_raises(self): + with self.assertRaises(SkyflowError): + validate_non_empty_string_list(None, [], "boom") + + def test_none_raises(self): + with self.assertRaises(SkyflowError): + validate_non_empty_string_list(None, None, "boom") + + def test_non_string_entry_raises(self): + with self.assertRaises(SkyflowError): + validate_non_empty_string_list(None, ["a", 1], "boom") + + def test_blank_string_entry_raises(self): + with self.assertRaises(SkyflowError): + validate_non_empty_string_list(None, ["a", " "], "boom") + + if __name__ == "__main__": unittest.main() diff --git a/common/utils/validations/__init__.py b/common/utils/validations/__init__.py index d49cc5de..f5100899 100644 --- a/common/utils/validations/__init__.py +++ b/common/utils/validations/__init__.py @@ -4,6 +4,7 @@ validate_credentials, validate_log_level, validate_keys, + validate_non_empty_string_list, validate_vault_config, validate_update_vault_config, ) diff --git a/common/utils/validations/_validations.py b/common/utils/validations/_validations.py index f600b6a0..2ce3bfdb 100644 --- a/common/utils/validations/_validations.py +++ b/common/utils/validations/_validations.py @@ -173,6 +173,11 @@ def validate_keys(logger, config, config_keys, messages=None): raise SkyflowError(messages.Error.INVALID_KEY.value.format(key), invalid_input_error_code) +def validate_non_empty_string_list(logger, value, error): + if not isinstance(value, list) or not value or not all(isinstance(item, str) and item.strip() for item in value): + raise SkyflowError(error, invalid_input_error_code) + + def validate_vault_config(logger, config, messages=None): messages = messages or SkyflowMessages log_info(messages.Info.VALIDATING_VAULT_CONFIG.value, logger) diff --git a/flowvault/CONTRACT_SHAPES.md b/flowvault/CONTRACT_SHAPES.md new file mode 100644 index 00000000..af6efa10 --- /dev/null +++ b/flowvault/CONTRACT_SHAPES.md @@ -0,0 +1,361 @@ +# flowvault Python SDK — Request / Response shapes + +JSON shapes as the **Python** flowvault SDK currently produces them, for manual comparison +against the Java FlowDB contract. Keys are **snake_case**. `tokens` and `hashed_data` are +normalized to typed lists; detokenize `metadata` is normalized to `{skyflow_id, table_name}`. + +> Requests below are shown as the JSON equivalent of the SDK request objects (constructor args). +> Request wire bodies are built separately by the generated client and are not shown here. +> `//` comments name the Python data class each object maps to (Python responses use plain dicts, +> so the Java equivalent class is noted for those). Blocks are `jsonc` (JSON + comments). + +**Parity status per op** + +| Operation | In Java contract? | Response shape | +|-----------|-------------------|----------------| +| insert / get / delete / detokenize / query | yes | new unified `records` list | +| bulk_insert / bulk_detokenize | yes | `summary` + `records` | +| update | **no (Python-only)** | old split `records` + `errors` | + +--- + +## Unary — insert + +**Request** — `InsertRequest(records: List[InsertRequestRecord], table_name=None, upsert=None)` +```jsonc +// InsertRequest +{ + "table_name": "cards", + "upsert": { "unique_columns": ["card_number"], "update_type": "UPDATE" }, // UpsertOptions(unique_columns, update_type) + "records": [ + // InsertRequestRecord(data, table_name=None, tokens=None, upsert=None) + { "data": { "card_number": "4111111111111111", "cardholder_name": "john doe" } } + ] +} +``` +Per-record table_name/upsert instead of request-level (exactly one level, never both); `tokens` is optional BYOT: +```jsonc +// InsertRequest +{ + "records": [ + // InsertRequestRecord + { "data": { "email": "jane@example.com" }, "table_name": "contacts", + "tokens": { "email": "my-own-token" }, + "upsert": { "unique_columns": ["email"], "update_type": "REPLACE" } } // upsert: UpsertOptions(unique_columns, update_type) + ] +} +``` +> `update_type` is an `UpsertType` enum (`.value` is the Java `"UPDATE"`/`"REPLACE"` string). + +**Response** — `InsertResponse(records)` +```jsonc +// InsertResponse +{ + "records": [ + // record: plain dict (Java: InsertResponseRecord) + { + "table_name": "cards", + "skyflow_id": "f1714ef8-8deb-489a-a18d-77e0e007f403", + "tokens": { + // each entry: plain dict (Java: Token) + "ssn": [ + { "token": "3340-9871-4511-3462", "token_group_name": "deterministic_string", "path": null }, + { "token": "7823-1234-5678-9012", "token_group_name": "random_string", "path": null } + ] + }, + "hashed_data": { "ssn": [ { "data": "2f3fd7b1d46c...", "hash_name": "hash1" } ] }, // entry Java: HashedValue + "http_code": 200, + "error": null + }, + { + "table_name": null, + "skyflow_id": null, + "tokens": null, + "hashed_data": null, + "http_code": 400, + "error": "Invalid request. Table name table not present for record. Specify a valid table name." + } + ] +} +``` +> Insert records omit `data` (unlike `get`, which includes it). + +--- + +## Unary — get + +Two **mutually exclusive** request modes. + +**Request — single-table mode** — `GetRequest(table, ids, unique_values, columns, column_redactions, limit, offset)` +```jsonc +// GetRequest +{ + "table": "persons", + "ids": ["9f5b8e6e-..."], + "unique_values": [ { "email": "john@example.com" } ], + "columns": ["name", "email"], + "column_redactions": [ { "column_name": "email", "redaction": "MASKED" } ], // entry: ColumnRedaction(column_name, redaction) + "limit": 25, + "offset": 0 +} +``` +**Request — multi-table batch mode** — `GetRequest(records=[GetRecordRequest(table, ids, columns, column_redactions: List[ColumnRedaction], unique_values)])` (no `limit`/`offset`; single-table fields must be unset) +```jsonc +// GetRequest +{ + "records": [ + // GetRecordRequest(table, ids=None, columns=None, column_redactions=None, unique_values=None) + { "table": "persons", "ids": ["9f5b8e6e-..."], "columns": ["name"], + "column_redactions": [], "unique_values": [] }, + { "table": "cards", "unique_values": [ { "email": "john@example.com" } ] } + ] +} +``` + +**Response** — `GetResponse(records)` (same per-record builder as insert, but `data` is included) +```jsonc +// GetResponse +{ + "records": [ + // record: plain dict (Java: GetResponseRecord / shared Record) + { + "table_name": "persons", + "skyflow_id": "9f5b8e6e-...", + "tokens": { "card_number": [ { "token": "5301-6390-5701-2392", "token_group_name": "det", "path": null } ] }, // entry Java: Token + "data": { "name": "John Doe", "email": "a1b2c3d4" }, + "hashed_data": { "email": [ { "data": "2f3fd7b1d46c...", "hash_name": "hash1" } ] }, // entry Java: HashedValue + "http_code": 200, + "error": null + }, + { + "table_name": null, + "skyflow_id": null, + "tokens": null, + "data": null, + "hashed_data": null, + "http_code": 404, + "error": "Record not found" + } + ] +} +``` + +--- + +## Unary — delete + +**Request** — `DeleteRequest(table, ids, unique_values)` +```jsonc +// DeleteRequest +{ "table": "persons", "ids": ["9f5b8e6e-..."], "unique_values": [ { "email": "john@example.com" } ] } +``` + +**Response** — `DeleteResponse(records)` +```jsonc +// DeleteResponse +{ + "records": [ + // record: plain dict (Java: DeleteResponseRecord / shared Record) + { "skyflow_id": "9f5b8e6e-...", "http_code": 200, "error": null }, + { "skyflow_id": null, "http_code": 404, "error": "Record not found" } + ] +} +``` + +--- + +## Unary — detokenize + +**Request** — `DetokenizeRequest(tokens, token_group_redactions)` +```jsonc +// DetokenizeRequest +{ + "tokens": ["12393023", "7c4a0139-9033-40ae-b41f-f3837976721"], + "token_group_redactions": [ { "token_group_name": "deterministic_string", "redaction": "MASKED" } ] // entry: dict {token_group_name, redaction} (Java: TokenGroupRedactions) +} +``` + +**Response** — `DetokenizeResponse(records)` +```jsonc +// DetokenizeResponse +{ + "records": [ + // record: plain dict (Java: DetokenizeResponseRecord) + { + "token": "12393023", + "token_group_name": "deterministic_string", + "value": "john@example.com", + "metadata": { "skyflow_id": "3ac0424e-fe45-43a9-9193-2e6d2913cbd2", "table_name": "table1" }, // dict (Java: DetokenizeMetadata) + "http_code": 200, + "error": null + }, + { + "token": "7c4a0139-9033-40ae-b41f-f3837976721", + "token_group_name": null, + "value": null, + "metadata": null, + "http_code": 404, + "error": "Detokenize failed. Token 7c4a0139-... is invalid. Specify a valid token." + } + ] +} +``` +> Note: the Java contract's new `DetokenizeResponseRecord` omits `value`; Python **keeps** it. + +--- + +## Unary — query + +**Request** — `QueryRequest(query)` +```jsonc +// QueryRequest +{ "query": "SELECT * FROM persons WHERE skyflow_id = '9f5b8e6e-...'" } +``` + +**Response** — `QueryResponse(records, metadata)` +```jsonc +// QueryResponse +{ + "records": [ + // record: plain dict {data} (Java: QueryResponseRecord) + { "data": { "skyflow_id": "9f5b8e6e-...", "name": "John Doe", "email": "a1b2c3d4" } } + ], + "metadata": { "columns": ["skyflow_id", "name", "email"] } // dict (Java: QueryResponseMetadata) +} +``` +On a failed call (`metadata` is `null`): +```jsonc +// QueryResponse +{ "records": [ { "data": null, "http_code": 400, "error": "bad query" } ], "metadata": null } +``` + +--- + +## Bulk — bulk_insert / bulk_insert_async + +**Request** — `BulkInsertRequest(records: List[BulkInsertRecord], table=None, upsert=None)` + +> Naming/shape difference vs Java and vs unary: the bulk record type is `BulkInsertRecord(data, table=None, upsert=None)` +> (**no** `tokens` field), whereas unary insert uses `InsertRequestRecord(data, table, tokens, upsert)`. +> Java has `BulkInsertRequestRecord extends InsertRequestRecord` (same fields, `tokens` included). +```jsonc +// BulkInsertRequest +{ + "table": "cards", + "upsert": { "unique_columns": ["card_number"], "update_type": "UPDATE" }, // UpsertOptions + "records": [ + // BulkInsertRecord(data, table=None, upsert: UpsertOptions=None) -- note: field is `table`, not `table_name` + { "data": { "card_number": "4111111111111111", "cardholder_name": "john doe" } }, + { "data": { "email": "jane@example.com" }, "table": "contacts", + "upsert": { "unique_columns": ["email"] } } // UpsertOptions + ] +} +``` + +**Response** — `BulkInsertResponse(summary, records)` +```jsonc +// BulkInsertResponse +{ + "summary": { "total_records": 2, "total_inserted": 1, "total_failed": 1 }, // BulkSummary + "records": [ + // record: plain dict (Java: BulkInsertResponseRecord) + { + "index": 0, + "request_id": null, + "table_name": "cards", + "skyflow_id": "9fac9201-7b8a-4446-93f8-5244e1213bd1", + "tokens": { "card_number": [ { "token": "5484-7829-1702-9110", "token_group_name": "card_number_cg", "path": null } ] }, // entry Java: Token + "data": { "card_number": "4111-1111-1111-1111" }, + "hashed_data": { "card_number": [ { "data": "b6e6d...c3f9", "hash_name": "hash1" } ] }, // entry Java: HashedValue + "http_code": 200, + "error": null + }, + { + "index": 1, + "request_id": "a1b2c3d4-...", + "table_name": null, + "skyflow_id": null, + "tokens": null, + "data": null, + "hashed_data": null, + "http_code": 400, + "error": "Insert failed. Column email is invalid." + } + ] +} +``` +> `response.records_to_retry()` returns the original `BulkInsertRecord`s whose `http_code` is 500–599 (excluding 529). Not part of the JSON. + +--- + +## Bulk — bulk_detokenize / bulk_detokenize_async + +**Request** — `BulkDetokenizeRequest(tokens, token_group_redactions)` +```jsonc +// BulkDetokenizeRequest +{ + "tokens": ["5479-4229-4622-1393", "a1b2c3d4-e5f6-7890-abcd-ef1234567890"], + "token_group_redactions": [ { "token_group_name": "card_number_cg", "redaction": "MASKED" } ] // entry: dict {token_group_name, redaction} (Java: TokenGroupRedactions) +} +``` + +**Response** — `BulkDetokenizeResponse(summary, records)` +```jsonc +// BulkDetokenizeResponse +{ + "summary": { "total_tokens": 2, "total_detokenized": 1, "total_failed": 1 }, // DetokenizeSummary + "records": [ + // record: plain dict (Java: BulkDetokenizeResponseRecord) + { + "index": 0, + "request_id": null, + "value": "4111111111111111", + "token_group_name": "card_number_cg", + "metadata": { "skyflow_id": "9fac9201-...", "table_name": "table1" }, // dict (Java: DetokenizeMetadata) + "http_code": 200, + "token": "5479-4229-4622-1393", + "error": null + }, + { + "index": 1, + "request_id": "a1b2c3d4-...", + "value": null, + "token_group_name": null, + "metadata": null, + "http_code": 404, + "token": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "error": "Token Not Found" + } + ] +} +``` +> `response.tokens_to_retry()` returns the original token strings whose `http_code` is 500–599 (excluding 529). + +--- + +## Python-only ops (NOT in the Java FlowDB contract — still the OLD split shape) + +### update + +**Request** — `UpdateRequest(records: list[dict], table_name=None, update_type=None)` +```jsonc +// UpdateRequest (records are plain dicts, not a typed record class) +{ + "table_name": "persons", + "update_type": "REPLACE", + "records": [ + { "skyflow_id": "9f5b8e6e-...", "data": { "name": "Jane" }, "tokens": { "ssn": "tok" }, "table_name": null } + ] +} +``` +> The regenerated `update_records` endpoint does not accept `update_type`; the field is validated +> but not forwarded. + +**Response** — `UpdateResponse(records, errors)` — old split shape (`records` = successes, plus `errors`) +```jsonc +// UpdateResponse +{ + "records": [ { "request_index": 0, "skyflow_id": "9f5b8e6e-...", "ssn": "tok1", "data": { "name": "Jane" } } ], + "errors": [ { "request_index": 1, "error": "not found", "code": 404, "request_id": "req-..." } ] +} +``` diff --git a/flowvault/MANIFEST.in b/flowvault/MANIFEST.in new file mode 100644 index 00000000..05007153 --- /dev/null +++ b/flowvault/MANIFEST.in @@ -0,0 +1 @@ +prune samples diff --git a/flowvault/README.md b/flowvault/README.md new file mode 100644 index 00000000..0a6a41d8 --- /dev/null +++ b/flowvault/README.md @@ -0,0 +1,297 @@ +# Skyflow FlowVault Python SDK + +`skyflow-flowvault` is the Skyflow Python SDK built for **Flow DB** vaults. It shares its client, +credentials, and configuration with the [skyvault SDK](../skyvault/README.md) (both depend on the +`common` module) but exposes its own surface: **unary** vault operations plus **bulk** (batched, +concurrent) insert and detokenize. + +> **`skyflow-flowvault` is versioned independently of `skyflow`.** It launched at `1.0.0` while +> `skyflow` is at `2.x`. The two are separate artifacts on separate version lines and cannot be +> installed into the same Python environment at once. + +## Table of Contents + +- [Overview](#overview) +- [Install](#install) +- [Quickstart](#quickstart) +- [Authenticate](#authenticate) +- [Initialize the client](#initialize-the-client) +- [Unary operations](#unary-operations) + - [Insert](#insert) · [Get](#get) · [Update](#update) · [Delete](#delete) · [Detokenize](#detokenize) · [Query](#query) +- [Bulk operations](#bulk-operations) + - [Bulk insert](#bulk-insert) · [Bulk detokenize](#bulk-detokenize) + - [Batching and concurrency](#batching-and-concurrency) +- [Error handling](#error-handling) +- [Logging](#logging) +- [Samples](#samples) +- [Request / response shapes](#request--response-shapes) + +## Overview + +- Authenticate with a Skyflow service account, an API key, or a bearer token. +- Perform **unary** operations — insert, get, update, delete, detokenize, query. +- Perform **bulk** operations — insert and detokenize — each with a synchronous and an async + variant, built for high-throughput Flow DB workloads. +- **Per-record reporting, not all-or-nothing.** A call succeeds as a call even when individual + records fail; each response reports the outcome of every record (its own `http_code` and `error`). + +## Install + +```bash +pip install skyflow-flowvault +``` + +Requirements: **Python 3.9+**. + +## Quickstart + +```python +from skyflow_flowvault import Skyflow, LogLevel, Env +from skyflow_flowvault.vault.data import InsertRequest, InsertRequestRecord + +credentials = {'api_key': ''} # or 'token' / 'path' / 'credentials_string' + +vault_config = { + 'vault_id': '', + 'cluster_id': '', # from the vault URL: https://{cluster_id}.vault.skyflowapis.com + 'env': Env.PROD, # DEV, STAGE, SANDBOX, or PROD (default) + 'credentials': credentials, +} + +skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) # default is ERROR + .build() +) + +vault = skyflow_client.vault('') + +response = vault.insert(InsertRequest( + table_name='cards', + records=[InsertRequestRecord(data={'card_number': '4111111111111111'})], +)) +print(response.records) +``` + +## Authenticate + +Requests are authorized with Skyflow credentials attached to the vault config's `credentials` dict. +Set **exactly one** of: + +| Key | What it is | +|---|---| +| `api_key` | A long-lived API key. Simplest option. | +| `token` | A short-lived bearer token you generate yourself. | +| `path` | Filesystem path to a service-account `credentials.json` — the SDK generates and refreshes tokens. | +| `credentials_string` | The contents of a `credentials.json` as a string — use when it comes from a secret store. | + +Credentials resolve **most specific first**: per-vault (`vault_config['credentials']`) → client-wide +(`Skyflow.builder().add_skyflow_credentials(...)`) → the `SKYFLOW_CREDENTIALS` environment variable. + +## Initialize the client + +Build the client once and keep it for your application's lifetime; get a controller from it with +`vault(...)`. + +```python +vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': {'path': ''}, +} + +skyflow_client = Skyflow.builder().add_vault_config(vault_config).build() +vault = skyflow_client.vault('') +``` + +## Unary operations + +Every unary response is a single **`records`** list — success and failure inline, one entry per +input, each carrying its own `http_code` and `error`. Exact JSON for each is in +[CONTRACT_SHAPES.md](CONTRACT_SHAPES.md). + +### Insert + +`table_name`/`upsert` go at **exactly one** level — on the request (applies to all records) or on +every record — never both. `upsert` is an `UpsertOptions`; `tokens` is optional BYOT. + +```python +from skyflow_flowvault.vault.data import InsertRequest, InsertRequestRecord, UpsertOptions +from skyflow_flowvault.utils.enums import UpsertType + +request = InsertRequest( + table_name='cards', + upsert=UpsertOptions(unique_columns=['card_number'], update_type=UpsertType.UPDATE), + records=[InsertRequestRecord(data={'card_number': '4111111111111111', 'cardholder_name': 'john doe'})], +) +response = vault.insert(request) +for r in response.records: + print(r['index'] if 'index' in r else '', r['skyflow_id'], r['tokens'], r['http_code'], r['error']) +``` +Each record: `{table_name, skyflow_id, tokens, hashed_data, http_code, error}` (no plaintext `data`). + +### Get + +Two mutually exclusive modes — single-table, or multi-table via `records=[GetRecordRequest(...)]`. +`column_redactions` entries are `ColumnRedaction` objects. + +```python +from skyflow_flowvault.vault.data import GetRequest, GetRecordRequest, ColumnRedaction + +# single-table +vault.get(GetRequest( + table='persons', ids=[''], columns=['name', 'email'], + column_redactions=[ColumnRedaction(column_name='email', redaction='MASKED')], +)) + +# multi-table batch +vault.get(GetRequest(records=[ + GetRecordRequest(table='persons', ids=[''], columns=['name']), + GetRecordRequest(table='cards', unique_values=[{'email': 'john@example.com'}]), +])) +``` +Each record: `{table_name, skyflow_id, tokens, data, hashed_data, http_code, error}`. + +### Update + +```python +from skyflow_flowvault.vault.data import UpdateRequest + +vault.update(UpdateRequest( + table_name='persons', + records=[{'skyflow_id': '', 'data': {'name': 'Jane'}}], +)) +``` + +### Delete + +```python +from skyflow_flowvault.vault.data import DeleteRequest + +vault.delete(DeleteRequest(table='persons', ids=[''])) +``` +Each record: `{skyflow_id, http_code, error}`. + +### Detokenize + +```python +from skyflow_flowvault.vault.data import DetokenizeRequest + +vault.detokenize(DetokenizeRequest( + tokens=[''], + token_group_redactions=[{'token_group_name': 'card_number_cg', 'redaction': 'MASKED'}], +)) +``` +Each record: `{token, token_group_name, value, metadata, http_code, error}`. + +### Query + +```python +from skyflow_flowvault.vault.data import QueryRequest + +response = vault.query(QueryRequest(query="SELECT * FROM persons WHERE skyflow_id = ''")) +print(response.records) # [{'data': {...}}, ...] +print(response.metadata) # {'columns': [...]} +``` + +## Bulk operations + +Bulk operations split the payload into batches sent **concurrently** and return a **`summary`** plus +a **`records`** list — one entry per submitted item, in input order, each tagged with its `index`. +A single bulk call accepts at most **10,000** items. + +### Bulk insert + +```python +from skyflow_flowvault.vault.data import BulkInsertRequest, BulkInsertRecord + +request = BulkInsertRequest(table='cards', records=[ + BulkInsertRecord(data={'card_number': '4111111111111111'}), + BulkInsertRecord(data={'card_number': '4222222222222222'}), +]) + +response = vault.bulk_insert(request) # synchronous +# response = await vault.bulk_insert_async(request) # async variant + +print(response.summary.total_records, response.summary.total_inserted, response.summary.total_failed) +for r in response.records: + print(r['index'], r['skyflow_id'], r['http_code'], r['error']) + +retry = response.records_to_retry() # original records whose http_code is 500-599 (excl. 529) +``` +> `BulkInsertRecord` uses the field name `table` (not `table_name`) and has no `tokens` field. + +### Bulk detokenize + +```python +from skyflow_flowvault.vault.data import BulkDetokenizeRequest + +request = BulkDetokenizeRequest(tokens=['', '']) + +response = vault.bulk_detokenize(request) # synchronous +# response = await vault.bulk_detokenize_async(request) # async variant + +retry_tokens = response.tokens_to_retry() +``` + +### Batching and concurrency + +Batch size and concurrency are configured **per operation** via environment variables, read from the +process environment first, then from a `.env` file in the working directory (via `python-dotenv`). + +| Operation | Batch size var | Default | Max | Concurrency var | Default | Max | +|---|---|---|---|---|---|---| +| Bulk insert | `INSERT_BATCH_SIZE` | 50 | 1000 | `INSERT_CONCURRENCY_LIMIT` | 1 | 10 | +| Bulk detokenize | `DETOKENIZE_BATCH_SIZE` | 50 | 1000 | `DETOKENIZE_CONCURRENCY_LIMIT` | 1 | 10 | + +Resolution: `batch_size = min(value, max)`; `concurrency = min(value, max, ceil(item_count / batch_size))` +— concurrency never exceeds the number of batches. Invalid values log a warning and fall back to the +default. The 10,000-item ceiling per call is fixed and not configurable. + +```dotenv +# .env +INSERT_BATCH_SIZE=100 +INSERT_CONCURRENCY_LIMIT=5 +``` + +Merging is by input order regardless of which batch finishes first, so `index` always matches an +item's position in your submitted payload. A per-batch failure only fails that batch's records. + +## Error handling + +Two layers: + +- **Request-level** — the call could not be made or wholly failed (invalid request, missing + credentials, auth failure, over the 10,000 ceiling): raised as a `SkyflowError`. +- **Record-level** — the call succeeded but individual records failed: returned in the response. + **Nothing is raised.** Each entry in `records` reports its own `http_code` and `error`. + +```python +from skyflow_flowvault.error import SkyflowError + +try: + response = vault.bulk_insert(request) # reaching here means the CALL succeeded + for r in response.records: + if r['error'] is not None: + print('row', r['index'], 'failed', r['http_code'], r['error']) +except SkyflowError as e: + print(e.http_code, e.message, e.details) +``` + +## Logging + +The SDK logs at `LogLevel.ERROR` by default. Change it with +`Skyflow.builder().set_log_level(LogLevel.INFO)` (`DEBUG` < `INFO` < `WARN` < `ERROR` < `OFF`). The +batching warnings above are emitted at `WARN`. + +## Samples + +Runnable examples live in [samples/](samples/) — one file per operation, with sync/async pairs for +the bulk ops. See [samples/README.md](samples/README.md) to run them. + +## Request / response shapes + +[CONTRACT_SHAPES.md](CONTRACT_SHAPES.md) documents the exact request and response JSON for every +operation (unary and bulk), for reference and comparison against the Java FlowDB contract. diff --git a/flowvault/samples/README.md b/flowvault/samples/README.md new file mode 100644 index 00000000..b9d0c586 --- /dev/null +++ b/flowvault/samples/README.md @@ -0,0 +1,55 @@ +# FlowVault Python samples + +Runnable examples for the `skyflow-flowvault` SDK — one file per operation. See the +[flowvault README](../README.md) for the full SDK guide. + +## Prerequisites + +- Python 3.9+ +- `pip install skyflow-flowvault` +- A Flow DB vault and Skyflow credentials (a service-account `credentials.json`, an API key, or a + bearer token). + +## Configure + +Each sample has placeholders near the top — replace them with your own values: + +```python +credentials = {'path': ''} # or 'api_key' / 'token' / 'credentials_string' +vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, +} +``` + +For the bulk samples you can tune batching/concurrency via env vars or a `.env` file in the working +directory — e.g. `INSERT_BATCH_SIZE=100`, `INSERT_CONCURRENCY_LIMIT=5`. + +## Run + +```bash +python flowvault/samples/vault_api/insert_records.py +python flowvault/samples/vault_api/bulk_insert_async.py # async samples run themselves via asyncio.run(...) +``` + +## Vault operations + +| Sample | Demonstrates | +|---|---| +| [insert_records.py](vault_api/insert_records.py) | Insert records (request-level and per-record `table_name`/`upsert`) | +| [get_records.py](vault_api/get_records.py) | Retrieve records by Skyflow ID | +| [update_record.py](vault_api/update_record.py) | Update a record | +| [delete_records.py](vault_api/delete_records.py) | Delete records | +| [detokenize_records.py](vault_api/detokenize_records.py) | Detokenize tokens | +| [query_records.py](vault_api/query_records.py) | Run a SQL `SELECT` query | + +## Bulk operations + +Each bulk operation ships a **sync** and an **async** variant. + +| Sample | Demonstrates | +|---|---| +| [bulk_insert_sync.py](vault_api/bulk_insert_sync.py) / [bulk_insert_async.py](vault_api/bulk_insert_async.py) | Batched, concurrent insert of many records; `summary`, per-record results, `records_to_retry()` | +| [bulk_detokenize_sync.py](vault_api/bulk_detokenize_sync.py) / [bulk_detokenize_async.py](vault_api/bulk_detokenize_async.py) | Batched, concurrent detokenize of many tokens; `tokens_to_retry()` | diff --git a/flowvault/samples/vault_api/bulk_detokenize_async.py b/flowvault/samples/vault_api/bulk_detokenize_async.py new file mode 100644 index 00000000..cdc30e29 --- /dev/null +++ b/flowvault/samples/vault_api/bulk_detokenize_async.py @@ -0,0 +1,49 @@ +import asyncio + +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import BulkDetokenizeRequest + + +async def perform_bulk_detokenize_async(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + detokenize_request = BulkDetokenizeRequest( + tokens=['', ''], + ) + + # Async variant -- batches are dispatched concurrently and awaited. + response = await skyflow_client.vault(vault_config.get('vault_id')).bulk_detokenize_async(detokenize_request) + + print('Summary: ', response.summary) + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +asyncio.run(perform_bulk_detokenize_async()) diff --git a/flowvault/samples/vault_api/bulk_detokenize_sync.py b/flowvault/samples/vault_api/bulk_detokenize_sync.py new file mode 100644 index 00000000..10cfe599 --- /dev/null +++ b/flowvault/samples/vault_api/bulk_detokenize_sync.py @@ -0,0 +1,59 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import BulkDetokenizeRequest + + +def perform_bulk_detokenize(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + # Batch size and concurrency are configured via env vars / a .env file: + # DETOKENIZE_BATCH_SIZE (default 50, max 1000), DETOKENIZE_CONCURRENCY_LIMIT (default 1, max 10). + # A single bulk call accepts at most 10,000 tokens. + detokenize_request = BulkDetokenizeRequest( + tokens=['', ''], + # optional per-group redaction override: + token_group_redactions=[{'token_group_name': '', 'redaction': 'MASKED'}], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).bulk_detokenize(detokenize_request) + + # response.summary: total_tokens / total_detokenized / total_failed + # response.records: one entry per input token, in order, each tagged with 'index': + # {'index': 0, 'request_id': None, 'value': '', 'token_group_name': '', + # 'metadata': {...}, 'http_code': 200, 'token': '', 'error': None} + print('Summary: ', response.summary) + print('Records: ', response.records) + + retry_tokens = response.tokens_to_retry() + if retry_tokens: + print(f'{len(retry_tokens)} token(s) worth retrying') + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_bulk_detokenize() diff --git a/flowvault/samples/vault_api/bulk_insert_async.py b/flowvault/samples/vault_api/bulk_insert_async.py new file mode 100644 index 00000000..9b742e22 --- /dev/null +++ b/flowvault/samples/vault_api/bulk_insert_async.py @@ -0,0 +1,53 @@ +import asyncio + +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import BulkInsertRequest, BulkInsertRecord + + +async def perform_bulk_insert_async(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + insert_request = BulkInsertRequest( + table='', + records=[ + BulkInsertRecord(data={'name': 'John Doe', 'email': 'john@example.com'}), + BulkInsertRecord(data={'name': 'Jane Doe', 'email': 'jane@example.com'}), + ], + ) + + # Async variant -- batches are dispatched concurrently and awaited. + response = await skyflow_client.vault(vault_config.get('vault_id')).bulk_insert_async(insert_request) + + print('Summary: ', response.summary) + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +asyncio.run(perform_bulk_insert_async()) diff --git a/flowvault/samples/vault_api/bulk_insert_sync.py b/flowvault/samples/vault_api/bulk_insert_sync.py new file mode 100644 index 00000000..9a9e89f0 --- /dev/null +++ b/flowvault/samples/vault_api/bulk_insert_sync.py @@ -0,0 +1,65 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import BulkInsertRequest, BulkInsertRecord, UpsertOptions +from skyflow_flowvault.utils.enums import UpsertType + + +def perform_bulk_insert(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + # Batch size and concurrency are configured via env vars / a .env file: + # INSERT_BATCH_SIZE (default 50, max 1000), INSERT_CONCURRENCY_LIMIT (default 1, max 10). + # A single bulk call accepts at most 10,000 records. + insert_request = BulkInsertRequest( + table='', + # upsert is optional; when present it sits at the same level as the table. + upsert=UpsertOptions(unique_columns=['email'], update_type=UpsertType.UPDATE), + records=[ + BulkInsertRecord(data={'name': 'John Doe', 'email': 'john@example.com'}), + BulkInsertRecord(data={'name': 'Jane Doe', 'email': 'jane@example.com'}), + ], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).bulk_insert(insert_request) + + # response.summary: total_records / total_inserted / total_failed + # response.records: one entry per input, in order, each tagged with 'index': + # {'index': 0, 'request_id': None, 'table_name': '', 'skyflow_id': '', + # 'tokens': {...}, 'data': {...}, 'hashed_data': {...}, 'http_code': 200, 'error': None} + print('Summary: ', response.summary) + print('Records: ', response.records) + + # Only server-side (5xx, excl. 529) failures are worth retrying: + retry = response.records_to_retry() + if retry: + print(f'{len(retry)} record(s) worth retrying') + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_bulk_insert() diff --git a/flowvault/samples/vault_api/delete_records.py b/flowvault/samples/vault_api/delete_records.py new file mode 100644 index 00000000..e61b5aa2 --- /dev/null +++ b/flowvault/samples/vault_api/delete_records.py @@ -0,0 +1,49 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import DeleteRequest + + +def perform_secure_data_deletion(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + delete_request = DeleteRequest( + table='', + ids=['', ''], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).delete(delete_request) + + # response.records (one entry per input, success + failure inline): + # {'skyflow_id': '', 'http_code': 200, 'error': None} + # {'skyflow_id': None, 'http_code': 404, 'error': ''} + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_secure_data_deletion() diff --git a/flowvault/samples/vault_api/detokenize_records.py b/flowvault/samples/vault_api/detokenize_records.py new file mode 100644 index 00000000..acd6f377 --- /dev/null +++ b/flowvault/samples/vault_api/detokenize_records.py @@ -0,0 +1,49 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import DetokenizeRequest + + +def perform_secure_detokenization(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + detokenize_request = DetokenizeRequest( + tokens=['', ''], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).detokenize(detokenize_request) + + # response.records (one entry per input, success + failure inline): + # {'token': '', 'token_group_name': '', 'value': '', + # 'metadata': {'skyflow_id': '', 'table_name': '
'}, 'http_code': 200, 'error': None} + # {'token': '', ..., 'http_code': 404, 'error': ''} + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_secure_detokenization() diff --git a/flowvault/samples/vault_api/get_records.py b/flowvault/samples/vault_api/get_records.py new file mode 100644 index 00000000..824fb8c0 --- /dev/null +++ b/flowvault/samples/vault_api/get_records.py @@ -0,0 +1,50 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import GetRequest + + +def perform_secure_data_retrieval(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + get_request = GetRequest( + table='', + ids=['', ''], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).get(get_request) + + # response.records (one entry per input, success + failure inline): + # {'table_name': 'persons', 'skyflow_id': '', 'tokens': {...}, 'data': {...}, + # 'hashed_data': {...}, 'http_code': 200, 'error': None} + # {'table_name': None, 'skyflow_id': None, ..., 'http_code': 404, 'error': 'Record not found'} + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_secure_data_retrieval() diff --git a/flowvault/samples/vault_api/insert_records.py b/flowvault/samples/vault_api/insert_records.py new file mode 100644 index 00000000..8930a51f --- /dev/null +++ b/flowvault/samples/vault_api/insert_records.py @@ -0,0 +1,66 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.utils.enums import UpsertType +from skyflow_flowvault.vault.data import InsertRequest, InsertRequestRecord, UpsertOptions + + +def perform_secure_data_insertion(): + try: + credentials = { + 'path': '', # or 'api_key' / 'token' / 'credentials_string' + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', # from the vault URL: https://{cluster_id}.vault.skyflowapis.com + 'env': Env.PROD, # DEV, STAGE, SANDBOX, or PROD (default) + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + # table_name/upsert are set at exactly ONE level -- on the request (applying to every + # record) OR on every record individually, never both. upsert is an UpsertOptions object. + records = [ + InsertRequestRecord(data={'name': 'John Doe', 'email': 'john@example.com'}), + # InsertRequestRecord( + # data={'name': 'Jane Doe', 'email': 'jane@example.com'}, + # table_name='', # per-record table override + # upsert=UpsertOptions(update_type=UpsertType.REPLACE, unique_columns=['email']), # per-record upsert override + # ), + ] + + insert_request = InsertRequest( + records=records, + table_name='', + upsert=UpsertOptions(update_type=UpsertType.UPDATE, unique_columns=['email']), + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).insert(insert_request) + + # response.records: [ + # {'table_name': '
', 'skyflow_id': '', + # 'tokens': {'email': [{'token': '', 'token_group_name': '', 'path': None}]}, + # 'hashed_data': {...}, 'http_code': 200, 'error': None}, + # {'table_name': None, 'skyflow_id': None, 'tokens': None, 'hashed_data': None, + # 'http_code': 400, 'error': ''} + # ] + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_secure_data_insertion() diff --git a/flowvault/samples/vault_api/query_records.py b/flowvault/samples/vault_api/query_records.py new file mode 100644 index 00000000..484858d8 --- /dev/null +++ b/flowvault/samples/vault_api/query_records.py @@ -0,0 +1,48 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import QueryRequest + + +def perform_secure_query(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + query_request = QueryRequest( + query="SELECT * FROM WHERE skyflow_id = ''", + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).query(query_request) + + # response.records: [{'data': {'skyflow_id': '', 'name': 'John Doe', 'email': ''}}, ...] + # response.metadata: {'columns': ['skyflow_id', 'name', 'email']} + print('Records: ', response.records) + print('Metadata: ', response.metadata) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_secure_query() diff --git a/flowvault/samples/vault_api/update_record.py b/flowvault/samples/vault_api/update_record.py new file mode 100644 index 00000000..f0f6bb37 --- /dev/null +++ b/flowvault/samples/vault_api/update_record.py @@ -0,0 +1,51 @@ +from skyflow_flowvault.error import SkyflowError +from skyflow_flowvault import Env +from skyflow_flowvault import Skyflow, LogLevel +from skyflow_flowvault.vault.data import UpdateRequest + + +def perform_secure_data_update(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + update_request = UpdateRequest( + records=[ + dict(skyflow_id='', data={'name': 'Jane Doe'}), + ], + table_name='', + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).update(update_request) + + # response.records: [{'request_index': 0, 'skyflow_id': '', 'name': ''}, ...] + # response.errors: [{'request_index': 0, 'error': '', 'code': 404, 'request_id': ''}, ...] + print('Records: ', response.records) + print('Errors: ', response.errors) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_secure_data_update() diff --git a/flowvault/setup.py b/flowvault/setup.py index e3fa571e..36e476a4 100644 --- a/flowvault/setup.py +++ b/flowvault/setup.py @@ -17,7 +17,7 @@ REPO_ROOT = os.path.dirname(HERE) COMMON_SRC = os.path.join(REPO_ROOT, 'common') -with open(os.path.join(REPO_ROOT, 'README.md'), 'r', encoding='utf-8') as f: +with open(os.path.join(HERE, 'README.md'), 'r', encoding='utf-8') as f: long_description = f.read() _COMMON_EXCLUDE_DIRS = {'__pycache__', '.pytest_cache', 'tests', '.mypy_cache'} @@ -36,7 +36,7 @@ def _ignore_common_files(_directory, names): class CustomBuildPy(_build_py): - """SK-2938 Option C bundling mechanism -- see v2/setup.py for full rationale. Bundles the + """SK-2938 Option C bundling mechanism -- see skyvault/setup.py for full rationale. Bundles the sibling common/ source tree into this variant's wheel; wheel builds only, not sdist.""" def run(self): diff --git a/flowvault/skyflow_flowvault/generated/rest/__init__.py b/flowvault/skyflow_flowvault/generated/rest/__init__.py index 14ae395a..d802d8d9 100644 --- a/flowvault/skyflow_flowvault/generated/rest/__init__.py +++ b/flowvault/skyflow_flowvault/generated/rest/__init__.py @@ -3,75 +3,84 @@ # isort: skip_file from .types import ( - FlowEnumUpdateType, - FlowTokenizeResponseObjectToken, - GoogleprotobufAny, - ProtobufNullValue, - RpcStatus, - V1ColumnRedactions, - V1DeleteResponse, - V1DeleteResponseObject, - V1DeleteTokenResponseObject, - V1ExecuteQueryRecordResponse, - V1ExecuteQueryResponse, - V1ExecuteQueryResponseMetadata, - V1FlowDeleteTokenResponse, - V1FlowDetokenizeResponse, - V1FlowDetokenizeResponseObject, - V1FlowTokenizeRequestObject, - V1FlowTokenizeResponse, - V1FlowTokenizeResponseObject, - V1FlowVaultMetricsData, - V1FlowVaultMetricsResponse, - V1GetRequestData, - V1GetResponse, - V1InsertRecordData, - V1InsertResponse, - V1RecordResponseObject, - V1TokenGroupRedactions, - V1UniqueValue, - V1UpdateRecordData, - V1UpdateResponse, - V1Upsert, + ColumnRedactions, + DeleteResponse, + DeleteResponseObject, + DetokenizeResponse, + DetokenizeResponseObject, + ErrorResponse, + ErrorResponseError, + ExecuteQueryRecordResponse, + ExecuteQueryResponse, + ExecuteQueryResponseMetadata, + GetRequestData, + GetResponse, + GetTokensFromValuesRequestObject, + GetTokensFromValuesResponse, + GoogleProtobufValue, + HttpCode, + InsertRecordData, + InsertResponse, + RecordResponseObject, + TokenGroupRedactions, + TokenizeResponseObject, + UniqueValue, + UpdateRecordData, + UpdateResponse, + Upsert, + UpsertUpdateType, ) -from . import flowservice, records +from .errors import ( + BadRequestError, + ForbiddenError, + InternalServerError, + NotFoundError, + TooManyRequestsError, + UnauthorizedError, +) +from . import query, records, tokens from .client import AsyncSkyflowAuth, SkyflowAuth +from .environment import SkyflowAuthEnvironment from .version import __version__ __all__ = [ "AsyncSkyflowAuth", - "FlowEnumUpdateType", - "FlowTokenizeResponseObjectToken", - "GoogleprotobufAny", - "ProtobufNullValue", - "RpcStatus", + "BadRequestError", + "ColumnRedactions", + "DeleteResponse", + "DeleteResponseObject", + "DetokenizeResponse", + "DetokenizeResponseObject", + "ErrorResponse", + "ErrorResponseError", + "ExecuteQueryRecordResponse", + "ExecuteQueryResponse", + "ExecuteQueryResponseMetadata", + "ForbiddenError", + "GetRequestData", + "GetResponse", + "GetTokensFromValuesRequestObject", + "GetTokensFromValuesResponse", + "GoogleProtobufValue", + "HttpCode", + "InsertRecordData", + "InsertResponse", + "InternalServerError", + "NotFoundError", + "RecordResponseObject", "SkyflowAuth", - "V1ColumnRedactions", - "V1DeleteResponse", - "V1DeleteResponseObject", - "V1DeleteTokenResponseObject", - "V1ExecuteQueryRecordResponse", - "V1ExecuteQueryResponse", - "V1ExecuteQueryResponseMetadata", - "V1FlowDeleteTokenResponse", - "V1FlowDetokenizeResponse", - "V1FlowDetokenizeResponseObject", - "V1FlowTokenizeRequestObject", - "V1FlowTokenizeResponse", - "V1FlowTokenizeResponseObject", - "V1FlowVaultMetricsData", - "V1FlowVaultMetricsResponse", - "V1GetRequestData", - "V1GetResponse", - "V1InsertRecordData", - "V1InsertResponse", - "V1RecordResponseObject", - "V1TokenGroupRedactions", - "V1UniqueValue", - "V1UpdateRecordData", - "V1UpdateResponse", - "V1Upsert", + "SkyflowAuthEnvironment", + "TokenGroupRedactions", + "TokenizeResponseObject", + "TooManyRequestsError", + "UnauthorizedError", + "UniqueValue", + "UpdateRecordData", + "UpdateResponse", + "Upsert", + "UpsertUpdateType", "__version__", - "flowservice", + "query", "records", + "tokens", ] diff --git a/flowvault/skyflow_flowvault/generated/rest/client.py b/flowvault/skyflow_flowvault/generated/rest/client.py index 0dc7a48c..0300ef32 100644 --- a/flowvault/skyflow_flowvault/generated/rest/client.py +++ b/flowvault/skyflow_flowvault/generated/rest/client.py @@ -4,8 +4,12 @@ import httpx from .core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from .flowservice.client import AsyncFlowserviceClient, FlowserviceClient +from .core.request_options import RequestOptions +from .environment import SkyflowAuthEnvironment +from .query.client import AsyncQueryClient, QueryClient +from .raw_client import AsyncRawSkyflowAuth, RawSkyflowAuth from .records.client import AsyncRecordsClient, RecordsClient +from .tokens.client import AsyncTokensClient, TokensClient class SkyflowAuth: @@ -14,9 +18,19 @@ class SkyflowAuth: Parameters ---------- - base_url : str + base_url : typing.Optional[str] The base url to use for requests from the client. + environment : SkyflowAuthEnvironment + The environment to use for requests from the client. from .environment import SkyflowAuthEnvironment + + + + Defaults to SkyflowAuthEnvironment.PRODUCTION + + + + token : typing.Optional[typing.Union[str, typing.Callable[[], str]]] headers : typing.Optional[typing.Dict[str, str]] Additional headers to send with every request. @@ -34,14 +48,16 @@ class SkyflowAuth: from skyflow import SkyflowAuth client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", + token="YOUR_TOKEN", ) """ def __init__( self, *, - base_url: str, + base_url: typing.Optional[str] = None, + environment: SkyflowAuthEnvironment = SkyflowAuthEnvironment.PRODUCTION, + token: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None, headers: typing.Optional[typing.Dict[str, str]] = None, timeout: typing.Optional[float] = None, follow_redirects: typing.Optional[bool] = True, @@ -51,7 +67,8 @@ def __init__( timeout if timeout is not None else 60 if httpx_client is None else httpx_client.timeout.read ) self._client_wrapper = SyncClientWrapper( - base_url=base_url, + base_url=_get_base_url(base_url=base_url, environment=environment), + token=token, headers=headers, httpx_client=httpx_client if httpx_client is not None @@ -60,8 +77,48 @@ def __init__( else httpx.Client(timeout=_defaulted_timeout), timeout=_defaulted_timeout, ) + self._raw_client = RawSkyflowAuth(client_wrapper=self._client_wrapper) + self.query = QueryClient(client_wrapper=self._client_wrapper) self.records = RecordsClient(client_wrapper=self._client_wrapper) - self.flowservice = FlowserviceClient(client_wrapper=self._client_wrapper) + self.tokens = TokensClient(client_wrapper=self._client_wrapper) + + @property + def with_raw_response(self) -> RawSkyflowAuth: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + RawSkyflowAuth + """ + return self._raw_client + + def patch_v2vaults_id( + self, vault_id: typing.Optional[str], *, request_options: typing.Optional[RequestOptions] = None + ) -> None: + """ + Parameters + ---------- + vault_id : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + None + + Examples + -------- + from skyflow import SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.patch_v2vaults_id() + """ + _response = self._raw_client.patch_v2vaults_id(vault_id, request_options=request_options) + return _response.data class AsyncSkyflowAuth: @@ -70,9 +127,19 @@ class AsyncSkyflowAuth: Parameters ---------- - base_url : str + base_url : typing.Optional[str] The base url to use for requests from the client. + environment : SkyflowAuthEnvironment + The environment to use for requests from the client. from .environment import SkyflowAuthEnvironment + + + + Defaults to SkyflowAuthEnvironment.PRODUCTION + + + + token : typing.Optional[typing.Union[str, typing.Callable[[], str]]] headers : typing.Optional[typing.Dict[str, str]] Additional headers to send with every request. @@ -90,14 +157,16 @@ class AsyncSkyflowAuth: from skyflow import AsyncSkyflowAuth client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", + token="YOUR_TOKEN", ) """ def __init__( self, *, - base_url: str, + base_url: typing.Optional[str] = None, + environment: SkyflowAuthEnvironment = SkyflowAuthEnvironment.PRODUCTION, + token: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None, headers: typing.Optional[typing.Dict[str, str]] = None, timeout: typing.Optional[float] = None, follow_redirects: typing.Optional[bool] = True, @@ -107,7 +176,8 @@ def __init__( timeout if timeout is not None else 60 if httpx_client is None else httpx_client.timeout.read ) self._client_wrapper = AsyncClientWrapper( - base_url=base_url, + base_url=_get_base_url(base_url=base_url, environment=environment), + token=token, headers=headers, httpx_client=httpx_client if httpx_client is not None @@ -116,5 +186,62 @@ def __init__( else httpx.AsyncClient(timeout=_defaulted_timeout), timeout=_defaulted_timeout, ) + self._raw_client = AsyncRawSkyflowAuth(client_wrapper=self._client_wrapper) + self.query = AsyncQueryClient(client_wrapper=self._client_wrapper) self.records = AsyncRecordsClient(client_wrapper=self._client_wrapper) - self.flowservice = AsyncFlowserviceClient(client_wrapper=self._client_wrapper) + self.tokens = AsyncTokensClient(client_wrapper=self._client_wrapper) + + @property + def with_raw_response(self) -> AsyncRawSkyflowAuth: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + AsyncRawSkyflowAuth + """ + return self._raw_client + + async def patch_v2vaults_id( + self, vault_id: typing.Optional[str], *, request_options: typing.Optional[RequestOptions] = None + ) -> None: + """ + Parameters + ---------- + vault_id : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + None + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.patch_v2vaults_id() + + + asyncio.run(main()) + """ + _response = await self._raw_client.patch_v2vaults_id(vault_id, request_options=request_options) + return _response.data + + +def _get_base_url(*, base_url: typing.Optional[str] = None, environment: SkyflowAuthEnvironment) -> str: + if base_url is not None: + return base_url + elif environment is not None: + return environment.value + else: + raise Exception("Please pass in either base_url or environment to construct the client") diff --git a/flowvault/skyflow_flowvault/generated/rest/core/client_wrapper.py b/flowvault/skyflow_flowvault/generated/rest/core/client_wrapper.py index 8f63f6ee..4b5ae221 100644 --- a/flowvault/skyflow_flowvault/generated/rest/core/client_wrapper.py +++ b/flowvault/skyflow_flowvault/generated/rest/core/client_wrapper.py @@ -10,10 +10,12 @@ class BaseClientWrapper: def __init__( self, *, + token: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None, headers: typing.Optional[typing.Dict[str, str]] = None, base_url: str, timeout: typing.Optional[float] = None, ): + self._token = token self._headers = headers self._base_url = base_url self._timeout = timeout @@ -22,11 +24,20 @@ def get_headers(self) -> typing.Dict[str, str]: headers: typing.Dict[str, str] = { "X-Fern-Language": "Python", "X-Fern-SDK-Name": "skyflow.generated.rest", - "X-Fern-SDK-Version": "0.0.10", + "X-Fern-SDK-Version": "0.0.19", **(self.get_custom_headers() or {}), } + token = self._get_token() + if token is not None: + headers["Authorization"] = f"Bearer {token}" return headers + def _get_token(self) -> typing.Optional[str]: + if isinstance(self._token, str) or self._token is None: + return self._token + else: + return self._token() + def get_custom_headers(self) -> typing.Optional[typing.Dict[str, str]]: return self._headers @@ -41,12 +52,13 @@ class SyncClientWrapper(BaseClientWrapper): def __init__( self, *, + token: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None, headers: typing.Optional[typing.Dict[str, str]] = None, base_url: str, timeout: typing.Optional[float] = None, httpx_client: httpx.Client, ): - super().__init__(headers=headers, base_url=base_url, timeout=timeout) + super().__init__(token=token, headers=headers, base_url=base_url, timeout=timeout) self.httpx_client = HttpClient( httpx_client=httpx_client, base_headers=self.get_headers, @@ -59,12 +71,13 @@ class AsyncClientWrapper(BaseClientWrapper): def __init__( self, *, + token: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None, headers: typing.Optional[typing.Dict[str, str]] = None, base_url: str, timeout: typing.Optional[float] = None, httpx_client: httpx.AsyncClient, ): - super().__init__(headers=headers, base_url=base_url, timeout=timeout) + super().__init__(token=token, headers=headers, base_url=base_url, timeout=timeout) self.httpx_client = AsyncHttpClient( httpx_client=httpx_client, base_headers=self.get_headers, diff --git a/flowvault/skyflow_flowvault/generated/rest/environment.py b/flowvault/skyflow_flowvault/generated/rest/environment.py new file mode 100644 index 00000000..d03788a0 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/environment.py @@ -0,0 +1,8 @@ +# This file was auto-generated by Fern from our API Definition. + +import enum + + +class SkyflowAuthEnvironment(enum.Enum): + PRODUCTION = "https://%7B%7Bvault_url%7D%7D" + SANDBOX = "https://%7B%7Bvault_url%7D%7D" diff --git a/flowvault/skyflow_flowvault/generated/rest/errors/__init__.py b/flowvault/skyflow_flowvault/generated/rest/errors/__init__.py new file mode 100644 index 00000000..7a0aed6c --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/errors/__init__.py @@ -0,0 +1,19 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +from .bad_request_error import BadRequestError +from .forbidden_error import ForbiddenError +from .internal_server_error import InternalServerError +from .not_found_error import NotFoundError +from .too_many_requests_error import TooManyRequestsError +from .unauthorized_error import UnauthorizedError + +__all__ = [ + "BadRequestError", + "ForbiddenError", + "InternalServerError", + "NotFoundError", + "TooManyRequestsError", + "UnauthorizedError", +] diff --git a/flowvault/skyflow_flowvault/generated/rest/errors/bad_request_error.py b/flowvault/skyflow_flowvault/generated/rest/errors/bad_request_error.py new file mode 100644 index 00000000..92449913 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/errors/bad_request_error.py @@ -0,0 +1,11 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.api_error import ApiError +from ..types.error_response import ErrorResponse + + +class BadRequestError(ApiError): + def __init__(self, body: ErrorResponse, headers: typing.Optional[typing.Dict[str, str]] = None): + super().__init__(status_code=400, headers=headers, body=body) diff --git a/flowvault/skyflow_flowvault/generated/rest/errors/forbidden_error.py b/flowvault/skyflow_flowvault/generated/rest/errors/forbidden_error.py new file mode 100644 index 00000000..0841085e --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/errors/forbidden_error.py @@ -0,0 +1,11 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.api_error import ApiError +from ..types.error_response import ErrorResponse + + +class ForbiddenError(ApiError): + def __init__(self, body: ErrorResponse, headers: typing.Optional[typing.Dict[str, str]] = None): + super().__init__(status_code=403, headers=headers, body=body) diff --git a/v2/skyflow/generated/rest/errors/internal_server_error.py b/flowvault/skyflow_flowvault/generated/rest/errors/internal_server_error.py similarity index 100% rename from v2/skyflow/generated/rest/errors/internal_server_error.py rename to flowvault/skyflow_flowvault/generated/rest/errors/internal_server_error.py diff --git a/flowvault/skyflow_flowvault/generated/rest/errors/not_found_error.py b/flowvault/skyflow_flowvault/generated/rest/errors/not_found_error.py new file mode 100644 index 00000000..3b2a2d1f --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/errors/not_found_error.py @@ -0,0 +1,11 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.api_error import ApiError +from ..types.error_response import ErrorResponse + + +class NotFoundError(ApiError): + def __init__(self, body: ErrorResponse, headers: typing.Optional[typing.Dict[str, str]] = None): + super().__init__(status_code=404, headers=headers, body=body) diff --git a/flowvault/skyflow_flowvault/generated/rest/errors/too_many_requests_error.py b/flowvault/skyflow_flowvault/generated/rest/errors/too_many_requests_error.py new file mode 100644 index 00000000..befc9d63 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/errors/too_many_requests_error.py @@ -0,0 +1,11 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.api_error import ApiError +from ..types.error_response import ErrorResponse + + +class TooManyRequestsError(ApiError): + def __init__(self, body: ErrorResponse, headers: typing.Optional[typing.Dict[str, str]] = None): + super().__init__(status_code=429, headers=headers, body=body) diff --git a/flowvault/skyflow_flowvault/generated/rest/errors/unauthorized_error.py b/flowvault/skyflow_flowvault/generated/rest/errors/unauthorized_error.py new file mode 100644 index 00000000..00a614a3 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/errors/unauthorized_error.py @@ -0,0 +1,11 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.api_error import ApiError +from ..types.error_response import ErrorResponse + + +class UnauthorizedError(ApiError): + def __init__(self, body: ErrorResponse, headers: typing.Optional[typing.Dict[str, str]] = None): + super().__init__(status_code=401, headers=headers, body=body) diff --git a/flowvault/skyflow_flowvault/generated/rest/flowservice/client.py b/flowvault/skyflow_flowvault/generated/rest/flowservice/client.py deleted file mode 100644 index 321f69bd..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/flowservice/client.py +++ /dev/null @@ -1,855 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from ..core.request_options import RequestOptions -from ..types.flow_enum_update_type import FlowEnumUpdateType -from ..types.v_1_column_redactions import V1ColumnRedactions -from ..types.v_1_delete_response import V1DeleteResponse -from ..types.v_1_flow_delete_token_response import V1FlowDeleteTokenResponse -from ..types.v_1_flow_detokenize_response import V1FlowDetokenizeResponse -from ..types.v_1_flow_tokenize_request_object import V1FlowTokenizeRequestObject -from ..types.v_1_flow_tokenize_response import V1FlowTokenizeResponse -from ..types.v_1_flow_vault_metrics_response import V1FlowVaultMetricsResponse -from ..types.v_1_get_request_data import V1GetRequestData -from ..types.v_1_get_response import V1GetResponse -from ..types.v_1_insert_record_data import V1InsertRecordData -from ..types.v_1_insert_response import V1InsertResponse -from ..types.v_1_token_group_redactions import V1TokenGroupRedactions -from ..types.v_1_unique_value import V1UniqueValue -from ..types.v_1_update_record_data import V1UpdateRecordData -from ..types.v_1_update_response import V1UpdateResponse -from ..types.v_1_upsert import V1Upsert -from .raw_client import AsyncRawFlowserviceClient, RawFlowserviceClient - -# this is used as the default value for optional parameters -OMIT = typing.cast(typing.Any, ...) - - -class FlowserviceClient: - def __init__(self, *, client_wrapper: SyncClientWrapper): - self._raw_client = RawFlowserviceClient(client_wrapper=client_wrapper) - - @property - def with_raw_response(self) -> RawFlowserviceClient: - """ - Retrieves a raw implementation of this client that returns raw responses. - - Returns - ------- - RawFlowserviceClient - """ - return self._raw_client - - def delete( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1DeleteResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being deleted - - table_name : typing.Optional[str] - Name of the table where data is being deleted - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be deleted - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1DeleteResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.delete() - """ - _response = self._raw_client.delete( - vault_id=vault_id, - table_name=table_name, - skyflow_i_ds=skyflow_i_ds, - unique_values=unique_values, - request_options=request_options, - ) - return _response.data - - def get( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - column_redactions: typing.Optional[typing.Sequence[V1ColumnRedactions]] = OMIT, - columns: typing.Optional[typing.Sequence[str]] = OMIT, - limit: typing.Optional[int] = OMIT, - offset: typing.Optional[int] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - records: typing.Optional[typing.Sequence[V1GetRequestData]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1GetResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being fetched - - table_name : typing.Optional[str] - Name of the table where data is being fetched - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be fetched - - column_redactions : typing.Optional[typing.Sequence[V1ColumnRedactions]] - List of columns to be redacted. - - columns : typing.Optional[typing.Sequence[str]] - List of columns to be fetched. - - limit : typing.Optional[int] - Limit for the number of records to be fetched - - offset : typing.Optional[int] - Offset for the number of records to be fetched - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - records : typing.Optional[typing.Sequence[V1GetRequestData]] - List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1GetResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.get() - """ - _response = self._raw_client.get( - vault_id=vault_id, - table_name=table_name, - skyflow_i_ds=skyflow_i_ds, - column_redactions=column_redactions, - columns=columns, - limit=limit, - offset=offset, - unique_values=unique_values, - records=records, - request_options=request_options, - ) - return _response.data - - def insert( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1InsertRecordData]] = OMIT, - upsert: typing.Optional[V1Upsert] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1InsertResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted - - table_name : typing.Optional[str] - Name of the table where data is being inserted - - records : typing.Optional[typing.Sequence[V1InsertRecordData]] - List of data row wise that is to be inserted in the vault - - upsert : typing.Optional[V1Upsert] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1InsertResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.insert() - """ - _response = self._raw_client.insert( - vault_id=vault_id, table_name=table_name, records=records, upsert=upsert, request_options=request_options - ) - return _response.data - - def update( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1UpdateRecordData]] = OMIT, - update_type: typing.Optional[FlowEnumUpdateType] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1UpdateResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being updated - - table_name : typing.Optional[str] - Name of the table where data is being updated - - records : typing.Optional[typing.Sequence[V1UpdateRecordData]] - List of data row wise that is to be updated in the vault - - update_type : typing.Optional[FlowEnumUpdateType] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1UpdateResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.update() - """ - _response = self._raw_client.update( - vault_id=vault_id, - table_name=table_name, - records=records, - update_type=update_type, - request_options=request_options, - ) - return _response.data - - def deletetoken( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1FlowDeleteTokenResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID - - tokens : typing.Optional[typing.Sequence[str]] - Token value - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowDeleteTokenResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.deletetoken() - """ - _response = self._raw_client.deletetoken(vault_id=vault_id, tokens=tokens, request_options=request_options) - return _response.data - - def detokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - token_group_redactions: typing.Optional[typing.Sequence[V1TokenGroupRedactions]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1FlowDetokenizeResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where detokenizing - - tokens : typing.Optional[typing.Sequence[str]] - Token to be detokenized - - token_group_redactions : typing.Optional[typing.Sequence[V1TokenGroupRedactions]] - List of token groups to be redacted. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowDetokenizeResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.detokenize() - """ - _response = self._raw_client.detokenize( - vault_id=vault_id, - tokens=tokens, - token_group_redactions=token_group_redactions, - request_options=request_options, - ) - return _response.data - - def tokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - data: typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1FlowTokenizeResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID. - - data : typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] - Data to be tokenized - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowTokenizeResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.tokenize() - """ - _response = self._raw_client.tokenize(vault_id=vault_id, data=data, request_options=request_options) - return _response.data - - def flowvaultmetrics( - self, *, vault_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None - ) -> V1FlowVaultMetricsResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault to get metrics for - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowVaultMetricsResponse - A successful response. - - Examples - -------- - from skyflow import SkyflowAuth - - client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - client.flowservice.flowvaultmetrics() - """ - _response = self._raw_client.flowvaultmetrics(vault_id=vault_id, request_options=request_options) - return _response.data - - -class AsyncFlowserviceClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._raw_client = AsyncRawFlowserviceClient(client_wrapper=client_wrapper) - - @property - def with_raw_response(self) -> AsyncRawFlowserviceClient: - """ - Retrieves a raw implementation of this client that returns raw responses. - - Returns - ------- - AsyncRawFlowserviceClient - """ - return self._raw_client - - async def delete( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1DeleteResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being deleted - - table_name : typing.Optional[str] - Name of the table where data is being deleted - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be deleted - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1DeleteResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.delete() - - - asyncio.run(main()) - """ - _response = await self._raw_client.delete( - vault_id=vault_id, - table_name=table_name, - skyflow_i_ds=skyflow_i_ds, - unique_values=unique_values, - request_options=request_options, - ) - return _response.data - - async def get( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - column_redactions: typing.Optional[typing.Sequence[V1ColumnRedactions]] = OMIT, - columns: typing.Optional[typing.Sequence[str]] = OMIT, - limit: typing.Optional[int] = OMIT, - offset: typing.Optional[int] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - records: typing.Optional[typing.Sequence[V1GetRequestData]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1GetResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being fetched - - table_name : typing.Optional[str] - Name of the table where data is being fetched - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be fetched - - column_redactions : typing.Optional[typing.Sequence[V1ColumnRedactions]] - List of columns to be redacted. - - columns : typing.Optional[typing.Sequence[str]] - List of columns to be fetched. - - limit : typing.Optional[int] - Limit for the number of records to be fetched - - offset : typing.Optional[int] - Offset for the number of records to be fetched - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - records : typing.Optional[typing.Sequence[V1GetRequestData]] - List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1GetResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.get() - - - asyncio.run(main()) - """ - _response = await self._raw_client.get( - vault_id=vault_id, - table_name=table_name, - skyflow_i_ds=skyflow_i_ds, - column_redactions=column_redactions, - columns=columns, - limit=limit, - offset=offset, - unique_values=unique_values, - records=records, - request_options=request_options, - ) - return _response.data - - async def insert( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1InsertRecordData]] = OMIT, - upsert: typing.Optional[V1Upsert] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1InsertResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted - - table_name : typing.Optional[str] - Name of the table where data is being inserted - - records : typing.Optional[typing.Sequence[V1InsertRecordData]] - List of data row wise that is to be inserted in the vault - - upsert : typing.Optional[V1Upsert] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1InsertResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.insert() - - - asyncio.run(main()) - """ - _response = await self._raw_client.insert( - vault_id=vault_id, table_name=table_name, records=records, upsert=upsert, request_options=request_options - ) - return _response.data - - async def update( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1UpdateRecordData]] = OMIT, - update_type: typing.Optional[FlowEnumUpdateType] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1UpdateResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being updated - - table_name : typing.Optional[str] - Name of the table where data is being updated - - records : typing.Optional[typing.Sequence[V1UpdateRecordData]] - List of data row wise that is to be updated in the vault - - update_type : typing.Optional[FlowEnumUpdateType] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1UpdateResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.update() - - - asyncio.run(main()) - """ - _response = await self._raw_client.update( - vault_id=vault_id, - table_name=table_name, - records=records, - update_type=update_type, - request_options=request_options, - ) - return _response.data - - async def deletetoken( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1FlowDeleteTokenResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID - - tokens : typing.Optional[typing.Sequence[str]] - Token value - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowDeleteTokenResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.deletetoken() - - - asyncio.run(main()) - """ - _response = await self._raw_client.deletetoken( - vault_id=vault_id, tokens=tokens, request_options=request_options - ) - return _response.data - - async def detokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - token_group_redactions: typing.Optional[typing.Sequence[V1TokenGroupRedactions]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1FlowDetokenizeResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where detokenizing - - tokens : typing.Optional[typing.Sequence[str]] - Token to be detokenized - - token_group_redactions : typing.Optional[typing.Sequence[V1TokenGroupRedactions]] - List of token groups to be redacted. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowDetokenizeResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.detokenize() - - - asyncio.run(main()) - """ - _response = await self._raw_client.detokenize( - vault_id=vault_id, - tokens=tokens, - token_group_redactions=token_group_redactions, - request_options=request_options, - ) - return _response.data - - async def tokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - data: typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> V1FlowTokenizeResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID. - - data : typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] - Data to be tokenized - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowTokenizeResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.tokenize() - - - asyncio.run(main()) - """ - _response = await self._raw_client.tokenize(vault_id=vault_id, data=data, request_options=request_options) - return _response.data - - async def flowvaultmetrics( - self, *, vault_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None - ) -> V1FlowVaultMetricsResponse: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault to get metrics for - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - V1FlowVaultMetricsResponse - A successful response. - - Examples - -------- - import asyncio - - from skyflow import AsyncSkyflowAuth - - client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.flowservice.flowvaultmetrics() - - - asyncio.run(main()) - """ - _response = await self._raw_client.flowvaultmetrics(vault_id=vault_id, request_options=request_options) - return _response.data diff --git a/flowvault/skyflow_flowvault/generated/rest/flowservice/raw_client.py b/flowvault/skyflow_flowvault/generated/rest/flowservice/raw_client.py deleted file mode 100644 index 7b005ea6..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/flowservice/raw_client.py +++ /dev/null @@ -1,1033 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing -from json.decoder import JSONDecodeError - -from ..core.api_error import ApiError -from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from ..core.http_response import AsyncHttpResponse, HttpResponse -from ..core.pydantic_utilities import parse_obj_as -from ..core.request_options import RequestOptions -from ..core.serialization import convert_and_respect_annotation_metadata -from ..types.flow_enum_update_type import FlowEnumUpdateType -from ..types.v_1_column_redactions import V1ColumnRedactions -from ..types.v_1_delete_response import V1DeleteResponse -from ..types.v_1_flow_delete_token_response import V1FlowDeleteTokenResponse -from ..types.v_1_flow_detokenize_response import V1FlowDetokenizeResponse -from ..types.v_1_flow_tokenize_request_object import V1FlowTokenizeRequestObject -from ..types.v_1_flow_tokenize_response import V1FlowTokenizeResponse -from ..types.v_1_flow_vault_metrics_response import V1FlowVaultMetricsResponse -from ..types.v_1_get_request_data import V1GetRequestData -from ..types.v_1_get_response import V1GetResponse -from ..types.v_1_insert_record_data import V1InsertRecordData -from ..types.v_1_insert_response import V1InsertResponse -from ..types.v_1_token_group_redactions import V1TokenGroupRedactions -from ..types.v_1_unique_value import V1UniqueValue -from ..types.v_1_update_record_data import V1UpdateRecordData -from ..types.v_1_update_response import V1UpdateResponse -from ..types.v_1_upsert import V1Upsert - -# this is used as the default value for optional parameters -OMIT = typing.cast(typing.Any, ...) - - -class RawFlowserviceClient: - def __init__(self, *, client_wrapper: SyncClientWrapper): - self._client_wrapper = client_wrapper - - def delete( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1DeleteResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being deleted - - table_name : typing.Optional[str] - Name of the table where data is being deleted - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be deleted - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1DeleteResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/records/delete", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "skyflowIDs": skyflow_i_ds, - "uniqueValues": convert_and_respect_annotation_metadata( - object_=unique_values, annotation=typing.Sequence[V1UniqueValue], direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1DeleteResponse, - parse_obj_as( - type_=V1DeleteResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - def get( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - column_redactions: typing.Optional[typing.Sequence[V1ColumnRedactions]] = OMIT, - columns: typing.Optional[typing.Sequence[str]] = OMIT, - limit: typing.Optional[int] = OMIT, - offset: typing.Optional[int] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - records: typing.Optional[typing.Sequence[V1GetRequestData]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1GetResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being fetched - - table_name : typing.Optional[str] - Name of the table where data is being fetched - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be fetched - - column_redactions : typing.Optional[typing.Sequence[V1ColumnRedactions]] - List of columns to be redacted. - - columns : typing.Optional[typing.Sequence[str]] - List of columns to be fetched. - - limit : typing.Optional[int] - Limit for the number of records to be fetched - - offset : typing.Optional[int] - Offset for the number of records to be fetched - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - records : typing.Optional[typing.Sequence[V1GetRequestData]] - List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1GetResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/records/get", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "skyflowIDs": skyflow_i_ds, - "columnRedactions": convert_and_respect_annotation_metadata( - object_=column_redactions, annotation=typing.Sequence[V1ColumnRedactions], direction="write" - ), - "columns": columns, - "limit": limit, - "offset": offset, - "uniqueValues": convert_and_respect_annotation_metadata( - object_=unique_values, annotation=typing.Sequence[V1UniqueValue], direction="write" - ), - "records": convert_and_respect_annotation_metadata( - object_=records, annotation=typing.Sequence[V1GetRequestData], direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1GetResponse, - parse_obj_as( - type_=V1GetResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - def insert( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1InsertRecordData]] = OMIT, - upsert: typing.Optional[V1Upsert] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1InsertResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted - - table_name : typing.Optional[str] - Name of the table where data is being inserted - - records : typing.Optional[typing.Sequence[V1InsertRecordData]] - List of data row wise that is to be inserted in the vault - - upsert : typing.Optional[V1Upsert] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1InsertResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/records/insert", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "records": convert_and_respect_annotation_metadata( - object_=records, annotation=typing.Sequence[V1InsertRecordData], direction="write" - ), - "upsert": convert_and_respect_annotation_metadata( - object_=upsert, annotation=V1Upsert, direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1InsertResponse, - parse_obj_as( - type_=V1InsertResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - def update( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1UpdateRecordData]] = OMIT, - update_type: typing.Optional[FlowEnumUpdateType] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1UpdateResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being updated - - table_name : typing.Optional[str] - Name of the table where data is being updated - - records : typing.Optional[typing.Sequence[V1UpdateRecordData]] - List of data row wise that is to be updated in the vault - - update_type : typing.Optional[FlowEnumUpdateType] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1UpdateResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/records/update", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "records": convert_and_respect_annotation_metadata( - object_=records, annotation=typing.Sequence[V1UpdateRecordData], direction="write" - ), - "updateType": update_type, - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1UpdateResponse, - parse_obj_as( - type_=V1UpdateResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - def deletetoken( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1FlowDeleteTokenResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID - - tokens : typing.Optional[typing.Sequence[str]] - Token value - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1FlowDeleteTokenResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/tokens/delete", - method="POST", - json={ - "vaultID": vault_id, - "tokens": tokens, - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowDeleteTokenResponse, - parse_obj_as( - type_=V1FlowDeleteTokenResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - def detokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - token_group_redactions: typing.Optional[typing.Sequence[V1TokenGroupRedactions]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1FlowDetokenizeResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where detokenizing - - tokens : typing.Optional[typing.Sequence[str]] - Token to be detokenized - - token_group_redactions : typing.Optional[typing.Sequence[V1TokenGroupRedactions]] - List of token groups to be redacted. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1FlowDetokenizeResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/tokens/detokenize", - method="POST", - json={ - "vaultID": vault_id, - "tokens": tokens, - "tokenGroupRedactions": convert_and_respect_annotation_metadata( - object_=token_group_redactions, - annotation=typing.Sequence[V1TokenGroupRedactions], - direction="write", - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowDetokenizeResponse, - parse_obj_as( - type_=V1FlowDetokenizeResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - def tokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - data: typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1FlowTokenizeResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID. - - data : typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] - Data to be tokenized - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1FlowTokenizeResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/tokens/tokenize", - method="POST", - json={ - "vaultID": vault_id, - "data": convert_and_respect_annotation_metadata( - object_=data, annotation=typing.Sequence[V1FlowTokenizeRequestObject], direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowTokenizeResponse, - parse_obj_as( - type_=V1FlowTokenizeResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - def flowvaultmetrics( - self, *, vault_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None - ) -> HttpResponse[V1FlowVaultMetricsResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault to get metrics for - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[V1FlowVaultMetricsResponse] - A successful response. - """ - _response = self._client_wrapper.httpx_client.request( - "v2/vaults/metrics", - method="POST", - json={ - "vaultID": vault_id, - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowVaultMetricsResponse, - parse_obj_as( - type_=V1FlowVaultMetricsResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - -class AsyncRawFlowserviceClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._client_wrapper = client_wrapper - - async def delete( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1DeleteResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being deleted - - table_name : typing.Optional[str] - Name of the table where data is being deleted - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be deleted - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1DeleteResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/records/delete", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "skyflowIDs": skyflow_i_ds, - "uniqueValues": convert_and_respect_annotation_metadata( - object_=unique_values, annotation=typing.Sequence[V1UniqueValue], direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1DeleteResponse, - parse_obj_as( - type_=V1DeleteResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - async def get( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, - column_redactions: typing.Optional[typing.Sequence[V1ColumnRedactions]] = OMIT, - columns: typing.Optional[typing.Sequence[str]] = OMIT, - limit: typing.Optional[int] = OMIT, - offset: typing.Optional[int] = OMIT, - unique_values: typing.Optional[typing.Sequence[V1UniqueValue]] = OMIT, - records: typing.Optional[typing.Sequence[V1GetRequestData]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1GetResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being fetched - - table_name : typing.Optional[str] - Name of the table where data is being fetched - - skyflow_i_ds : typing.Optional[typing.Sequence[str]] - Skyflow ID for the record to be fetched - - column_redactions : typing.Optional[typing.Sequence[V1ColumnRedactions]] - List of columns to be redacted. - - columns : typing.Optional[typing.Sequence[str]] - List of columns to be fetched. - - limit : typing.Optional[int] - Limit for the number of records to be fetched - - offset : typing.Optional[int] - Offset for the number of records to be fetched - - unique_values : typing.Optional[typing.Sequence[V1UniqueValue]] - List of unique constraint values to query records by data - - records : typing.Optional[typing.Sequence[V1GetRequestData]] - List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1GetResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/records/get", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "skyflowIDs": skyflow_i_ds, - "columnRedactions": convert_and_respect_annotation_metadata( - object_=column_redactions, annotation=typing.Sequence[V1ColumnRedactions], direction="write" - ), - "columns": columns, - "limit": limit, - "offset": offset, - "uniqueValues": convert_and_respect_annotation_metadata( - object_=unique_values, annotation=typing.Sequence[V1UniqueValue], direction="write" - ), - "records": convert_and_respect_annotation_metadata( - object_=records, annotation=typing.Sequence[V1GetRequestData], direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1GetResponse, - parse_obj_as( - type_=V1GetResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - async def insert( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1InsertRecordData]] = OMIT, - upsert: typing.Optional[V1Upsert] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1InsertResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted - - table_name : typing.Optional[str] - Name of the table where data is being inserted - - records : typing.Optional[typing.Sequence[V1InsertRecordData]] - List of data row wise that is to be inserted in the vault - - upsert : typing.Optional[V1Upsert] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1InsertResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/records/insert", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "records": convert_and_respect_annotation_metadata( - object_=records, annotation=typing.Sequence[V1InsertRecordData], direction="write" - ), - "upsert": convert_and_respect_annotation_metadata( - object_=upsert, annotation=V1Upsert, direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1InsertResponse, - parse_obj_as( - type_=V1InsertResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - async def update( - self, - *, - vault_id: typing.Optional[str] = OMIT, - table_name: typing.Optional[str] = OMIT, - records: typing.Optional[typing.Sequence[V1UpdateRecordData]] = OMIT, - update_type: typing.Optional[FlowEnumUpdateType] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1UpdateResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being updated - - table_name : typing.Optional[str] - Name of the table where data is being updated - - records : typing.Optional[typing.Sequence[V1UpdateRecordData]] - List of data row wise that is to be updated in the vault - - update_type : typing.Optional[FlowEnumUpdateType] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1UpdateResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/records/update", - method="POST", - json={ - "vaultID": vault_id, - "tableName": table_name, - "records": convert_and_respect_annotation_metadata( - object_=records, annotation=typing.Sequence[V1UpdateRecordData], direction="write" - ), - "updateType": update_type, - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1UpdateResponse, - parse_obj_as( - type_=V1UpdateResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - async def deletetoken( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1FlowDeleteTokenResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID - - tokens : typing.Optional[typing.Sequence[str]] - Token value - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1FlowDeleteTokenResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/tokens/delete", - method="POST", - json={ - "vaultID": vault_id, - "tokens": tokens, - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowDeleteTokenResponse, - parse_obj_as( - type_=V1FlowDeleteTokenResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - async def detokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - tokens: typing.Optional[typing.Sequence[str]] = OMIT, - token_group_redactions: typing.Optional[typing.Sequence[V1TokenGroupRedactions]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1FlowDetokenizeResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault where detokenizing - - tokens : typing.Optional[typing.Sequence[str]] - Token to be detokenized - - token_group_redactions : typing.Optional[typing.Sequence[V1TokenGroupRedactions]] - List of token groups to be redacted. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1FlowDetokenizeResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/tokens/detokenize", - method="POST", - json={ - "vaultID": vault_id, - "tokens": tokens, - "tokenGroupRedactions": convert_and_respect_annotation_metadata( - object_=token_group_redactions, - annotation=typing.Sequence[V1TokenGroupRedactions], - direction="write", - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowDetokenizeResponse, - parse_obj_as( - type_=V1FlowDetokenizeResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - async def tokenize( - self, - *, - vault_id: typing.Optional[str] = OMIT, - data: typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1FlowTokenizeResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - Vault ID. - - data : typing.Optional[typing.Sequence[V1FlowTokenizeRequestObject]] - Data to be tokenized - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1FlowTokenizeResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/tokens/tokenize", - method="POST", - json={ - "vaultID": vault_id, - "data": convert_and_respect_annotation_metadata( - object_=data, annotation=typing.Sequence[V1FlowTokenizeRequestObject], direction="write" - ), - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowTokenizeResponse, - parse_obj_as( - type_=V1FlowTokenizeResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - - async def flowvaultmetrics( - self, *, vault_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None - ) -> AsyncHttpResponse[V1FlowVaultMetricsResponse]: - """ - Parameters - ---------- - vault_id : typing.Optional[str] - ID of the vault to get metrics for - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[V1FlowVaultMetricsResponse] - A successful response. - """ - _response = await self._client_wrapper.httpx_client.request( - "v2/vaults/metrics", - method="POST", - json={ - "vaultID": vault_id, - }, - headers={ - "content-type": "application/json", - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - V1FlowVaultMetricsResponse, - parse_obj_as( - type_=V1FlowVaultMetricsResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) diff --git a/flowvault/skyflow_flowvault/generated/rest/flowservice/__init__.py b/flowvault/skyflow_flowvault/generated/rest/query/__init__.py similarity index 100% rename from flowvault/skyflow_flowvault/generated/rest/flowservice/__init__.py rename to flowvault/skyflow_flowvault/generated/rest/query/__init__.py diff --git a/flowvault/skyflow_flowvault/generated/rest/query/client.py b/flowvault/skyflow_flowvault/generated/rest/query/client.py new file mode 100644 index 00000000..113d34a9 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/query/client.py @@ -0,0 +1,139 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.request_options import RequestOptions +from ..types.execute_query_response import ExecuteQueryResponse +from .raw_client import AsyncRawQueryClient, RawQueryClient + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class QueryClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._raw_client = RawQueryClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> RawQueryClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + RawQueryClient + """ + return self._raw_client + + def execute_query( + self, *, vault_id: str, query: str, request_options: typing.Optional[RequestOptions] = None + ) -> ExecuteQueryResponse: + """ + Returns records for a valid SQL query. This endpoint + - Can return masked record values. + - Supports only the `SELECT` command. + - Returns a maximum of 25 records. To return additional records, perform another query using the `OFFSET` keyword. + - Can't modify the vault or perform transactions. + - Can't return tokens. + - Can't return file download or render URLs. + + Parameters + ---------- + vault_id : str + ID of the vault where the query is being performed. + + query : str + Query to perform. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + ExecuteQueryResponse + OK + + Examples + -------- + from skyflow import SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.query.execute_query( + vault_id="d408485953784308a000f8dcf81901ef", + query="query", + ) + """ + _response = self._raw_client.execute_query(vault_id=vault_id, query=query, request_options=request_options) + return _response.data + + +class AsyncQueryClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._raw_client = AsyncRawQueryClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> AsyncRawQueryClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + AsyncRawQueryClient + """ + return self._raw_client + + async def execute_query( + self, *, vault_id: str, query: str, request_options: typing.Optional[RequestOptions] = None + ) -> ExecuteQueryResponse: + """ + Returns records for a valid SQL query. This endpoint + - Can return masked record values. + - Supports only the `SELECT` command. + - Returns a maximum of 25 records. To return additional records, perform another query using the `OFFSET` keyword. + - Can't modify the vault or perform transactions. + - Can't return tokens. + - Can't return file download or render URLs. + + Parameters + ---------- + vault_id : str + ID of the vault where the query is being performed. + + query : str + Query to perform. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + ExecuteQueryResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.query.execute_query( + vault_id="d408485953784308a000f8dcf81901ef", + query="query", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.execute_query( + vault_id=vault_id, query=query, request_options=request_options + ) + return _response.data diff --git a/flowvault/skyflow_flowvault/generated/rest/query/raw_client.py b/flowvault/skyflow_flowvault/generated/rest/query/raw_client.py new file mode 100644 index 00000000..e0d88bd0 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/query/raw_client.py @@ -0,0 +1,229 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from ..core.api_error import ApiError +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.http_response import AsyncHttpResponse, HttpResponse +from ..core.pydantic_utilities import parse_obj_as +from ..core.request_options import RequestOptions +from ..errors.bad_request_error import BadRequestError +from ..errors.internal_server_error import InternalServerError +from ..errors.not_found_error import NotFoundError +from ..errors.unauthorized_error import UnauthorizedError +from ..types.error_response import ErrorResponse +from ..types.execute_query_response import ExecuteQueryResponse + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class RawQueryClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def execute_query( + self, *, vault_id: str, query: str, request_options: typing.Optional[RequestOptions] = None + ) -> HttpResponse[ExecuteQueryResponse]: + """ + Returns records for a valid SQL query. This endpoint + - Can return masked record values. + - Supports only the `SELECT` command. + - Returns a maximum of 25 records. To return additional records, perform another query using the `OFFSET` keyword. + - Can't modify the vault or perform transactions. + - Can't return tokens. + - Can't return file download or render URLs. + + Parameters + ---------- + vault_id : str + ID of the vault where the query is being performed. + + query : str + Query to perform. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[ExecuteQueryResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/query", + method="POST", + json={ + "vaultID": vault_id, + "query": query, + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + ExecuteQueryResponse, + parse_obj_as( + type_=ExecuteQueryResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + +class AsyncRawQueryClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def execute_query( + self, *, vault_id: str, query: str, request_options: typing.Optional[RequestOptions] = None + ) -> AsyncHttpResponse[ExecuteQueryResponse]: + """ + Returns records for a valid SQL query. This endpoint + - Can return masked record values. + - Supports only the `SELECT` command. + - Returns a maximum of 25 records. To return additional records, perform another query using the `OFFSET` keyword. + - Can't modify the vault or perform transactions. + - Can't return tokens. + - Can't return file download or render URLs. + + Parameters + ---------- + vault_id : str + ID of the vault where the query is being performed. + + query : str + Query to perform. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[ExecuteQueryResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/query", + method="POST", + json={ + "vaultID": vault_id, + "query": query, + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + ExecuteQueryResponse, + parse_obj_as( + type_=ExecuteQueryResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) diff --git a/flowvault/skyflow_flowvault/generated/rest/raw_client.py b/flowvault/skyflow_flowvault/generated/rest/raw_client.py new file mode 100644 index 00000000..d6e9c34d --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/raw_client.py @@ -0,0 +1,76 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from .core.api_error import ApiError +from .core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from .core.http_response import AsyncHttpResponse, HttpResponse +from .core.jsonable_encoder import jsonable_encoder +from .core.request_options import RequestOptions + + +class RawSkyflowAuth: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def patch_v2vaults_id( + self, vault_id: typing.Optional[str], *, request_options: typing.Optional[RequestOptions] = None + ) -> HttpResponse[None]: + """ + Parameters + ---------- + vault_id : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[None] + """ + _response = self._client_wrapper.httpx_client.request( + f"v2/vaults/{jsonable_encoder(vault_id)}", + method="PATCH", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return HttpResponse(response=_response, data=None) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + +class AsyncRawSkyflowAuth: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def patch_v2vaults_id( + self, vault_id: typing.Optional[str], *, request_options: typing.Optional[RequestOptions] = None + ) -> AsyncHttpResponse[None]: + """ + Parameters + ---------- + vault_id : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[None] + """ + _response = await self._client_wrapper.httpx_client.request( + f"v2/vaults/{jsonable_encoder(vault_id)}", + method="PATCH", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return AsyncHttpResponse(response=_response, data=None) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) diff --git a/flowvault/skyflow_flowvault/generated/rest/records/client.py b/flowvault/skyflow_flowvault/generated/rest/records/client.py index 0503a99e..2c30277a 100644 --- a/flowvault/skyflow_flowvault/generated/rest/records/client.py +++ b/flowvault/skyflow_flowvault/generated/rest/records/client.py @@ -4,7 +4,16 @@ from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper from ..core.request_options import RequestOptions -from ..types.v_1_execute_query_response import V1ExecuteQueryResponse +from ..types.column_redactions import ColumnRedactions +from ..types.delete_response import DeleteResponse +from ..types.get_request_data import GetRequestData +from ..types.get_response import GetResponse +from ..types.insert_record_data import InsertRecordData +from ..types.insert_response import InsertResponse +from ..types.unique_value import UniqueValue +from ..types.update_record_data import UpdateRecordData +from ..types.update_response import UpdateResponse +from ..types.upsert import Upsert from .raw_client import AsyncRawRecordsClient, RawRecordsClient # this is used as the default value for optional parameters @@ -26,43 +35,295 @@ def with_raw_response(self) -> RawRecordsClient: """ return self._raw_client - def flow_service_execute_query( + def delete_records( self, *, - vault_id: typing.Optional[str] = OMIT, - query: typing.Optional[str] = OMIT, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, request_options: typing.Optional[RequestOptions] = None, - ) -> V1ExecuteQueryResponse: + ) -> DeleteResponse: """ - Executes a query on the specified vault. + Deletes records from a vault. Parameters ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted + vault_id : str + ID of the vault. - query : typing.Optional[str] - Query to execute. + table_name : typing.Optional[str] + Name of the table. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to delete. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DeleteResponse + OK + + Examples + -------- + from skyflow import SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.records.delete_records( + vault_id="d408485953784308a000f8dcf81901ef", + ) + """ + _response = self._raw_client.delete_records( + vault_id=vault_id, + table_name=table_name, + skyflow_i_ds=skyflow_i_ds, + unique_values=unique_values, + request_options=request_options, + ) + return _response.data + + def get_records( + self, + *, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + column_redactions: typing.Optional[typing.Sequence[ColumnRedactions]] = OMIT, + columns: typing.Optional[typing.Sequence[str]] = OMIT, + limit: typing.Optional[int] = OMIT, + offset: typing.Optional[int] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, + records: typing.Optional[typing.Sequence[GetRequestData]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> GetResponse: + """ + Returns the specified records from a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : typing.Optional[str] + Name of the table to perform the operation on. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to return. Either `skyflowIDs` or `uniqueValues` are required. If both are provided, the request fails. + + column_redactions : typing.Optional[typing.Sequence[ColumnRedactions]] + List of columns to redact. + + columns : typing.Optional[typing.Sequence[str]] + List of columns to return. + + limit : typing.Optional[int] + Limit for the number of records to be fetched. + + offset : typing.Optional[int] + Offset for the number of records to be fetched. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + records : typing.Optional[typing.Sequence[GetRequestData]] + List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - V1ExecuteQueryResponse - A successful response. + GetResponse + OK Examples -------- from skyflow import SkyflowAuth client = SkyflowAuth( - base_url="https://yourhost.com/path/to/api", + token="YOUR_TOKEN", + ) + client.records.get_records( + vault_id="d408485953784308a000f8dcf81901ef", ) - client.records.flow_service_execute_query() """ - _response = self._raw_client.flow_service_execute_query( - vault_id=vault_id, query=query, request_options=request_options + _response = self._raw_client.get_records( + vault_id=vault_id, + table_name=table_name, + skyflow_i_ds=skyflow_i_ds, + column_redactions=column_redactions, + columns=columns, + limit=limit, + offset=offset, + unique_values=unique_values, + records=records, + request_options=request_options, + ) + return _response.data + + def insert_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[InsertRecordData], + upsert: typing.Optional[Upsert] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> InsertResponse: + """ + Inserts new records into a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to perform the operation on. Can be defined at both the request body level and individual record level. If provided at both levels, the record-level `tableName` takes precedence. + + records : typing.Sequence[InsertRecordData] + Data to insert as a list of records. + + upsert : typing.Optional[Upsert] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + InsertResponse + OK + + Examples + -------- + from skyflow import InsertRecordData, SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.records.insert_records( + vault_id="d408485953784308a000f8dcf81901ef", + table_name="employees", + records=[ + InsertRecordData( + data={ + "name": "Vivek1", + "email": "vivek.varshney@skyflow.com", + "age": 23, + "adult": True, + "address": { + "street": "Bata Gali", + "city": "Aligarh", + "state": "UP", + "postal_code": 202001, + "country": "India", + "phone_numbers": [ + {"type": "home", "number": [1234, 5678]}, + {"type": "work", "number": [4321, 8765]}, + ], + }, + }, + ), + InsertRecordData( + data={ + "name": "Asad1", + "email": "asad.public@gmail.com", + "age": 16, + "adult": False, + "address": { + "street": "Sarojini", + "city": "Deoria", + "state": "UP", + "postal_code": 274001, + "country": "India", + "phone_numbers": [ + {"type": "home", "number": [7890, 3456]}, + {"type": "work", "number": [9087, 6543]}, + ], + }, + }, + ), + ], + ) + """ + _response = self._raw_client.insert_records( + vault_id=vault_id, table_name=table_name, records=records, upsert=upsert, request_options=request_options + ) + return _response.data + + def update_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[UpdateRecordData], + request_options: typing.Optional[RequestOptions] = None, + ) -> UpdateResponse: + """ + Updates the specified records in a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table. + + records : typing.Sequence[UpdateRecordData] + Data to update as a list of records. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + UpdateResponse + OK + + Examples + -------- + from skyflow import SkyflowAuth, UpdateRecordData + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.records.update_records( + vault_id="d408485953784308a000f8dcf81901ef", + table_name="employees", + records=[ + UpdateRecordData( + skyflow_id="97cdd1af-02ac-47eb-ab0d-8339dbef6ccb", + data={ + "name": "Vivek", + "email": "asad.public5@gmail.com", + "age": 25, + "adult": True, + "address": {"city": "aligarh", "country": "India"}, + }, + ), + UpdateRecordData( + skyflow_id="aed32bbc-e7a4-4c7b-8c88-7f52ce8cb066", + data={ + "name": "Asad", + "email": "asad.public7@gmail.com", + "age": 24, + "adult": False, + "address": {"city": "deoria", "country": "India"}, + }, + ), + ], + ) + """ + _response = self._raw_client.update_records( + vault_id=vault_id, table_name=table_name, records=records, request_options=request_options ) return _response.data @@ -82,31 +343,121 @@ def with_raw_response(self) -> AsyncRawRecordsClient: """ return self._raw_client - async def flow_service_execute_query( + async def delete_records( + self, + *, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> DeleteResponse: + """ + Deletes records from a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : typing.Optional[str] + Name of the table. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to delete. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DeleteResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.records.delete_records( + vault_id="d408485953784308a000f8dcf81901ef", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.delete_records( + vault_id=vault_id, + table_name=table_name, + skyflow_i_ds=skyflow_i_ds, + unique_values=unique_values, + request_options=request_options, + ) + return _response.data + + async def get_records( self, *, - vault_id: typing.Optional[str] = OMIT, - query: typing.Optional[str] = OMIT, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + column_redactions: typing.Optional[typing.Sequence[ColumnRedactions]] = OMIT, + columns: typing.Optional[typing.Sequence[str]] = OMIT, + limit: typing.Optional[int] = OMIT, + offset: typing.Optional[int] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, + records: typing.Optional[typing.Sequence[GetRequestData]] = OMIT, request_options: typing.Optional[RequestOptions] = None, - ) -> V1ExecuteQueryResponse: + ) -> GetResponse: """ - Executes a query on the specified vault. + Returns the specified records from a vault. Parameters ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted + vault_id : str + ID of the vault. - query : typing.Optional[str] - Query to execute. + table_name : typing.Optional[str] + Name of the table to perform the operation on. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to return. Either `skyflowIDs` or `uniqueValues` are required. If both are provided, the request fails. + + column_redactions : typing.Optional[typing.Sequence[ColumnRedactions]] + List of columns to redact. + + columns : typing.Optional[typing.Sequence[str]] + List of columns to return. + + limit : typing.Optional[int] + Limit for the number of records to be fetched. + + offset : typing.Optional[int] + Offset for the number of records to be fetched. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + records : typing.Optional[typing.Sequence[GetRequestData]] + List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - V1ExecuteQueryResponse - A successful response. + GetResponse + OK Examples -------- @@ -115,17 +466,203 @@ async def flow_service_execute_query( from skyflow import AsyncSkyflowAuth client = AsyncSkyflowAuth( - base_url="https://yourhost.com/path/to/api", + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.records.get_records( + vault_id="d408485953784308a000f8dcf81901ef", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.get_records( + vault_id=vault_id, + table_name=table_name, + skyflow_i_ds=skyflow_i_ds, + column_redactions=column_redactions, + columns=columns, + limit=limit, + offset=offset, + unique_values=unique_values, + records=records, + request_options=request_options, + ) + return _response.data + + async def insert_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[InsertRecordData], + upsert: typing.Optional[Upsert] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> InsertResponse: + """ + Inserts new records into a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to perform the operation on. Can be defined at both the request body level and individual record level. If provided at both levels, the record-level `tableName` takes precedence. + + records : typing.Sequence[InsertRecordData] + Data to insert as a list of records. + + upsert : typing.Optional[Upsert] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + InsertResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth, InsertRecordData + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.records.insert_records( + vault_id="d408485953784308a000f8dcf81901ef", + table_name="employees", + records=[ + InsertRecordData( + data={ + "name": "Vivek1", + "email": "vivek.varshney@skyflow.com", + "age": 23, + "adult": True, + "address": { + "street": "Bata Gali", + "city": "Aligarh", + "state": "UP", + "postal_code": 202001, + "country": "India", + "phone_numbers": [ + {"type": "home", "number": [1234, 5678]}, + {"type": "work", "number": [4321, 8765]}, + ], + }, + }, + ), + InsertRecordData( + data={ + "name": "Asad1", + "email": "asad.public@gmail.com", + "age": 16, + "adult": False, + "address": { + "street": "Sarojini", + "city": "Deoria", + "state": "UP", + "postal_code": 274001, + "country": "India", + "phone_numbers": [ + {"type": "home", "number": [7890, 3456]}, + {"type": "work", "number": [9087, 6543]}, + ], + }, + }, + ), + ], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.insert_records( + vault_id=vault_id, table_name=table_name, records=records, upsert=upsert, request_options=request_options + ) + return _response.data + + async def update_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[UpdateRecordData], + request_options: typing.Optional[RequestOptions] = None, + ) -> UpdateResponse: + """ + Updates the specified records in a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table. + + records : typing.Sequence[UpdateRecordData] + Data to update as a list of records. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + UpdateResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth, UpdateRecordData + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", ) async def main() -> None: - await client.records.flow_service_execute_query() + await client.records.update_records( + vault_id="d408485953784308a000f8dcf81901ef", + table_name="employees", + records=[ + UpdateRecordData( + skyflow_id="97cdd1af-02ac-47eb-ab0d-8339dbef6ccb", + data={ + "name": "Vivek", + "email": "asad.public5@gmail.com", + "age": 25, + "adult": True, + "address": {"city": "aligarh", "country": "India"}, + }, + ), + UpdateRecordData( + skyflow_id="aed32bbc-e7a4-4c7b-8c88-7f52ce8cb066", + data={ + "name": "Asad", + "email": "asad.public7@gmail.com", + "age": 24, + "adult": False, + "address": {"city": "deoria", "country": "India"}, + }, + ), + ], + ) asyncio.run(main()) """ - _response = await self._raw_client.flow_service_execute_query( - vault_id=vault_id, query=query, request_options=request_options + _response = await self._raw_client.update_records( + vault_id=vault_id, table_name=table_name, records=records, request_options=request_options ) return _response.data diff --git a/flowvault/skyflow_flowvault/generated/rest/records/raw_client.py b/flowvault/skyflow_flowvault/generated/rest/records/raw_client.py index 98a1365a..e65e6559 100644 --- a/flowvault/skyflow_flowvault/generated/rest/records/raw_client.py +++ b/flowvault/skyflow_flowvault/generated/rest/records/raw_client.py @@ -8,7 +8,22 @@ from ..core.http_response import AsyncHttpResponse, HttpResponse from ..core.pydantic_utilities import parse_obj_as from ..core.request_options import RequestOptions -from ..types.v_1_execute_query_response import V1ExecuteQueryResponse +from ..core.serialization import convert_and_respect_annotation_metadata +from ..errors.bad_request_error import BadRequestError +from ..errors.internal_server_error import InternalServerError +from ..errors.not_found_error import NotFoundError +from ..errors.unauthorized_error import UnauthorizedError +from ..types.column_redactions import ColumnRedactions +from ..types.delete_response import DeleteResponse +from ..types.error_response import ErrorResponse +from ..types.get_request_data import GetRequestData +from ..types.get_response import GetResponse +from ..types.insert_record_data import InsertRecordData +from ..types.insert_response import InsertResponse +from ..types.unique_value import UniqueValue +from ..types.update_record_data import UpdateRecordData +from ..types.update_response import UpdateResponse +from ..types.upsert import Upsert # this is used as the default value for optional parameters OMIT = typing.cast(typing.Any, ...) @@ -18,38 +33,189 @@ class RawRecordsClient: def __init__(self, *, client_wrapper: SyncClientWrapper): self._client_wrapper = client_wrapper - def flow_service_execute_query( + def delete_records( self, *, - vault_id: typing.Optional[str] = OMIT, - query: typing.Optional[str] = OMIT, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[V1ExecuteQueryResponse]: + ) -> HttpResponse[DeleteResponse]: """ - Executes a query on the specified vault. + Deletes records from a vault. Parameters ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted + vault_id : str + ID of the vault. - query : typing.Optional[str] - Query to execute. + table_name : typing.Optional[str] + Name of the table. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to delete. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[DeleteResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/records/delete", + method="POST", + json={ + "vaultID": vault_id, + "tableName": table_name, + "skyflowIDs": skyflow_i_ds, + "uniqueValues": convert_and_respect_annotation_metadata( + object_=unique_values, annotation=typing.Sequence[UniqueValue], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DeleteResponse, + parse_obj_as( + type_=DeleteResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + def get_records( + self, + *, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + column_redactions: typing.Optional[typing.Sequence[ColumnRedactions]] = OMIT, + columns: typing.Optional[typing.Sequence[str]] = OMIT, + limit: typing.Optional[int] = OMIT, + offset: typing.Optional[int] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, + records: typing.Optional[typing.Sequence[GetRequestData]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[GetResponse]: + """ + Returns the specified records from a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : typing.Optional[str] + Name of the table to perform the operation on. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to return. Either `skyflowIDs` or `uniqueValues` are required. If both are provided, the request fails. + + column_redactions : typing.Optional[typing.Sequence[ColumnRedactions]] + List of columns to redact. + + columns : typing.Optional[typing.Sequence[str]] + List of columns to return. + + limit : typing.Optional[int] + Limit for the number of records to be fetched. + + offset : typing.Optional[int] + Offset for the number of records to be fetched. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + records : typing.Optional[typing.Sequence[GetRequestData]] + List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[V1ExecuteQueryResponse] - A successful response. + HttpResponse[GetResponse] + OK """ _response = self._client_wrapper.httpx_client.request( - "v2/query", + "v2/records/get", method="POST", json={ "vaultID": vault_id, - "query": query, + "tableName": table_name, + "skyflowIDs": skyflow_i_ds, + "columnRedactions": convert_and_respect_annotation_metadata( + object_=column_redactions, annotation=typing.Sequence[ColumnRedactions], direction="write" + ), + "columns": columns, + "limit": limit, + "offset": offset, + "uniqueValues": convert_and_respect_annotation_metadata( + object_=unique_values, annotation=typing.Sequence[UniqueValue], direction="write" + ), + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[GetRequestData], direction="write" + ), }, headers={ "content-type": "application/json", @@ -60,13 +226,260 @@ def flow_service_execute_query( try: if 200 <= _response.status_code < 300: _data = typing.cast( - V1ExecuteQueryResponse, + GetResponse, parse_obj_as( - type_=V1ExecuteQueryResponse, # type: ignore + type_=GetResponse, # type: ignore object_=_response.json(), ), ) return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + def insert_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[InsertRecordData], + upsert: typing.Optional[Upsert] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[InsertResponse]: + """ + Inserts new records into a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to perform the operation on. Can be defined at both the request body level and individual record level. If provided at both levels, the record-level `tableName` takes precedence. + + records : typing.Sequence[InsertRecordData] + Data to insert as a list of records. + + upsert : typing.Optional[Upsert] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[InsertResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/records/insert", + method="POST", + json={ + "vaultID": vault_id, + "tableName": table_name, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[InsertRecordData], direction="write" + ), + "upsert": convert_and_respect_annotation_metadata(object_=upsert, annotation=Upsert, direction="write"), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + InsertResponse, + parse_obj_as( + type_=InsertResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + def update_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[UpdateRecordData], + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[UpdateResponse]: + """ + Updates the specified records in a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table. + + records : typing.Sequence[UpdateRecordData] + Data to update as a list of records. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[UpdateResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/records/update", + method="POST", + json={ + "vaultID": vault_id, + "tableName": table_name, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[UpdateRecordData], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + UpdateResponse, + parse_obj_as( + type_=UpdateResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) _response_json = _response.json() except JSONDecodeError: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) @@ -77,38 +490,189 @@ class AsyncRawRecordsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper - async def flow_service_execute_query( + async def delete_records( self, *, - vault_id: typing.Optional[str] = OMIT, - query: typing.Optional[str] = OMIT, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[V1ExecuteQueryResponse]: + ) -> AsyncHttpResponse[DeleteResponse]: """ - Executes a query on the specified vault. + Deletes records from a vault. Parameters ---------- - vault_id : typing.Optional[str] - ID of the vault where data is being inserted + vault_id : str + ID of the vault. - query : typing.Optional[str] - Query to execute. + table_name : typing.Optional[str] + Name of the table. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to delete. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[DeleteResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/records/delete", + method="POST", + json={ + "vaultID": vault_id, + "tableName": table_name, + "skyflowIDs": skyflow_i_ds, + "uniqueValues": convert_and_respect_annotation_metadata( + object_=unique_values, annotation=typing.Sequence[UniqueValue], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DeleteResponse, + parse_obj_as( + type_=DeleteResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def get_records( + self, + *, + vault_id: str, + table_name: typing.Optional[str] = OMIT, + skyflow_i_ds: typing.Optional[typing.Sequence[str]] = OMIT, + column_redactions: typing.Optional[typing.Sequence[ColumnRedactions]] = OMIT, + columns: typing.Optional[typing.Sequence[str]] = OMIT, + limit: typing.Optional[int] = OMIT, + offset: typing.Optional[int] = OMIT, + unique_values: typing.Optional[typing.Sequence[UniqueValue]] = OMIT, + records: typing.Optional[typing.Sequence[GetRequestData]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[GetResponse]: + """ + Returns the specified records from a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : typing.Optional[str] + Name of the table to perform the operation on. + + skyflow_i_ds : typing.Optional[typing.Sequence[str]] + Skyflow IDs of the records to return. Either `skyflowIDs` or `uniqueValues` are required. If both are provided, the request fails. + + column_redactions : typing.Optional[typing.Sequence[ColumnRedactions]] + List of columns to redact. + + columns : typing.Optional[typing.Sequence[str]] + List of columns to return. + + limit : typing.Optional[int] + Limit for the number of records to be fetched. + + offset : typing.Optional[int] + Offset for the number of records to be fetched. + + unique_values : typing.Optional[typing.Sequence[UniqueValue]] + List of unique constraint values to query records by data. + + records : typing.Optional[typing.Sequence[GetRequestData]] + List of records to be fetched. This field contains tableName and skyflowIDs belonging to the table. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[V1ExecuteQueryResponse] - A successful response. + AsyncHttpResponse[GetResponse] + OK """ _response = await self._client_wrapper.httpx_client.request( - "v2/query", + "v2/records/get", method="POST", json={ "vaultID": vault_id, - "query": query, + "tableName": table_name, + "skyflowIDs": skyflow_i_ds, + "columnRedactions": convert_and_respect_annotation_metadata( + object_=column_redactions, annotation=typing.Sequence[ColumnRedactions], direction="write" + ), + "columns": columns, + "limit": limit, + "offset": offset, + "uniqueValues": convert_and_respect_annotation_metadata( + object_=unique_values, annotation=typing.Sequence[UniqueValue], direction="write" + ), + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[GetRequestData], direction="write" + ), }, headers={ "content-type": "application/json", @@ -119,13 +683,260 @@ async def flow_service_execute_query( try: if 200 <= _response.status_code < 300: _data = typing.cast( - V1ExecuteQueryResponse, + GetResponse, parse_obj_as( - type_=V1ExecuteQueryResponse, # type: ignore + type_=GetResponse, # type: ignore object_=_response.json(), ), ) return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def insert_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[InsertRecordData], + upsert: typing.Optional[Upsert] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[InsertResponse]: + """ + Inserts new records into a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to perform the operation on. Can be defined at both the request body level and individual record level. If provided at both levels, the record-level `tableName` takes precedence. + + records : typing.Sequence[InsertRecordData] + Data to insert as a list of records. + + upsert : typing.Optional[Upsert] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[InsertResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/records/insert", + method="POST", + json={ + "vaultID": vault_id, + "tableName": table_name, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[InsertRecordData], direction="write" + ), + "upsert": convert_and_respect_annotation_metadata(object_=upsert, annotation=Upsert, direction="write"), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + InsertResponse, + parse_obj_as( + type_=InsertResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def update_records( + self, + *, + vault_id: str, + table_name: str, + records: typing.Sequence[UpdateRecordData], + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[UpdateResponse]: + """ + Updates the specified records in a vault. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table. + + records : typing.Sequence[UpdateRecordData] + Data to update as a list of records. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[UpdateResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/records/update", + method="POST", + json={ + "vaultID": vault_id, + "tableName": table_name, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[UpdateRecordData], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + UpdateResponse, + parse_obj_as( + type_=UpdateResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) _response_json = _response.json() except JSONDecodeError: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) diff --git a/v2/skyflow/generated/rest/authentication/__init__.py b/flowvault/skyflow_flowvault/generated/rest/tokens/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/authentication/__init__.py rename to flowvault/skyflow_flowvault/generated/rest/tokens/__init__.py diff --git a/flowvault/skyflow_flowvault/generated/rest/tokens/client.py b/flowvault/skyflow_flowvault/generated/rest/tokens/client.py new file mode 100644 index 00000000..1e6b0643 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/tokens/client.py @@ -0,0 +1,246 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.request_options import RequestOptions +from ..types.detokenize_response import DetokenizeResponse +from ..types.get_tokens_from_values_request_object import GetTokensFromValuesRequestObject +from ..types.get_tokens_from_values_response import GetTokensFromValuesResponse +from ..types.token_group_redactions import TokenGroupRedactions +from .raw_client import AsyncRawTokensClient, RawTokensClient + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class TokensClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._raw_client = RawTokensClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> RawTokensClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + RawTokensClient + """ + return self._raw_client + + def get_tokens( + self, + *, + vault_id: str, + records: typing.Sequence[GetTokensFromValuesRequestObject], + request_options: typing.Optional[RequestOptions] = None, + ) -> GetTokensFromValuesResponse: + """ + Returns the deterministic token previously issued for each supplied plaintext value within the specified token group. Only applicable to deterministic tokengroups. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[GetTokensFromValuesRequestObject] + Array of value/token-group pairs to look up. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + GetTokensFromValuesResponse + OK + + Examples + -------- + from skyflow import SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.tokens.get_tokens( + vault_id="d408485953784308a000f8dcf81901ef", + records=[], + ) + """ + _response = self._raw_client.get_tokens(vault_id=vault_id, records=records, request_options=request_options) + return _response.data + + def detokenize( + self, + *, + vault_id: str, + tokens: typing.Sequence[str], + token_group_redactions: typing.Optional[typing.Sequence[TokenGroupRedactions]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> DetokenizeResponse: + """ + Returns values that correspond to the specified tokens. + + Parameters + ---------- + vault_id : str + ID of the vault. + + tokens : typing.Sequence[str] + Token to be detokenized + + token_group_redactions : typing.Optional[typing.Sequence[TokenGroupRedactions]] + List of token groups to redact. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DetokenizeResponse + OK + + Examples + -------- + from skyflow import SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.tokens.detokenize( + vault_id="d408485953784308a000f8dcf81901ef", + tokens=["RYtQoeJdSQ", "8bd036b6-8fe0-4176-945b-3a5a63e8fd18"], + ) + """ + _response = self._raw_client.detokenize( + vault_id=vault_id, + tokens=tokens, + token_group_redactions=token_group_redactions, + request_options=request_options, + ) + return _response.data + + +class AsyncTokensClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._raw_client = AsyncRawTokensClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> AsyncRawTokensClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + AsyncRawTokensClient + """ + return self._raw_client + + async def get_tokens( + self, + *, + vault_id: str, + records: typing.Sequence[GetTokensFromValuesRequestObject], + request_options: typing.Optional[RequestOptions] = None, + ) -> GetTokensFromValuesResponse: + """ + Returns the deterministic token previously issued for each supplied plaintext value within the specified token group. Only applicable to deterministic tokengroups. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[GetTokensFromValuesRequestObject] + Array of value/token-group pairs to look up. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + GetTokensFromValuesResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.tokens.get_tokens( + vault_id="d408485953784308a000f8dcf81901ef", + records=[], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.get_tokens( + vault_id=vault_id, records=records, request_options=request_options + ) + return _response.data + + async def detokenize( + self, + *, + vault_id: str, + tokens: typing.Sequence[str], + token_group_redactions: typing.Optional[typing.Sequence[TokenGroupRedactions]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> DetokenizeResponse: + """ + Returns values that correspond to the specified tokens. + + Parameters + ---------- + vault_id : str + ID of the vault. + + tokens : typing.Sequence[str] + Token to be detokenized + + token_group_redactions : typing.Optional[typing.Sequence[TokenGroupRedactions]] + List of token groups to redact. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DetokenizeResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.tokens.detokenize( + vault_id="d408485953784308a000f8dcf81901ef", + tokens=["RYtQoeJdSQ", "8bd036b6-8fe0-4176-945b-3a5a63e8fd18"], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.detokenize( + vault_id=vault_id, + tokens=tokens, + token_group_redactions=token_group_redactions, + request_options=request_options, + ) + return _response.data diff --git a/flowvault/skyflow_flowvault/generated/rest/tokens/raw_client.py b/flowvault/skyflow_flowvault/generated/rest/tokens/raw_client.py new file mode 100644 index 00000000..d4e30ab7 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/tokens/raw_client.py @@ -0,0 +1,489 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from ..core.api_error import ApiError +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.http_response import AsyncHttpResponse, HttpResponse +from ..core.pydantic_utilities import parse_obj_as +from ..core.request_options import RequestOptions +from ..core.serialization import convert_and_respect_annotation_metadata +from ..errors.bad_request_error import BadRequestError +from ..errors.forbidden_error import ForbiddenError +from ..errors.internal_server_error import InternalServerError +from ..errors.not_found_error import NotFoundError +from ..errors.too_many_requests_error import TooManyRequestsError +from ..errors.unauthorized_error import UnauthorizedError +from ..types.detokenize_response import DetokenizeResponse +from ..types.error_response import ErrorResponse +from ..types.get_tokens_from_values_request_object import GetTokensFromValuesRequestObject +from ..types.get_tokens_from_values_response import GetTokensFromValuesResponse +from ..types.token_group_redactions import TokenGroupRedactions + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class RawTokensClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def get_tokens( + self, + *, + vault_id: str, + records: typing.Sequence[GetTokensFromValuesRequestObject], + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[GetTokensFromValuesResponse]: + """ + Returns the deterministic token previously issued for each supplied plaintext value within the specified token group. Only applicable to deterministic tokengroups. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[GetTokensFromValuesRequestObject] + Array of value/token-group pairs to look up. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[GetTokensFromValuesResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/records/getTokens", + method="POST", + json={ + "vaultID": vault_id, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[GetTokensFromValuesRequestObject], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + GetTokensFromValuesResponse, + parse_obj_as( + type_=GetTokensFromValuesResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + def detokenize( + self, + *, + vault_id: str, + tokens: typing.Sequence[str], + token_group_redactions: typing.Optional[typing.Sequence[TokenGroupRedactions]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[DetokenizeResponse]: + """ + Returns values that correspond to the specified tokens. + + Parameters + ---------- + vault_id : str + ID of the vault. + + tokens : typing.Sequence[str] + Token to be detokenized + + token_group_redactions : typing.Optional[typing.Sequence[TokenGroupRedactions]] + List of token groups to redact. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[DetokenizeResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/tokens/detokenize", + method="POST", + json={ + "vaultID": vault_id, + "tokens": tokens, + "tokenGroupRedactions": convert_and_respect_annotation_metadata( + object_=token_group_redactions, annotation=typing.Sequence[TokenGroupRedactions], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DetokenizeResponse, + parse_obj_as( + type_=DetokenizeResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + +class AsyncRawTokensClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def get_tokens( + self, + *, + vault_id: str, + records: typing.Sequence[GetTokensFromValuesRequestObject], + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[GetTokensFromValuesResponse]: + """ + Returns the deterministic token previously issued for each supplied plaintext value within the specified token group. Only applicable to deterministic tokengroups. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[GetTokensFromValuesRequestObject] + Array of value/token-group pairs to look up. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[GetTokensFromValuesResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/records/getTokens", + method="POST", + json={ + "vaultID": vault_id, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[GetTokensFromValuesRequestObject], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + GetTokensFromValuesResponse, + parse_obj_as( + type_=GetTokensFromValuesResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def detokenize( + self, + *, + vault_id: str, + tokens: typing.Sequence[str], + token_group_redactions: typing.Optional[typing.Sequence[TokenGroupRedactions]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[DetokenizeResponse]: + """ + Returns values that correspond to the specified tokens. + + Parameters + ---------- + vault_id : str + ID of the vault. + + tokens : typing.Sequence[str] + Token to be detokenized + + token_group_redactions : typing.Optional[typing.Sequence[TokenGroupRedactions]] + List of token groups to redact. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[DetokenizeResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/tokens/detokenize", + method="POST", + json={ + "vaultID": vault_id, + "tokens": tokens, + "tokenGroupRedactions": convert_and_respect_annotation_metadata( + object_=token_group_redactions, annotation=typing.Sequence[TokenGroupRedactions], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DetokenizeResponse, + parse_obj_as( + type_=DetokenizeResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) diff --git a/flowvault/skyflow_flowvault/generated/rest/types/__init__.py b/flowvault/skyflow_flowvault/generated/rest/types/__init__.py index b088dc4c..980df206 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/__init__.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/__init__.py @@ -2,66 +2,58 @@ # isort: skip_file -from .flow_enum_update_type import FlowEnumUpdateType -from .flow_tokenize_response_object_token import FlowTokenizeResponseObjectToken -from .googleprotobuf_any import GoogleprotobufAny -from .protobuf_null_value import ProtobufNullValue -from .rpc_status import RpcStatus -from .v_1_column_redactions import V1ColumnRedactions -from .v_1_delete_response import V1DeleteResponse -from .v_1_delete_response_object import V1DeleteResponseObject -from .v_1_delete_token_response_object import V1DeleteTokenResponseObject -from .v_1_execute_query_record_response import V1ExecuteQueryRecordResponse -from .v_1_execute_query_response import V1ExecuteQueryResponse -from .v_1_execute_query_response_metadata import V1ExecuteQueryResponseMetadata -from .v_1_flow_delete_token_response import V1FlowDeleteTokenResponse -from .v_1_flow_detokenize_response import V1FlowDetokenizeResponse -from .v_1_flow_detokenize_response_object import V1FlowDetokenizeResponseObject -from .v_1_flow_tokenize_request_object import V1FlowTokenizeRequestObject -from .v_1_flow_tokenize_response import V1FlowTokenizeResponse -from .v_1_flow_tokenize_response_object import V1FlowTokenizeResponseObject -from .v_1_flow_vault_metrics_data import V1FlowVaultMetricsData -from .v_1_flow_vault_metrics_response import V1FlowVaultMetricsResponse -from .v_1_get_request_data import V1GetRequestData -from .v_1_get_response import V1GetResponse -from .v_1_insert_record_data import V1InsertRecordData -from .v_1_insert_response import V1InsertResponse -from .v_1_record_response_object import V1RecordResponseObject -from .v_1_token_group_redactions import V1TokenGroupRedactions -from .v_1_unique_value import V1UniqueValue -from .v_1_update_record_data import V1UpdateRecordData -from .v_1_update_response import V1UpdateResponse -from .v_1_upsert import V1Upsert +from .column_redactions import ColumnRedactions +from .delete_response import DeleteResponse +from .delete_response_object import DeleteResponseObject +from .detokenize_response import DetokenizeResponse +from .detokenize_response_object import DetokenizeResponseObject +from .error_response import ErrorResponse +from .error_response_error import ErrorResponseError +from .execute_query_record_response import ExecuteQueryRecordResponse +from .execute_query_response import ExecuteQueryResponse +from .execute_query_response_metadata import ExecuteQueryResponseMetadata +from .get_request_data import GetRequestData +from .get_response import GetResponse +from .get_tokens_from_values_request_object import GetTokensFromValuesRequestObject +from .get_tokens_from_values_response import GetTokensFromValuesResponse +from .google_protobuf_value import GoogleProtobufValue +from .http_code import HttpCode +from .insert_record_data import InsertRecordData +from .insert_response import InsertResponse +from .record_response_object import RecordResponseObject +from .token_group_redactions import TokenGroupRedactions +from .tokenize_response_object import TokenizeResponseObject +from .unique_value import UniqueValue +from .update_record_data import UpdateRecordData +from .update_response import UpdateResponse +from .upsert import Upsert +from .upsert_update_type import UpsertUpdateType __all__ = [ - "FlowEnumUpdateType", - "FlowTokenizeResponseObjectToken", - "GoogleprotobufAny", - "ProtobufNullValue", - "RpcStatus", - "V1ColumnRedactions", - "V1DeleteResponse", - "V1DeleteResponseObject", - "V1DeleteTokenResponseObject", - "V1ExecuteQueryRecordResponse", - "V1ExecuteQueryResponse", - "V1ExecuteQueryResponseMetadata", - "V1FlowDeleteTokenResponse", - "V1FlowDetokenizeResponse", - "V1FlowDetokenizeResponseObject", - "V1FlowTokenizeRequestObject", - "V1FlowTokenizeResponse", - "V1FlowTokenizeResponseObject", - "V1FlowVaultMetricsData", - "V1FlowVaultMetricsResponse", - "V1GetRequestData", - "V1GetResponse", - "V1InsertRecordData", - "V1InsertResponse", - "V1RecordResponseObject", - "V1TokenGroupRedactions", - "V1UniqueValue", - "V1UpdateRecordData", - "V1UpdateResponse", - "V1Upsert", + "ColumnRedactions", + "DeleteResponse", + "DeleteResponseObject", + "DetokenizeResponse", + "DetokenizeResponseObject", + "ErrorResponse", + "ErrorResponseError", + "ExecuteQueryRecordResponse", + "ExecuteQueryResponse", + "ExecuteQueryResponseMetadata", + "GetRequestData", + "GetResponse", + "GetTokensFromValuesRequestObject", + "GetTokensFromValuesResponse", + "GoogleProtobufValue", + "HttpCode", + "InsertRecordData", + "InsertResponse", + "RecordResponseObject", + "TokenGroupRedactions", + "TokenizeResponseObject", + "UniqueValue", + "UpdateRecordData", + "UpdateResponse", + "Upsert", + "UpsertUpdateType", ] diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_column_redactions.py b/flowvault/skyflow_flowvault/generated/rest/types/column_redactions.py similarity index 61% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_column_redactions.py rename to flowvault/skyflow_flowvault/generated/rest/types/column_redactions.py index 65d089a5..7fc7f92d 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_column_redactions.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/column_redactions.py @@ -8,17 +8,15 @@ from ..core.serialization import FieldMetadata -class V1ColumnRedactions(UniversalBaseModel): - column_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="columnName")] = pydantic.Field( - default=None - ) +class ColumnRedactions(UniversalBaseModel): + column_name: typing_extensions.Annotated[str, FieldMetadata(alias="columnName")] = pydantic.Field() """ - Name of the column to be redacted + Name of the column to redact. """ - redaction: typing.Optional[str] = pydantic.Field(default=None) + redaction: str = pydantic.Field() """ - Name of the redaction. Eg: `plain_text`, `redacted`, `mask1` + Name of the redaction type. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_response.py b/flowvault/skyflow_flowvault/generated/rest/types/delete_response.py similarity index 72% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_response.py rename to flowvault/skyflow_flowvault/generated/rest/types/delete_response.py index 9b281978..317d4b7e 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_response.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/delete_response.py @@ -4,11 +4,11 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_delete_response_object import V1DeleteResponseObject +from .delete_response_object import DeleteResponseObject -class V1DeleteResponse(UniversalBaseModel): - records: typing.Optional[typing.List[V1DeleteResponseObject]] = pydantic.Field(default=None) +class DeleteResponse(UniversalBaseModel): + records: typing.List[DeleteResponseObject] = pydantic.Field() """ List of deleted records with skyflow ID and any partial errors. """ diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_token_response_object.py b/flowvault/skyflow_flowvault/generated/rest/types/delete_response_object.py similarity index 64% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_token_response_object.py rename to flowvault/skyflow_flowvault/generated/rest/types/delete_response_object.py index 2a482ec0..e4c746b5 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_token_response_object.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/delete_response_object.py @@ -8,22 +8,20 @@ from ..core.serialization import FieldMetadata -class V1DeleteTokenResponseObject(UniversalBaseModel): - value: typing.Optional[str] = pydantic.Field(default=None) +class DeleteResponseObject(UniversalBaseModel): + skyflow_id: typing_extensions.Annotated[str, FieldMetadata(alias="skyflowID")] = pydantic.Field() """ - Token value + Skyflow ID of the deleted record. """ error: typing.Optional[str] = pydantic.Field(default=None) """ - Error if deletion failed + Error message, if any. """ - http_code: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="httpCode")] = pydantic.Field( - default=None - ) + http_code: typing_extensions.Annotated[int, FieldMetadata(alias="httpCode")] = pydantic.Field() """ - HTTP status code of the response + HTTP status code of the response. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_detokenize_response.py b/flowvault/skyflow_flowvault/generated/rest/types/detokenize_response.py similarity index 67% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_detokenize_response.py rename to flowvault/skyflow_flowvault/generated/rest/types/detokenize_response.py index 47ab50dd..beac966d 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_detokenize_response.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/detokenize_response.py @@ -4,11 +4,11 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_flow_detokenize_response_object import V1FlowDetokenizeResponseObject +from .detokenize_response_object import DetokenizeResponseObject -class V1FlowDetokenizeResponse(UniversalBaseModel): - response: typing.Optional[typing.List[V1FlowDetokenizeResponseObject]] = pydantic.Field(default=None) +class DetokenizeResponse(UniversalBaseModel): + response: typing.List[DetokenizeResponseObject] = pydantic.Field() """ Detokenized data """ diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_detokenize_response_object.py b/flowvault/skyflow_flowvault/generated/rest/types/detokenize_response_object.py similarity index 64% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_detokenize_response_object.py rename to flowvault/skyflow_flowvault/generated/rest/types/detokenize_response_object.py index 382a2b1a..e035dcb0 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_detokenize_response_object.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/detokenize_response_object.py @@ -6,41 +6,36 @@ import typing_extensions from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel from ..core.serialization import FieldMetadata +from .google_protobuf_value import GoogleProtobufValue -class V1FlowDetokenizeResponseObject(UniversalBaseModel): - token: typing.Optional[str] = pydantic.Field(default=None) +class DetokenizeResponseObject(UniversalBaseModel): + token: str = pydantic.Field() """ - Token to be detokenized + Token that was detokenized. """ - value: typing.Optional[typing.Optional[typing.Any]] = pydantic.Field(default=None) + value: GoogleProtobufValue + token_group_name: typing_extensions.Annotated[str, FieldMetadata(alias="tokenGroupName")] = pydantic.Field() """ - Detokenized value for the token - """ - - token_group_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tokenGroupName")] = ( - pydantic.Field(default=None) - ) - """ - Token group name + Name of the token group. """ error: typing.Optional[str] = pydantic.Field(default=None) """ - Error if detokenization failed + Error message, if any. """ http_code: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="httpCode")] = pydantic.Field( default=None ) """ - HTTP status code of the response + HTTP status code of the response. """ metadata: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) """ - Additional metadata associated with the token, such as tableName or skyflowID + Additional metadata associated with the token, such as tableName or skyflowID. """ if IS_PYDANTIC_V2: diff --git a/v2/skyflow/generated/rest/types/error_response.py b/flowvault/skyflow_flowvault/generated/rest/types/error_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/error_response.py rename to flowvault/skyflow_flowvault/generated/rest/types/error_response.py diff --git a/v2/skyflow/generated/rest/types/error_response_error.py b/flowvault/skyflow_flowvault/generated/rest/types/error_response_error.py similarity index 100% rename from v2/skyflow/generated/rest/types/error_response_error.py rename to flowvault/skyflow_flowvault/generated/rest/types/error_response_error.py diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_unique_value.py b/flowvault/skyflow_flowvault/generated/rest/types/execute_query_record_response.py similarity index 86% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_unique_value.py rename to flowvault/skyflow_flowvault/generated/rest/types/execute_query_record_response.py index e0cfa021..676e96e8 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_unique_value.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/execute_query_record_response.py @@ -6,10 +6,10 @@ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -class V1UniqueValue(UniversalBaseModel): +class ExecuteQueryRecordResponse(UniversalBaseModel): data: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) """ - Columns names and values for unique value entry + Fields and values for the record. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_response.py b/flowvault/skyflow_flowvault/generated/rest/types/execute_query_response.py similarity index 58% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_response.py rename to flowvault/skyflow_flowvault/generated/rest/types/execute_query_response.py index 17caef33..c8f4c115 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_response.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/execute_query_response.py @@ -4,17 +4,17 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_execute_query_record_response import V1ExecuteQueryRecordResponse -from .v_1_execute_query_response_metadata import V1ExecuteQueryResponseMetadata +from .execute_query_record_response import ExecuteQueryRecordResponse +from .execute_query_response_metadata import ExecuteQueryResponseMetadata -class V1ExecuteQueryResponse(UniversalBaseModel): - records: typing.Optional[typing.List[V1ExecuteQueryRecordResponse]] = pydantic.Field(default=None) +class ExecuteQueryResponse(UniversalBaseModel): + records: typing.Optional[typing.List[ExecuteQueryRecordResponse]] = pydantic.Field(default=None) """ Records corresponding to the specified query. """ - metadata: typing.Optional[V1ExecuteQueryResponseMetadata] = None + metadata: typing.Optional[ExecuteQueryResponseMetadata] = None if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_response_metadata.py b/flowvault/skyflow_flowvault/generated/rest/types/execute_query_response_metadata.py similarity index 80% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_response_metadata.py rename to flowvault/skyflow_flowvault/generated/rest/types/execute_query_response_metadata.py index 3eb0e86c..73934b01 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_response_metadata.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/execute_query_response_metadata.py @@ -6,10 +6,14 @@ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -class V1ExecuteQueryResponseMetadata(UniversalBaseModel): +class ExecuteQueryResponseMetadata(UniversalBaseModel): + """ + Metadata for the query. + """ + columns: typing.Optional[typing.List[str]] = pydantic.Field(default=None) """ - Return columns for the query + Columns returned for the query. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/flow_enum_update_type.py b/flowvault/skyflow_flowvault/generated/rest/types/flow_enum_update_type.py deleted file mode 100644 index 01b2bab9..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/flow_enum_update_type.py +++ /dev/null @@ -1,5 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -FlowEnumUpdateType = typing.Union[typing.Literal["UPDATE", "REPLACE"], typing.Any] diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_get_request_data.py b/flowvault/skyflow_flowvault/generated/rest/types/get_request_data.py similarity index 50% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_get_request_data.py rename to flowvault/skyflow_flowvault/generated/rest/types/get_request_data.py index caf815b6..92a0efae 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_get_request_data.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/get_request_data.py @@ -6,42 +6,38 @@ import typing_extensions from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel from ..core.serialization import FieldMetadata -from .v_1_column_redactions import V1ColumnRedactions -from .v_1_unique_value import V1UniqueValue +from .column_redactions import ColumnRedactions +from .unique_value import UniqueValue -class V1GetRequestData(UniversalBaseModel): - table_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tableName")] = pydantic.Field( - default=None - ) +class GetRequestData(UniversalBaseModel): + table_name: typing_extensions.Annotated[str, FieldMetadata(alias="tableName")] = pydantic.Field() """ - Name of the table where data is being fetched + Name of the table. """ - skyflow_i_ds: typing_extensions.Annotated[typing.Optional[typing.List[str]], FieldMetadata(alias="skyflowIDs")] = ( - pydantic.Field(default=None) - ) + skyflow_i_ds: typing_extensions.Annotated[typing.List[str], FieldMetadata(alias="skyflowIDs")] = pydantic.Field() """ - Skyflow ID for the record to be fetched + Skyflow IDs of the records to return. """ column_redactions: typing_extensions.Annotated[ - typing.Optional[typing.List[V1ColumnRedactions]], FieldMetadata(alias="columnRedactions") + typing.Optional[typing.List[ColumnRedactions]], FieldMetadata(alias="columnRedactions") ] = pydantic.Field(default=None) """ - List of columns to be redacted. + List of columns to redact. """ columns: typing.Optional[typing.List[str]] = pydantic.Field(default=None) """ - List of columns to be fetched. + List of columns to return. """ unique_values: typing_extensions.Annotated[ - typing.Optional[typing.List[V1UniqueValue]], FieldMetadata(alias="uniqueValues") + typing.Optional[typing.List[UniqueValue]], FieldMetadata(alias="uniqueValues") ] = pydantic.Field(default=None) """ - List of unique constraint values to query records by data + List of unique constraint values to query records by data. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/get_response.py b/flowvault/skyflow_flowvault/generated/rest/types/get_response.py new file mode 100644 index 00000000..5b348ae3 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/types/get_response.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .record_response_object import RecordResponseObject + + +class GetResponse(UniversalBaseModel): + records: typing.List[RecordResponseObject] = pydantic.Field() + """ + List of fetched records. For file columns, the value in the data map is an object containing: fileName, mimeType, sizeBytes, fileStatus (PENDING | READY | FAILED | SCAN_ERROR), fileFailureReason (non-null only when fileStatus is FAILED), url (pre-signed download URL, non-null only when fileStatus is READY), and urlExpiresAt (UTC expiry of the URL). + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/get_tokens_from_values_request_object.py b/flowvault/skyflow_flowvault/generated/rest/types/get_tokens_from_values_request_object.py new file mode 100644 index 00000000..ee9e1c56 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/types/get_tokens_from_values_request_object.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata +from .google_protobuf_value import GoogleProtobufValue + + +class GetTokensFromValuesRequestObject(UniversalBaseModel): + value: GoogleProtobufValue + token_group_name: typing_extensions.Annotated[str, FieldMetadata(alias="tokenGroupName")] = pydantic.Field() + """ + Name of the deterministic token group. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_record_response.py b/flowvault/skyflow_flowvault/generated/rest/types/get_tokens_from_values_response.py similarity index 63% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_record_response.py rename to flowvault/skyflow_flowvault/generated/rest/types/get_tokens_from_values_response.py index 30de3867..bc57229c 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_execute_query_record_response.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/get_tokens_from_values_response.py @@ -4,12 +4,13 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .tokenize_response_object import TokenizeResponseObject -class V1ExecuteQueryRecordResponse(UniversalBaseModel): - data: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) +class GetTokensFromValuesResponse(UniversalBaseModel): + records: typing.List[TokenizeResponseObject] = pydantic.Field() """ - Fields and values for the record. For example, `{'field_1':'value_1', 'field_2':'value_2'}`. + Array of token result objects, one per input entry, in the same order as the request. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/protobuf_null_value.py b/flowvault/skyflow_flowvault/generated/rest/types/google_protobuf_value.py similarity index 61% rename from flowvault/skyflow_flowvault/generated/rest/types/protobuf_null_value.py rename to flowvault/skyflow_flowvault/generated/rest/types/google_protobuf_value.py index 7a4d590f..37b377d9 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/protobuf_null_value.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/google_protobuf_value.py @@ -2,4 +2,4 @@ import typing -ProtobufNullValue = typing.Literal["NULL_VALUE"] +GoogleProtobufValue = typing.Optional[typing.Any] diff --git a/flowvault/skyflow_flowvault/generated/rest/types/googleprotobuf_any.py b/flowvault/skyflow_flowvault/generated/rest/types/googleprotobuf_any.py deleted file mode 100644 index aebcc5b9..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/googleprotobuf_any.py +++ /dev/null @@ -1,139 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from ..core.serialization import FieldMetadata - - -class GoogleprotobufAny(UniversalBaseModel): - """ - `Any` contains an arbitrary serialized protocol buffer message along with a - URL that describes the type of the serialized message. - - Protobuf library provides support to pack/unpack Any values in the form - of utility functions or additional generated methods of the Any type. - - Example 1: Pack and unpack a message in C++. - - Foo foo = ...; - Any any; - any.PackFrom(foo); - ... - if (any.UnpackTo(&foo)) { - ... - } - - Example 2: Pack and unpack a message in Java. - - Foo foo = ...; - Any any = Any.pack(foo); - ... - if (any.is(Foo.class)) { - foo = any.unpack(Foo.class); - } - // or ... - if (any.isSameTypeAs(Foo.getDefaultInstance())) { - foo = any.unpack(Foo.getDefaultInstance()); - } - - Example 3: Pack and unpack a message in Python. - - foo = Foo(...) - any = Any() - any.Pack(foo) - ... - if any.Is(Foo.DESCRIPTOR): - any.Unpack(foo) - ... - - Example 4: Pack and unpack a message in Go - - foo := &pb.Foo{...} - any, err := anypb.New(foo) - if err != nil { - ... - } - ... - foo := &pb.Foo{} - if err := any.UnmarshalTo(foo); err != nil { - ... - } - - The pack methods provided by protobuf library will by default use - 'type.googleapis.com/full.type.name' as the type URL and the unpack - methods only use the fully qualified type name after the last '/' - in the type URL, for example "foo.bar.com/x/y.z" will yield type - name "y.z". - - JSON - ==== - The JSON representation of an `Any` value uses the regular - representation of the deserialized, embedded message, with an - additional field `@type` which contains the type URL. Example: - - package google.profile; - message Person { - string first_name = 1; - string last_name = 2; - } - - { - "@type": "type.googleapis.com/google.profile.Person", - "firstName": , - "lastName": - } - - If the embedded message type is well-known and has a custom JSON - representation, that representation will be embedded adding a field - `value` which holds the custom JSON in addition to the `@type` - field. Example (for message [google.protobuf.Duration][]): - - { - "@type": "type.googleapis.com/google.protobuf.Duration", - "value": "1.212s" - } - """ - - type: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="@type")] = pydantic.Field(default=None) - """ - A URL/resource name that uniquely identifies the type of the serialized - protocol buffer message. This string must contain at least - one "/" character. The last segment of the URL's path must represent - the fully qualified name of the type (as in - `path/google.protobuf.Duration`). The name should be in a canonical form - (e.g., leading "." is not accepted). - - In practice, teams usually precompile into the binary all types that they - expect it to use in the context of Any. However, for URLs which use the - scheme `http`, `https`, or no scheme, one can optionally set up a type - server that maps type URLs to message definitions as follows: - - * If no scheme is provided, `https` is assumed. - * An HTTP GET on the URL must yield a [google.protobuf.Type][] - value in binary format, or produce an error. - * Applications are allowed to cache lookup results based on the - URL, or have them precompiled into a binary to avoid any - lookup. Therefore, binary compatibility needs to be preserved - on changes to types. (Use versioned type names to manage - breaking changes.) - - Note: this functionality is not currently available in the official - protobuf release, and it is not used for type URLs beginning with - type.googleapis.com. As of May 2023, there are no widely used type server - implementations and no plans to implement one. - - Schemes other than `http`, `https` (or the empty scheme) might be - used with implementation specific semantics. - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/v2/skyflow/generated/rest/types/http_code.py b/flowvault/skyflow_flowvault/generated/rest/types/http_code.py similarity index 100% rename from v2/skyflow/generated/rest/types/http_code.py rename to flowvault/skyflow_flowvault/generated/rest/types/http_code.py diff --git a/flowvault/skyflow_flowvault/generated/rest/types/insert_record_data.py b/flowvault/skyflow_flowvault/generated/rest/types/insert_record_data.py new file mode 100644 index 00000000..ac7050a6 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/types/insert_record_data.py @@ -0,0 +1,34 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata +from .upsert import Upsert + + +class InsertRecordData(UniversalBaseModel): + data: typing.Dict[str, typing.Optional[typing.Any]] = pydantic.Field() + """ + Columns and values for the record. + """ + + table_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tableName")] = pydantic.Field( + default=None + ) + """ + Name of the table to insert data into. + """ + + upsert: typing.Optional[Upsert] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_vault_metrics_data.py b/flowvault/skyflow_flowvault/generated/rest/types/insert_response.py similarity index 71% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_vault_metrics_data.py rename to flowvault/skyflow_flowvault/generated/rest/types/insert_response.py index f8611c37..f110fdb8 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_vault_metrics_data.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/insert_response.py @@ -4,12 +4,13 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .record_response_object import RecordResponseObject -class V1FlowVaultMetricsData(UniversalBaseModel): - tables: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) +class InsertResponse(UniversalBaseModel): + records: typing.List[RecordResponseObject] = pydantic.Field() """ - Map of table names to their metrics + List of inserted records. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_record_response_object.py b/flowvault/skyflow_flowvault/generated/rest/types/record_response_object.py similarity index 61% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_record_response_object.py rename to flowvault/skyflow_flowvault/generated/rest/types/record_response_object.py index 0f02a93f..72d8ec1f 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_record_response_object.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/record_response_object.py @@ -8,48 +8,44 @@ from ..core.serialization import FieldMetadata -class V1RecordResponseObject(UniversalBaseModel): - skyflow_id: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="skyflowID")] = pydantic.Field( - default=None - ) +class RecordResponseObject(UniversalBaseModel): + skyflow_id: typing_extensions.Annotated[str, FieldMetadata(alias="skyflowID")] = pydantic.Field() """ Skyflow ID for the inserted record """ tokens: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) """ - Tokens data for the columns if any + Columns and tokens for the record. """ data: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) """ - Columns names and values + Columns and values for the record. For file columns, the value is an object containing file metadata: fileName, mimeType, sizeBytes, fileStatus (PENDING | READY | FAILED | SCAN_ERROR), fileFailureReason (populated only when fileStatus is FAILED), url (pre-signed download URL, populated only when fileStatus is READY), and urlExpiresAt (UTC expiry of the URL). """ hashed_data: typing_extensions.Annotated[ typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]], FieldMetadata(alias="hashedData") ] = pydantic.Field(default=None) """ - Hashed Data for the columns if any + Columns and hashed values for the record. """ error: typing.Optional[str] = pydantic.Field(default=None) """ - Partial Error message if any + Error message, if any. """ - http_code: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="httpCode")] = pydantic.Field( - default=None - ) + http_code: typing_extensions.Annotated[int, FieldMetadata(alias="httpCode")] = pydantic.Field() """ - HTTP status code of the response + HTTP status code of the response. """ table_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tableName")] = pydantic.Field( default=None ) """ - Name of the table record belongs to + Name of the table that the record belongs to. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_token_group_redactions.py b/flowvault/skyflow_flowvault/generated/rest/types/token_group_redactions.py similarity index 83% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_token_group_redactions.py rename to flowvault/skyflow_flowvault/generated/rest/types/token_group_redactions.py index 69263a19..3213712b 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_token_group_redactions.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/token_group_redactions.py @@ -8,17 +8,17 @@ from ..core.serialization import FieldMetadata -class V1TokenGroupRedactions(UniversalBaseModel): +class TokenGroupRedactions(UniversalBaseModel): token_group_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tokenGroupName")] = ( pydantic.Field(default=None) ) """ - Name of the token group to be redacted + Name of the token group to redact. """ redaction: typing.Optional[str] = pydantic.Field(default=None) """ - Name of the redaction. Eg: `plain_text`, `redacted`, `mask1` + Name of the redaction to perform. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/flow_tokenize_response_object_token.py b/flowvault/skyflow_flowvault/generated/rest/types/tokenize_response_object.py similarity index 66% rename from flowvault/skyflow_flowvault/generated/rest/types/flow_tokenize_response_object_token.py rename to flowvault/skyflow_flowvault/generated/rest/types/tokenize_response_object.py index 928a6606..a80c12d0 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/flow_tokenize_response_object_token.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/tokenize_response_object.py @@ -6,31 +6,31 @@ import typing_extensions from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel from ..core.serialization import FieldMetadata +from .google_protobuf_value import GoogleProtobufValue -class FlowTokenizeResponseObjectToken(UniversalBaseModel): - token_group_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tokenGroupName")] = ( - pydantic.Field(default=None) - ) +class TokenizeResponseObject(UniversalBaseModel): + token: str = pydantic.Field() """ - Token group Name + Token that was generated. """ - token: typing.Optional[str] = pydantic.Field(default=None) + value: GoogleProtobufValue + token_group_name: typing_extensions.Annotated[str, FieldMetadata(alias="tokenGroupName")] = pydantic.Field() """ - Token value + Name of the token group. """ error: typing.Optional[str] = pydantic.Field(default=None) """ - Error if tokenization failed + Error message, if any. """ http_code: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="httpCode")] = pydantic.Field( default=None ) """ - HTTP status code of the response + HTTP status code of the response. """ if IS_PYDANTIC_V2: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/rpc_status.py b/flowvault/skyflow_flowvault/generated/rest/types/unique_value.py similarity index 66% rename from flowvault/skyflow_flowvault/generated/rest/types/rpc_status.py rename to flowvault/skyflow_flowvault/generated/rest/types/unique_value.py index cf324547..3a984173 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/rpc_status.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/unique_value.py @@ -4,13 +4,13 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .googleprotobuf_any import GoogleprotobufAny -class RpcStatus(UniversalBaseModel): - code: typing.Optional[int] = None - message: typing.Optional[str] = None - details: typing.Optional[typing.List[GoogleprotobufAny]] = None +class UniqueValue(UniversalBaseModel): + data: typing.Dict[str, typing.Optional[typing.Any]] = pydantic.Field() + """ + Columns names and values for the unique value entry. + """ if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_insert_record_data.py b/flowvault/skyflow_flowvault/generated/rest/types/update_record_data.py similarity index 61% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_insert_record_data.py rename to flowvault/skyflow_flowvault/generated/rest/types/update_record_data.py index 063626d3..d90713d3 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_insert_record_data.py +++ b/flowvault/skyflow_flowvault/generated/rest/types/update_record_data.py @@ -6,29 +6,26 @@ import typing_extensions from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel from ..core.serialization import FieldMetadata -from .v_1_upsert import V1Upsert -class V1InsertRecordData(UniversalBaseModel): - data: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) +class UpdateRecordData(UniversalBaseModel): + skyflow_id: typing_extensions.Annotated[str, FieldMetadata(alias="skyflowID")] = pydantic.Field() """ - Columns names and values + Skyflow ID of the record to update. """ - tokens: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) + data: typing.Dict[str, typing.Optional[typing.Any]] = pydantic.Field() """ - undocumented_field; Tokens data for the columns if any + Columns and values for the record. """ table_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tableName")] = pydantic.Field( default=None ) """ - Table name for the record + Name of the table to update data in. """ - upsert: typing.Optional[V1Upsert] = None - if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 else: diff --git a/flowvault/skyflow_flowvault/generated/rest/types/update_response.py b/flowvault/skyflow_flowvault/generated/rest/types/update_response.py new file mode 100644 index 00000000..4c1738ec --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/types/update_response.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .record_response_object import RecordResponseObject + + +class UpdateResponse(UniversalBaseModel): + records: typing.List[RecordResponseObject] = pydantic.Field() + """ + List of updated records. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/upsert.py b/flowvault/skyflow_flowvault/generated/rest/types/upsert.py new file mode 100644 index 00000000..2b487955 --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/types/upsert.py @@ -0,0 +1,38 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata +from .upsert_update_type import UpsertUpdateType + + +class Upsert(UniversalBaseModel): + """ + Upsert details. + """ + + update_type: typing_extensions.Annotated[typing.Optional[UpsertUpdateType], FieldMetadata(alias="updateType")] = ( + pydantic.Field(default=None) + ) + """ + Type of update operation to perform. + """ + + unique_columns: typing_extensions.Annotated[typing.List[str], FieldMetadata(alias="uniqueColumns")] = ( + pydantic.Field() + ) + """ + List of unique columns in the table that upsert operations use to identify if a record with matching values exists. If a matching record exists, the record updates with the specified values. If a matching record doesn't exist, the upsert operation inserts a new record. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/upsert_update_type.py b/flowvault/skyflow_flowvault/generated/rest/types/upsert_update_type.py new file mode 100644 index 00000000..0fd1129a --- /dev/null +++ b/flowvault/skyflow_flowvault/generated/rest/types/upsert_update_type.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +UpsertUpdateType = typing.Union[typing.Literal["UPDATE", "REPLACE"], typing.Any] diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_response_object.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_response_object.py deleted file mode 100644 index eda4c5ab..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_delete_response_object.py +++ /dev/null @@ -1,38 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from ..core.serialization import FieldMetadata - - -class V1DeleteResponseObject(UniversalBaseModel): - skyflow_id: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="skyflowID")] = pydantic.Field( - default=None - ) - """ - Skyflow ID for the deleted record - """ - - error: typing.Optional[str] = pydantic.Field(default=None) - """ - Partial Error message if any - """ - - http_code: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="httpCode")] = pydantic.Field( - default=None - ) - """ - HTTP status code of the response - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_request_object.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_request_object.py deleted file mode 100644 index 42a926ee..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_request_object.py +++ /dev/null @@ -1,36 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from ..core.serialization import FieldMetadata - - -class V1FlowTokenizeRequestObject(UniversalBaseModel): - value: typing.Optional[typing.Optional[typing.Any]] = pydantic.Field(default=None) - """ - Token Value - """ - - token_group_names: typing_extensions.Annotated[ - typing.Optional[typing.List[str]], FieldMetadata(alias="tokenGroupNames") - ] = pydantic.Field(default=None) - """ - List of token group names - """ - - token: typing.Optional[typing.Optional[typing.Any]] = pydantic.Field(default=None) - """ - Token for the value, in case of BYOT. - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_response.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_response.py deleted file mode 100644 index 88616410..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_response.py +++ /dev/null @@ -1,23 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_flow_tokenize_response_object import V1FlowTokenizeResponseObject - - -class V1FlowTokenizeResponse(UniversalBaseModel): - response: typing.Optional[typing.List[V1FlowTokenizeResponseObject]] = pydantic.Field(default=None) - """ - Tokenized data - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_response_object.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_response_object.py deleted file mode 100644 index e77e153b..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_tokenize_response_object.py +++ /dev/null @@ -1,28 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .flow_tokenize_response_object_token import FlowTokenizeResponseObjectToken - - -class V1FlowTokenizeResponseObject(UniversalBaseModel): - value: typing.Optional[typing.Optional[typing.Any]] = pydantic.Field(default=None) - """ - Value of token - """ - - tokens: typing.Optional[typing.List[FlowTokenizeResponseObjectToken]] = pydantic.Field(default=None) - """ - Token value - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_vault_metrics_response.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_vault_metrics_response.py deleted file mode 100644 index 5234fd91..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_vault_metrics_response.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_flow_vault_metrics_data import V1FlowVaultMetricsData - - -class V1FlowVaultMetricsResponse(UniversalBaseModel): - data: typing.Optional[V1FlowVaultMetricsData] = None - error: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) - """ - Error information, if any - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_get_response.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_get_response.py deleted file mode 100644 index ab966469..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_get_response.py +++ /dev/null @@ -1,23 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_record_response_object import V1RecordResponseObject - - -class V1GetResponse(UniversalBaseModel): - records: typing.Optional[typing.List[V1RecordResponseObject]] = pydantic.Field(default=None) - """ - List of fetched records with skyflow ID, tokens, data, and any partial errors - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_insert_response.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_insert_response.py deleted file mode 100644 index bac58b52..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_insert_response.py +++ /dev/null @@ -1,23 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_record_response_object import V1RecordResponseObject - - -class V1InsertResponse(UniversalBaseModel): - records: typing.Optional[typing.List[V1RecordResponseObject]] = pydantic.Field(default=None) - """ - List of inserted records with skyflow ID, tokens, data, and any partial errors. - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_update_record_data.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_update_record_data.py deleted file mode 100644 index 19622eab..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_update_record_data.py +++ /dev/null @@ -1,43 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from ..core.serialization import FieldMetadata - - -class V1UpdateRecordData(UniversalBaseModel): - skyflow_id: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="skyflowID")] = pydantic.Field( - default=None - ) - """ - Skyflow ID for the record to be updated - """ - - data: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) - """ - List of data row wise that is to be updated in the vault - """ - - tokens: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None) - """ - undocumented_field; Tokens data for the columns if any - """ - - table_name: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="tableName")] = pydantic.Field( - default=None - ) - """ - Table name for the record - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_update_response.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_update_response.py deleted file mode 100644 index 4f4eb228..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_update_response.py +++ /dev/null @@ -1,23 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_record_response_object import V1RecordResponseObject - - -class V1UpdateResponse(UniversalBaseModel): - records: typing.Optional[typing.List[V1RecordResponseObject]] = pydantic.Field(default=None) - """ - List of updated records with skyflow ID, tokens, data, and any partial errors - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_upsert.py b/flowvault/skyflow_flowvault/generated/rest/types/v_1_upsert.py deleted file mode 100644 index f9531a37..00000000 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_upsert.py +++ /dev/null @@ -1,30 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from ..core.serialization import FieldMetadata -from .flow_enum_update_type import FlowEnumUpdateType - - -class V1Upsert(UniversalBaseModel): - update_type: typing_extensions.Annotated[typing.Optional[FlowEnumUpdateType], FieldMetadata(alias="updateType")] = ( - None - ) - unique_columns: typing_extensions.Annotated[ - typing.Optional[typing.List[str]], FieldMetadata(alias="uniqueColumns") - ] = pydantic.Field(default=None) - """ - Name of a unique columns in the table. Uses upsert operations to check if a record exists based on the unique column's value. If a matching record exists, the record updates with the values you provide. If a matching record doesn't exist, the upsert operation inserts a new record. - """ - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/utils/_batching.py b/flowvault/skyflow_flowvault/utils/_batching.py new file mode 100644 index 00000000..39877428 --- /dev/null +++ b/flowvault/skyflow_flowvault/utils/_batching.py @@ -0,0 +1,78 @@ +import math +import os + +from dotenv import dotenv_values, find_dotenv + +from common.utils.logger import log_warn +from skyflow_flowvault.utils._skyflow_messages import SkyflowMessages + +MAX_BULK_DATA_SIZE = 10000 + +DEFAULT_BATCH_SIZE = 50 +MAX_BATCH_SIZE = 1000 +DEFAULT_CONCURRENCY = 1 +MAX_CONCURRENCY = 10 + +INSERT_BATCH_SIZE_KEY = "INSERT_BATCH_SIZE" +INSERT_CONCURRENCY_LIMIT_KEY = "INSERT_CONCURRENCY_LIMIT" +DETOKENIZE_BATCH_SIZE_KEY = "DETOKENIZE_BATCH_SIZE" +DETOKENIZE_CONCURRENCY_LIMIT_KEY = "DETOKENIZE_CONCURRENCY_LIMIT" + + +def _resolve_setting(key): + value = os.getenv(key) + if value is None: + try: + path = find_dotenv(usecwd=True) + if path: + value = dotenv_values(path).get(key) + except Exception: + value = None + return value + + +def _resolve_batch_size(batch_env_key, logger): + raw = _resolve_setting(batch_env_key) + if raw is None: + return DEFAULT_BATCH_SIZE + try: + parsed = int(raw) + except (ValueError, TypeError): + log_warn(SkyflowMessages.Error.INVALID_BATCH_SIZE.value, logger) + return DEFAULT_BATCH_SIZE + if parsed > MAX_BATCH_SIZE: + log_warn(SkyflowMessages.Error.BATCH_SIZE_EXCEEDS_MAX.value, logger) + capped = min(parsed, MAX_BATCH_SIZE) + if capped > 0: + return capped + log_warn(SkyflowMessages.Error.INVALID_BATCH_SIZE.value, logger) + return DEFAULT_BATCH_SIZE + + +def _resolve_concurrency(conc_env_key, item_count, batch_size, logger): + batch_count = max(1, math.ceil(item_count / batch_size)) if batch_size > 0 else 1 + raw = _resolve_setting(conc_env_key) + if raw is None: + return min(DEFAULT_CONCURRENCY, batch_count) + try: + parsed = int(raw) + except (ValueError, TypeError): + log_warn(SkyflowMessages.Error.INVALID_CONCURRENCY_LIMIT.value, logger) + return min(DEFAULT_CONCURRENCY, batch_count) + if parsed > MAX_CONCURRENCY: + log_warn(SkyflowMessages.Error.CONCURRENCY_EXCEEDS_MAX.value, logger) + capped = min(parsed, MAX_CONCURRENCY) + if capped > 0: + return min(capped, batch_count) + log_warn(SkyflowMessages.Error.INVALID_CONCURRENCY_LIMIT.value, logger) + return min(DEFAULT_CONCURRENCY, batch_count) + + +def resolve_batch_config(batch_env_key, conc_env_key, item_count, logger=None): + batch_size = _resolve_batch_size(batch_env_key, logger) + concurrency = _resolve_concurrency(conc_env_key, item_count, batch_size, logger) + return batch_size, concurrency + + +def create_batches(items, batch_size): + return [items[i:i + batch_size] for i in range(0, len(items), batch_size)] diff --git a/flowvault/skyflow_flowvault/utils/_response_parsing.py b/flowvault/skyflow_flowvault/utils/_response_parsing.py new file mode 100644 index 00000000..d42c454c --- /dev/null +++ b/flowvault/skyflow_flowvault/utils/_response_parsing.py @@ -0,0 +1,64 @@ +def parse_tokens(raw): + if raw is None: + return None + parsed = {} + for column, value in raw.items(): + entries = _parse_entries(value, _to_token) + if entries is not None: + parsed[column] = entries + return parsed + + +def parse_hashed_data(raw): + if raw is None: + return None + parsed = {} + for column, value in raw.items(): + entries = _parse_entries(value, _to_hash) + if entries is not None: + parsed[column] = entries + return parsed + + +def parse_metadata(raw): + if raw is None: + return None + return { + 'skyflow_id': raw.get('skyflowID', raw.get('skyflowId', raw.get('skyflow_id'))), + 'table_name': raw.get('table', raw.get('tableName', raw.get('table_name'))), + } + + +def _parse_entries(raw_value, to_entry): + if raw_value is None: + return None + items = raw_value if isinstance(raw_value, list) else [raw_value] + entries = [] + for item in items: + entry = to_entry(item) + if entry is not None: + entries.append(entry) + return entries + + +def _to_token(entry): + if isinstance(entry, dict): + return { + 'token': entry.get('token'), + 'token_group_name': entry.get('tokenGroupName', entry.get('token_group_name')), + 'path': entry.get('path'), + } + if entry is not None: + return {'token': entry, 'token_group_name': None, 'path': None} + return None + + +def _to_hash(entry): + if isinstance(entry, dict): + return { + 'data': entry.get('data'), + 'hash_name': entry.get('hashName', entry.get('hash_name')), + } + if entry is not None: + return {'data': entry, 'hash_name': None} + return None diff --git a/flowvault/skyflow_flowvault/utils/_skyflow_messages.py b/flowvault/skyflow_flowvault/utils/_skyflow_messages.py index 35596296..5cbc41cd 100644 --- a/flowvault/skyflow_flowvault/utils/_skyflow_messages.py +++ b/flowvault/skyflow_flowvault/utils/_skyflow_messages.py @@ -17,10 +17,10 @@ class SkyflowMessages: class Error(Enum): EMPTY_RECORDS_IN_INSERT = f"{error_prefix} Insert failed. Specify at least one record to insert." - INVALID_RECORDS_TYPE_IN_INSERT = f"{error_prefix} Insert failed. 'records' must be a list of dicts." - INVALID_RECORD_DATA_IN_INSERT = f"{error_prefix} Insert failed. Each record's 'values' must be a non-empty dict." - INVALID_TABLE_NAME_IN_INSERT = f"{error_prefix} Insert failed. 'table' must be a non-empty string." - INVALID_UPSERT_TYPE_IN_INSERT = f"{error_prefix} Insert failed. 'upsert' must be a dict." + INVALID_RECORDS_TYPE_IN_INSERT = f"{error_prefix} Insert failed. 'records' must be a list of InsertRequestRecord objects." + INVALID_RECORD_DATA_IN_INSERT = f"{error_prefix} Validation error. Each record's 'values' must be a non-empty dict." + INVALID_TABLE_NAME_IN_INSERT = f"{error_prefix} Validation error. 'table' must be a non-empty string." + INVALID_UPSERT_TYPE_IN_INSERT = f"{error_prefix} Insert failed. 'upsert' must be an UpsertOptions object." INVALID_UPSERT_UNIQUE_COLUMNS_IN_INSERT = f"{error_prefix} Insert failed. Upsert's 'unique_columns' must be a non-empty list of strings." INVALID_UPSERT_UPDATE_TYPE_IN_INSERT = f"{error_prefix} Insert failed. Upsert's 'update_type' must be an UpsertType value." TOO_MANY_RECORDS_IN_INSERT = f"{error_prefix} Insert failed. A single insert request cannot contain more than 10000 records." @@ -44,14 +44,102 @@ class Error(Enum): "provided per-record -- InsertRequest's request-level 'upsert' cannot be used while " "'table' is set on individual records." ) - EMPTY_KEY_IN_INSERT_DATA = f"{error_prefix} Insert failed. Each record's 'values' must not contain a null or empty key." + EMPTY_KEY_IN_INSERT_DATA = f"{error_prefix} Validation error. Each record's 'values' must not contain a null or empty key." EMPTY_VALUE_IN_INSERT_DATA = f"{error_prefix} Insert failed. Each record's 'values' must not contain a null or empty value." + MISSING_TABLE_NAME_IN_GET = f"{error_prefix} Get failed. Specify a table name." + MISSING_IDS_OR_UNIQUE_VALUES_IN_GET = f"{error_prefix} Get failed. Specify at least one of 'ids' or 'unique_values'." + INVALID_IDS_IN_GET = f"{error_prefix} Get failed. 'ids' must be a non-empty list of strings." + INVALID_RECORDS_TYPE_IN_GET = f"{error_prefix} Get failed. 'records' must be a non-empty list of GetRecordRequest objects." + GET_MODE_CONFLICT = f"{error_prefix} Get failed. Use either 'records' (multi-table) or the single-table fields (table/ids/unique_values/columns/column_redactions/limit/offset), not both." + + EMPTY_RECORDS_IN_UPDATE = f"{error_prefix} Update failed. Specify at least one record to update." + INVALID_RECORDS_TYPE_IN_UPDATE = f"{error_prefix} Update failed. 'records' must be a list of dicts." + MISSING_SKYFLOW_ID_IN_UPDATE = f"{error_prefix} Update failed. Each record must specify a non-empty 'skyflow_id'." + INVALID_UPDATE_TYPE_IN_UPDATE = f"{error_prefix} Update failed. 'update_type' must be an UpsertType value." + TABLE_NAME_IN_BOTH_PLACES_IN_UPDATE = ( + f"{error_prefix} Update failed. 'table' cannot be set on UpdateRequest at the same " + "time as any record's 'table' -- specify a table name outside the records " + "(request-level, applying to all of them) or inside each record, but not both at once." + ) + TABLE_NAME_MISSING_IN_UPDATE = ( + f"{error_prefix} Update failed. 'table' is not set on UpdateRequest, so every record " + "must set its own 'table' -- either set 'table' once at the request level, or set it " + "individually on every record." + ) + + MISSING_TABLE_NAME_IN_DELETE = f"{error_prefix} Delete failed. Specify a table name." + MISSING_IDS_OR_UNIQUE_VALUES_IN_DELETE = f"{error_prefix} Delete failed. Specify at least one of 'ids' or 'unique_values'." + INVALID_IDS_IN_DELETE = f"{error_prefix} Delete failed. 'ids' must be a non-empty list of strings." + + EMPTY_TOKENS_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. Specify at least one token to detokenize." + INVALID_TOKENS_TYPE_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. 'tokens' must be a non-empty list of strings." + INVALID_TOKEN_GROUP_REDACTIONS_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. 'token_group_redactions' must be a list of dicts with 'token_group_name' and 'redaction' keys." + + INVALID_QUERY_IN_QUERY = f"{error_prefix} Query failed. 'query' must be a non-empty string." + + + EMPTY_RECORDS_IN_BULK_INSERT = f"{error_prefix} Bulk insert failed. Specify at least one record to insert." + INVALID_RECORDS_TYPE_IN_BULK_INSERT = f"{error_prefix} Bulk insert failed. 'records' must be a list of BulkInsertRecord objects." + INVALID_RECORD_IN_BULK_INSERT = f"{error_prefix} Bulk insert failed. Each record must be a BulkInsertRecord object." + TOO_MANY_RECORDS_IN_BULK_INSERT = f"{error_prefix} Bulk insert failed. A single bulk insert request cannot contain more than 10000 records." + TOO_MANY_TOKENS_IN_BULK_DETOKENIZE = f"{error_prefix} Bulk detokenize failed. A single bulk detokenize request cannot contain more than 10000 tokens." + + INVALID_BATCH_SIZE = f"{error_prefix} Invalid batch size provided. Falling back to the default batch size." + BATCH_SIZE_EXCEEDS_MAX = f"{error_prefix} Batch size exceeds the maximum allowed. Using the maximum batch size." + INVALID_CONCURRENCY_LIMIT = f"{error_prefix} Invalid concurrency limit provided. Falling back to the default concurrency limit." + CONCURRENCY_EXCEEDS_MAX = f"{error_prefix} Concurrency limit exceeds the maximum allowed. Using the maximum concurrency limit." + class Info(Enum): VALIDATE_INSERT_REQUEST = f"{INFO}: [{error_prefix}] Validating insert request." INSERT_TRIGGERED = f"{INFO}: [{error_prefix}] Insert method triggered." INSERT_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Insert request resolved." INSERT_SUCCESS = f"{INFO}: [{error_prefix}] Data inserted." + VALIDATE_GET_REQUEST = f"{INFO}: [{error_prefix}] Validating get request." + GET_TRIGGERED = f"{INFO}: [{error_prefix}] Get method triggered." + GET_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Get request resolved." + GET_SUCCESS = f"{INFO}: [{error_prefix}] Data fetched." + + VALIDATE_UPDATE_REQUEST = f"{INFO}: [{error_prefix}] Validating update request." + UPDATE_TRIGGERED = f"{INFO}: [{error_prefix}] Update method triggered." + UPDATE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Update request resolved." + UPDATE_SUCCESS = f"{INFO}: [{error_prefix}] Data updated." + + VALIDATE_DELETE_REQUEST = f"{INFO}: [{error_prefix}] Validating delete request." + DELETE_TRIGGERED = f"{INFO}: [{error_prefix}] Delete method triggered." + DELETE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Delete request resolved." + DELETE_SUCCESS = f"{INFO}: [{error_prefix}] Data deleted." + + VALIDATE_DETOKENIZE_REQUEST = f"{INFO}: [{error_prefix}] Validating detokenize request." + DETOKENIZE_TRIGGERED = f"{INFO}: [{error_prefix}] Detokenize method triggered." + DETOKENIZE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Detokenize request resolved." + DETOKENIZE_SUCCESS = f"{INFO}: [{error_prefix}] Tokens detokenized." + + VALIDATE_QUERY_REQUEST = f"{INFO}: [{error_prefix}] Validating query request." + QUERY_TRIGGERED = f"{INFO}: [{error_prefix}] Query method triggered." + QUERY_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Query request resolved." + QUERY_SUCCESS = f"{INFO}: [{error_prefix}] Query executed." + + + VALIDATE_BULK_INSERT_REQUEST = f"{INFO}: [{error_prefix}] Validating bulk insert request." + BULK_INSERT_TRIGGERED = f"{INFO}: [{error_prefix}] Bulk insert method triggered." + BULK_INSERT_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Bulk insert request resolved." + BULK_INSERT_SUCCESS = f"{INFO}: [{error_prefix}] Bulk insert completed." + + VALIDATE_BULK_DETOKENIZE_REQUEST = f"{INFO}: [{error_prefix}] Validating bulk detokenize request." + BULK_DETOKENIZE_TRIGGERED = f"{INFO}: [{error_prefix}] Bulk detokenize method triggered." + BULK_DETOKENIZE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Bulk detokenize request resolved." + BULK_DETOKENIZE_SUCCESS = f"{INFO}: [{error_prefix}] Bulk detokenize completed." + + PROCESSING_BATCHES = f"{INFO}: [{error_prefix}] Processing batches." + class ErrorLogs(Enum): INSERT_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Insert call resulted in failure." + GET_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Get call resulted in failure." + UPDATE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Update call resulted in failure." + DELETE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Delete call resulted in failure." + DETOKENIZE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Detokenize call resulted in failure." + QUERY_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Query call resulted in failure." + BULK_INSERT_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Bulk insert batch resulted in failure." + BULK_DETOKENIZE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Bulk detokenize batch resulted in failure." diff --git a/flowvault/skyflow_flowvault/utils/validations/__init__.py b/flowvault/skyflow_flowvault/utils/validations/__init__.py index 499bed61..5adadc53 100644 --- a/flowvault/skyflow_flowvault/utils/validations/__init__.py +++ b/flowvault/skyflow_flowvault/utils/validations/__init__.py @@ -1 +1,12 @@ -from ._validations import validate_vault_config, validate_update_vault_config, validate_insert_request +from ._validations import ( + validate_vault_config, + validate_update_vault_config, + validate_insert_request, + validate_get_request, + validate_update_request, + validate_delete_request, + validate_detokenize_request, + validate_query_request, + validate_bulk_insert_request, + validate_bulk_detokenize_request, +) diff --git a/flowvault/skyflow_flowvault/utils/validations/_validations.py b/flowvault/skyflow_flowvault/utils/validations/_validations.py index a8ea74df..31dcb43e 100644 --- a/flowvault/skyflow_flowvault/utils/validations/_validations.py +++ b/flowvault/skyflow_flowvault/utils/validations/_validations.py @@ -3,14 +3,15 @@ from common.utils.validations import ( validate_keys, validate_credentials, + validate_non_empty_string_list, validate_vault_config, validate_update_vault_config, ) from skyflow_flowvault.utils import SkyflowMessages from skyflow_flowvault.utils.enums import UpsertType +from skyflow_flowvault.vault.data import GetRecordRequest, BulkInsertRecord, InsertRequestRecord, UpsertOptions -VALID_INSERT_RECORD_KEYS = ["values", "table", "upsert"] -VALID_UPSERT_KEYS = ["update_type", "unique_columns"] +VALID_UPDATE_RECORD_KEYS = ["skyflow_id", "data", "tokens", "table_name"] invalid_input_error_code = CommonMessages.ErrorCodes.INVALID_INPUT.value @@ -22,15 +23,13 @@ def _validate_upsert(logger, upsert): if upsert is None: return - if not isinstance(upsert, dict): + if not isinstance(upsert, UpsertOptions): raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_TYPE_IN_INSERT.value, invalid_input_error_code) - validate_keys(logger, upsert, VALID_UPSERT_KEYS) - unique_columns = upsert.get("unique_columns") + unique_columns = upsert.unique_columns if (not isinstance(unique_columns, list) or not unique_columns or not all(isinstance(c, str) for c in unique_columns)): raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_UNIQUE_COLUMNS_IN_INSERT.value, invalid_input_error_code) - update_type = upsert.get("update_type") - if update_type is not None and not isinstance(update_type, UpsertType): + if upsert.update_type is not None and not isinstance(upsert.update_type, UpsertType): raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_UPDATE_TYPE_IN_INSERT.value, invalid_input_error_code) @@ -38,42 +37,173 @@ def _validate_upsert(logger, upsert): def validate_insert_request(logger, request): - if not isinstance(request.values, list) or not all(isinstance(r, dict) for r in request.values): + if not isinstance(request.records, list) or not all(isinstance(r, InsertRequestRecord) for r in request.records): raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_INSERT.value, invalid_input_error_code) - if not request.values: + if not request.records: raise SkyflowError(SkyflowMessages.Error.EMPTY_RECORDS_IN_INSERT.value, invalid_input_error_code) - if len(request.values) > MAX_INSERT_RECORDS: + if len(request.records) > MAX_INSERT_RECORDS: raise SkyflowError(SkyflowMessages.Error.TOO_MANY_RECORDS_IN_INSERT.value, invalid_input_error_code) - # request.table/record["table"] format and record["values"] emptiness/key/value validity are - # checked by the controller via the shared BaseVaultController._validate_table_name_if_present() - # / _validate_field_values() -- not here, to avoid duplicating that logic. + # record.table format and record.data emptiness/key/value validity are checked by the controller + # via BaseVaultController._validate_table_name_if_present() / _validate_field_values(). _validate_upsert(logger, request.upsert) - - for record in request.values: - validate_keys(logger, record, VALID_INSERT_RECORD_KEYS) - _validate_upsert(logger, record.get("upsert")) + for record in request.records: + _validate_upsert(logger, record.upsert) # table must be set in exactly one place -- request-level (every record) or per-record (no # partial mix) -- and upsert must live at that same place (mirrors Java's v3 Validations). - table_at_request_level = request.table is not None + table_at_request_level = request.table_name is not None if table_at_request_level: - for record in request.values: - if record.get("table") is not None: + for record in request.records: + if record.table_name is not None: raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_IN_BOTH_PLACES_IN_INSERT.value, invalid_input_error_code) else: - for record in request.values: - if record.get("table") is None: + for record in request.records: + if record.table_name is None: raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_MISSING_IN_INSERT.value, invalid_input_error_code) if table_at_request_level: - for record in request.values: - if record.get("upsert") is not None: + for record in request.records: + if record.upsert is not None: raise SkyflowError(SkyflowMessages.Error.RECORD_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code) + elif request.upsert is not None: + raise SkyflowError(SkyflowMessages.Error.REQUEST_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code) + + +def validate_get_request(logger, request): + # Two mutually exclusive modes: multi-table batch (request.records) vs single-table. + if request.records is not None: + single_table_fields_set = ( + request.table or request.ids or request.unique_values or request.columns + or request.column_redactions or request.limit is not None or request.offset is not None + ) + if single_table_fields_set: + raise SkyflowError(SkyflowMessages.Error.GET_MODE_CONFLICT.value, invalid_input_error_code) + if (not isinstance(request.records, list) or not request.records + or not all(isinstance(r, GetRecordRequest) for r in request.records)): + raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_GET.value, invalid_input_error_code) + for record in request.records: + if not record.table: + raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_GET.value, invalid_input_error_code) + if not record.ids and not record.unique_values: + raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_GET.value, invalid_input_error_code) + if record.ids is not None: + validate_non_empty_string_list(logger, record.ids, SkyflowMessages.Error.INVALID_IDS_IN_GET.value) + return + + if not request.table: + raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_GET.value, invalid_input_error_code) + + if not request.ids and not request.unique_values: + raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_GET.value, invalid_input_error_code) + + if request.ids is not None: + validate_non_empty_string_list(logger, request.ids, SkyflowMessages.Error.INVALID_IDS_IN_GET.value) + + +def validate_update_request(logger, request): + if not isinstance(request.records, list) or not all(isinstance(r, dict) for r in request.records): + raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_UPDATE.value, invalid_input_error_code) + + if not request.records: + raise SkyflowError(SkyflowMessages.Error.EMPTY_RECORDS_IN_UPDATE.value, invalid_input_error_code) + + if request.update_type is not None and not isinstance(request.update_type, UpsertType): + raise SkyflowError(SkyflowMessages.Error.INVALID_UPDATE_TYPE_IN_UPDATE.value, invalid_input_error_code) + + for record in request.records: + validate_keys(logger, record, VALID_UPDATE_RECORD_KEYS) + skyflow_id = record.get("skyflow_id") + if not isinstance(skyflow_id, str) or not skyflow_id.strip(): + raise SkyflowError(SkyflowMessages.Error.MISSING_SKYFLOW_ID_IN_UPDATE.value, invalid_input_error_code) + + table_at_request_level = request.table_name is not None + + if table_at_request_level: + for record in request.records: + if record.get("table_name") is not None: + raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_IN_BOTH_PLACES_IN_UPDATE.value, invalid_input_error_code) + else: + for record in request.records: + if record.get("table_name") is None: + raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_MISSING_IN_UPDATE.value, invalid_input_error_code) + + +def validate_delete_request(logger, request): + if not request.table: + raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_DELETE.value, invalid_input_error_code) + + if not request.ids and not request.unique_values: + raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_DELETE.value, invalid_input_error_code) + + if request.ids is not None: + validate_non_empty_string_list(logger, request.ids, SkyflowMessages.Error.INVALID_IDS_IN_DELETE.value) + + +def validate_detokenize_request(logger, request): + if ( + not isinstance(request.tokens, list) or not all(isinstance(t, str) and t.strip() for t in request.tokens) + ): + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_TYPE_IN_DETOKENIZE.value, invalid_input_error_code) + + if not request.tokens: + raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENS_IN_DETOKENIZE.value, invalid_input_error_code) + + if request.token_group_redactions is not None: + valid = ( + isinstance(request.token_group_redactions, list) + and all( + isinstance(entry, dict) and isinstance(entry.get("token_group_name"), str) and entry.get("token_group_name").strip() + for entry in request.token_group_redactions + ) + ) + if not valid: + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_GROUP_REDACTIONS_IN_DETOKENIZE.value, invalid_input_error_code) + + +def validate_query_request(logger, request): + if not isinstance(request.query, str) or not request.query.strip(): + raise SkyflowError(SkyflowMessages.Error.INVALID_QUERY_IN_QUERY.value, invalid_input_error_code) + + +def validate_bulk_insert_request(logger, request): + if not isinstance(request.records, list) or not all(isinstance(r, BulkInsertRecord) for r in request.records): + raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_BULK_INSERT.value, invalid_input_error_code) + + if not request.records: + raise SkyflowError(SkyflowMessages.Error.EMPTY_RECORDS_IN_BULK_INSERT.value, invalid_input_error_code) + + if len(request.records) > MAX_INSERT_RECORDS: + raise SkyflowError(SkyflowMessages.Error.TOO_MANY_RECORDS_IN_BULK_INSERT.value, invalid_input_error_code) + + _validate_upsert(logger, request.upsert) + for record in request.records: + _validate_upsert(logger, record.upsert) + + table_at_request_level = request.table is not None + + if table_at_request_level: + for record in request.records: + if record.table is not None: + raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_IN_BOTH_PLACES_IN_INSERT.value, invalid_input_error_code) else: - if request.upsert is not None: - raise SkyflowError(SkyflowMessages.Error.REQUEST_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code) + for record in request.records: + if record.table is None: + raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_MISSING_IN_INSERT.value, invalid_input_error_code) + + if table_at_request_level: + for record in request.records: + if record.upsert is not None: + raise SkyflowError(SkyflowMessages.Error.RECORD_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code) + elif request.upsert is not None: + raise SkyflowError(SkyflowMessages.Error.REQUEST_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code) + + +def validate_bulk_detokenize_request(logger, request): + validate_detokenize_request(logger, request) + if len(request.tokens) > MAX_INSERT_RECORDS: + raise SkyflowError(SkyflowMessages.Error.TOO_MANY_TOKENS_IN_BULK_DETOKENIZE.value, invalid_input_error_code) diff --git a/flowvault/skyflow_flowvault/vault/client/client.py b/flowvault/skyflow_flowvault/vault/client/client.py index 5dc4c47d..b328354e 100644 --- a/flowvault/skyflow_flowvault/vault/client/client.py +++ b/flowvault/skyflow_flowvault/vault/client/client.py @@ -1,5 +1,5 @@ from common.vault.base_vault_client import BaseVaultClient -from skyflow_flowvault.generated.rest.client import SkyflowAuth +from skyflow_flowvault.generated.rest.client import SkyflowAuth, AsyncSkyflowAuth from skyflow_flowvault.utils import get_vault_url @@ -8,7 +8,20 @@ def resolve_vault_url(self, cluster_id, env, vault_id, logger=None): return get_vault_url(cluster_id, env, vault_id, logger=logger) def initialize_api_client(self, vault_url, bearer_token): - self._api_client = SkyflowAuth(base_url=vault_url) + self._api_client = SkyflowAuth(base_url=vault_url, token=bearer_token or "") + self._async_api_client = AsyncSkyflowAuth(base_url=vault_url, token=bearer_token or "") - def get_insert_api(self): - return self._api_client.flowservice + def get_records_api(self): + return self._api_client.records + + def get_tokens_api(self): + return self._api_client.tokens + + def get_query_api(self): + return self._api_client.query + + def get_async_records_api(self): + return self._async_api_client.records + + def get_async_tokens_api(self): + return self._async_api_client.tokens diff --git a/flowvault/skyflow_flowvault/vault/controller/_vault.py b/flowvault/skyflow_flowvault/vault/controller/_vault.py index a6abaa49..742bd909 100644 --- a/flowvault/skyflow_flowvault/vault/controller/_vault.py +++ b/flowvault/skyflow_flowvault/vault/controller/_vault.py @@ -1,14 +1,63 @@ +import asyncio import json +from concurrent.futures import ThreadPoolExecutor +from functools import partial from common.utils import SkyflowMessages as CommonMessages from common.utils.constants import SKY_META_DATA_HEADER from common.utils.logger import log_info, log_error_log from common.vault.base_vault_controller import BaseVaultController -from skyflow_flowvault.generated.rest import V1InsertRecordData, V1Upsert +from skyflow_flowvault.generated.rest import ( + ColumnRedactions, + GetRequestData, + InsertRecordData, + TokenGroupRedactions, + UniqueValue, + UpdateRecordData, + Upsert, +) from skyflow_flowvault.generated.rest.core import ApiError from skyflow_flowvault.utils import SkyflowMessages, get_metrics -from skyflow_flowvault.utils.validations import validate_insert_request -from skyflow_flowvault.vault.data import InsertRequest, InsertResponse +from skyflow_flowvault.utils._response_parsing import parse_tokens, parse_hashed_data, parse_metadata +from skyflow_flowvault.utils._batching import ( + resolve_batch_config, + create_batches, + INSERT_BATCH_SIZE_KEY, + INSERT_CONCURRENCY_LIMIT_KEY, + DETOKENIZE_BATCH_SIZE_KEY, + DETOKENIZE_CONCURRENCY_LIMIT_KEY, +) +from skyflow_flowvault.utils.validations import ( + validate_insert_request, + validate_get_request, + validate_update_request, + validate_delete_request, + validate_detokenize_request, + validate_query_request, + validate_bulk_insert_request, + validate_bulk_detokenize_request, +) +from skyflow_flowvault.vault.data import ( + InsertRequest, + InsertResponse, + GetRequest, + GetRecordRequest, + GetResponse, + UpdateRequest, + UpdateResponse, + DeleteRequest, + DeleteResponse, + DetokenizeRequest, + DetokenizeResponse, + QueryRequest, + QueryResponse, + BulkInsertRequest, + BulkInsertResponse, + BulkSummary, + BulkDetokenizeRequest, + BulkDetokenizeResponse, + DetokenizeSummary, +) REQUEST_ID_HEADER = "x-request-id" @@ -22,66 +71,473 @@ def __init__(self, vault_client): def insert(self, request: InsertRequest) -> InsertResponse: log_info(SkyflowMessages.Info.VALIDATE_INSERT_REQUEST.value, self._vault_client.get_logger()) validate_insert_request(self._vault_client.get_logger(), request) - self._validate_table_name_if_present(request.table) - for record in request.values: - self._validate_table_name_if_present(record.get("table")) - self._validate_field_values(record.get("values")) + self._validate_table_name_if_present(request.table_name) + for record in request.records: + self._validate_table_name_if_present(record.table_name) + self._validate_field_values(record.data) log_info(SkyflowMessages.Info.INSERT_REQUEST_RESOLVED.value, self._vault_client.get_logger()) self._vault_client.initialize_client_configuration() - insert_api = self._vault_client.get_insert_api() + records_api = self._vault_client.get_records_api() - needs_per_record_table = any(r.get("table") is not None for r in request.values) - needs_per_record_upsert = any(r.get("upsert") is not None for r in request.values) + needs_per_record_table = any(r.table_name is not None for r in request.records) + needs_per_record_upsert = any(r.upsert is not None for r in request.records) wire_records = [ self.__build_wire_record(record, request, needs_per_record_table, needs_per_record_upsert) - for record in request.values + for record in request.records ] try: log_info(SkyflowMessages.Info.INSERT_TRIGGERED.value, self._vault_client.get_logger()) headers = self.__build_headers() - top_level_kwargs = self.__omit_none( - table_name=None if needs_per_record_table else request.table, - upsert=None if needs_per_record_upsert else self.__to_v1_upsert(request.upsert), + upsert_kwargs = self.__omit_none( + upsert=None if needs_per_record_upsert else self.__to_upsert(request.upsert), ) - raw_response = insert_api.with_raw_response.insert( + raw_response = records_api.with_raw_response.insert_records( vault_id=self._vault_client.get_vault_id(), + table_name=request.table_name, records=wire_records, request_options={'additional_headers': headers}, - **top_level_kwargs, + **upsert_kwargs, ) - request_id = self.__extract_request_id(raw_response.headers) - inserted_fields, errors = self.__split_success_and_errors(raw_response.data.records or [], 0, request_id) + records = [self.__record_row(record, include_data=False) for record in (raw_response.data.records or [])] except Exception as e: log_error_log(SkyflowMessages.ErrorLogs.INSERT_RECORDS_REJECTED.value, self._vault_client.get_logger()) - inserted_fields, errors = [], self.__errors_from_exception(e, request.values, 0) + records = self.__unary_error_records(e, len(request.records), partial(self.__record_error_row, include_data=False)) log_info(SkyflowMessages.Info.INSERT_SUCCESS.value, self._vault_client.get_logger()) - return InsertResponse(inserted_fields=inserted_fields, errors=errors if errors else None) + return InsertResponse(records=records) + + def get(self, request: GetRequest) -> GetResponse: + log_info(SkyflowMessages.Info.VALIDATE_GET_REQUEST.value, self._vault_client.get_logger()) + validate_get_request(self._vault_client.get_logger(), request) + self._validate_table_name_if_present(request.table) + log_info(SkyflowMessages.Info.GET_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + records_api = self._vault_client.get_records_api() + + if request.records is not None: + call_kwargs = {'records': self.__to_get_request_data(request.records)} + error_count = len(request.records) + else: + call_kwargs = { + 'table_name': request.table, + 'skyflow_i_ds': request.ids, + 'unique_values': self.__to_unique_values(request.unique_values), + 'columns': request.columns, + 'column_redactions': self.__to_column_redactions(request.column_redactions), + 'limit': request.limit, + 'offset': request.offset, + } + error_count = len(request.ids or request.unique_values or []) + + try: + log_info(SkyflowMessages.Info.GET_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = records_api.with_raw_response.get_records( + vault_id=self._vault_client.get_vault_id(), + request_options={'additional_headers': self.__build_headers()}, + **call_kwargs, + ) + records = [self.__record_row(record, include_data=True) for record in (raw_response.data.records or [])] + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.GET_RECORDS_REJECTED.value, self._vault_client.get_logger()) + records = self.__unary_error_records(e, error_count or 1, partial(self.__record_error_row, include_data=True)) + + log_info(SkyflowMessages.Info.GET_SUCCESS.value, self._vault_client.get_logger()) + return GetResponse(records=records) + + def update(self, request: UpdateRequest) -> UpdateResponse: + log_info(SkyflowMessages.Info.VALIDATE_UPDATE_REQUEST.value, self._vault_client.get_logger()) + validate_update_request(self._vault_client.get_logger(), request) + self._validate_table_name_if_present(request.table_name) + for record in request.records: + self._validate_table_name_if_present(record.get("table_name")) + if record.get("data") is not None: + self._validate_field_values(record.get("data")) + log_info(SkyflowMessages.Info.UPDATE_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + records_api = self._vault_client.get_records_api() + + needs_per_record_table = any(r.get("table_name") is not None for r in request.records) + + wire_records = [ + self.__build_update_wire_record(record, request, needs_per_record_table) + for record in request.records + ] + + try: + log_info(SkyflowMessages.Info.UPDATE_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = records_api.with_raw_response.update_records( + vault_id=self._vault_client.get_vault_id(), + table_name=request.table_name, + records=wire_records, + request_options={'additional_headers': self.__build_headers()}, + ) + request_id = self.__extract_request_id(raw_response.headers) + records, errors = self.__split_success_and_errors( + raw_response.data.records or [], 0, request_id, include_data=True, + ) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.UPDATE_RECORDS_REJECTED.value, self._vault_client.get_logger()) + records, errors = [], self.__errors_from_exception(e, request.records, 0) + + log_info(SkyflowMessages.Info.UPDATE_SUCCESS.value, self._vault_client.get_logger()) + return UpdateResponse(records=records, errors=errors if errors else None) + + def delete(self, request: DeleteRequest) -> DeleteResponse: + log_info(SkyflowMessages.Info.VALIDATE_DELETE_REQUEST.value, self._vault_client.get_logger()) + validate_delete_request(self._vault_client.get_logger(), request) + self._validate_table_name_if_present(request.table) + log_info(SkyflowMessages.Info.DELETE_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + records_api = self._vault_client.get_records_api() + items = request.ids or request.unique_values or [] + + try: + log_info(SkyflowMessages.Info.DELETE_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = records_api.with_raw_response.delete_records( + vault_id=self._vault_client.get_vault_id(), + table_name=request.table, + skyflow_i_ds=request.ids, + unique_values=self.__to_unique_values(request.unique_values), + request_options={'additional_headers': self.__build_headers()}, + ) + records = [self.__delete_row(record) for record in (raw_response.data.records or [])] + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.DELETE_RECORDS_REJECTED.value, self._vault_client.get_logger()) + records = self.__unary_error_records(e, len(items) or 1, self.__delete_error_row) + + log_info(SkyflowMessages.Info.DELETE_SUCCESS.value, self._vault_client.get_logger()) + return DeleteResponse(records=records) + + def query(self, request: QueryRequest) -> QueryResponse: + log_info(SkyflowMessages.Info.VALIDATE_QUERY_REQUEST.value, self._vault_client.get_logger()) + validate_query_request(self._vault_client.get_logger(), request) + log_info(SkyflowMessages.Info.QUERY_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + query_api = self._vault_client.get_query_api() + + try: + log_info(SkyflowMessages.Info.QUERY_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = query_api.with_raw_response.execute_query( + vault_id=self._vault_client.get_vault_id(), + query=request.query, + request_options={'additional_headers': self.__build_headers()}, + ) + records = [{'data': getattr(record, 'data', None)} for record in (raw_response.data.records or [])] + metadata = self.__query_metadata(raw_response.data) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.QUERY_RECORDS_REJECTED.value, self._vault_client.get_logger()) + records = self.__unary_error_records(e, 1, self.__query_error_row) + metadata = None + + log_info(SkyflowMessages.Info.QUERY_SUCCESS.value, self._vault_client.get_logger()) + return QueryResponse(records=records, metadata=metadata) + + def detokenize(self, request: DetokenizeRequest) -> DetokenizeResponse: + log_info(SkyflowMessages.Info.VALIDATE_DETOKENIZE_REQUEST.value, self._vault_client.get_logger()) + validate_detokenize_request(self._vault_client.get_logger(), request) + log_info(SkyflowMessages.Info.DETOKENIZE_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + tokens_api = self._vault_client.get_tokens_api() + + try: + log_info(SkyflowMessages.Info.DETOKENIZE_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = tokens_api.with_raw_response.detokenize( + vault_id=self._vault_client.get_vault_id(), + tokens=request.tokens, + token_group_redactions=self.__to_token_group_redactions(request.token_group_redactions), + request_options={'additional_headers': self.__build_headers()}, + ) + records = [self.__detokenize_row(resp) for resp in (raw_response.data.response or [])] + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.DETOKENIZE_RECORDS_REJECTED.value, self._vault_client.get_logger()) + records = self.__unary_error_records(e, len(request.tokens), self.__detokenize_error_row) + + log_info(SkyflowMessages.Info.DETOKENIZE_SUCCESS.value, self._vault_client.get_logger()) + return DetokenizeResponse(records=records) + + def bulk_insert(self, request: BulkInsertRequest) -> BulkInsertResponse: + batches, concurrency, top_kwargs = self.__prepare_bulk_insert(request) + records_api = self._vault_client.get_records_api() + logger = self._vault_client.get_logger() + log_info(SkyflowMessages.Info.BULK_INSERT_TRIGGERED.value, logger) + + def call_batch(batch, start_index): + try: + raw_response = records_api.with_raw_response.insert_records( + vault_id=self._vault_client.get_vault_id(), + table_name=request.table, + records=batch, + request_options={'additional_headers': self.__build_headers()}, + **top_kwargs, + ) + return self.__format_bulk_insert_batch(raw_response.data.records or [], start_index, raw_response.headers) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.BULK_INSERT_RECORDS_REJECTED.value, logger) + return self.__bulk_insert_batch_error_rows(e, len(batch), start_index) + + records = self.__run_batches_sync(batches, call_batch, concurrency) + log_info(SkyflowMessages.Info.BULK_INSERT_SUCCESS.value, logger) + return self.__build_bulk_insert_response(records, request.records) + + async def bulk_insert_async(self, request: BulkInsertRequest) -> BulkInsertResponse: + batches, concurrency, top_kwargs = self.__prepare_bulk_insert(request) + records_api = self._vault_client.get_async_records_api() + logger = self._vault_client.get_logger() + log_info(SkyflowMessages.Info.BULK_INSERT_TRIGGERED.value, logger) + + async def call_batch(batch, start_index): + try: + raw_response = await records_api.with_raw_response.insert_records( + vault_id=self._vault_client.get_vault_id(), + table_name=request.table, + records=batch, + request_options={'additional_headers': self.__build_headers()}, + **top_kwargs, + ) + return self.__format_bulk_insert_batch(raw_response.data.records or [], start_index, raw_response.headers) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.BULK_INSERT_RECORDS_REJECTED.value, logger) + return self.__bulk_insert_batch_error_rows(e, len(batch), start_index) + + records = await self.__run_batches_async(batches, call_batch, concurrency) + log_info(SkyflowMessages.Info.BULK_INSERT_SUCCESS.value, logger) + return self.__build_bulk_insert_response(records, request.records) + + def bulk_detokenize(self, request: BulkDetokenizeRequest) -> BulkDetokenizeResponse: + batches, concurrency, redactions = self.__prepare_bulk_detokenize(request) + tokens_api = self._vault_client.get_tokens_api() + logger = self._vault_client.get_logger() + log_info(SkyflowMessages.Info.BULK_DETOKENIZE_TRIGGERED.value, logger) - def get(self, request): - raise NotImplementedError("VaultController.get is not implemented yet") + def call_batch(batch, start_index): + try: + raw_response = tokens_api.with_raw_response.detokenize( + vault_id=self._vault_client.get_vault_id(), + tokens=batch, + token_group_redactions=redactions, + request_options={'additional_headers': self.__build_headers()}, + ) + return self.__format_bulk_detokenize_batch(raw_response.data.response or [], start_index, raw_response.headers) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.BULK_DETOKENIZE_RECORDS_REJECTED.value, logger) + return self.__bulk_detokenize_batch_error_rows(e, len(batch), start_index) - def update(self, request): - raise NotImplementedError("VaultController.update is not implemented yet") + records = self.__run_batches_sync(batches, call_batch, concurrency) + log_info(SkyflowMessages.Info.BULK_DETOKENIZE_SUCCESS.value, logger) + return self.__build_bulk_detokenize_response(records, request.tokens) - def delete(self, request): - raise NotImplementedError("VaultController.delete is not implemented yet") + async def bulk_detokenize_async(self, request: BulkDetokenizeRequest) -> BulkDetokenizeResponse: + batches, concurrency, redactions = self.__prepare_bulk_detokenize(request) + tokens_api = self._vault_client.get_async_tokens_api() + logger = self._vault_client.get_logger() + log_info(SkyflowMessages.Info.BULK_DETOKENIZE_TRIGGERED.value, logger) - def query(self, request): - raise NotImplementedError("VaultController.query is not implemented yet") + async def call_batch(batch, start_index): + try: + raw_response = await tokens_api.with_raw_response.detokenize( + vault_id=self._vault_client.get_vault_id(), + tokens=batch, + token_group_redactions=redactions, + request_options={'additional_headers': self.__build_headers()}, + ) + return self.__format_bulk_detokenize_batch(raw_response.data.response or [], start_index, raw_response.headers) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.BULK_DETOKENIZE_RECORDS_REJECTED.value, logger) + return self.__bulk_detokenize_batch_error_rows(e, len(batch), start_index) - def detokenize(self, request): - raise NotImplementedError("VaultController.detokenize is not implemented yet") + records = await self.__run_batches_async(batches, call_batch, concurrency) + log_info(SkyflowMessages.Info.BULK_DETOKENIZE_SUCCESS.value, logger) + return self.__build_bulk_detokenize_response(records, request.tokens) + + def __prepare_bulk_insert(self, request): + logger = self._vault_client.get_logger() + log_info(SkyflowMessages.Info.VALIDATE_BULK_INSERT_REQUEST.value, logger) + validate_bulk_insert_request(logger, request) + self._validate_table_name_if_present(request.table) + for record in request.records: + self._validate_table_name_if_present(record.table) + self._validate_field_values(record.data) + log_info(SkyflowMessages.Info.BULK_INSERT_REQUEST_RESOLVED.value, logger) + self._vault_client.initialize_client_configuration() + + batch_size, concurrency = resolve_batch_config( + INSERT_BATCH_SIZE_KEY, INSERT_CONCURRENCY_LIMIT_KEY, len(request.records), logger, + ) + needs_per_record_table = any(r.table is not None for r in request.records) + needs_per_record_upsert = any(r.upsert is not None for r in request.records) + wire_records = [ + self.__build_bulk_insert_wire_record(r, request, needs_per_record_table, needs_per_record_upsert) + for r in request.records + ] + batches = self.__index_batches(wire_records, batch_size) + top_kwargs = self.__omit_none( + upsert=None if needs_per_record_upsert else self.__to_upsert(request.upsert), + ) + log_info(SkyflowMessages.Info.PROCESSING_BATCHES.value, logger) + return batches, concurrency, top_kwargs + + def __prepare_bulk_detokenize(self, request): + logger = self._vault_client.get_logger() + log_info(SkyflowMessages.Info.VALIDATE_BULK_DETOKENIZE_REQUEST.value, logger) + validate_bulk_detokenize_request(logger, request) + log_info(SkyflowMessages.Info.BULK_DETOKENIZE_REQUEST_RESOLVED.value, logger) + self._vault_client.initialize_client_configuration() + + batch_size, concurrency = resolve_batch_config( + DETOKENIZE_BATCH_SIZE_KEY, DETOKENIZE_CONCURRENCY_LIMIT_KEY, len(request.tokens), logger, + ) + batches = self.__index_batches(request.tokens, batch_size) + redactions = self.__to_token_group_redactions(request.token_group_redactions) + log_info(SkyflowMessages.Info.PROCESSING_BATCHES.value, logger) + return batches, concurrency, redactions + + def __index_batches(self, items, batch_size): + batches, start_index, indexed = create_batches(items, batch_size), 0, [] + for batch in batches: + indexed.append((batch, start_index)) + start_index += len(batch) + return indexed + + def __run_batches_sync(self, batches, call_batch, concurrency): + with ThreadPoolExecutor(max_workers=max(1, concurrency)) as executor: + futures = [executor.submit(call_batch, batch, start_index) for batch, start_index in batches] + merged = [] + for future in futures: + merged.extend(future.result()) + return merged + + async def __run_batches_async(self, batches, call_batch, concurrency): + semaphore = asyncio.Semaphore(max(1, concurrency)) + + async def guarded(batch, start_index): + async with semaphore: + return await call_batch(batch, start_index) + + results = await asyncio.gather(*(guarded(batch, start_index) for batch, start_index in batches)) + merged = [] + for result in results: + merged.extend(result) + return merged + + def __build_bulk_insert_wire_record(self, record, request, needs_per_record_table, needs_per_record_upsert): + return InsertRecordData(data=record.data, **self.__omit_none( + table_name=(record.table or request.table) if needs_per_record_table else None, + upsert=self.__to_upsert(record.upsert or request.upsert) if needs_per_record_upsert else None, + )) + + def __format_bulk_insert_batch(self, records, start_index, headers): + request_id = self.__extract_request_id(headers) + rows = [] + for offset, record in enumerate(records): + error = getattr(record, 'error', None) + rows.append({ + 'index': start_index + offset, + 'request_id': request_id if error is not None else None, + 'table_name': getattr(record, 'table_name', None), + 'skyflow_id': getattr(record, 'skyflow_id', None), + 'tokens': parse_tokens(getattr(record, 'tokens', None)), + 'data': getattr(record, 'data', None), + 'hashed_data': parse_hashed_data(getattr(record, 'hashed_data', None)), + 'http_code': getattr(record, 'http_code', None), + 'error': error, + }) + return rows + + def __format_bulk_detokenize_batch(self, responses, start_index, headers): + request_id = self.__extract_request_id(headers) + rows = [] + for offset, resp in enumerate(responses): + error = getattr(resp, 'error', None) + rows.append({ + 'index': start_index + offset, + 'request_id': request_id if error is not None else None, + 'value': getattr(resp, 'value', None), + 'token_group_name': getattr(resp, 'token_group_name', None), + 'metadata': parse_metadata(getattr(resp, 'metadata', None)), + 'http_code': getattr(resp, 'http_code', None), + 'token': getattr(resp, 'token', None), + 'error': error, + }) + return rows + + def __bulk_batch_error_tuples(self, e, count, start_index): + if isinstance(e, ApiError): + request_id = self.__extract_request_id(e.headers) + status = e.status_code + body = e.body if isinstance(e.body, dict) else None + if body and isinstance(body.get('records'), list) and body['records']: + tuples = [ + (start_index + offset, request_id, + record.get('error', record.get('message', 'Unknown error')), + record.get('http_code', record.get('httpCode', record.get('statusCode', status)))) + for offset, record in enumerate(body['records']) if isinstance(record, dict) + ] + if tuples: + return tuples + if body and body.get('error') is not None and not isinstance(body['error'], dict): + message = str(body['error']) + else: + message = str(e) + return [(start_index + i, request_id, message, status) for i in range(count)] + message = str(e) if e else CommonMessages.Error.GENERIC_API_ERROR.value + return [(start_index + i, None, message, None) for i in range(count)] + + def __bulk_insert_batch_error_rows(self, e, count, start_index): + return [ + {'index': idx, 'request_id': request_id, 'table_name': None, 'skyflow_id': None, + 'tokens': None, 'data': None, 'hashed_data': None, 'http_code': code, 'error': message} + for idx, request_id, message, code in self.__bulk_batch_error_tuples(e, count, start_index) + ] + + def __bulk_detokenize_batch_error_rows(self, e, count, start_index): + return [ + {'index': idx, 'request_id': request_id, 'value': None, 'token_group_name': None, + 'metadata': None, 'http_code': code, 'token': None, 'error': message} + for idx, request_id, message, code in self.__bulk_batch_error_tuples(e, count, start_index) + ] + + def __build_bulk_insert_response(self, records, original_records): + total_failed = sum(1 for record in records if record.get('error') is not None) + summary = BulkSummary( + total_records=len(original_records), + total_inserted=len(records) - total_failed, + total_failed=total_failed, + ) + return BulkInsertResponse(summary=summary, records=records, _original_records=original_records) + + def __build_bulk_detokenize_response(self, records, original_tokens): + total_failed = sum(1 for record in records if record.get('error') is not None) + summary = DetokenizeSummary( + total_tokens=len(original_tokens), + total_detokenized=len(records) - total_failed, + total_failed=total_failed, + ) + return BulkDetokenizeResponse(summary=summary, records=records, _original_tokens=original_tokens) def __build_wire_record(self, record, request, needs_per_record_table, needs_per_record_upsert): - return V1InsertRecordData(data=record["values"], **self.__omit_none( - table_name=(record.get("table") or request.table) if needs_per_record_table else None, - upsert=self.__to_v1_upsert(record.get("upsert") or request.upsert) if needs_per_record_upsert else None, + return InsertRecordData(data=record.data, **self.__omit_none( + tokens=record.tokens, + table_name=(record.table_name or request.table_name) if needs_per_record_table else None, + upsert=self.__to_upsert(record.upsert or request.upsert) if needs_per_record_upsert else None, )) + def __build_update_wire_record(self, record, request, needs_per_record_table): + return UpdateRecordData( + skyflow_id=record.get("skyflow_id"), + data=record.get("data"), + **self.__omit_none( + table_name=(record.get("table_name") or request.table_name) if needs_per_record_table else None, + ), + ) + def __omit_none(self, **kwargs): return {k: v for k, v in kwargs.items() if v is not None} @@ -92,20 +548,113 @@ def __build_headers(self): headers['Authorization'] = f'Bearer {token}' return headers - def __to_v1_upsert(self, upsert): + def __to_upsert(self, upsert): if upsert is None: return None - update_type = upsert.get("update_type") - return V1Upsert( + update_type = upsert.update_type + return Upsert( update_type=update_type.value if update_type else None, - unique_columns=upsert.get("unique_columns"), + unique_columns=upsert.unique_columns, ) + def __to_unique_values(self, unique_values): + if unique_values is None: + return None + return [UniqueValue(data=value) for value in unique_values] + + def __to_column_redactions(self, column_redactions): + if column_redactions is None: + return None + return [ + ColumnRedactions(column_name=entry.column_name, redaction=entry.redaction) + for entry in column_redactions + ] + + def __to_token_group_redactions(self, token_group_redactions): + if token_group_redactions is None: + return None + return [ + TokenGroupRedactions(token_group_name=entry.get("token_group_name"), redaction=entry.get("redaction")) + for entry in token_group_redactions + ] + def __extract_request_id(self, headers): return headers.get(REQUEST_ID_HEADER) if headers else None - def __split_success_and_errors(self, records, start_index, request_id): - + def __record_row(self, record, include_data): + row = { + 'table_name': getattr(record, 'table_name', None), + 'skyflow_id': getattr(record, 'skyflow_id', None), + 'tokens': parse_tokens(getattr(record, 'tokens', None)), + 'hashed_data': parse_hashed_data(getattr(record, 'hashed_data', None)), + 'http_code': getattr(record, 'http_code', None), + 'error': getattr(record, 'error', None), + } + if include_data: + row['data'] = getattr(record, 'data', None) + return row + + def __record_error_row(self, message, code, include_data): + row = {'table_name': None, 'skyflow_id': None, 'tokens': None, + 'hashed_data': None, 'http_code': code, 'error': message} + if include_data: + row['data'] = None + return row + + def __to_get_request_data(self, records): + return [ + GetRequestData( + table_name=record.table, + skyflow_i_ds=record.ids or [], + **self.__omit_none( + columns=record.columns, + column_redactions=self.__to_column_redactions(record.column_redactions), + unique_values=self.__to_unique_values(record.unique_values), + ), + ) + for record in records + ] + + def __delete_row(self, record): + return { + 'skyflow_id': getattr(record, 'skyflow_id', None), + 'http_code': getattr(record, 'http_code', None), + 'error': getattr(record, 'error', None), + } + + def __delete_error_row(self, message, code): + return {'skyflow_id': None, 'http_code': code, 'error': message} + + def __detokenize_row(self, resp): + return { + 'token': getattr(resp, 'token', None), + 'token_group_name': getattr(resp, 'token_group_name', None), + 'value': getattr(resp, 'value', None), + 'metadata': parse_metadata(getattr(resp, 'metadata', None)), + 'http_code': getattr(resp, 'http_code', None), + 'error': getattr(resp, 'error', None), + } + + def __detokenize_error_row(self, message, code): + return {'token': None, 'token_group_name': None, 'value': None, 'metadata': None, + 'http_code': code, 'error': message} + + def __query_metadata(self, data): + meta = getattr(data, 'metadata', None) + if meta is None: + return None + return {'columns': getattr(meta, 'columns', None)} + + def __query_error_row(self, message, code): + return {'data': None, 'http_code': code, 'error': message} + + def __unary_error_records(self, e, count, row_builder): + return [ + row_builder(message, code) + for _, _, message, code in self.__bulk_batch_error_tuples(e, max(count, 1), 0) + ] + + def __split_success_and_errors(self, records, start_index, request_id, include_data=False): successes, errors = [], [] for offset, record in enumerate(records): request_index = start_index + offset @@ -116,7 +665,14 @@ def __split_success_and_errors(self, records, start_index, request_id): 'request_index': request_index, 'skyflow_id': record.skyflow_id, } - success.update(self.__flatten_tokens(record.tokens)) + success.update(self.__flatten_tokens(getattr(record, 'tokens', None))) + if include_data: + data = getattr(record, 'data', None) + if data: + success['data'] = data + hashed_data = getattr(record, 'hashed_data', None) + if hashed_data: + success['hashed_data'] = hashed_data successes.append(success) return successes, errors diff --git a/flowvault/skyflow_flowvault/vault/data/__init__.py b/flowvault/skyflow_flowvault/vault/data/__init__.py index 62ae85cc..6017ad77 100644 --- a/flowvault/skyflow_flowvault/vault/data/__init__.py +++ b/flowvault/skyflow_flowvault/vault/data/__init__.py @@ -1,3 +1,23 @@ +from ._upsert_options import UpsertOptions +from ._column_redaction import ColumnRedaction +from ._insert_request_record import InsertRequestRecord from ._insert_request import InsertRequest from ._insert_response import InsertResponse -from ._upsert import Upsert +from ._get_record_request import GetRecordRequest +from ._get_request import GetRequest +from ._get_response import GetResponse +from ._update_request import UpdateRequest +from ._update_response import UpdateResponse +from ._delete_request import DeleteRequest +from ._delete_response import DeleteResponse +from ._detokenize_request import DetokenizeRequest +from ._detokenize_response import DetokenizeResponse +from ._query_request import QueryRequest +from ._query_response import QueryResponse +from ._bulk_insert_record import BulkInsertRecord +from ._bulk_insert_request import BulkInsertRequest +from ._bulk_summary import BulkSummary +from ._bulk_insert_response import BulkInsertResponse +from ._bulk_detokenize_request import BulkDetokenizeRequest +from ._detokenize_summary import DetokenizeSummary +from ._bulk_detokenize_response import BulkDetokenizeResponse diff --git a/flowvault/skyflow_flowvault/vault/data/_bulk_detokenize_request.py b/flowvault/skyflow_flowvault/vault/data/_bulk_detokenize_request.py new file mode 100644 index 00000000..9520110f --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_bulk_detokenize_request.py @@ -0,0 +1,4 @@ +class BulkDetokenizeRequest: + def __init__(self, tokens: list, token_group_redactions: list = None): + self.tokens = tokens + self.token_group_redactions = token_group_redactions diff --git a/flowvault/skyflow_flowvault/vault/data/_bulk_detokenize_response.py b/flowvault/skyflow_flowvault/vault/data/_bulk_detokenize_response.py new file mode 100644 index 00000000..78d7f740 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_bulk_detokenize_response.py @@ -0,0 +1,24 @@ +def _is_retryable(http_code): + return isinstance(http_code, int) and 500 <= http_code <= 599 and http_code != 529 + + +class BulkDetokenizeResponse: + def __init__(self, summary=None, records=None, _original_tokens=None): + self.summary = summary + self.records = records + self._original_tokens = _original_tokens + + def tokens_to_retry(self): + if not self._original_tokens: + return [] + return [ + self._original_tokens[record["index"]] + for record in (self.records or []) + if _is_retryable(record.get("http_code")) and 0 <= record.get("index", -1) < len(self._original_tokens) + ] + + def __repr__(self): + return f"BulkDetokenizeResponse(summary={self.summary}, records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_bulk_insert_record.py b/flowvault/skyflow_flowvault/vault/data/_bulk_insert_record.py new file mode 100644 index 00000000..685c631e --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_bulk_insert_record.py @@ -0,0 +1,8 @@ +from ._upsert_options import UpsertOptions + + +class BulkInsertRecord: + def __init__(self, data: dict, table: str = None, upsert: UpsertOptions = None): + self.data = data + self.table = table + self.upsert = upsert diff --git a/flowvault/skyflow_flowvault/vault/data/_bulk_insert_request.py b/flowvault/skyflow_flowvault/vault/data/_bulk_insert_request.py new file mode 100644 index 00000000..5b39baf6 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_bulk_insert_request.py @@ -0,0 +1,11 @@ +from typing import List + +from ._bulk_insert_record import BulkInsertRecord +from ._upsert_options import UpsertOptions + + +class BulkInsertRequest: + def __init__(self, records: List[BulkInsertRecord], table: str = None, upsert: UpsertOptions = None): + self.records = records + self.table = table + self.upsert = upsert diff --git a/flowvault/skyflow_flowvault/vault/data/_bulk_insert_response.py b/flowvault/skyflow_flowvault/vault/data/_bulk_insert_response.py new file mode 100644 index 00000000..17894995 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_bulk_insert_response.py @@ -0,0 +1,24 @@ +def _is_retryable(http_code): + return isinstance(http_code, int) and 500 <= http_code <= 599 and http_code != 529 + + +class BulkInsertResponse: + def __init__(self, summary=None, records=None, _original_records=None): + self.summary = summary + self.records = records + self._original_records = _original_records + + def records_to_retry(self): + if not self._original_records: + return [] + return [ + self._original_records[record["index"]] + for record in (self.records or []) + if _is_retryable(record.get("http_code")) and 0 <= record.get("index", -1) < len(self._original_records) + ] + + def __repr__(self): + return f"BulkInsertResponse(summary={self.summary}, records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_bulk_summary.py b/flowvault/skyflow_flowvault/vault/data/_bulk_summary.py new file mode 100644 index 00000000..62be2877 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_bulk_summary.py @@ -0,0 +1,12 @@ +class BulkSummary: + def __init__(self, total_records=0, total_inserted=0, total_failed=0): + self.total_records = total_records + self.total_inserted = total_inserted + self.total_failed = total_failed + + def __repr__(self): + return (f"BulkSummary(total_records={self.total_records}, " + f"total_inserted={self.total_inserted}, total_failed={self.total_failed})") + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_column_redaction.py b/flowvault/skyflow_flowvault/vault/data/_column_redaction.py new file mode 100644 index 00000000..d5b911af --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_column_redaction.py @@ -0,0 +1,4 @@ +class ColumnRedaction: + def __init__(self, column_name: str, redaction: str = None): + self.column_name = column_name + self.redaction = redaction diff --git a/flowvault/skyflow_flowvault/vault/data/_delete_request.py b/flowvault/skyflow_flowvault/vault/data/_delete_request.py new file mode 100644 index 00000000..b985fef2 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_delete_request.py @@ -0,0 +1,5 @@ +class DeleteRequest: + def __init__(self, table: str, ids: list = None, unique_values: list = None): + self.table = table + self.ids = ids + self.unique_values = unique_values diff --git a/flowvault/skyflow_flowvault/vault/data/_delete_response.py b/flowvault/skyflow_flowvault/vault/data/_delete_response.py new file mode 100644 index 00000000..685a195f --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_delete_response.py @@ -0,0 +1,9 @@ +class DeleteResponse: + def __init__(self, records=None): + self.records = records + + def __repr__(self): + return f"DeleteResponse(records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_detokenize_request.py b/flowvault/skyflow_flowvault/vault/data/_detokenize_request.py new file mode 100644 index 00000000..fae72711 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_detokenize_request.py @@ -0,0 +1,4 @@ +class DetokenizeRequest: + def __init__(self, tokens: list, token_group_redactions: list = None): + self.tokens = tokens + self.token_group_redactions = token_group_redactions diff --git a/flowvault/skyflow_flowvault/vault/data/_detokenize_response.py b/flowvault/skyflow_flowvault/vault/data/_detokenize_response.py new file mode 100644 index 00000000..ea4dd1a4 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_detokenize_response.py @@ -0,0 +1,9 @@ +class DetokenizeResponse: + def __init__(self, records=None): + self.records = records + + def __repr__(self): + return f"DetokenizeResponse(records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_detokenize_summary.py b/flowvault/skyflow_flowvault/vault/data/_detokenize_summary.py new file mode 100644 index 00000000..08c9f428 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_detokenize_summary.py @@ -0,0 +1,12 @@ +class DetokenizeSummary: + def __init__(self, total_tokens=0, total_detokenized=0, total_failed=0): + self.total_tokens = total_tokens + self.total_detokenized = total_detokenized + self.total_failed = total_failed + + def __repr__(self): + return (f"DetokenizeSummary(total_tokens={self.total_tokens}, " + f"total_detokenized={self.total_detokenized}, total_failed={self.total_failed})") + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_get_record_request.py b/flowvault/skyflow_flowvault/vault/data/_get_record_request.py new file mode 100644 index 00000000..d7a8b247 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_get_record_request.py @@ -0,0 +1,13 @@ +from typing import List + +from ._column_redaction import ColumnRedaction + + +class GetRecordRequest: + def __init__(self, table: str, ids: list = None, columns: list = None, + column_redactions: List[ColumnRedaction] = None, unique_values: list = None): + self.table = table + self.ids = ids + self.columns = columns + self.column_redactions = column_redactions + self.unique_values = unique_values diff --git a/flowvault/skyflow_flowvault/vault/data/_get_request.py b/flowvault/skyflow_flowvault/vault/data/_get_request.py new file mode 100644 index 00000000..90858fba --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_get_request.py @@ -0,0 +1,17 @@ +from typing import List + +from ._column_redaction import ColumnRedaction + + +class GetRequest: + def __init__(self, table: str = None, ids: list = None, unique_values: list = None, columns: list = None, + column_redactions: List[ColumnRedaction] = None, limit: int = None, offset: int = None, + records: list = None): + self.table = table + self.ids = ids + self.unique_values = unique_values + self.columns = columns + self.column_redactions = column_redactions + self.limit = limit + self.offset = offset + self.records = records diff --git a/flowvault/skyflow_flowvault/vault/data/_get_response.py b/flowvault/skyflow_flowvault/vault/data/_get_response.py new file mode 100644 index 00000000..3e5e6452 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_get_response.py @@ -0,0 +1,9 @@ +class GetResponse: + def __init__(self, records=None): + self.records = records + + def __repr__(self): + return f"GetResponse(records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_insert_request.py b/flowvault/skyflow_flowvault/vault/data/_insert_request.py index a6da5c11..27471b57 100644 --- a/flowvault/skyflow_flowvault/vault/data/_insert_request.py +++ b/flowvault/skyflow_flowvault/vault/data/_insert_request.py @@ -1,7 +1,11 @@ -from common.vault.data import BaseInsertRequest -from skyflow_flowvault.vault.data._upsert import Upsert +from typing import List +from ._insert_request_record import InsertRequestRecord +from ._upsert_options import UpsertOptions -class InsertRequest(BaseInsertRequest): - def __init__(self, values: list, table: str = None, upsert: Upsert = None): - super().__init__(table, values, upsert=upsert) + +class InsertRequest: + def __init__(self, records: List[InsertRequestRecord], table_name: str = None, upsert: UpsertOptions = None): + self.records = records + self.table_name = table_name + self.upsert = upsert diff --git a/flowvault/skyflow_flowvault/vault/data/_insert_request_record.py b/flowvault/skyflow_flowvault/vault/data/_insert_request_record.py new file mode 100644 index 00000000..c4511a01 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_insert_request_record.py @@ -0,0 +1,9 @@ +from ._upsert_options import UpsertOptions + + +class InsertRequestRecord: + def __init__(self, data: dict, table_name: str = None, tokens: dict = None, upsert: UpsertOptions = None): + self.data = data + self.table_name = table_name + self.tokens = tokens + self.upsert = upsert diff --git a/flowvault/skyflow_flowvault/vault/data/_insert_response.py b/flowvault/skyflow_flowvault/vault/data/_insert_response.py index ddb87134..9662f997 100644 --- a/flowvault/skyflow_flowvault/vault/data/_insert_response.py +++ b/flowvault/skyflow_flowvault/vault/data/_insert_response.py @@ -1,6 +1,9 @@ -from common.vault.data import BaseInsertResponse +class InsertResponse: + def __init__(self, records=None): + self.records = records + def __repr__(self): + return f"InsertResponse(records={self.records})" -class InsertResponse(BaseInsertResponse): - """flowvault's own insert() response class -- currently identical to the shared base, kept as - its own subclass so flowvault-specific fields can be added later without touching PDB.""" + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_query_request.py b/flowvault/skyflow_flowvault/vault/data/_query_request.py new file mode 100644 index 00000000..11ac0b4c --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_query_request.py @@ -0,0 +1,3 @@ +class QueryRequest: + def __init__(self, query: str): + self.query = query diff --git a/flowvault/skyflow_flowvault/vault/data/_query_response.py b/flowvault/skyflow_flowvault/vault/data/_query_response.py new file mode 100644 index 00000000..8748a6c5 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_query_response.py @@ -0,0 +1,10 @@ +class QueryResponse: + def __init__(self, records=None, metadata=None): + self.records = records + self.metadata = metadata + + def __repr__(self): + return f"QueryResponse(records={self.records}, metadata={self.metadata})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_update_request.py b/flowvault/skyflow_flowvault/vault/data/_update_request.py new file mode 100644 index 00000000..b5ed1134 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_update_request.py @@ -0,0 +1,5 @@ +class UpdateRequest: + def __init__(self, records: list, table_name: str = None, update_type=None): + self.records = records + self.table_name = table_name + self.update_type = update_type diff --git a/flowvault/skyflow_flowvault/vault/data/_update_response.py b/flowvault/skyflow_flowvault/vault/data/_update_response.py new file mode 100644 index 00000000..2b07fe35 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_update_response.py @@ -0,0 +1,10 @@ +class UpdateResponse: + def __init__(self, records=None, errors=None): + self.records = records + self.errors = errors + + def __repr__(self): + return f"UpdateResponse(records={self.records}, errors={self.errors})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow_flowvault/vault/data/_upsert.py b/flowvault/skyflow_flowvault/vault/data/_upsert.py deleted file mode 100644 index 27d8fdab..00000000 --- a/flowvault/skyflow_flowvault/vault/data/_upsert.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Optional, TypedDict -from skyflow_flowvault.utils.enums import UpsertType - -class Upsert(TypedDict, total=False): - update_type: Optional[UpsertType] - unique_columns: list diff --git a/flowvault/skyflow_flowvault/vault/data/_upsert_options.py b/flowvault/skyflow_flowvault/vault/data/_upsert_options.py new file mode 100644 index 00000000..b36b4219 --- /dev/null +++ b/flowvault/skyflow_flowvault/vault/data/_upsert_options.py @@ -0,0 +1,4 @@ +class UpsertOptions: + def __init__(self, unique_columns: list = None, update_type=None): + self.unique_columns = unique_columns + self.update_type = update_type diff --git a/flowvault/tests/utils/test__batching.py b/flowvault/tests/utils/test__batching.py new file mode 100644 index 00000000..f38879dc --- /dev/null +++ b/flowvault/tests/utils/test__batching.py @@ -0,0 +1,74 @@ +import os +import unittest +from unittest.mock import patch + +from skyflow_flowvault.utils import _batching +from skyflow_flowvault.utils._batching import ( + resolve_batch_config, + create_batches, + DEFAULT_BATCH_SIZE, + MAX_BATCH_SIZE, + MAX_CONCURRENCY, + INSERT_BATCH_SIZE_KEY, + INSERT_CONCURRENCY_LIMIT_KEY, +) + + +def _with_settings(mapping): + return patch.object(_batching, "_resolve_setting", lambda key: mapping.get(key)) + + +class TestCreateBatches(unittest.TestCase): + def test_contiguous_slices(self): + self.assertEqual(create_batches([1, 2, 3, 4, 5], 2), [[1, 2], [3, 4], [5]]) + + def test_single_batch_when_size_exceeds_count(self): + self.assertEqual(create_batches([1, 2, 3], 10), [[1, 2, 3]]) + + def test_empty(self): + self.assertEqual(create_batches([], 5), []) + + +class TestResolveBatchConfig(unittest.TestCase): + def test_defaults_when_unset(self): + with _with_settings({}): + batch_size, concurrency = resolve_batch_config(INSERT_BATCH_SIZE_KEY, INSERT_CONCURRENCY_LIMIT_KEY, 500) + self.assertEqual(batch_size, DEFAULT_BATCH_SIZE) + self.assertEqual(concurrency, 1) + + def test_batch_size_capped_at_max(self): + with _with_settings({INSERT_BATCH_SIZE_KEY: "5000"}): + batch_size, _ = resolve_batch_config(INSERT_BATCH_SIZE_KEY, INSERT_CONCURRENCY_LIMIT_KEY, 10) + self.assertEqual(batch_size, MAX_BATCH_SIZE) + + def test_invalid_batch_size_falls_back(self): + for raw in ("abc", "0", "-5"): + with _with_settings({INSERT_BATCH_SIZE_KEY: raw}): + batch_size, _ = resolve_batch_config(INSERT_BATCH_SIZE_KEY, INSERT_CONCURRENCY_LIMIT_KEY, 10) + self.assertEqual(batch_size, DEFAULT_BATCH_SIZE) + + def test_concurrency_capped_by_batch_count(self): + with _with_settings({INSERT_BATCH_SIZE_KEY: "100", INSERT_CONCURRENCY_LIMIT_KEY: "10"}): + batch_size, concurrency = resolve_batch_config(INSERT_BATCH_SIZE_KEY, INSERT_CONCURRENCY_LIMIT_KEY, 500) + self.assertEqual(batch_size, 100) + self.assertEqual(concurrency, 5) # 5 batches, so min(10, max, 5) = 5 + + def test_concurrency_capped_at_max(self): + with _with_settings({INSERT_BATCH_SIZE_KEY: "1", INSERT_CONCURRENCY_LIMIT_KEY: "999"}): + _, concurrency = resolve_batch_config(INSERT_BATCH_SIZE_KEY, INSERT_CONCURRENCY_LIMIT_KEY, 100) + self.assertEqual(concurrency, MAX_CONCURRENCY) + + def test_invalid_concurrency_falls_back(self): + with _with_settings({INSERT_CONCURRENCY_LIMIT_KEY: "abc"}): + _, concurrency = resolve_batch_config(INSERT_BATCH_SIZE_KEY, INSERT_CONCURRENCY_LIMIT_KEY, 500) + self.assertEqual(concurrency, 1) + + +class TestResolveSetting(unittest.TestCase): + def test_reads_process_env_first(self): + with patch.dict(os.environ, {INSERT_BATCH_SIZE_KEY: "77"}): + self.assertEqual(_batching._resolve_setting(INSERT_BATCH_SIZE_KEY), "77") + + +if __name__ == "__main__": + unittest.main() diff --git a/flowvault/tests/utils/test__response_parsing.py b/flowvault/tests/utils/test__response_parsing.py new file mode 100644 index 00000000..f1756d74 --- /dev/null +++ b/flowvault/tests/utils/test__response_parsing.py @@ -0,0 +1,60 @@ +import unittest + +from skyflow_flowvault.utils._response_parsing import parse_tokens, parse_hashed_data, parse_metadata + + +class TestParseTokens(unittest.TestCase): + def test_list_of_entries_normalized_to_snake_case(self): + raw = {"ssn": [ + {"token": "t1", "tokenGroupName": "g1", "path": "p1"}, + {"token": "t2", "tokenGroupName": "g2"}, + ]} + self.assertEqual(parse_tokens(raw), {"ssn": [ + {"token": "t1", "token_group_name": "g1", "path": "p1"}, + {"token": "t2", "token_group_name": "g2", "path": None}, + ]}) + + def test_single_unwrapped_entry_becomes_a_list(self): + self.assertEqual( + parse_tokens({"ssn": {"token": "t1", "tokenGroupName": "g1"}}), + {"ssn": [{"token": "t1", "token_group_name": "g1", "path": None}]}, + ) + + def test_bare_value_becomes_a_token(self): + self.assertEqual( + parse_tokens({"ssn": "bare"}), + {"ssn": [{"token": "bare", "token_group_name": None, "path": None}]}, + ) + + def test_none_returns_none(self): + self.assertIsNone(parse_tokens(None)) + + +class TestParseHashedData(unittest.TestCase): + def test_list_of_hash_entries(self): + raw = {"ssn": [{"data": "h", "hashName": "hash1"}]} + self.assertEqual(parse_hashed_data(raw), {"ssn": [{"data": "h", "hash_name": "hash1"}]}) + + def test_bare_value_wrapped(self): + self.assertEqual( + parse_hashed_data({"email": "abc"}), + {"email": [{"data": "abc", "hash_name": None}]}, + ) + + def test_none_returns_none(self): + self.assertIsNone(parse_hashed_data(None)) + + +class TestParseMetadata(unittest.TestCase): + def test_reads_both_casings(self): + self.assertEqual(parse_metadata({"skyflowID": "id", "tableName": "t1"}), + {"skyflow_id": "id", "table_name": "t1"}) + self.assertEqual(parse_metadata({"skyflowId": "id", "table": "t1"}), + {"skyflow_id": "id", "table_name": "t1"}) + + def test_none_returns_none(self): + self.assertIsNone(parse_metadata(None)) + + +if __name__ == "__main__": + unittest.main() diff --git a/flowvault/tests/utils/validations/test__validations.py b/flowvault/tests/utils/validations/test__validations.py index 10b36b6b..7098a49f 100644 --- a/flowvault/tests/utils/validations/test__validations.py +++ b/flowvault/tests/utils/validations/test__validations.py @@ -3,13 +3,32 @@ from common.errors import SkyflowError from common.utils.enums import Env from skyflow_flowvault.utils.enums import UpsertType -from skyflow_flowvault.utils.validations import validate_insert_request, validate_vault_config -from skyflow_flowvault.vault.data import InsertRequest +from skyflow_flowvault.utils.validations import ( + validate_insert_request, + validate_get_request, + validate_update_request, + validate_delete_request, + validate_detokenize_request, + validate_query_request, + validate_vault_config, +) +from skyflow_flowvault.vault.data import ( + UpsertOptions, + ColumnRedaction, + InsertRequestRecord, + InsertRequest, + GetRequest, + GetRecordRequest, + UpdateRequest, + DeleteRequest, + DetokenizeRequest, + QueryRequest, +) class TestValidateInsertRequest(unittest.TestCase): def test_valid_minimal_request(self): - request = InsertRequest(values=[dict(values={"a": 1})], table="t1") + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name="t1") validate_insert_request(None, request) # should not raise def test_valid_rich_request_with_per_record_overrides(self): @@ -19,9 +38,9 @@ def test_valid_rich_request_with_per_record_overrides(self): partial mix is invalid -- see test_table_missing_from_one_record_raises), so both records set their own here.""" request = InsertRequest( - values=[ - dict(values={"a": 1}, table="t2"), - dict(values={"a": 2}, table="t2", upsert={"update_type": UpsertType.REPLACE, "unique_columns": ["a"]}), + records=[ + InsertRequestRecord(data={"a": 1}, table_name="t2"), + InsertRequestRecord(data={"a": 2}, table_name="t2", upsert=UpsertOptions(update_type= UpsertType.REPLACE, unique_columns= ["a"])), ], ) validate_insert_request(None, request) # should not raise @@ -30,16 +49,16 @@ def test_table_in_both_places_raises(self): """Confirmed directly against a real vault: 'Table name should be present outside the records or inside each record. Should be present at one place.'""" request = InsertRequest( - values=[dict(values={"a": 1}, table="t2")], - table="t1", + records=[InsertRequestRecord(data={"a": 1}, table_name="t2")], + table_name="t1", ) with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_table_in_both_places_raises_even_if_only_one_record_sets_it(self): request = InsertRequest( - values=[dict(values={"a": 1}, table="t2"), dict(values={"a": 2})], - table="t1", + records=[InsertRequestRecord(data={"a": 1}, table_name="t2"), InsertRequestRecord(data={"a": 2})], + table_name="t1", ) with self.assertRaises(SkyflowError): validate_insert_request(None, request) @@ -49,34 +68,34 @@ def test_record_level_upsert_forbidden_when_table_is_at_request_level(self): request level, so a record-level upsert is rejected even though this record's own table placement (none) is fine.""" request = InsertRequest( - values=[dict(values={"a": 1}, upsert={"unique_columns": ["b"]})], - table="t1", - upsert={"unique_columns": ["a"]}, + records=[InsertRequestRecord(data={"a": 1}, upsert=UpsertOptions(unique_columns= ["b"]))], + table_name="t1", + upsert=UpsertOptions(unique_columns= ["a"]), ) with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_request_level_upsert_forbidden_when_table_is_per_record(self): request = InsertRequest( - values=[dict(values={"a": 1}, table="t1")], - upsert={"unique_columns": ["a"]}, + records=[InsertRequestRecord(data={"a": 1}, table_name="t1")], + upsert=UpsertOptions(unique_columns= ["a"]), ) with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_too_many_records_raises(self): - request = InsertRequest(values=[dict(values={"a": 1}) for _ in range(10001)], table="t1") + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1}) for _ in range(10001)], table_name="t1") with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_exactly_max_records_is_valid(self): - request = InsertRequest(values=[dict(values={"a": 1}) for _ in range(10000)], table="t1") + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1}) for _ in range(10000)], table_name="t1") validate_insert_request(None, request) # should not raise def test_table_missing_from_one_record_raises(self): """Java parity: when there's no request-level table, EVERY record must set its own -- a partial mix (some records with a table, some without) is invalid.""" - request = InsertRequest(values=[dict(values={"a": 1}, table="t1"), dict(values={"a": 2})]) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1}, table_name="t1"), InsertRequestRecord(data={"a": 2})]) with self.assertRaises(SkyflowError): validate_insert_request(None, request) @@ -89,34 +108,34 @@ def test_table_missing_from_one_record_raises(self): def test_falsy_non_string_values_are_valid(self): """0, False, [], {} are all legitimate values -- only None/empty-string should raise (mirrors Java's value.toString().trim().isEmpty(), which is non-empty for all of these).""" - request = InsertRequest(values=[dict(values={"a": 0, "b": False, "c": [], "d": {}})], table="t1") + request = InsertRequest(records=[InsertRequestRecord(data={"a": 0, "b": False, "c": [], "d": {}})], table_name="t1") validate_insert_request(None, request) # should not raise def test_request_level_table_alone_is_valid(self): - request = InsertRequest(values=[dict(values={"a": 1}), dict(values={"a": 2})], table="t1") + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1}), InsertRequestRecord(data={"a": 2})], table_name="t1") validate_insert_request(None, request) # should not raise def test_per_record_table_alone_is_valid(self): - request = InsertRequest(values=[dict(values={"a": 1}, table="t1"), dict(values={"a": 2}, table="t2")]) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1}, table_name="t1"), InsertRequestRecord(data={"a": 2}, table_name="t2")]) validate_insert_request(None, request) # should not raise def test_records_must_be_a_list(self): - request = InsertRequest(values="not-a-list", table="t1") + request = InsertRequest(records="not-a-list", table_name="t1") with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_records_must_be_dicts(self): - request = InsertRequest(values=["not-a-dict"], table="t1") + request = InsertRequest(records=["not-a-dict"], table_name="t1") with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_record_with_unknown_key_raises(self): - request = InsertRequest(values=[{"a": 1}], table="t1") + request = InsertRequest(records=[{"a": 1}], table_name="t1") with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_records_must_not_be_empty(self): - request = InsertRequest(values=[], table="t1") + request = InsertRequest(records=[], table_name="t1") with self.assertRaises(SkyflowError): validate_insert_request(None, request) @@ -126,36 +145,248 @@ def test_records_must_not_be_empty(self): # common/tests/vault/test_base_vault_controller.py for the shared helper's own unit tests. def test_table_is_optional_when_every_record_has_its_own(self): - request = InsertRequest(values=[dict(values={"a": 1}, table="t2")]) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1}, table_name="t2")]) validate_insert_request(None, request) # should not raise def test_upsert_must_be_a_dict(self): - request = InsertRequest(values=[dict(values={"a": 1})], table="t1", upsert="not-an-upsert") + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name="t1", upsert="not-an-upsert") with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_upsert_unique_columns_must_be_non_empty_list_of_strings(self): - request = InsertRequest(values=[dict(values={"a": 1})], table="t1", upsert={"unique_columns": []}) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name="t1", upsert=UpsertOptions(unique_columns= [])) with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_upsert_update_type_must_be_upsert_type_enum(self): request = InsertRequest( - values=[dict(values={"a": 1})], table="t1", - upsert={"update_type": "REPLACE", "unique_columns": ["a"]}, # plain string, not the enum + records=[InsertRequestRecord(data={"a": 1})], table_name="t1", + upsert=UpsertOptions(update_type= "REPLACE", unique_columns= ["a"]), # plain string, not the enum ) with self.assertRaises(SkyflowError): validate_insert_request(None, request) def test_per_record_upsert_is_also_validated(self): request = InsertRequest( - values=[dict(values={"a": 1}, upsert={"unique_columns": []})], - table="t1", + records=[InsertRequestRecord(data={"a": 1}, upsert=UpsertOptions(unique_columns= []))], + table_name="t1", ) with self.assertRaises(SkyflowError): validate_insert_request(None, request) +class TestValidateGetRequest(unittest.TestCase): + def test_valid_request_with_ids(self): + request = GetRequest(table="t1", ids=["id1"]) + validate_get_request(None, request) # should not raise + + def test_valid_request_with_unique_values(self): + request = GetRequest(table="t1", unique_values=[{"email": "a@b.com"}]) + validate_get_request(None, request) # should not raise + + def test_missing_table_raises(self): + request = GetRequest(table=None, ids=["id1"]) + with self.assertRaises(SkyflowError): + validate_get_request(None, request) + + def test_empty_table_raises(self): + request = GetRequest(table="", ids=["id1"]) + with self.assertRaises(SkyflowError): + validate_get_request(None, request) + + def test_missing_ids_and_unique_values_raises(self): + request = GetRequest(table="t1") + with self.assertRaises(SkyflowError): + validate_get_request(None, request) + + def test_ids_must_be_a_list(self): + request = GetRequest(table="t1", ids="not-a-list") + with self.assertRaises(SkyflowError): + validate_get_request(None, request) + + def test_ids_must_be_non_empty(self): + request = GetRequest(table="t1", ids=[]) + with self.assertRaises(SkyflowError): + validate_get_request(None, request) + + def test_ids_must_be_strings(self): + request = GetRequest(table="t1", ids=[123]) + with self.assertRaises(SkyflowError): + validate_get_request(None, request) + + +class TestValidateUpdateRequest(unittest.TestCase): + def test_valid_request_with_request_level_table(self): + request = UpdateRequest(records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1") + validate_update_request(None, request) # should not raise + + def test_valid_request_with_per_record_table(self): + request = UpdateRequest(records=[{"skyflow_id": "id1", "data": {"a": 1}, "table_name": "t1"}]) + validate_update_request(None, request) # should not raise + + def test_records_must_be_a_list(self): + request = UpdateRequest(records="not-a-list", table_name="t1") + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_records_must_not_be_empty(self): + request = UpdateRequest(records=[], table_name="t1") + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_missing_skyflow_id_raises(self): + request = UpdateRequest(records=[{"data": {"a": 1}}], table_name="t1") + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_empty_skyflow_id_raises(self): + request = UpdateRequest(records=[{"skyflow_id": " ", "data": {"a": 1}}], table_name="t1") + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_record_with_unknown_key_raises(self): + request = UpdateRequest(records=[{"skyflow_id": "id1", "unexpected": 1}], table_name="t1") + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_table_in_both_places_raises(self): + request = UpdateRequest(records=[{"skyflow_id": "id1", "data": {"a": 1}, "table_name": "t2"}], table_name="t1") + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_table_missing_from_one_record_raises(self): + request = UpdateRequest(records=[ + {"skyflow_id": "id1", "data": {"a": 1}, "table_name": "t1"}, + {"skyflow_id": "id2", "data": {"a": 2}}, + ]) + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_invalid_update_type_raises(self): + request = UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1", update_type="REPLACE", + ) + with self.assertRaises(SkyflowError): + validate_update_request(None, request) + + def test_valid_update_type_enum_is_valid(self): + request = UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1", update_type=UpsertType.REPLACE, + ) + validate_update_request(None, request) # should not raise + + +class TestValidateDeleteRequest(unittest.TestCase): + def test_valid_request_with_ids(self): + request = DeleteRequest(table="t1", ids=["id1"]) + validate_delete_request(None, request) # should not raise + + def test_valid_request_with_unique_values(self): + request = DeleteRequest(table="t1", unique_values=[{"email": "a@b.com"}]) + validate_delete_request(None, request) # should not raise + + def test_missing_table_raises(self): + request = DeleteRequest(table=None, ids=["id1"]) + with self.assertRaises(SkyflowError): + validate_delete_request(None, request) + + def test_missing_ids_and_unique_values_raises(self): + request = DeleteRequest(table="t1") + with self.assertRaises(SkyflowError): + validate_delete_request(None, request) + + def test_ids_must_be_non_empty(self): + request = DeleteRequest(table="t1", ids=[]) + with self.assertRaises(SkyflowError): + validate_delete_request(None, request) + + def test_ids_must_be_strings(self): + request = DeleteRequest(table="t1", ids=[123]) + with self.assertRaises(SkyflowError): + validate_delete_request(None, request) + + +class TestValidateDetokenizeRequest(unittest.TestCase): + def test_valid_request(self): + request = DetokenizeRequest(tokens=["tok1", "tok2"]) + validate_detokenize_request(None, request) # should not raise + + def test_valid_request_with_token_group_redactions(self): + request = DetokenizeRequest( + tokens=["tok1"], token_group_redactions=[{"token_group_name": "g1", "redaction": "mask1"}], + ) + validate_detokenize_request(None, request) # should not raise + + def test_tokens_must_be_a_list(self): + request = DetokenizeRequest(tokens="not-a-list") + with self.assertRaises(SkyflowError): + validate_detokenize_request(None, request) + + def test_tokens_must_not_be_empty(self): + request = DetokenizeRequest(tokens=[]) + with self.assertRaises(SkyflowError): + validate_detokenize_request(None, request) + + def test_tokens_must_be_strings(self): + request = DetokenizeRequest(tokens=[123]) + with self.assertRaises(SkyflowError): + validate_detokenize_request(None, request) + + def test_empty_string_token_raises(self): + request = DetokenizeRequest(tokens=[" "]) + with self.assertRaises(SkyflowError): + validate_detokenize_request(None, request) + + def test_invalid_token_group_redactions_raises(self): + request = DetokenizeRequest(tokens=["tok1"], token_group_redactions=["not-a-dict"]) + with self.assertRaises(SkyflowError): + validate_detokenize_request(None, request) + + def test_token_group_redactions_missing_name_raises(self): + request = DetokenizeRequest(tokens=["tok1"], token_group_redactions=[{"redaction": "mask1"}]) + with self.assertRaises(SkyflowError): + validate_detokenize_request(None, request) + + +class TestValidateQueryRequest(unittest.TestCase): + def test_valid_request(self): + validate_query_request(None, QueryRequest(query="SELECT * FROM t1")) # should not raise + + def test_query_must_be_a_string(self): + with self.assertRaises(SkyflowError): + validate_query_request(None, QueryRequest(query=123)) + + def test_query_must_not_be_empty(self): + with self.assertRaises(SkyflowError): + validate_query_request(None, QueryRequest(query=" ")) + + +class TestValidateGetRequestMultiTable(unittest.TestCase): + def test_valid_multi_table_request(self): + request = GetRequest(records=[GetRecordRequest(table="persons", ids=["id1"])]) + validate_get_request(None, request) # should not raise + + def test_records_must_be_get_record_request_objects(self): + with self.assertRaises(SkyflowError): + validate_get_request(None, GetRequest(records=[{"table": "persons", "ids": ["id1"]}])) + + def test_records_must_not_be_empty(self): + with self.assertRaises(SkyflowError): + validate_get_request(None, GetRequest(records=[])) + + def test_records_and_single_table_fields_are_mutually_exclusive(self): + with self.assertRaises(SkyflowError): + validate_get_request(None, GetRequest(table="persons", records=[GetRecordRequest(table="persons", ids=["id1"])])) + + def test_each_record_needs_a_table(self): + with self.assertRaises(SkyflowError): + validate_get_request(None, GetRequest(records=[GetRecordRequest(table=None, ids=["id1"])])) + + def test_each_record_needs_ids_or_unique_values(self): + with self.assertRaises(SkyflowError): + validate_get_request(None, GetRequest(records=[GetRecordRequest(table="persons")])) + + class TestValidateVaultConfig(unittest.TestCase): def test_valid_config(self): config = { diff --git a/flowvault/tests/vault/client/test__client.py b/flowvault/tests/vault/client/test__client.py index 2accba2b..d2e3e2ad 100644 --- a/flowvault/tests/vault/client/test__client.py +++ b/flowvault/tests/vault/client/test__client.py @@ -36,19 +36,37 @@ def test_resolve_vault_url_uses_v3_skyvault_domain_stage(self): self.assertEqual(url, "https://qhdmceurtnlz.skyvault.skyflowapis.tech") @patch("skyflow_flowvault.vault.client.client.SkyflowAuth") - def test_initialize_api_client_does_not_pass_token(self, mock_skyflow_auth): - """v3's generated client has no `token` param at all -- unlike v2, nothing should be - baked in at construction time; auth is injected per-call instead (see Vault._build_headers).""" + def test_initialize_api_client_passes_base_url_and_token(self, mock_skyflow_auth): self.vault_client.initialize_api_client("https://test-vault-url.com", "some_bearer_token") _, kwargs = mock_skyflow_auth.call_args self.assertEqual(kwargs.get("base_url"), "https://test-vault-url.com") - self.assertNotIn("token", kwargs) + self.assertEqual(kwargs.get("token"), "some_bearer_token") - def test_get_insert_api_returns_flowservice(self): + def test_get_records_api_returns_records(self): self.vault_client._api_client = MagicMock() - result = self.vault_client.get_insert_api() - self.assertEqual(result, self.vault_client._api_client.flowservice) + result = self.vault_client.get_records_api() + self.assertEqual(result, self.vault_client._api_client.records) + + def test_get_tokens_api_returns_tokens(self): + self.vault_client._api_client = MagicMock() + result = self.vault_client.get_tokens_api() + self.assertEqual(result, self.vault_client._api_client.tokens) + + def test_get_query_api_returns_query(self): + self.vault_client._api_client = MagicMock() + result = self.vault_client.get_query_api() + self.assertEqual(result, self.vault_client._api_client.query) + + def test_get_async_records_api_returns_records(self): + self.vault_client._async_api_client = MagicMock() + result = self.vault_client.get_async_records_api() + self.assertEqual(result, self.vault_client._async_api_client.records) + + def test_get_async_tokens_api_returns_tokens(self): + self.vault_client._async_api_client = MagicMock() + result = self.vault_client.get_async_tokens_api() + self.assertEqual(result, self.vault_client._async_api_client.tokens) if __name__ == "__main__": diff --git a/flowvault/tests/vault/controller/test__vault.py b/flowvault/tests/vault/controller/test__vault.py index 449ede9d..7dc33467 100644 --- a/flowvault/tests/vault/controller/test__vault.py +++ b/flowvault/tests/vault/controller/test__vault.py @@ -1,30 +1,70 @@ +import asyncio +import os import unittest -from unittest.mock import MagicMock, Mock, patch +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, Mock, patch from common.errors import SkyflowError from skyflow_flowvault.generated.rest.core import ApiError from skyflow_flowvault.vault.controller import VaultController -from skyflow_flowvault.vault.data import InsertRequest +from skyflow_flowvault.vault.data import ( + UpsertOptions, + ColumnRedaction, + InsertRequestRecord, + InsertRequest, + GetRequest, + GetRecordRequest, + UpdateRequest, + DeleteRequest, + DetokenizeRequest, + QueryRequest, + BulkInsertRecord, + BulkInsertRequest, + BulkDetokenizeRequest, +) from skyflow_flowvault.utils.enums import UpsertType +class FakeExecuteQueryRecord: + def __init__(self, data=None): + self.data = data + + class FakeRecordResponseObject: - def __init__(self, skyflow_id=None, tokens=None, data=None, error=None, http_code=None, table_name=None): + def __init__(self, skyflow_id=None, tokens=None, data=None, hashed_data=None, error=None, http_code=None, table_name=None): self.skyflow_id = skyflow_id self.tokens = tokens self.data = data + self.hashed_data = hashed_data self.error = error self.http_code = http_code self.table_name = table_name +class FakeDeleteResponseObject: + def __init__(self, skyflow_id=None, error=None, http_code=None): + self.skyflow_id = skyflow_id + self.error = error + self.http_code = http_code + + +class FakeDetokenizeResponseObject: + def __init__(self, token=None, value=None, token_group_name=None, error=None, http_code=None, metadata=None): + self.token = token + self.value = value + self.token_group_name = token_group_name + self.error = error + self.http_code = http_code + self.metadata = metadata + + class FakeV1InsertResponse: def __init__(self, records): self.records = records class FakeRawResponse: - """Stands in for the HttpResponse wrapper returned by with_raw_response.insert(...) -- + """Stands in for the HttpResponse wrapper returned by with_raw_response.insert_records(...) -- exposes .data (the parsed V1InsertResponse) and .headers, mirroring the real generated client's RawFlowserviceClient.""" @@ -40,7 +80,7 @@ def setUp(self): self.vault_client.get_logger.return_value = Mock() self.vault_client.get_current_bearer_token.return_value = None self.insert_api = MagicMock() - self.vault_client.get_insert_api.return_value = self.insert_api + self.vault_client.get_records_api.return_value = self.insert_api self.vault = VaultController(self.vault_client) # ------------------------------------------------------------------ # @@ -49,8 +89,8 @@ def setUp(self): @patch("skyflow_flowvault.vault.controller._vault.validate_insert_request") def test_insert_validates_before_initializing_client(self, mock_validate): - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) - request = InsertRequest(values=[dict(values={"a": 1})], table="t1") + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name="t1") self.vault.insert(request) @@ -59,7 +99,7 @@ def test_insert_validates_before_initializing_client(self, mock_validate): def test_insert_raises_for_invalid_request(self): with self.assertRaises(SkyflowError): - self.vault.insert(InsertRequest(values=[], table="t1")) + self.vault.insert(InsertRequest(records=[], table_name="t1")) self.vault_client.initialize_client_configuration.assert_not_called() # ------------------------------------------------------------------ # @@ -69,29 +109,29 @@ def test_insert_raises_for_invalid_request(self): def test_insert_raises_on_empty_key(self): with self.assertRaises(SkyflowError): - self.vault.insert(InsertRequest(values=[dict(values={"": "value"})], table="t1")) - self.insert_api.with_raw_response.insert.assert_not_called() + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"": "value"})], table_name="t1")) + self.insert_api.with_raw_response.insert_records.assert_not_called() def test_insert_allows_empty_value(self): - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) - self.vault.insert(InsertRequest(values=[dict(values={"a": ""})], table="t1")) - self.insert_api.with_raw_response.insert.assert_called_once() + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"a": ""})], table_name="t1")) + self.insert_api.with_raw_response.insert_records.assert_called_once() def test_insert_raises_on_non_dict_values(self): with self.assertRaises(SkyflowError): - self.vault.insert(InsertRequest(values=[dict(values=["not", "a", "dict"])], table="t1")) + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data=["not", "a", "dict"])], table_name="t1")) def test_insert_raises_on_empty_values_dict(self): with self.assertRaises(SkyflowError): - self.vault.insert(InsertRequest(values=[dict(values={})], table="t1")) + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={})], table_name="t1")) def test_insert_raises_on_invalid_request_level_table_name(self): with self.assertRaises(SkyflowError): - self.vault.insert(InsertRequest(values=[dict(values={"a": 1})], table=" ")) + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name=" ")) def test_insert_raises_on_invalid_per_record_table_name(self): with self.assertRaises(SkyflowError): - self.vault.insert(InsertRequest(values=[dict(values={"a": 1}, table=" ")])) + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"a": 1}, table_name=" ")])) # ------------------------------------------------------------------ # # request -> wire field mapping @@ -101,16 +141,16 @@ def test_maps_request_level_table_and_upsert(self): """When no record sets its own table/upsert, both go ONLY at the request level -- the vault rejects sending table_name/upsert in both places (see the validation tests), so the wire records must NOT also carry a resolved copy.""" - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) request = InsertRequest( - values=[dict(values={"a": 1})], - table="t1", - upsert={"update_type": UpsertType.REPLACE, "unique_columns": ["a"]}, + records=[InsertRequestRecord(data={"a": 1})], + table_name="t1", + upsert=UpsertOptions(update_type= UpsertType.REPLACE, unique_columns= ["a"]), ) self.vault.insert(request) - _, kwargs = self.insert_api.with_raw_response.insert.call_args + _, kwargs = self.insert_api.with_raw_response.insert_records.call_args self.assertEqual(kwargs["vault_id"], "vault123") self.assertEqual(kwargs["table_name"], "t1") self.assertEqual(len(kwargs["records"]), 1) @@ -124,61 +164,57 @@ def test_setting_table_at_both_request_and_record_level_raises(self): real vault. validate_insert_request (tested separately) is what actually raises this; this test just confirms insert() surfaces it rather than silently choosing one.""" request = InsertRequest( - values=[dict(values={"a": 1}, table="t2")], - table="t1", + records=[InsertRequestRecord(data={"a": 1}, table_name="t2")], + table_name="t1", ) with self.assertRaises(SkyflowError): self.vault.insert(request) - self.insert_api.with_raw_response.insert.assert_not_called() + self.insert_api.with_raw_response.insert_records.assert_not_called() def test_per_record_table_and_upsert_used_when_request_level_unset(self): """Legitimate per-record use: no request-level table/upsert at all -- Java parity requires EVERY record to set its own table in this mode (see validation tests), so both records do; only the second also sets its own upsert.""" - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) - request = InsertRequest(values=[ - dict(values={"a": 1}, table="t2", upsert={"unique_columns": ["b"]}), - dict(values={"a": 2}, table="t2"), + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) + request = InsertRequest(records=[ + InsertRequestRecord(data={"a": 1}, table_name="t2", upsert=UpsertOptions(unique_columns= ["b"])), + InsertRequestRecord(data={"a": 2}, table_name="t2"), ]) self.vault.insert(request) - _, kwargs = self.insert_api.with_raw_response.insert.call_args - self.assertNotIn("table_name", kwargs) + _, kwargs = self.insert_api.with_raw_response.insert_records.call_args + self.assertIsNone(kwargs["table_name"]) self.assertNotIn("upsert", kwargs) self.assertEqual(kwargs["records"][0].table_name, "t2") self.assertEqual(kwargs["records"][0].upsert.unique_columns, ["b"]) self.assertEqual(kwargs["records"][1].table_name, "t2") self.assertIsNone(kwargs["records"][1].upsert) - def test_no_request_level_table_is_omitted_not_sent_as_none(self): - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) - request = InsertRequest(values=[dict(values={"a": 1}, table="t2")]) # no request-level table + def test_no_request_level_table_passed_as_none(self): + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1}, table_name="t2")]) # no request-level table self.vault.insert(request) - _, kwargs = self.insert_api.with_raw_response.insert.call_args - self.assertNotIn("table_name", kwargs) + _, kwargs = self.insert_api.with_raw_response.insert_records.call_args + self.assertIsNone(kwargs["table_name"]) def test_wire_shape_matches_confirmed_working_request(self): - """Regression pin for a real bug: a request with only per-record table/upsert (no - request-level table/upsert at all) previously sent explicit `"tableName": null` / - `"upsert": null` at the top level, which diverged from a hand-verified working request - against a real vault (confirmed to have neither key present when unset).""" - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) - request = InsertRequest(values=[ - dict( - values={"name": "saileshwar", "email": "nanana@gmail.com"}, - table="table1", - upsert={"update_type": UpsertType.UPDATE, "unique_columns": ["email"]}, + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) + request = InsertRequest(records=[ + InsertRequestRecord( + data={"name": "saileshwar", "email": "nanana@gmail.com"}, + table_name="table1", + upsert=UpsertOptions(update_type= UpsertType.UPDATE, unique_columns= ["email"]), ), ]) self.vault.insert(request) - _, kwargs = self.insert_api.with_raw_response.insert.call_args - self.assertNotIn("table_name", kwargs) + _, kwargs = self.insert_api.with_raw_response.insert_records.call_args + self.assertIsNone(kwargs["table_name"]) self.assertNotIn("upsert", kwargs) self.assertEqual(kwargs["records"][0].table_name, "table1") self.assertEqual(kwargs["records"][0].upsert.update_type, "UPDATE") @@ -187,42 +223,44 @@ def test_wire_shape_matches_confirmed_working_request(self): def test_no_upsert_is_omitted_not_sent_as_none(self): """upsert must be OMITTED from the wire call entirely when unset, not passed as None -- a real vault confirmed a working request never includes a null upsert/tableName key.""" - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) - request = InsertRequest(values=[dict(values={"a": 1})], table="t1") + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name="t1") self.vault.insert(request) - _, kwargs = self.insert_api.with_raw_response.insert.call_args + _, kwargs = self.insert_api.with_raw_response.insert_records.call_args self.assertNotIn("upsert", kwargs) self.assertIsNone(kwargs["records"][0].upsert) # ------------------------------------------------------------------ # - # response shape -- mirrors PDB's InsertResponse (inserted_fields/errors) + # response shape -- unified records list (FlowDB contract), tokens normalized # ------------------------------------------------------------------ # - def test_successful_records_go_to_inserted_fields(self): - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([ + def test_successful_record_carries_normalized_fields(self): + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([ FakeRecordResponseObject( skyflow_id="id1", - tokens={"name": [{"token": "tok1", "tokenGroupName": "deterministic_string"}]}, + tokens={"name": [{"token": "tok1", "tokenGroupName": "deterministic_string", "path": "p"}]}, data={"name": "john doe"}, + hashed_data={"name": [{"data": "h", "hashName": "hash1"}]}, table_name="table1", + http_code=200, ), ], headers={"x-request-id": "req-1"}) - response = self.vault.insert(InsertRequest(values=[dict(values={"name": "john doe"})], table="table1")) - - self.assertEqual(len(response.inserted_fields), 1) - inserted = response.inserted_fields[0] - self.assertEqual(inserted["request_index"], 0) - self.assertEqual(inserted["skyflow_id"], "id1") - self.assertEqual(inserted["name"], "tok1") - self.assertNotIn("data", inserted) - self.assertNotIn("table", inserted) - self.assertNotIn("tokens", inserted) - self.assertIsNone(response.errors) - - def test_multiple_token_groups_for_one_field_flatten_to_a_list(self): - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([ + response = self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"name": "john doe"})], table_name="table1")) + + self.assertEqual(len(response.records), 1) + record = response.records[0] + self.assertEqual(record["skyflow_id"], "id1") + self.assertEqual(record["table_name"], "table1") + self.assertEqual(record["tokens"], {"name": [{"token": "tok1", "token_group_name": "deterministic_string", "path": "p"}]}) + self.assertNotIn("data", record) # insert response omits data + self.assertEqual(record["hashed_data"], {"name": [{"data": "h", "hash_name": "hash1"}]}) + self.assertEqual(record["http_code"], 200) + self.assertIsNone(record["error"]) + + def test_tokens_normalized_to_typed_list_per_group(self): + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([ FakeRecordResponseObject( skyflow_id="id1", tokens={"email": [ @@ -231,66 +269,46 @@ def test_multiple_token_groups_for_one_field_flatten_to_a_list(self): ]}, ), ]) - response = self.vault.insert(InsertRequest(values=[dict(values={"email": "a@b.com"})], table="t1")) + response = self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"email": "a@b.com"})], table_name="t1")) - self.assertEqual(response.inserted_fields[0]["email"], ["tok-det", "tok-nondet"]) + self.assertEqual(response.records[0]["tokens"]["email"], [ + {"token": "tok-det", "token_group_name": "deterministic_string", "path": None}, + {"token": "tok-nondet", "token_group_name": "nondeterministic_string", "path": None}, + ]) - def test_mixed_success_and_error_records_are_split(self): - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([ - FakeRecordResponseObject(skyflow_id="id1", tokens=None), + def test_success_and_error_records_in_one_list(self): + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([ + FakeRecordResponseObject(skyflow_id="id1", tokens=None, http_code=200), FakeRecordResponseObject(error="bad row", http_code=400, table_name="t1"), ], headers={"x-request-id": "req-2"}) response = self.vault.insert(InsertRequest( - values=[dict(values={"a": 1}), dict(values={"a": 2})], table="t1", + records=[InsertRequestRecord(data={"a": 1}), InsertRequestRecord(data={"a": 2})], table_name="t1", )) - self.assertEqual(len(response.inserted_fields), 1) - self.assertEqual(response.inserted_fields[0]["request_index"], 0) - self.assertEqual(response.inserted_fields[0]["skyflow_id"], "id1") - self.assertEqual(len(response.errors), 1) - self.assertEqual(response.errors[0]["request_index"], 1) - self.assertEqual(response.errors[0]["error"], "bad row") - self.assertEqual(response.errors[0]["code"], 400) - self.assertEqual(response.errors[0]["request_id"], "req-2") - - def test_error_record_identified_by_error_field_alone(self): - """Mirrors Java's Utils.formatResponse exactly: a record is an error purely by .error - being present -- http_code is read onto the error dict's 'code' key but is not itself - part of the success/error decision.""" - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([ - FakeRecordResponseObject(skyflow_id="id1", http_code=200), - ]) - response = self.vault.insert(InsertRequest(values=[dict(values={"a": 1})], table="t1")) - - self.assertEqual(len(response.inserted_fields), 1) - self.assertIsNone(response.errors) + self.assertEqual(len(response.records), 2) + self.assertEqual(response.records[0]["skyflow_id"], "id1") + self.assertIsNone(response.records[0]["error"]) + self.assertEqual(response.records[1]["error"], "bad row") + self.assertEqual(response.records[1]["http_code"], 400) + self.assertIsNone(response.records[1]["skyflow_id"]) # ------------------------------------------------------------------ # # no batching -- every insert is exactly one API call # ------------------------------------------------------------------ # def test_all_records_sent_in_a_single_api_call_regardless_of_count(self): - self.insert_api.with_raw_response.insert.side_effect = lambda **kwargs: FakeRawResponse( + self.insert_api.with_raw_response.insert_records.side_effect = lambda **kwargs: FakeRawResponse( [FakeRecordResponseObject(skyflow_id=f"id-{i}") for i in range(len(kwargs["records"]))] ) - records = [dict(values={"a": i}) for i in range(4)] + records = [InsertRequestRecord(data={"a": i}) for i in range(4)] - response = self.vault.insert(InsertRequest(values=records, table="t1")) + response = self.vault.insert(InsertRequest(records=records, table_name="t1")) - self.insert_api.with_raw_response.insert.assert_called_once() - call_size = len(self.insert_api.with_raw_response.insert.call_args.kwargs["records"]) + self.insert_api.with_raw_response.insert_records.assert_called_once() + call_size = len(self.insert_api.with_raw_response.insert_records.call_args.kwargs["records"]) self.assertEqual(call_size, 4) - self.assertEqual(len(response.inserted_fields), 4) - - def test_request_index_matches_position_in_the_original_records_list(self): - self.insert_api.with_raw_response.insert.side_effect = lambda **kwargs: FakeRawResponse( - [FakeRecordResponseObject(skyflow_id=f"id-{i}") for i in range(len(kwargs["records"]))] - ) - records = [dict(values={"a": i}) for i in range(4)] - - response = self.vault.insert(InsertRequest(values=records, table="t1")) - - self.assertEqual(sorted(s["request_index"] for s in response.inserted_fields), [0, 1, 2, 3]) + self.assertEqual(len(response.records), 4) + self.assertEqual([r["skyflow_id"] for r in response.records], ["id-0", "id-1", "id-2", "id-3"]) # ------------------------------------------------------------------ # # transport failure @@ -299,22 +317,17 @@ def test_request_index_matches_position_in_the_original_records_list(self): def test_transport_exception_marks_every_record_as_an_error(self): """Without batching, one API call carries every record -- a transport-level exception on that single call means every record in the request fails, not just some.""" - self.insert_api.with_raw_response.insert.side_effect = Exception("network blip") - records = [dict(values={"a": 1}), dict(values={"a": 2})] + self.insert_api.with_raw_response.insert_records.side_effect = Exception("network blip") + records = [InsertRequestRecord(data={"a": 1}), InsertRequestRecord(data={"a": 2})] - response = self.vault.insert(InsertRequest(values=records, table="t1")) + response = self.vault.insert(InsertRequest(records=records, table_name="t1")) - self.insert_api.with_raw_response.insert.assert_called_once() - self.assertEqual(len(response.inserted_fields), 0) - self.assertEqual(len(response.errors), 2) - self.assertTrue(all("network blip" in e["error"] for e in response.errors)) - self.assertEqual([e["request_index"] for e in response.errors], [0, 1]) + self.insert_api.with_raw_response.insert_records.assert_called_once() + self.assertEqual(len(response.records), 2) + self.assertTrue(all("network blip" in r["error"] for r in response.records)) + self.assertTrue(all(r["skyflow_id"] is None for r in response.records)) def test_api_error_with_structured_per_record_body_splits_into_one_error_per_row(self): - """Mirrors Java's Utils.handleBatchException: a structured error body (a 'records' list) - is split into individual error dicts instead of repeating one flat message for the - whole batch -- shaped after a real vault's actual 400 response for a partial-batch - failure (e.g. a NOT NULL column violation on one row).""" api_error = ApiError( status_code=400, headers={"x-request-id": "req-3"}, @@ -323,28 +336,25 @@ def test_api_error_with_structured_per_record_body_splits_into_one_error_per_row "httpCode": 400}, ]}, ) - self.insert_api.with_raw_response.insert.side_effect = api_error + self.insert_api.with_raw_response.insert_records.side_effect = api_error - response = self.vault.insert(InsertRequest(values=[dict(values={"name": "a"})], table="t1")) + response = self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"name": "a"})], table_name="t1")) - self.assertEqual(len(response.errors), 1) - self.assertIn("notNull", response.errors[0]["error"]) - self.assertEqual(response.errors[0]["code"], 400) - self.assertEqual(response.errors[0]["request_id"], "req-3") - self.assertEqual(response.errors[0]["request_index"], 0) + self.assertEqual(len(response.records), 1) + self.assertIn("notNull", response.records[0]["error"]) + self.assertEqual(response.records[0]["http_code"], 400) def test_api_error_with_flat_body_falls_back_to_one_error_per_record(self): api_error = ApiError(status_code=500, headers={}, body={"error": "internal error"}) - self.insert_api.with_raw_response.insert.side_effect = api_error + self.insert_api.with_raw_response.insert_records.side_effect = api_error response = self.vault.insert(InsertRequest( - values=[dict(values={"a": 1}), dict(values={"a": 2})], table="t1", + records=[InsertRequestRecord(data={"a": 1}), InsertRequestRecord(data={"a": 2})], table_name="t1", )) - self.assertEqual(len(response.errors), 2) - self.assertTrue(all(e["error"] == "internal error" for e in response.errors)) - self.assertTrue(all(e["code"] == 500 for e in response.errors)) - self.assertEqual([e["request_index"] for e in response.errors], [0, 1]) + self.assertEqual(len(response.records), 2) + self.assertTrue(all(r["error"] == "internal error" for r in response.records)) + self.assertTrue(all(r["http_code"] == 500 for r in response.records)) # ------------------------------------------------------------------ # # per-call Authorization header injection @@ -352,24 +362,1027 @@ def test_api_error_with_flat_body_falls_back_to_one_error_per_record(self): def test_injects_authorization_header_from_current_bearer_token(self): self.vault_client.get_current_bearer_token.return_value = "the-current-token" - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) - self.vault.insert(InsertRequest(values=[dict(values={"a": 1})], table="t1")) + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name="t1")) - _, kwargs = self.insert_api.with_raw_response.insert.call_args + _, kwargs = self.insert_api.with_raw_response.insert_records.call_args headers = kwargs["request_options"]["additional_headers"] self.assertEqual(headers.get("Authorization"), "Bearer the-current-token") def test_no_authorization_header_when_no_token_available(self): self.vault_client.get_current_bearer_token.return_value = None - self.insert_api.with_raw_response.insert.return_value = FakeRawResponse([]) + self.insert_api.with_raw_response.insert_records.return_value = FakeRawResponse([]) - self.vault.insert(InsertRequest(values=[dict(values={"a": 1})], table="t1")) + self.vault.insert(InsertRequest(records=[InsertRequestRecord(data={"a": 1})], table_name="t1")) - _, kwargs = self.insert_api.with_raw_response.insert.call_args + _, kwargs = self.insert_api.with_raw_response.insert_records.call_args headers = kwargs["request_options"]["additional_headers"] self.assertNotIn("Authorization", headers) +def fake_get_raw_response(records, headers=None): + return SimpleNamespace(data=SimpleNamespace(records=records), headers=headers or {}) + + +class TestVaultGet(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.get_api = MagicMock() + self.vault_client.get_records_api.return_value = self.get_api + self.vault = VaultController(self.vault_client) + + # ------------------------------------------------------------------ # + # validation / initialization sequencing + # ------------------------------------------------------------------ # + + @patch("skyflow_flowvault.vault.controller._vault.validate_get_request") + def test_get_validates_before_initializing_client(self, mock_validate): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([]) + request = GetRequest(table="t1", ids=["id1"]) + + self.vault.get(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_get_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.get(GetRequest(table="t1")) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_get_raises_on_invalid_table_name(self): + with self.assertRaises(SkyflowError): + self.vault.get(GetRequest(table=" ", ids=["id1"])) + self.get_api.with_raw_response.get_records.assert_not_called() + + # ------------------------------------------------------------------ # + # request -> wire field mapping + # ------------------------------------------------------------------ # + + def test_maps_table_and_ids(self): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([]) + + self.vault.get(GetRequest(table="t1", ids=["id1", "id2"])) + + _, kwargs = self.get_api.with_raw_response.get_records.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(kwargs["table_name"], "t1") + self.assertEqual(kwargs["skyflow_i_ds"], ["id1", "id2"]) + + def test_maps_unique_values(self): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([]) + + self.vault.get(GetRequest(table="t1", unique_values=[{"email": "a@b.com"}])) + + _, kwargs = self.get_api.with_raw_response.get_records.call_args + self.assertEqual(len(kwargs["unique_values"]), 1) + self.assertEqual(kwargs["unique_values"][0].data, {"email": "a@b.com"}) + + def test_multi_table_mode_sends_records_and_omits_single_table_fields(self): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([]) + + self.vault.get(GetRequest(records=[ + GetRecordRequest(table="persons", ids=["id1"], columns=["name"]), + GetRecordRequest(table="cards", unique_values=[{"email": "a@b.com"}]), + ])) + + _, kwargs = self.get_api.with_raw_response.get_records.call_args + self.assertNotIn("table_name", kwargs) + self.assertNotIn("skyflow_i_ds", kwargs) + self.assertEqual(len(kwargs["records"]), 2) + self.assertEqual(kwargs["records"][0].table_name, "persons") + self.assertEqual(kwargs["records"][0].skyflow_i_ds, ["id1"]) + self.assertEqual(kwargs["records"][0].columns, ["name"]) + self.assertEqual(kwargs["records"][1].table_name, "cards") + self.assertEqual(kwargs["records"][1].skyflow_i_ds, []) + self.assertEqual(kwargs["records"][1].unique_values[0].data, {"email": "a@b.com"}) + + def test_maps_column_redactions(self): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([]) + + self.vault.get(GetRequest( + table="t1", ids=["id1"], column_redactions=[ColumnRedaction(column_name="ssn", redaction="mask1")], + )) + + _, kwargs = self.get_api.with_raw_response.get_records.call_args + self.assertEqual(len(kwargs["column_redactions"]), 1) + self.assertEqual(kwargs["column_redactions"][0].column_name, "ssn") + self.assertEqual(kwargs["column_redactions"][0].redaction, "mask1") + + def test_maps_limit_offset_columns(self): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([]) + + self.vault.get(GetRequest(table="t1", ids=["id1"], columns=["a", "b"], limit=10, offset=5)) + + _, kwargs = self.get_api.with_raw_response.get_records.call_args + self.assertEqual(kwargs["columns"], ["a", "b"]) + self.assertEqual(kwargs["limit"], 10) + self.assertEqual(kwargs["offset"], 5) + + # ------------------------------------------------------------------ # + # response shape -- includes data, unlike insert + # ------------------------------------------------------------------ # + + def test_successful_record_carries_data_hashed_data_and_tokens(self): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([ + FakeRecordResponseObject( + skyflow_id="id1", + tokens={"name": [{"token": "tok1", "tokenGroupName": "deterministic_string"}]}, + data={"name": "john doe"}, + hashed_data={"email": [{"data": "a1b2c3", "hashName": "hash1"}]}, + table_name="t1", + http_code=200, + ), + ], headers={"x-request-id": "req-1"}) + + response = self.vault.get(GetRequest(table="t1", ids=["id1"])) + + self.assertEqual(len(response.records), 1) + record = response.records[0] + self.assertEqual(record["skyflow_id"], "id1") + self.assertEqual(record["table_name"], "t1") + self.assertEqual(record["data"], {"name": "john doe"}) + self.assertEqual(record["hashed_data"], {"email": [{"data": "a1b2c3", "hash_name": "hash1"}]}) + self.assertEqual(record["tokens"], {"name": [{"token": "tok1", "token_group_name": "deterministic_string", "path": None}]}) + self.assertEqual(record["http_code"], 200) + self.assertIsNone(record["error"]) + + def test_success_and_error_records_in_one_list(self): + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([ + FakeRecordResponseObject(skyflow_id="id1", data={"a": 1}, http_code=200), + FakeRecordResponseObject(error="not found", http_code=404), + ], headers={"x-request-id": "req-2"}) + + response = self.vault.get(GetRequest(table="t1", ids=["id1", "id2"])) + + self.assertEqual(len(response.records), 2) + self.assertEqual(response.records[0]["data"], {"a": 1}) + self.assertIsNone(response.records[0]["error"]) + self.assertEqual(response.records[1]["error"], "not found") + self.assertEqual(response.records[1]["http_code"], 404) + + # ------------------------------------------------------------------ # + # transport failure + # ------------------------------------------------------------------ # + + def test_transport_exception_marks_every_id_as_an_error(self): + self.get_api.with_raw_response.get_records.side_effect = Exception("network blip") + + response = self.vault.get(GetRequest(table="t1", ids=["id1", "id2"])) + + self.assertEqual(len(response.records), 2) + self.assertTrue(all("network blip" in r["error"] for r in response.records)) + + def test_api_error_with_structured_body_splits_into_one_error_per_row(self): + api_error = ApiError( + status_code=404, + headers={"x-request-id": "req-3"}, + body={"records": [{"error": "not found", "httpCode": 404}]}, + ) + self.get_api.with_raw_response.get_records.side_effect = api_error + + response = self.vault.get(GetRequest(table="t1", ids=["id1"])) + + self.assertEqual(len(response.records), 1) + self.assertEqual(response.records[0]["error"], "not found") + self.assertEqual(response.records[0]["http_code"], 404) + + # ------------------------------------------------------------------ # + # per-call Authorization header injection + # ------------------------------------------------------------------ # + + def test_injects_authorization_header_from_current_bearer_token(self): + self.vault_client.get_current_bearer_token.return_value = "the-current-token" + self.get_api.with_raw_response.get_records.return_value = fake_get_raw_response([]) + + self.vault.get(GetRequest(table="t1", ids=["id1"])) + + _, kwargs = self.get_api.with_raw_response.get_records.call_args + headers = kwargs["request_options"]["additional_headers"] + self.assertEqual(headers.get("Authorization"), "Bearer the-current-token") + + +def fake_update_raw_response(records, headers=None): + return SimpleNamespace(data=SimpleNamespace(records=records), headers=headers or {}) + + +class TestVaultUpdate(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.update_api = MagicMock() + self.vault_client.get_records_api.return_value = self.update_api + self.vault = VaultController(self.vault_client) + + # ------------------------------------------------------------------ # + # validation / initialization sequencing + # ------------------------------------------------------------------ # + + @patch("skyflow_flowvault.vault.controller._vault.validate_update_request") + def test_update_validates_before_initializing_client(self, mock_validate): + self.update_api.with_raw_response.update_records.return_value = fake_update_raw_response([]) + request = UpdateRequest(records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1") + + self.vault.update(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_update_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.update(UpdateRequest(records=[], table_name="t1")) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_update_raises_on_empty_key(self): + with self.assertRaises(SkyflowError): + self.vault.update(UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"": "value"}}], table_name="t1", + )) + self.update_api.with_raw_response.update_records.assert_not_called() + + def test_update_raises_on_invalid_table_name(self): + with self.assertRaises(SkyflowError): + self.vault.update(UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name=" ", + )) + + # ------------------------------------------------------------------ # + # request -> wire field mapping + # ------------------------------------------------------------------ # + + def test_maps_request_level_table(self): + self.update_api.with_raw_response.update_records.return_value = fake_update_raw_response([]) + request = UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1", + ) + + self.vault.update(request) + + _, kwargs = self.update_api.with_raw_response.update_records.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(kwargs["table_name"], "t1") + self.assertEqual(len(kwargs["records"]), 1) + self.assertEqual(kwargs["records"][0].skyflow_id, "id1") + self.assertEqual(kwargs["records"][0].data, {"a": 1}) + self.assertIsNone(kwargs["records"][0].table_name) + + def test_maps_per_record_table_when_request_level_unset(self): + self.update_api.with_raw_response.update_records.return_value = fake_update_raw_response([]) + request = UpdateRequest(records=[ + {"skyflow_id": "id1", "data": {"a": 1}, "table_name": "t2"}, + ]) + + self.vault.update(request) + + _, kwargs = self.update_api.with_raw_response.update_records.call_args + self.assertIsNone(kwargs["table_name"]) + self.assertEqual(kwargs["records"][0].table_name, "t2") + + def test_update_type_is_not_sent_to_the_update_endpoint(self): + self.update_api.with_raw_response.update_records.return_value = fake_update_raw_response([]) + request = UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1", update_type=UpsertType.REPLACE, + ) + + self.vault.update(request) + + _, kwargs = self.update_api.with_raw_response.update_records.call_args + self.assertNotIn("update_type", kwargs) + + # ------------------------------------------------------------------ # + # response shape -- includes data, like get + # ------------------------------------------------------------------ # + + def test_successful_records_include_data_and_tokens(self): + self.update_api.with_raw_response.update_records.return_value = fake_update_raw_response([ + FakeRecordResponseObject( + skyflow_id="id1", + tokens={"name": [{"token": "tok1", "tokenGroupName": "deterministic_string"}]}, + data={"name": "john doe"}, + ), + ], headers={"x-request-id": "req-1"}) + + response = self.vault.update(UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"name": "john doe"}}], table_name="t1", + )) + + self.assertEqual(len(response.records), 1) + record = response.records[0] + self.assertEqual(record["skyflow_id"], "id1") + self.assertEqual(record["name"], "tok1") + self.assertEqual(record["data"], {"name": "john doe"}) + self.assertIsNone(response.errors) + + def test_mixed_success_and_error_records_are_split(self): + self.update_api.with_raw_response.update_records.return_value = fake_update_raw_response([ + FakeRecordResponseObject(skyflow_id="id1", data={"a": 1}), + FakeRecordResponseObject(error="not found", http_code=404), + ], headers={"x-request-id": "req-2"}) + + response = self.vault.update(UpdateRequest(records=[ + {"skyflow_id": "id1", "data": {"a": 1}}, + {"skyflow_id": "id2", "data": {"a": 2}}, + ], table_name="t1")) + + self.assertEqual(len(response.records), 1) + self.assertEqual(len(response.errors), 1) + self.assertEqual(response.errors[0]["error"], "not found") + self.assertEqual(response.errors[0]["code"], 404) + + # ------------------------------------------------------------------ # + # transport failure + # ------------------------------------------------------------------ # + + def test_transport_exception_marks_every_record_as_an_error(self): + self.update_api.with_raw_response.update_records.side_effect = Exception("network blip") + records = [{"skyflow_id": "id1", "data": {"a": 1}}, {"skyflow_id": "id2", "data": {"a": 2}}] + + response = self.vault.update(UpdateRequest(records=records, table_name="t1")) + + self.assertEqual(len(response.records), 0) + self.assertEqual(len(response.errors), 2) + self.assertTrue(all("network blip" in e["error"] for e in response.errors)) + + def test_api_error_with_structured_body_splits_into_one_error_per_row(self): + api_error = ApiError( + status_code=404, + headers={"x-request-id": "req-3"}, + body={"records": [{"error": "not found", "httpCode": 404}]}, + ) + self.update_api.with_raw_response.update_records.side_effect = api_error + + response = self.vault.update(UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1", + )) + + self.assertEqual(len(response.errors), 1) + self.assertEqual(response.errors[0]["error"], "not found") + self.assertEqual(response.errors[0]["code"], 404) + self.assertEqual(response.errors[0]["request_id"], "req-3") + + # ------------------------------------------------------------------ # + # per-call Authorization header injection + # ------------------------------------------------------------------ # + + def test_injects_authorization_header_from_current_bearer_token(self): + self.vault_client.get_current_bearer_token.return_value = "the-current-token" + self.update_api.with_raw_response.update_records.return_value = fake_update_raw_response([]) + + self.vault.update(UpdateRequest(records=[{"skyflow_id": "id1", "data": {"a": 1}}], table_name="t1")) + + _, kwargs = self.update_api.with_raw_response.update_records.call_args + headers = kwargs["request_options"]["additional_headers"] + self.assertEqual(headers.get("Authorization"), "Bearer the-current-token") + + +def fake_delete_raw_response(records, headers=None): + return SimpleNamespace(data=SimpleNamespace(records=records), headers=headers or {}) + + +class TestVaultDelete(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.delete_api = MagicMock() + self.vault_client.get_records_api.return_value = self.delete_api + self.vault = VaultController(self.vault_client) + + # ------------------------------------------------------------------ # + # validation / initialization sequencing + # ------------------------------------------------------------------ # + + @patch("skyflow_flowvault.vault.controller._vault.validate_delete_request") + def test_delete_validates_before_initializing_client(self, mock_validate): + self.delete_api.with_raw_response.delete_records.return_value = fake_delete_raw_response([]) + request = DeleteRequest(table="t1", ids=["id1"]) + + self.vault.delete(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_delete_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.delete(DeleteRequest(table="t1")) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_delete_raises_on_invalid_table_name(self): + with self.assertRaises(SkyflowError): + self.vault.delete(DeleteRequest(table=" ", ids=["id1"])) + self.delete_api.with_raw_response.delete_records.assert_not_called() + + # ------------------------------------------------------------------ # + # request -> wire field mapping + # ------------------------------------------------------------------ # + + def test_maps_table_and_ids(self): + self.delete_api.with_raw_response.delete_records.return_value = fake_delete_raw_response([]) + + self.vault.delete(DeleteRequest(table="t1", ids=["id1", "id2"])) + + _, kwargs = self.delete_api.with_raw_response.delete_records.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(kwargs["table_name"], "t1") + self.assertEqual(kwargs["skyflow_i_ds"], ["id1", "id2"]) + + def test_maps_unique_values(self): + self.delete_api.with_raw_response.delete_records.return_value = fake_delete_raw_response([]) + + self.vault.delete(DeleteRequest(table="t1", unique_values=[{"email": "a@b.com"}])) + + _, kwargs = self.delete_api.with_raw_response.delete_records.call_args + self.assertEqual(len(kwargs["unique_values"]), 1) + self.assertEqual(kwargs["unique_values"][0].data, {"email": "a@b.com"}) + + # ------------------------------------------------------------------ # + # response shape -- unified records list; delete rows carry only skyflow_id/http_code/error + # ------------------------------------------------------------------ # + + def test_successful_record_carries_skyflow_id_and_http_code(self): + self.delete_api.with_raw_response.delete_records.return_value = fake_delete_raw_response([ + FakeDeleteResponseObject(skyflow_id="id1", http_code=200), + ], headers={"x-request-id": "req-1"}) + + response = self.vault.delete(DeleteRequest(table="t1", ids=["id1"])) + + self.assertEqual(len(response.records), 1) + record = response.records[0] + self.assertEqual(record["skyflow_id"], "id1") + self.assertEqual(record["http_code"], 200) + self.assertIsNone(record["error"]) + self.assertNotIn("data", record) + self.assertNotIn("tokens", record) + + def test_success_and_error_records_in_one_list(self): + self.delete_api.with_raw_response.delete_records.return_value = fake_delete_raw_response([ + FakeDeleteResponseObject(skyflow_id="id1", http_code=200), + FakeDeleteResponseObject(error="not found", http_code=404), + ], headers={"x-request-id": "req-2"}) + + response = self.vault.delete(DeleteRequest(table="t1", ids=["id1", "id2"])) + + self.assertEqual(len(response.records), 2) + self.assertEqual(response.records[0]["skyflow_id"], "id1") + self.assertEqual(response.records[1]["error"], "not found") + self.assertEqual(response.records[1]["http_code"], 404) + + # ------------------------------------------------------------------ # + # transport failure + # ------------------------------------------------------------------ # + + def test_transport_exception_marks_every_id_as_an_error(self): + self.delete_api.with_raw_response.delete_records.side_effect = Exception("network blip") + + response = self.vault.delete(DeleteRequest(table="t1", ids=["id1", "id2"])) + + self.assertEqual(len(response.records), 2) + self.assertTrue(all("network blip" in r["error"] for r in response.records)) + + def test_api_error_with_structured_body_splits_into_one_error_per_row(self): + api_error = ApiError( + status_code=404, + headers={"x-request-id": "req-3"}, + body={"records": [{"error": "not found", "httpCode": 404}]}, + ) + self.delete_api.with_raw_response.delete_records.side_effect = api_error + + response = self.vault.delete(DeleteRequest(table="t1", ids=["id1"])) + + self.assertEqual(len(response.records), 1) + self.assertEqual(response.records[0]["error"], "not found") + self.assertEqual(response.records[0]["http_code"], 404) + + # ------------------------------------------------------------------ # + # per-call Authorization header injection + # ------------------------------------------------------------------ # + + def test_injects_authorization_header_from_current_bearer_token(self): + self.vault_client.get_current_bearer_token.return_value = "the-current-token" + self.delete_api.with_raw_response.delete_records.return_value = fake_delete_raw_response([]) + + self.vault.delete(DeleteRequest(table="t1", ids=["id1"])) + + _, kwargs = self.delete_api.with_raw_response.delete_records.call_args + headers = kwargs["request_options"]["additional_headers"] + self.assertEqual(headers.get("Authorization"), "Bearer the-current-token") + + +def fake_detokenize_raw_response(response, headers=None): + return SimpleNamespace(data=SimpleNamespace(response=response), headers=headers or {}) + + +class TestVaultDetokenize(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.detokenize_api = MagicMock() + self.vault_client.get_tokens_api.return_value = self.detokenize_api + self.vault = VaultController(self.vault_client) + + # ------------------------------------------------------------------ # + # validation / initialization sequencing + # ------------------------------------------------------------------ # + + @patch("skyflow_flowvault.vault.controller._vault.validate_detokenize_request") + def test_detokenize_validates_before_initializing_client(self, mock_validate): + self.detokenize_api.with_raw_response.detokenize.return_value = fake_detokenize_raw_response([]) + request = DetokenizeRequest(tokens=["tok1"]) + + self.vault.detokenize(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_detokenize_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.detokenize(DetokenizeRequest(tokens=[])) + self.vault_client.initialize_client_configuration.assert_not_called() + + # ------------------------------------------------------------------ # + # request -> wire field mapping + # ------------------------------------------------------------------ # + + def test_maps_tokens(self): + self.detokenize_api.with_raw_response.detokenize.return_value = fake_detokenize_raw_response([]) + + self.vault.detokenize(DetokenizeRequest(tokens=["tok1", "tok2"])) + + _, kwargs = self.detokenize_api.with_raw_response.detokenize.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(kwargs["tokens"], ["tok1", "tok2"]) + + def test_maps_token_group_redactions(self): + self.detokenize_api.with_raw_response.detokenize.return_value = fake_detokenize_raw_response([]) + + self.vault.detokenize(DetokenizeRequest( + tokens=["tok1"], token_group_redactions=[{"token_group_name": "g1", "redaction": "mask1"}], + )) + + _, kwargs = self.detokenize_api.with_raw_response.detokenize.call_args + self.assertEqual(len(kwargs["token_group_redactions"]), 1) + self.assertEqual(kwargs["token_group_redactions"][0].token_group_name, "g1") + self.assertEqual(kwargs["token_group_redactions"][0].redaction, "mask1") + + # ------------------------------------------------------------------ # + # response shape -- unified records list; metadata normalized to snake_case + # ------------------------------------------------------------------ # + + def test_successful_record_carries_value_group_and_metadata(self): + self.detokenize_api.with_raw_response.detokenize.return_value = fake_detokenize_raw_response([ + FakeDetokenizeResponseObject( + token="tok1", value="john doe", token_group_name="deterministic_string", + http_code=200, metadata={"skyflowID": "sid", "tableName": "t1"}, + ), + ], headers={"x-request-id": "req-1"}) + + response = self.vault.detokenize(DetokenizeRequest(tokens=["tok1"])) + + self.assertEqual(len(response.records), 1) + record = response.records[0] + self.assertEqual(record["token"], "tok1") + self.assertEqual(record["value"], "john doe") + self.assertEqual(record["token_group_name"], "deterministic_string") + self.assertEqual(record["metadata"], {"skyflow_id": "sid", "table_name": "t1"}) + self.assertEqual(record["http_code"], 200) + self.assertIsNone(record["error"]) + + def test_success_and_error_records_in_one_list(self): + self.detokenize_api.with_raw_response.detokenize.return_value = fake_detokenize_raw_response([ + FakeDetokenizeResponseObject(token="tok1", value="john doe", http_code=200), + FakeDetokenizeResponseObject(token="tok2", error="invalid token", http_code=404), + ], headers={"x-request-id": "req-2"}) + + response = self.vault.detokenize(DetokenizeRequest(tokens=["tok1", "tok2"])) + + self.assertEqual(len(response.records), 2) + self.assertEqual(response.records[0]["value"], "john doe") + self.assertEqual(response.records[1]["token"], "tok2") + self.assertEqual(response.records[1]["error"], "invalid token") + self.assertEqual(response.records[1]["http_code"], 404) + + # ------------------------------------------------------------------ # + # transport failure + # ------------------------------------------------------------------ # + + def test_transport_exception_marks_every_token_as_an_error(self): + self.detokenize_api.with_raw_response.detokenize.side_effect = Exception("network blip") + + response = self.vault.detokenize(DetokenizeRequest(tokens=["tok1", "tok2"])) + + self.assertEqual(len(response.records), 2) + self.assertTrue(all("network blip" in r["error"] for r in response.records)) + + def test_api_error_with_structured_body_splits_into_one_error_per_row(self): + api_error = ApiError( + status_code=404, + headers={"x-request-id": "req-3"}, + body={"records": [{"error": "invalid token", "httpCode": 404}]}, + ) + self.detokenize_api.with_raw_response.detokenize.side_effect = api_error + + response = self.vault.detokenize(DetokenizeRequest(tokens=["tok1"])) + + self.assertEqual(len(response.records), 1) + self.assertEqual(response.records[0]["error"], "invalid token") + self.assertEqual(response.records[0]["http_code"], 404) + + # ------------------------------------------------------------------ # + # per-call Authorization header injection + # ------------------------------------------------------------------ # + + def test_injects_authorization_header_from_current_bearer_token(self): + self.vault_client.get_current_bearer_token.return_value = "the-current-token" + self.detokenize_api.with_raw_response.detokenize.return_value = fake_detokenize_raw_response([]) + + self.vault.detokenize(DetokenizeRequest(tokens=["tok1"])) + + _, kwargs = self.detokenize_api.with_raw_response.detokenize.call_args + headers = kwargs["request_options"]["additional_headers"] + self.assertEqual(headers.get("Authorization"), "Bearer the-current-token") + + +def fake_query_raw_response(records, headers=None, metadata=None): + return SimpleNamespace(data=SimpleNamespace(records=records, metadata=metadata), headers=headers or {}) + + +class TestVaultQuery(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.query_api = MagicMock() + self.vault_client.get_query_api.return_value = self.query_api + self.vault = VaultController(self.vault_client) + + @patch("skyflow_flowvault.vault.controller._vault.validate_query_request") + def test_query_validates_before_initializing_client(self, mock_validate): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response([]) + request = QueryRequest(query="SELECT * FROM t1") + + self.vault.query(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_query_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.query(QueryRequest(query=" ")) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_maps_query(self): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response([]) + + self.vault.query(QueryRequest(query="SELECT * FROM t1 WHERE a = 1")) + + _, kwargs = self.query_api.with_raw_response.execute_query.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(kwargs["query"], "SELECT * FROM t1 WHERE a = 1") + + def test_records_carry_data_and_metadata_columns(self): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response( + [FakeExecuteQueryRecord(data={"a": 1}), FakeExecuteQueryRecord(data={"a": 2})], + headers={"x-request-id": "req-1"}, + metadata=SimpleNamespace(columns=["a"]), + ) + + response = self.vault.query(QueryRequest(query="SELECT * FROM t1")) + + self.assertEqual(len(response.records), 2) + self.assertEqual(response.records[0], {"data": {"a": 1}}) + self.assertEqual(response.records[1], {"data": {"a": 2}}) + self.assertEqual(response.metadata, {"columns": ["a"]}) + + def test_transport_exception_produces_a_single_error_record(self): + self.query_api.with_raw_response.execute_query.side_effect = Exception("network blip") + + response = self.vault.query(QueryRequest(query="SELECT * FROM t1")) + + self.assertEqual(len(response.records), 1) + self.assertIn("network blip", response.records[0]["error"]) + self.assertIsNone(response.metadata) + + def test_api_error_with_flat_body_surfaces_the_error(self): + api_error = ApiError(status_code=400, headers={"x-request-id": "req-3"}, body={"error": "bad query"}) + self.query_api.with_raw_response.execute_query.side_effect = api_error + + response = self.vault.query(QueryRequest(query="SELECT bad")) + + self.assertEqual(len(response.records), 1) + self.assertEqual(response.records[0]["error"], "bad query") + self.assertEqual(response.records[0]["http_code"], 400) + + def test_injects_authorization_header_from_current_bearer_token(self): + self.vault_client.get_current_bearer_token.return_value = "the-current-token" + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response([]) + + self.vault.query(QueryRequest(query="SELECT * FROM t1")) + + _, kwargs = self.query_api.with_raw_response.execute_query.call_args + headers = kwargs["request_options"]["additional_headers"] + self.assertEqual(headers.get("Authorization"), "Bearer the-current-token") + + +def fake_bulk_insert_call(**kwargs): + records = [ + FakeRecordResponseObject(skyflow_id=f"id-{i}", http_code=200) + for i in range(len(kwargs["records"])) + ] + return FakeRawResponse(records, headers={"x-request-id": "req"}) + + +def fake_bulk_detokenize_call(**kwargs): + response = [ + FakeDetokenizeResponseObject(token=t, value=f"v-{t}", token_group_name="g", http_code=200) + for t in kwargs["tokens"] + ] + return SimpleNamespace(data=SimpleNamespace(response=response), headers={"x-request-id": "req"}) + + +class TestVaultBulkInsert(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.records_api = MagicMock() + self.vault_client.get_records_api.return_value = self.records_api + self.vault = VaultController(self.vault_client) + env = patch.dict(os.environ, {"INSERT_BATCH_SIZE": "2", "INSERT_CONCURRENCY_LIMIT": "1"}) + env.start() + self.addCleanup(env.stop) + + def _request(self, n): + return BulkInsertRequest(records=[BulkInsertRecord(data={"a": i}) for i in range(n)], table="t1") + + def test_splits_into_batches_and_merges_in_order(self): + self.records_api.with_raw_response.insert_records.side_effect = fake_bulk_insert_call + + response = self.vault.bulk_insert(self._request(3)) + + self.assertEqual(self.records_api.with_raw_response.insert_records.call_count, 2) + self.assertEqual([r["index"] for r in response.records], [0, 1, 2]) + self.assertEqual(response.summary.total_records, 3) + self.assertEqual(response.summary.total_inserted, 3) + self.assertEqual(response.summary.total_failed, 0) + + def test_large_payload_indexing_is_contiguous_and_aligned_under_concurrency(self): + # Each wire record's data['a'] IS its original input index, so the fake echoes it back + # as skyflow_id -- letting us assert every merged record's index lines up with the exact + # input it came from, even across 10 concurrent batches completing in any order. + def side_effect(**kwargs): + return FakeRawResponse( + [FakeRecordResponseObject(skyflow_id=f"id-{rec.data['a']}", http_code=200) for rec in kwargs["records"]] + ) + + self.records_api.with_raw_response.insert_records.side_effect = side_effect + request = BulkInsertRequest(records=[BulkInsertRecord(data={"a": i}) for i in range(500)], table="t1") + + with patch.dict(os.environ, {"INSERT_BATCH_SIZE": "50", "INSERT_CONCURRENCY_LIMIT": "10"}): + response = self.vault.bulk_insert(request) + + self.assertEqual(self.records_api.with_raw_response.insert_records.call_count, 10) # 500 / 50 + self.assertEqual([r["index"] for r in response.records], list(range(500))) # contiguous, in order, no gaps/dupes + self.assertTrue(all(r["skyflow_id"] == f"id-{r['index']}" for r in response.records)) # index aligns with input + self.assertEqual(response.summary.total_records, 500) + self.assertEqual(response.summary.total_inserted, 500) + self.assertEqual(response.summary.total_failed, 0) + + def test_large_payload_failing_middle_batch_keeps_correct_indices(self): + # The batch covering indices 200..249 fails wholesale; every other batch succeeds. + def side_effect(**kwargs): + if kwargs["records"][0].data["a"] == 200: + raise ApiError(status_code=500, headers={"x-request-id": "req-err"}, body={"error": "boom"}) + return FakeRawResponse( + [FakeRecordResponseObject(skyflow_id=f"id-{rec.data['a']}", http_code=200) for rec in kwargs["records"]] + ) + + self.records_api.with_raw_response.insert_records.side_effect = side_effect + request = BulkInsertRequest(records=[BulkInsertRecord(data={"a": i}) for i in range(500)], table="t1") + + with patch.dict(os.environ, {"INSERT_BATCH_SIZE": "50", "INSERT_CONCURRENCY_LIMIT": "10"}): + response = self.vault.bulk_insert(request) + + self.assertEqual([r["index"] for r in response.records], list(range(500))) + failed = [r["index"] for r in response.records if r["error"] is not None] + self.assertEqual(failed, list(range(200, 250))) # exactly the failed batch's indices + self.assertTrue(all(response.records[i]["http_code"] == 500 for i in range(200, 250))) + self.assertEqual(response.summary.total_failed, 50) + self.assertEqual(response.summary.total_inserted, 450) + # 500 is retryable -> exactly the original records at those indices come back, in order + retry = response.records_to_retry() + self.assertEqual([r.data["a"] for r in retry], list(range(200, 250))) + + def test_tokens_and_hashed_data_are_normalized(self): + self.records_api.with_raw_response.insert_records.return_value = FakeRawResponse([ + FakeRecordResponseObject( + skyflow_id="id0", + tokens={"ssn": [{"token": "t1", "tokenGroupName": "g1", "path": "p"}]}, + hashed_data={"ssn": [{"data": "h", "hashName": "hash1"}]}, + http_code=200, + ), + ], headers={"x-request-id": "req"}) + + response = self.vault.bulk_insert(self._request(1)) + + record = response.records[0] + self.assertEqual(record["tokens"], {"ssn": [{"token": "t1", "token_group_name": "g1", "path": "p"}]}) + self.assertEqual(record["hashed_data"], {"ssn": [{"data": "h", "hash_name": "hash1"}]}) + + def test_failed_batch_marks_its_records_and_reports_summary(self): + calls = {"n": 0} + + def side_effect(**kwargs): + calls["n"] += 1 + if calls["n"] == 2: + raise ApiError(status_code=500, headers={"x-request-id": "req-err"}, body={"error": "boom"}) + return fake_bulk_insert_call(**kwargs) + + self.records_api.with_raw_response.insert_records.side_effect = side_effect + + response = self.vault.bulk_insert(self._request(3)) + + self.assertEqual(response.summary.total_records, 3) + self.assertEqual(response.summary.total_inserted, 2) + self.assertEqual(response.summary.total_failed, 1) + failed = [r for r in response.records if r["error"] is not None] + self.assertEqual(len(failed), 1) + self.assertEqual(failed[0]["index"], 2) + self.assertEqual(failed[0]["http_code"], 500) + self.assertEqual(failed[0]["request_id"], "req-err") + # 500 is retryable -> the original record at index 2 comes back + retry = response.records_to_retry() + self.assertEqual(len(retry), 1) + self.assertEqual(retry[0].data, {"a": 2}) + + def test_client_error_batch_is_not_retryable(self): + def side_effect(**kwargs): + raise ApiError(status_code=400, headers={}, body={"error": "bad"}) + + self.records_api.with_raw_response.insert_records.side_effect = side_effect + + response = self.vault.bulk_insert(self._request(2)) + self.assertEqual(response.summary.total_failed, 2) + self.assertEqual(response.records_to_retry(), []) + + def test_validation_error_raises_without_api_call(self): + with self.assertRaises(SkyflowError): + self.vault.bulk_insert(BulkInsertRequest(records=[], table="t1")) + self.records_api.with_raw_response.insert_records.assert_not_called() + + def test_injects_authorization_header(self): + self.vault_client.get_current_bearer_token.return_value = "the-token" + self.records_api.with_raw_response.insert_records.side_effect = fake_bulk_insert_call + + self.vault.bulk_insert(self._request(1)) + + _, kwargs = self.records_api.with_raw_response.insert_records.call_args + headers = kwargs["request_options"]["additional_headers"] + self.assertEqual(headers.get("Authorization"), "Bearer the-token") + + +class TestVaultBulkDetokenize(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.tokens_api = MagicMock() + self.vault_client.get_tokens_api.return_value = self.tokens_api + self.vault = VaultController(self.vault_client) + env = patch.dict(os.environ, {"DETOKENIZE_BATCH_SIZE": "2", "DETOKENIZE_CONCURRENCY_LIMIT": "1"}) + env.start() + self.addCleanup(env.stop) + + def test_splits_into_batches_and_merges_in_order(self): + self.tokens_api.with_raw_response.detokenize.side_effect = fake_bulk_detokenize_call + + response = self.vault.bulk_detokenize(BulkDetokenizeRequest(tokens=["t0", "t1", "t2"])) + + self.assertEqual(self.tokens_api.with_raw_response.detokenize.call_count, 2) + self.assertEqual([r["index"] for r in response.records], [0, 1, 2]) + self.assertEqual(response.records[0]["token"], "t0") + self.assertEqual(response.records[0]["value"], "v-t0") + self.assertEqual(response.summary.total_tokens, 3) + self.assertEqual(response.summary.total_detokenized, 3) + self.assertEqual(response.summary.total_failed, 0) + + def test_large_payload_indexing_is_contiguous_and_aligned_under_concurrency(self): + self.tokens_api.with_raw_response.detokenize.side_effect = fake_bulk_detokenize_call + tokens = [f"t{i}" for i in range(300)] + + with patch.dict(os.environ, {"DETOKENIZE_BATCH_SIZE": "50", "DETOKENIZE_CONCURRENCY_LIMIT": "10"}): + response = self.vault.bulk_detokenize(BulkDetokenizeRequest(tokens=tokens)) + + self.assertEqual(self.tokens_api.with_raw_response.detokenize.call_count, 6) # 300 / 50 + self.assertEqual([r["index"] for r in response.records], list(range(300))) + # each merged record's token/value line up with its original input position + self.assertTrue(all(r["token"] == f"t{r['index']}" for r in response.records)) + self.assertTrue(all(r["value"] == f"v-t{r['index']}" for r in response.records)) + self.assertEqual(response.summary.total_tokens, 300) + self.assertEqual(response.summary.total_detokenized, 300) + + def test_failed_batch_is_retryable_on_5xx(self): + calls = {"n": 0} + + def side_effect(**kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise ApiError(status_code=503, headers={"x-request-id": "req-err"}, body={"error": "boom"}) + return fake_bulk_detokenize_call(**kwargs) + + self.tokens_api.with_raw_response.detokenize.side_effect = side_effect + + response = self.vault.bulk_detokenize(BulkDetokenizeRequest(tokens=["t0", "t1", "t2"])) + + self.assertEqual(response.summary.total_failed, 2) # first batch (t0, t1) failed + self.assertEqual(sorted(response.tokens_to_retry()), ["t0", "t1"]) + + def test_validation_error_raises_without_api_call(self): + with self.assertRaises(SkyflowError): + self.vault.bulk_detokenize(BulkDetokenizeRequest(tokens=[])) + self.tokens_api.with_raw_response.detokenize.assert_not_called() + + +class TestVaultBulkInsertAsync(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.records_api = MagicMock() + self.records_api.with_raw_response.insert_records = AsyncMock(side_effect=fake_bulk_insert_call) + self.vault_client.get_async_records_api.return_value = self.records_api + self.vault = VaultController(self.vault_client) + env = patch.dict(os.environ, {"INSERT_BATCH_SIZE": "2", "INSERT_CONCURRENCY_LIMIT": "2"}) + env.start() + self.addCleanup(env.stop) + + async def test_bulk_insert_async_batches_and_merges(self): + request = BulkInsertRequest(records=[BulkInsertRecord(data={"a": i}) for i in range(3)], table="t1") + + response = await self.vault.bulk_insert_async(request) + + self.assertEqual(self.records_api.with_raw_response.insert_records.await_count, 2) + self.assertEqual([r["index"] for r in response.records], [0, 1, 2]) + self.assertEqual(response.summary.total_records, 3) + self.assertEqual(response.summary.total_inserted, 3) + + async def test_large_payload_indexing_survives_out_of_order_completion(self): + # Earlier batches sleep longest, so batches COMPLETE in reverse order -- proves the merge + # is by submission order (index), not completion order, across 10 concurrent batches. + async def side_effect(**kwargs): + first = kwargs["records"][0].data["a"] + await asyncio.sleep((500 - first) / 100000.0) + return FakeRawResponse( + [FakeRecordResponseObject(skyflow_id=f"id-{rec.data['a']}", http_code=200) for rec in kwargs["records"]] + ) + + self.records_api.with_raw_response.insert_records = AsyncMock(side_effect=side_effect) + self.vault_client.get_async_records_api.return_value = self.records_api + request = BulkInsertRequest(records=[BulkInsertRecord(data={"a": i}) for i in range(500)], table="t1") + + with patch.dict(os.environ, {"INSERT_BATCH_SIZE": "50", "INSERT_CONCURRENCY_LIMIT": "10"}): + response = await self.vault.bulk_insert_async(request) + + self.assertEqual(self.records_api.with_raw_response.insert_records.await_count, 10) + self.assertEqual([r["index"] for r in response.records], list(range(500))) + self.assertTrue(all(r["skyflow_id"] == f"id-{r['index']}" for r in response.records)) + self.assertEqual(response.summary.total_inserted, 500) + + +class TestVaultBulkDetokenizeAsync(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.tokens_api = MagicMock() + self.tokens_api.with_raw_response.detokenize = AsyncMock(side_effect=fake_bulk_detokenize_call) + self.vault_client.get_async_tokens_api.return_value = self.tokens_api + self.vault = VaultController(self.vault_client) + env = patch.dict(os.environ, {"DETOKENIZE_BATCH_SIZE": "2", "DETOKENIZE_CONCURRENCY_LIMIT": "2"}) + env.start() + self.addCleanup(env.stop) + + async def test_bulk_detokenize_async_batches_and_merges(self): + response = await self.vault.bulk_detokenize_async(BulkDetokenizeRequest(tokens=["t0", "t1", "t2"])) + + self.assertEqual(self.tokens_api.with_raw_response.detokenize.await_count, 2) + self.assertEqual([r["index"] for r in response.records], [0, 1, 2]) + self.assertEqual(response.summary.total_tokens, 3) + self.assertEqual(response.summary.total_detokenized, 3) + + if __name__ == "__main__": unittest.main() diff --git a/flowvault/tests/vault/data/test_data_classes.py b/flowvault/tests/vault/data/test_data_classes.py index 33b274ca..c892368b 100644 --- a/flowvault/tests/vault/data/test_data_classes.py +++ b/flowvault/tests/vault/data/test_data_classes.py @@ -1,56 +1,347 @@ import unittest -from common.vault.data import BaseInsertRequest, BaseInsertResponse from skyflow_flowvault.utils.enums import UpsertType -from skyflow_flowvault.vault.data import InsertRequest, InsertResponse +from skyflow_flowvault.vault.data import ( + UpsertOptions, + ColumnRedaction, + InsertRequestRecord, + InsertRequest, + InsertResponse, + GetRequest, + GetResponse, + UpdateRequest, + UpdateResponse, + DeleteRequest, + DeleteResponse, + DetokenizeRequest, + DetokenizeResponse, + QueryRequest, + QueryResponse, + GetRecordRequest, + BulkInsertRecord, + BulkInsertRequest, + BulkInsertResponse, + BulkSummary, + BulkDetokenizeRequest, + BulkDetokenizeResponse, + DetokenizeSummary, +) class TestInsertRequest(unittest.TestCase): - def test_is_a_base_insert_request(self): - request = InsertRequest(values=[{"values": {"a": 1}}], table="t1") - self.assertIsInstance(request, BaseInsertRequest) - self.assertEqual(request.table, "t1") + def test_records_and_table_stored(self): + records = [InsertRequestRecord(data={"a": 1})] + request = InsertRequest(records=records, table_name="t1") + self.assertIs(request.records, records) + self.assertEqual(request.table_name, "t1") - def test_records_are_plain_dicts_supporting_per_record_overrides(self): - upsert = {"update_type": UpsertType.REPLACE, "unique_columns": ["a"]} - record = {"values": {"a": 1}, "table": "t2", "upsert": upsert} - request = InsertRequest(values=[record]) - self.assertEqual(request.values[0]["values"], {"a": 1}) - self.assertEqual(request.values[0]["table"], "t2") - self.assertIs(request.values[0]["upsert"], upsert) + def test_record_fields_and_per_record_overrides(self): + upsert = UpsertOptions(update_type=UpsertType.REPLACE, unique_columns=["a"]) + record = InsertRequestRecord(data={"a": 1}, table_name="t2", tokens={"a": "tok"}, upsert=upsert) + request = InsertRequest(records=[record]) + self.assertEqual(request.records[0].data, {"a": 1}) + self.assertEqual(request.records[0].table_name, "t2") + self.assertEqual(request.records[0].tokens, {"a": "tok"}) + self.assertIs(request.records[0].upsert, upsert) def test_table_and_upsert_are_optional_defaults(self): - request = InsertRequest(values=[{"values": {"a": 1}}]) - self.assertIsNone(request.table) + request = InsertRequest(records=[InsertRequestRecord(data={"a": 1})]) + self.assertIsNone(request.table_name) self.assertIsNone(request.upsert) - def test_no_v2_only_fields_exist(self): - request = InsertRequest(values=[{"values": {"a": 1}}]) - for legacy_field in ("tokens", "homogeneous", "continue_on_error", "token_mode", "return_tokens"): - self.assertFalse(hasattr(request, legacy_field), f"v3 InsertRequest should not have '{legacy_field}'") + def test_record_optional_defaults(self): + record = InsertRequestRecord(data={"a": 1}) + self.assertIsNone(record.table_name) + self.assertIsNone(record.tokens) + self.assertIsNone(record.upsert) + + +class TestUpsertOptions(unittest.TestCase): + def test_fields_stored(self): + opts = UpsertOptions(unique_columns=["email"], update_type=UpsertType.UPDATE) + self.assertEqual(opts.unique_columns, ["email"]) + self.assertEqual(opts.update_type, UpsertType.UPDATE) + + def test_update_type_optional(self): + opts = UpsertOptions(unique_columns=["email"]) + self.assertIsNone(opts.update_type) + + +class TestColumnRedaction(unittest.TestCase): + def test_fields_stored(self): + cr = ColumnRedaction(column_name="email", redaction="MASKED") + self.assertEqual(cr.column_name, "email") + self.assertEqual(cr.redaction, "MASKED") class TestInsertResponse(unittest.TestCase): - """Shared shape with PDB's InsertResponse -- inserted_fields/errors, each entry tagged - request_index -- plain dicts/list-of-dicts, not custom classes.""" + def test_shape(self): + records = [{"skyflow_id": "id1", "http_code": 200, "error": None}] + response = InsertResponse(records=records) + self.assertIs(response.records, records) + + def test_defaults(self): + response = InsertResponse() + self.assertIsNone(response.records) + def test_repr_does_not_raise(self): + response = InsertResponse(records=[{"skyflow_id": "id1"}]) + self.assertIn("InsertResponse", repr(response)) + self.assertIn("InsertResponse", str(response)) + + +class TestGetRequest(unittest.TestCase): + def test_required_and_optional_defaults(self): + request = GetRequest(table="t1", ids=["id1"]) + self.assertEqual(request.table, "t1") + self.assertEqual(request.ids, ["id1"]) + self.assertIsNone(request.unique_values) + self.assertIsNone(request.columns) + self.assertIsNone(request.column_redactions) + self.assertIsNone(request.limit) + self.assertIsNone(request.offset) + + def test_all_fields_stored(self): + request = GetRequest( + table="t1", ids=["id1"], unique_values=[{"email": "a@b.com"}], columns=["a", "b"], + column_redactions=[ColumnRedaction(column_name="a", redaction="mask1")], limit=10, offset=5, + ) + self.assertEqual(request.unique_values, [{"email": "a@b.com"}]) + self.assertEqual(request.columns, ["a", "b"]) + self.assertEqual(request.column_redactions[0].column_name, "a") + self.assertEqual(request.column_redactions[0].redaction, "mask1") + self.assertEqual(request.limit, 10) + self.assertEqual(request.offset, 5) + + +class TestGetResponse(unittest.TestCase): def test_shape(self): - inserted_fields = [{"request_index": 0, "skyflow_id": "id1"}] - response = InsertResponse(inserted_fields=inserted_fields, errors=[]) + records = [{"skyflow_id": "id1", "data": {"a": 1}, "http_code": 200}] + response = GetResponse(records=records) + self.assertIs(response.records, records) + + def test_defaults(self): + response = GetResponse() + self.assertIsNone(response.records) + + def test_repr_and_str_do_not_raise(self): + response = GetResponse(records=[]) + self.assertIn("GetResponse", repr(response)) + self.assertIn("GetResponse", str(response)) - self.assertIs(response.inserted_fields, inserted_fields) + +class TestUpdateRequest(unittest.TestCase): + def test_required_and_optional_defaults(self): + request = UpdateRequest(records=[{"skyflow_id": "id1", "data": {"a": 1}}]) + self.assertEqual(request.records, [{"skyflow_id": "id1", "data": {"a": 1}}]) + self.assertIsNone(request.table_name) + self.assertIsNone(request.update_type) + + def test_all_fields_stored(self): + request = UpdateRequest( + records=[{"skyflow_id": "id1", "data": {"a": 1}, "tokens": {"a": "tok"}, "table_name": "t2"}], + table_name="t1", update_type=UpsertType.REPLACE, + ) + self.assertEqual(request.table_name, "t1") + self.assertEqual(request.update_type, UpsertType.REPLACE) + self.assertEqual(request.records[0]["tokens"], {"a": "tok"}) + + +class TestUpdateResponse(unittest.TestCase): + def test_shape(self): + records = [{"request_index": 0, "skyflow_id": "id1"}] + response = UpdateResponse(records=records, errors=[]) + self.assertIs(response.records, records) self.assertEqual(response.errors, []) - def test_is_a_base_insert_response(self): - response = InsertResponse(inserted_fields=[], errors=None) - self.assertIsInstance(response, BaseInsertResponse) + def test_defaults(self): + response = UpdateResponse() + self.assertIsNone(response.records) + self.assertIsNone(response.errors) - def test_repr_does_not_raise(self): - response = InsertResponse( - inserted_fields=[], - errors=[{"request_index": 0, "error": "boom", "code": 500, "request_id": None}], + def test_repr_and_str_do_not_raise(self): + response = UpdateResponse(records=[], errors=[{"request_index": 0, "error": "boom"}]) + self.assertIn("UpdateResponse", repr(response)) + self.assertIn("UpdateResponse", str(response)) + + +class TestDeleteRequest(unittest.TestCase): + def test_required_and_optional_defaults(self): + request = DeleteRequest(table="t1", ids=["id1"]) + self.assertEqual(request.table, "t1") + self.assertEqual(request.ids, ["id1"]) + self.assertIsNone(request.unique_values) + + def test_unique_values_stored(self): + request = DeleteRequest(table="t1", unique_values=[{"email": "a@b.com"}]) + self.assertEqual(request.unique_values, [{"email": "a@b.com"}]) + + +class TestDeleteResponse(unittest.TestCase): + def test_shape(self): + records = [{"skyflow_id": "id1", "http_code": 200, "error": None}] + response = DeleteResponse(records=records) + self.assertIs(response.records, records) + + def test_defaults(self): + response = DeleteResponse() + self.assertIsNone(response.records) + + def test_repr_and_str_do_not_raise(self): + response = DeleteResponse(records=[]) + self.assertIn("DeleteResponse", repr(response)) + self.assertIn("DeleteResponse", str(response)) + + +class TestDetokenizeRequest(unittest.TestCase): + def test_required_and_optional_defaults(self): + request = DetokenizeRequest(tokens=["tok1", "tok2"]) + self.assertEqual(request.tokens, ["tok1", "tok2"]) + self.assertIsNone(request.token_group_redactions) + + def test_token_group_redactions_stored(self): + request = DetokenizeRequest( + tokens=["tok1"], token_group_redactions=[{"token_group_name": "g1", "redaction": "mask1"}], ) - self.assertIn("InsertResponse", repr(response)) + self.assertEqual(request.token_group_redactions, [{"token_group_name": "g1", "redaction": "mask1"}]) + + +class TestDetokenizeResponse(unittest.TestCase): + def test_shape(self): + records = [{"token": "tok1", "value": "john", "http_code": 200, "error": None}] + response = DetokenizeResponse(records=records) + self.assertIs(response.records, records) + + def test_defaults(self): + response = DetokenizeResponse() + self.assertIsNone(response.records) + + def test_repr_and_str_do_not_raise(self): + response = DetokenizeResponse(records=[]) + self.assertIn("DetokenizeResponse", repr(response)) + self.assertIn("DetokenizeResponse", str(response)) + + +class TestQueryRequest(unittest.TestCase): + def test_query_stored(self): + request = QueryRequest(query="SELECT * FROM t1") + self.assertEqual(request.query, "SELECT * FROM t1") + + +class TestQueryResponse(unittest.TestCase): + def test_shape(self): + records = [{"data": {"a": 1}}] + response = QueryResponse(records=records, metadata={"columns": ["a"]}) + self.assertIs(response.records, records) + self.assertEqual(response.metadata, {"columns": ["a"]}) + + def test_defaults(self): + response = QueryResponse() + self.assertIsNone(response.records) + self.assertIsNone(response.metadata) + + def test_repr_and_str_do_not_raise(self): + response = QueryResponse(records=[], metadata=None) + self.assertIn("QueryResponse", repr(response)) + self.assertIn("QueryResponse", str(response)) + + +class TestGetRecordRequest(unittest.TestCase): + def test_fields_stored(self): + record = GetRecordRequest(table="t1", ids=["id1"], columns=["a"], + column_redactions=[ColumnRedaction(column_name="a", redaction="MASKED")], + unique_values=[{"email": "a@b.com"}]) + self.assertEqual(record.table, "t1") + self.assertEqual(record.ids, ["id1"]) + self.assertEqual(record.columns, ["a"]) + self.assertEqual(record.column_redactions[0].column_name, "a") + self.assertEqual(record.column_redactions[0].redaction, "MASKED") + self.assertEqual(record.unique_values, [{"email": "a@b.com"}]) + + def test_optional_defaults(self): + record = GetRecordRequest(table="t1") + self.assertIsNone(record.ids) + self.assertIsNone(record.columns) + self.assertIsNone(record.column_redactions) + self.assertIsNone(record.unique_values) + + +class TestBulkInsertRecord(unittest.TestCase): + def test_fields_stored(self): + record = BulkInsertRecord(data={"a": 1}, table="t1", upsert=UpsertOptions(unique_columns=["a"])) + self.assertEqual(record.data, {"a": 1}) + self.assertEqual(record.table, "t1") + self.assertEqual(record.upsert.unique_columns, ["a"]) + + def test_optional_defaults(self): + record = BulkInsertRecord(data={"a": 1}) + self.assertIsNone(record.table) + self.assertIsNone(record.upsert) + + +class TestBulkInsertRequest(unittest.TestCase): + def test_fields_stored(self): + records = [BulkInsertRecord(data={"a": 1})] + request = BulkInsertRequest(records=records, table="t1") + self.assertIs(request.records, records) + self.assertEqual(request.table, "t1") + self.assertIsNone(request.upsert) + + +class TestBulkSummary(unittest.TestCase): + def test_fields_and_repr(self): + summary = BulkSummary(total_records=3, total_inserted=2, total_failed=1) + self.assertEqual((summary.total_records, summary.total_inserted, summary.total_failed), (3, 2, 1)) + self.assertIn("BulkSummary", repr(summary)) + + +class TestBulkInsertResponse(unittest.TestCase): + def test_records_to_retry_only_server_5xx_except_529(self): + records = [ + {"index": 0, "http_code": 200}, + {"index": 1, "http_code": 500}, + {"index": 2, "http_code": 529}, + {"index": 3, "http_code": 400}, + {"index": 4, "http_code": 503}, + ] + originals = ["r0", "r1", "r2", "r3", "r4"] + response = BulkInsertResponse(summary=None, records=records, _original_records=originals) + self.assertEqual(response.records_to_retry(), ["r1", "r4"]) + + def test_records_to_retry_empty_without_originals(self): + response = BulkInsertResponse(summary=None, records=[{"index": 0, "http_code": 500}]) + self.assertEqual(response.records_to_retry(), []) + + def test_repr_does_not_raise(self): + self.assertIn("BulkInsertResponse", repr(BulkInsertResponse(summary=BulkSummary(), records=[]))) + + +class TestBulkDetokenizeRequest(unittest.TestCase): + def test_fields_stored(self): + request = BulkDetokenizeRequest(tokens=["t1", "t2"], token_group_redactions=[{"token_group_name": "g", "redaction": "MASKED"}]) + self.assertEqual(request.tokens, ["t1", "t2"]) + self.assertEqual(request.token_group_redactions, [{"token_group_name": "g", "redaction": "MASKED"}]) + + +class TestDetokenizeSummary(unittest.TestCase): + def test_fields_and_repr(self): + summary = DetokenizeSummary(total_tokens=2, total_detokenized=1, total_failed=1) + self.assertEqual((summary.total_tokens, summary.total_detokenized, summary.total_failed), (2, 1, 1)) + self.assertIn("DetokenizeSummary", repr(summary)) + + +class TestBulkDetokenizeResponse(unittest.TestCase): + def test_tokens_to_retry_only_server_5xx_except_529(self): + records = [ + {"index": 0, "http_code": 200}, + {"index": 1, "http_code": 500}, + {"index": 2, "http_code": 529}, + ] + response = BulkDetokenizeResponse(summary=None, records=records, _original_tokens=["a", "b", "c"]) + self.assertEqual(response.tokens_to_retry(), ["b"]) + + def test_repr_does_not_raise(self): + self.assertIn("BulkDetokenizeResponse", repr(BulkDetokenizeResponse(summary=DetokenizeSummary(), records=[]))) if __name__ == "__main__": diff --git a/skyvault/MANIFEST.in b/skyvault/MANIFEST.in new file mode 100644 index 00000000..05007153 --- /dev/null +++ b/skyvault/MANIFEST.in @@ -0,0 +1 @@ +prune samples diff --git a/skyvault/README.md b/skyvault/README.md new file mode 100644 index 00000000..79700e31 --- /dev/null +++ b/skyvault/README.md @@ -0,0 +1,1016 @@ +# Skyflow Python SDK + +> **This is the current, recommended version of the Skyflow SDK.** V2.1.0 brings flexible auth, multi-vault support, native data types, and rich error diagnostics. +> +> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 is in maintenance mode and will reach End of Life on October 31, 2026. + +The Skyflow Python SDK is designed to help with integrating Skyflow into a Python backend. + +## Table of Contents + +- [Skyflow Python SDK](#skyflow-python-sdk) + - [Table of Contents](#table-of-contents) + - [Overview](#overview) + - [Installation](#installation) + - [Require](#require) + - [Configuration](#configuration) + - [Quickstart](#quickstart) + - [Authenticate](#authenticate) + - [API Key](#api-key) + - [Bearer Token (static)](#bearer-token-static) + - [Initialize the client](#initialize-the-client) + - [Insert data into the vault, get tokens back](#insert-data-into-the-vault-get-tokens-back) + - [Upgrade from v1 to v2](#upgrade-from-v1-to-v2) + - [Vault](#vault) + - [Insert and tokenize data: `.insert(request)`](#insert-and-tokenize-data-insertrequest) + - [Insert example with `continue_on_error` option](#insert-example-with-continue_on_error-option) + - [Upsert request](#upsert-request) + - [Detokenize: `.detokenize(request, options)`](#detokenize-detokenizerequest-options) + - [Construct a detokenize request](#construct-a-detokenize-request) + - [Get Record(s): `.get(request)`](#get-records-getrequest) + - [Construct a get request](#construct-a-get-request) + - [Get by Skyflow IDs](#get-by-skyflow-ids) + - [Get tokens for records](#get-tokens-for-records) + - [Get by column name and column values](#get-by-column-name-and-column-values) + - [Redaction Types](#redaction-types) + - [Update Records](#update-records) + - [Construct an update request](#construct-an-update-request) + - [Delete Records](#delete-records) + - [Query](#query) + - [Upload File](#upload-file) + - [Retrieve Existing Tokens: `.tokenize(request)`](#retrieve-existing-tokens-tokenizerequest) + - [Construct a `.tokenize()` request](#construct-a-tokenize-request) + - [Detect](#detect) + - [De-identify Text: `.deidentify_text(request)`](#de-identify-text-deidentify_textrequest) + - [Re-identify Text: `.reidentify_text(request)`](#re-identify-text-reidentify_textrequest) + - [De-identify File: `.deidentify_file(request)`](#de-identify-file-deidentify_filerequest) + - [Get Run: `.get_detect_run(request)`](#get-run-get_detect_runrequest) + - [Connections](#connections) + - [Invoke a connection](#invoke-a-connection) + - [Construct an invoke connection request](#construct-an-invoke-connection-request) + - [Authentication & authorization](#authentication--authorization) + - [Types of `credentials`](#types-of-credentials) + - [Generate bearer tokens for authentication & authorization](#generate-bearer-tokens-for-authentication--authorization) + - [Generate a bearer token](#generate-a-bearer-token) + - [`generate_bearer_token(filepath)`](#generate_bearer_tokenfilepath) + - [`generate_bearer_token_from_creds(credentials)`](#generate_bearer_token_from_credscredentials) + - [Generate bearer tokens scoped to certain roles](#generate-bearer-tokens-scoped-to-certain-roles) + - [Generate bearer tokens with `ctx` for context-aware authorization](#generate-bearer-tokens-with-ctx-for-context-aware-authorization) + - [Generate signed data tokens: `generate_signed_data_tokens(filepath, options)`](#generate-signed-data-tokens-generate_signed_data_tokensfilepath-options) + - [Logging](#logging) + - [Example: Setting LogLevel to INFO](#example-setting-loglevel-to-info) + - [Error handling](#error-handling) + - [Catching `SkyflowError` instances](#catching-skyflowerror-instances) + - [Bearer token expiration edge cases](#bearer-token-expiration-edge-cases) + - [Security](#security) + - [Reporting a Vulnerability](#reporting-a-vulnerability) + +## Overview + +The Skyflow SDK enables you to connect to your Skyflow Vault(s) to securely handle sensitive data at rest, in-transit, and in-use. + +> [!TIP] +> Looking for the full list of request parameters, response object attributes, enums, client-management methods, and Detect helper classes? See the **[API Reference](../docs/api_reference.md)**. + +> [!IMPORTANT] +> This readme documents SDK version 2. +> For version 1 see the [v1.16.0 README](https://github.com/skyflowapi/skyflow-python/tree/v1). +> For more information on how to migrate see [MIGRATE_TO_V2.md](../docs/migrate_to_v2.md). + +## Installation + +### Require + +- Python 3.9 and above (tested with Python 3.9) + +### Configuration + +The package can be installed using pip: + +```bash +pip install skyflow +``` + +## Quickstart + +Get started quickly with the essential steps: authenticate, initialize the client, and perform a basic vault operation. This section shows you a minimal working example. + +### Before you begin + +To run the examples below, you need a Skyflow account and a few values from the Skyflow Studio console. If you don't have an account yet, [request a demo](https://www.skyflow.com/get-demo). + +| Value | Where to find it | +|-------|------------------| +| `vault_id` | Your vault's details page in Skyflow Studio. | +| `cluster_id` | The first segment of your vault URL: `https://{cluster_id}.vault.skyflowapis.com`. | +| `env` | The environment your vault runs in — `Env.PROD`, `Env.SANDBOX`, `Env.DEV`, or `Env.STAGE` (defaults to `PROD`). | +| Credentials | Create a **service account** in Studio. Choose **API key** during creation for the simplest setup, or download the service-account `credentials.json` for token-based auth. See [Authentication & authorization](#authentication--authorization). | + +The quickstart below assumes a table named `table1` with `card_number` and `cardholder_name` columns. Create a matching table (or adjust the table/column names to your schema) in your vault before running it. See the [Skyflow docs](https://docs.skyflow.com/) for creating vaults, tables, and service accounts. + +### Authenticate + +You can use an API key or a personal bearer token to directly authenticate and authorize requests with the SDK. Use API keys for long-term service authentication. Use bearer tokens for optimal security. + +### API Key + +```python +credentials = { + "api_key": "" +} +``` + +### Bearer Token (static) + +```python +credentials = { + "token": "" +} +``` + +For authenticating via generated bearer tokens including support for scoped tokens, context-aware access tokens, and more, refer to the [Authentication & Authorization](#authentication--authorization) section. + +### Initialize the client + +Initialize the Skyflow client first. You can specify different credential types during initialization. + +```python +from skyflow import Skyflow, LogLevel, Env + +# Configure vault +config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': { + 'api_key': '' + } +} + +# Initialize Skyflow client +skyflow_client = ( + Skyflow.builder() + .add_vault_config(config) + .set_log_level(LogLevel.ERROR) + .build() +) +``` + +See [docs/advanced_initialization.md](../docs/advanced_initialization.md) for advanced initialization examples including multiple vaults and different credential types. + +### Insert data into the vault, get tokens back + +Insert data into your vault using the `insert` method. Set `return_tokens=True` in the request to ensure values are tokenized in the response. + +Create an insert request with the [`InsertRequest`](../docs/api_reference.md#insertrequest) class, which includes the values to be inserted as a list of records. + +Below is a simple example to get started. See the [Insert and tokenize data](#insert-and-tokenize-data-insertrequest) section for advanced options. + +```python +from skyflow.vault.data import InsertRequest + +# Insert sensitive data into the vault +insert_data = [ + { 'card_number': '4111111111111111', 'cardholder_name': 'John Doe' }, +] + +insert_request = InsertRequest( + table='table1', + values=insert_data, + return_tokens=True +) + +insert_response = skyflow_client.vault('').insert(insert_request) +print('Insert response:', insert_response) +``` + +Returns an [`InsertResponse`](../docs/api_reference.md#insertresponse) (`inserted_fields`, `errors`). With `return_tokens=True`, each entry includes the `skyflow_id` and a token per column: + +```text +Insert response: InsertResponse(inserted_fields=[{'skyflow_id': 'a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', 'card_number': '5391-4629-3722-7102', 'cardholder_name': '0f6b8a2c-90ab-4cde-9def-567890abcdef'}], errors=None) +``` + +## Upgrade from v1 to v2 + +Upgrade from `skyflow-python` v1 using the dedicated guide in [docs/migrate_to_v2.md](../docs/migrate_to_v2.md). + +## Vault + +The [Vault](https://docs.skyflow.com/docs/vaults) performs operations on the vault, including inserting records, detokenizing tokens, and retrieving tokens associated with a skyflow_id. + +### Insert and tokenize data: `.insert(request)` + +Pass options to the `insert` method to enable additional functionality such as returning tokenized data, upserting records, or allowing bulk operations to continue despite errors. See [Quickstart](#quickstart) for a basic example. + +```python +from skyflow.vault.data import InsertRequest + +insert_request = InsertRequest( + table='table1', + values=[ + { + '': '', + '': '' + }, + { + '': '', + '': '' + } + ], + return_tokens=True +) + +response = skyflow_client.vault('').insert(insert_request) +print('Insert response:', response) +``` + +Returns an [`InsertResponse`](../docs/api_reference.md#insertresponse): + +```text +Insert response: InsertResponse(inserted_fields=[{'skyflow_id': 'a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', '': '', '': ''}], errors=None) +``` + +> With `continue_on_error=True`, each entry also carries a `request_index`, and `errors` is a list of `{request_index, request_id, error, http_code}` for the rows that failed. + +#### Insert example with `continue_on_error` option + +Set the `continue_on_error` flag to `True` to allow insert operations to proceed despite encountering partial errors. + +> [!TIP] +> See the full example in the samples directory: [insert_records.py](samples/vault_api/insert_records.py) + +#### Upsert request + +Turn an insert into an 'update-or-insert' operation using the upsert option. The vault checks for an existing record with the same value in the specified column. If a match exists, the record updates; otherwise, a new record inserts. + +```python +# Specify the column to use as the index for the upsert. +# Note: The column must have the `unique` constraint configured in the vault. +insert_request = InsertRequest( + table='table1', + values=insert_data, + upsert='' +) +``` + +### Detokenize: `.detokenize(request, options)` + +Convert tokens back into plaintext values (or masked values) using the `.detokenize()` method. Detokenization accepts tokens and returns values. + +Create a detokenization request with the [`DetokenizeRequest`](../docs/api_reference.md#detokenizerequest) class, which requires a list of tokens and column groups as input. + +Provide optional parameters such as the redaction type and the option to continue on error. + +#### Construct a detokenize request + +```python +from skyflow.vault.tokens import DetokenizeRequest +from skyflow.utils.enums import RedactionType + +detokenize_request = DetokenizeRequest( + data=[ + {'token': 'token1', 'redaction_type': RedactionType.PLAIN_TEXT}, + {'token': 'token2', 'redaction_type': RedactionType.PLAIN_TEXT} + ], + continue_on_error=True +) + +response = skyflow_client.vault('').detokenize(detokenize_request) +print('Detokenization response:', response) +``` + +Returns a [`DetokenizeResponse`](../docs/api_reference.md#detokenizeresponse) (`detokenized_fields`, `errors`); each field has `token`, `value`, and `type`: + +```text +Detokenization response: DetokenizeResponse(detokenized_fields=[{'token': 'token1', 'value': '4111111111111111', 'type': 'STRING'}, {'token': 'token2', 'value': 'John Doe', 'type': 'STRING'}], errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [detokenize_records.py](samples/vault_api/detokenize_records.py) + +### Get Record(s): `.get(request)` + +Retrieve data using Skyflow IDs or unique column values with the `get` method. Create a get request with the [`GetRequest`](../docs/api_reference.md#getrequest) class, specifying parameters such as the table name, redaction type, Skyflow IDs, column names, and column values. + +> [!NOTE] +> You can't use both Skyflow IDs and column name/value pairs in the same request. + +#### Construct a get request + +```python +from skyflow.vault.data import GetRequest +from skyflow.utils.enums import RedactionType + +get_request = GetRequest( + table='table1', + ids=['', ''], + redaction_type=RedactionType.PLAIN_TEXT, + return_tokens=False +) + +response = skyflow_client.vault('').get(get_request) +print('Get response:', response) +``` + +Returns a [`GetResponse`](../docs/api_reference.md#getresponse) (`data`, `errors`), where `data` is a list of record dicts: + +```text +Get response: GetResponse(data=[{'skyflow_id': 'a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', 'card_number': '4111111111111111', 'cardholder_name': 'John Doe'}], errors=None) +``` + +#### Get by Skyflow IDs + +Retrieve specific records using Skyflow IDs. Use this method when you know the exact record IDs. + +```python +from skyflow.vault.data import GetRequest +from skyflow.utils.enums import RedactionType + +get_request = GetRequest( + table='table1', + ids=['', ''], + redaction_type=RedactionType.PLAIN_TEXT +) + +response = skyflow_client.vault('').get(get_request) + +print('Data retrieval successful:', response) +``` + +```text +Data retrieval successful: GetResponse(data=[{'skyflow_id': '', 'card_number': '4111111111111111', 'cardholder_name': 'John Doe'}], errors=None) +``` + +#### Get tokens for records + +Return tokens for records to securely process sensitive data while maintaining data privacy. + +```python +get_request = GetRequest( + table='table1', + ids=[''], + return_tokens=True # Set to `True` to get tokens +) +``` + +> [!TIP] +> See the full example in the samples directory: [get_records.py](samples/vault_api/get_records.py) + +#### Get by column name and column values + +Retrieve records by unique column values when you don't know the Skyflow IDs. Use this method to query data with alternate unique identifiers. + +```python +get_request = GetRequest( + table='table1', + column_name='email', + column_values=['user@email.com'], # Column values of the records to return +) +``` + +> [!TIP] +> See the full example in the samples directory: [get_column_values.py](samples/vault_api/get_column_values.py) + +#### Redaction Types + +Use redaction types to control how sensitive data displays when retrieved from the vault. + +**Available Redaction Types** + +- `DEFAULT`: Applies the vault-configured default redaction setting. +- `REDACTED`: Completely removes sensitive data from view. +- `MASKED`: Partially obscures sensitive information. +- `PLAIN_TEXT`: Displays the full, unmasked data. + +**Choosing the Right Redaction Type** + +- Use `REDACTED` for scenarios requiring maximum data protection to prevent exposure of sensitive information. +- Use `MASKED` to provide partial visibility of sensitive data for less critical use cases. +- Use `PLAIN_TEXT` for internal, authorized access where full data visibility is necessary. + +### Update Records + +Update data in your vault using the `update` method. Create an update request with the [`UpdateRequest`](../docs/api_reference.md#updaterequest) class, specifying parameters such as the table name and data (as a dictionary). + +You can pass options like `return_tokens` directly to the request. When `True`, Skyflow returns tokens for the updated records. When `False`, it returns IDs. + +#### Construct an update request + +```python +from skyflow.vault.data import UpdateRequest + +update_request = UpdateRequest( + table='table1', + data={ + 'skyflow_id': '', + '': '', + '': '' + } +) + +response = skyflow_client.vault('').update(update_request) +print('Update response:', response) +``` + +Returns an [`UpdateResponse`](../docs/api_reference.md#updateresponse) (`updated_field`, `errors`). With the default `return_tokens=False`, only the `skyflow_id` is returned; with `return_tokens=True`, tokens for the updated columns are included: + +```text +Update response: UpdateResponse(updated_field={'skyflow_id': ''}, errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [update_record.py](samples/vault_api/update_record.py) + +### Delete Records + +Delete records using Skyflow IDs with the `delete` method. Create a delete request with the [`DeleteRequest`](../docs/api_reference.md#deleterequest) class, which accepts a list of Skyflow IDs: + +```python +from skyflow.vault.data import DeleteRequest + +delete_request = DeleteRequest( + table='', + ids=['', '', ''] +) + +response = skyflow_client.vault('').delete(delete_request) +print('Delete response:', response) +``` + +Returns a [`DeleteResponse`](../docs/api_reference.md#deleteresponse) (`deleted_ids`, `errors`): + +```text +Delete response: DeleteResponse(deleted_ids=['', '', ''], errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [delete_records.py](samples/vault_api/delete_records.py) + +### Query + +Retrieve data with SQL queries using the `query` method. Create a query request with the [`QueryRequest`](../docs/api_reference.md#queryrequest) class, which takes the `query` parameter as follows: + +```python +from skyflow.vault.data import QueryRequest + +query_request = QueryRequest( + query="SELECT * FROM table1 WHERE column1 = 'value'" +) + +response = skyflow_client.vault('').query(query_request) +print('Query response:', response) +``` + +Returns a [`QueryResponse`](../docs/api_reference.md#queryresponse) (`fields`, `errors`), where `fields` is a list of matching record dicts (each also includes a `tokenized_data` map): + +```text +Query response: QueryResponse(fields=[{'card_number': '4111111111111111', 'cardholder_name': 'John Doe', 'tokenized_data': {}}], errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [query_records.py](samples/vault_api/query_records.py) + +Refer to [Query your data](https://docs.skyflow.com/query-data/) and [Execute Query](https://docs.skyflow.com/record/#QueryService_ExecuteQuery) for guidelines and restrictions on supported SQL statements, operators, and keywords. + +### Upload File + +Upload files to a Skyflow vault using the `upload_file` method. Create a file upload request with the [`FileUploadRequest`](../docs/api_reference.md#fileuploadrequest) class. + +**Upload a file to an existing record:** + +```python +from skyflow.vault.data import FileUploadRequest + +# Open the file in binary read mode +with open('path/to/file.pdf', 'rb') as file_obj: + upload_request = FileUploadRequest( + table='', + column_name='', + skyflow_id='', + file_object=file_obj + ) + + response = skyflow_client.vault('').upload_file(upload_request) + print('File upload:', response) +``` + +**Upload a file and create a new record (omit `skyflow_id`):** + +```python +with open('path/to/file.pdf', 'rb') as file_obj: + upload_request = FileUploadRequest( + table='documents', + column_name='attachment', + file_object=file_obj + ) + + response = skyflow_client.vault('').upload_file(upload_request) + print('File upload:', response) +``` + +Both forms return a [`FileUploadResponse`](../docs/api_reference.md#fileuploadresponse) (`skyflow_id`, `errors`) with the ID of the record the file was attached to (or the newly created record): + +```text +File upload: FileUploadResponse(skyflow_id='a8f0c2e1-7b3d-4f9a-8c21-1d2e3f4a5b6c', errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [upload_file.py](samples/vault_api/upload_file.py) + +### Retrieve Existing Tokens: `.tokenize(request)` + +Retrieve tokens for values that already exist in the vault using the `.tokenize()` method. This method returns existing tokens only and does not generate new tokens. Build the request with the [`TokenizeRequest`](../docs/api_reference.md#tokenizerequest) class. + +#### Construct a `.tokenize()` request + +```python +from skyflow.vault.tokens import TokenizeRequest + +tokenize_request = TokenizeRequest( + values=[ + {"value": "", "column_group": ""}, + {"value": "", "column_group": ""} + ] +) + +response = skyflow_client.vault('').tokenize(tokenize_request) +print('Tokenization result:', response) +``` + +Returns a [`TokenizeResponse`](../docs/api_reference.md#tokenizeresponse) (`tokenized_fields`, `errors`); each field carries its `token`: + +```text +Tokenization result: TokenizeResponse(tokenized_fields=[{'token': 'a1b2c3d4-...'}, {'token': 'e5f6g7h8-...'}], errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [tokenize_records.py](samples/vault_api/tokenize_records.py) + +## Detect + +De-identify and reidentify sensitive data in text and files using Skyflow Detect, which supports advanced privacy-preserving workflows. + +### De-identify Text: `.deidentify_text(request)` + +De-identify or anonymize text using the `deidentify_text` method. + +Create a de-identify text request with the [`DeidentifyTextRequest`](../docs/api_reference.md#deidentifytextrequest) class. + +```python +from skyflow.vault.detect import DeidentifyTextRequest, TokenFormat, Transformations, DateTransformation +from skyflow.utils.enums import DetectEntities, TokenType + +request = DeidentifyTextRequest( + text="", + entities=[DetectEntities.SSN, DetectEntities.CREDIT_CARD], + token_format=TokenFormat(default=TokenType.VAULT_TOKEN), + transformations=Transformations( + shift_dates=DateTransformation( + max_days=30, # Maximum days to shift + min_days=10, # Minimum days to shift + entities=[DetectEntities.DOB] + ) + ) +) + +response = skyflow_client.detect('').deidentify_text(request) +print('De-identify Text Response:', response) +``` + +Returns a [`DeidentifyTextResponse`](../docs/api_reference.md#deidentifytextresponse) (`processed_text`, `entities`, `word_count`, `char_count`, `errors`). `entities` is a list of [`EntityInfo`](../docs/api_reference.md#entityinfo) describing each detected entity: + +```text +De-identify Text Response: DeidentifyTextResponse(processed_text='My SSN is [SSN_1].', entities=[...], word_count=4, char_count=18, errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [deidentify_text.py](samples/detect_api/deidentify_text.py) + +### Re-identify Text: `.reidentify_text(request)` + +Re-identify text using the `reidentify_text` method. Create a reidentify text request with the [`ReidentifyTextRequest`](../docs/api_reference.md#reidentifytextrequest) class, which includes the redacted or de-identified text to be re-identified. + +```python +from skyflow.vault.detect import ReidentifyTextRequest +from skyflow.utils.enums import DetectEntities + +request = ReidentifyTextRequest( + text="", + redacted_entities=[DetectEntities.SSN], # Keep redacted + masked_entities=[DetectEntities.CREDIT_CARD], # Mask + plain_text_entities=[DetectEntities.NAME] # Reveal +) + +response = skyflow_client.detect().reidentify_text(request) +print('Re-identify Text Response:', response) +``` + +Returns a [`ReidentifyTextResponse`](../docs/api_reference.md#reidentifytextresponse) (`processed_text`, `errors`): + +```text +Re-identify Text Response: ReidentifyTextResponse(processed_text='John lives in NYC', errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [reidentify_text.py](samples/detect_api/reidentify_text.py) + +### De-identify File: `.deidentify_file(request)` + +De-identify files using the `deidentify_file` method. Create a request with the [`DeidentifyFileRequest`](../docs/api_reference.md#deidentifyfilerequest) class, which includes the file to be deidentified. Provide optional parameters to control how entities are detected and deidentified. + +```python +from skyflow.vault.detect import DeidentifyFileRequest, TokenFormat, FileInput +from skyflow.utils.enums import DetectEntities, TokenType + +# Open file in binary mode +with open('path/to/file.pdf', 'rb') as file_obj: + request = DeidentifyFileRequest( + file=FileInput(file_obj), + entities=[DetectEntities.SSN, DetectEntities.CREDIT_CARD], + token_format=TokenFormat(default=TokenType.ENTITY_ONLY), + output_directory='', + wait_time=64 + ) + + response = skyflow_client.detect().deidentify_file(request) + print('De-identify File Response:', response) +``` + +Returns a [`DeidentifyFileResponse`](../docs/api_reference.md#deidentifyfileresponse) with the processed file plus metadata (`file`, `type`, `extension`, `word_count`, `char_count`, `size_in_kb`, `entities`, `run_id`, `status`, `errors`, and more — see the [API Reference](../docs/api_reference.md#response-objects)). If processing exceeds `wait_time`, only `run_id` and `status` are returned (poll with `get_detect_run`): + +```text +De-identify File Response: DeidentifyFileResponse(file_base64=None, file=, type='application/pdf', extension='pdf', ..., run_id='r-9c1f2a3b', status='SUCCESS', errors=None) +``` + +**Supported file types:** + +- Documents: `doc`, `docx`, `pdf` +- PDFs: `pdf` +- Images: `bmp`, `jpeg`, `jpg`, `png`, `tif`, `tiff` +- Structured text: `json`, `xml` +- Spreadsheets: `csv`, `xls`, `xlsx` +- Presentations: `ppt`, `pptx` +- Audio: `mp3`, `wav` + +**Notes:** + +- Transformations can't be applied to Documents, Images, or PDFs file formats. +- The `wait_time` option must be ≤ 64 seconds; otherwise, an error is thrown. +- If the API takes more than 64 seconds to process the file, it will return only the `run_id` and `status` in the response. + +> [!TIP] +> See the full example in the samples directory: [deidentify_file.py](samples/detect_api/deidentify_file.py) + +### Get Run: `.get_detect_run(request)` + +Retrieve the results of a previously started file de-identification operation using the `get_detect_run` method. Build the request with the [`GetDetectRunRequest`](../docs/api_reference.md#getdetectrunrequest) class, initialized with the `run_id` returned from a prior `deidentify_file` call. + +```python +from skyflow.vault.detect import GetDetectRunRequest + +request = GetDetectRunRequest( + run_id='' +) + +response = skyflow_client.detect().get_detect_run(request) +print('Get Detect Run Response:', response) +``` + +Returns a [`DeidentifyFileResponse`](../docs/api_reference.md#deidentifyfileresponse) with the current `status` for the run (and the processed file once `status` is complete): + +```text +Get Detect Run Response: DeidentifyFileResponse(file_base64=None, file=None, ..., run_id='r-9c1f2a3b', status='IN_PROGRESS', errors=None) +``` + +> [!TIP] +> See the full example in the samples directory: [get_detect_run.py](samples/detect_api/get_detect_run.py) + +## Connections + +Securely send and receive data between your systems and first- or third-party services using Skyflow Connections. The [connections](https://github.com/skyflowapi/skyflow-python/tree/v2/skyflow/vault/connection) module invokes both inbound and/or outbound connections. + +- **Inbound connections**: Act as intermediaries between your client and server, tokenizing sensitive data before it reaches your backend, ensuring downstream services handle only tokenized data. +- **Outbound connections**: Enable secure extraction of data from the vault and transfer it to third-party services via your backend server, such as processing checkout or card issuance flows. + +### Invoke a connection + +To invoke a connection, use the `invoke` method of the Skyflow client. Build the request with the [`InvokeConnectionRequest`](../docs/api_reference.md#invokeconnectionrequest) class. + +#### Construct an invoke connection request + +```python +from skyflow.vault.connection import InvokeConnectionRequest +from skyflow.utils.enums import RequestMethod + +invoke_request = InvokeConnectionRequest( + method=RequestMethod.POST, + body={ '': '' }, + headers={ '': '' }, + path_params={ '': '' }, + query_params={ '': '' } +) + +response = skyflow_client.connection().invoke(invoke_request) +print('Connection response:', response) +``` + +Returns an [`InvokeConnectionResponse`](../docs/api_reference.md#invokeconnectionresponse) (`data`, `metadata`, `errors`), where `data` is the connection's response body: + +```text +Connection response: InvokeConnectionResponse(data={'message': 'success'}, metadata={'request_id': 'b7d3...'}, errors=None) +``` + +`method` supports the following methods (see [`RequestMethod`](../docs/api_reference.md#requestmethod)): + +- `GET` +- `POST` +- `PUT` +- `DELETE` + +**path_params, query_params, header, body** are the JSON objects represented as dictionaries that will be sent through the connection integration url. + +> [!TIP] +> See the full example in the samples directory: [invoke_connection.py](samples/vault_api/invoke_connection.py) +> See [docs.skyflow.com](https://docs.skyflow.com) for more details on integrations with Connections, Functions, and Pipelines. + +## Authentication & authorization + +### Types of `credentials` + +The SDK accepts one of several types of credentials object. + +1. **API keys** + A unique identifier used to authenticate and authorize requests to an API. Use for long-term service authentication. To create an API key, first create a 'Service Account' in Skyflow and choose the 'API key' option during creation. + + ```python + credentials = { + "api_key": "" + } + ``` + +2. **Bearer tokens** + A temporary access token used to authenticate API requests. Use for optimal security. As a developer with the right access, you can generate a temporary personal bearer token in Skyflow in the user menu. + + ```python + credentials = { + "token": "" + } + ``` + +3. **Service account credentials file path** + The file path pointing to a JSON file containing credentials for a service account. Use when credentials are managed externally or stored in secure file systems. + + ```python + credentials = { + "path": "" + } + ``` + +4. **Service account credentials string** + JSON-formatted string containing service account credentials. Use when integrating with secret management systems or when credentials are passed programmatically. + + ```python + import os + + credentials = { + "credentials_string": os.getenv("SKYFLOW_CREDENTIALS") + } + ``` + +5. **Environment variables** + If no credentials are explicitly provided, the SDK automatically looks for the SKYFLOW_CREDENTIALS environment variable. Use to avoid hardcoding credentials in source code. This variable must return an object like one of the examples above. + +> [!NOTE] +> Only one type of credential can be used at a time. If multiple credentials are provided, the last one added will take precedence. + +### Generate bearer tokens for authentication & authorization + +Generate and manage bearer tokens to authenticate API calls. This section covers options for scoping to certain roles, passing context, and signing data tokens. + +#### Generate a bearer token + +Generate service account tokens using the [Service Account](https://github.com/skyflowapi/skyflow-python/tree/main/skyflow/service_account) Python package with a service account credentials file provided when a service account is created. Tokens generated by this module are valid for 60 minutes and can be used to make API calls to the [Data](https://docs.skyflow.com/record/) and [Management](https://docs.skyflow.com/management/) APIs, depending on the permissions assigned to the service account. + +##### `generate_bearer_token(filepath)` + +The `generate_bearer_token(filepath)` function takes the `credentials.json` file path for token generation. + +```python +from skyflow.service_account import generate_bearer_token + +token, _ = generate_bearer_token('path/to/credentials.json') +print("Bearer Token:", token) +``` + +##### `generate_bearer_token_from_creds(credentials)` + +Alternatively, you can also send the entire credentials as string by using `generate_bearer_token_from_creds(string)`. + +> [!TIP] +> See the full example in the samples directory: [token_generation_example.py](https://github.com/skyflowapi/skyflow-python/blob/main/samples/service_account/token_generation_example.py) + +#### Generate bearer tokens scoped to certain roles + +Generate bearer tokens with access limited to a specific role by specifying the appropriate roleID when using a service account with multiple roles. Use this to limit access for services with multiple responsibilities, such as segregating access for billing and analytics. Generated bearer tokens are valid for 60 minutes and can only execute operations permitted by the permissions associated with the designated role. + +```python +options = { + 'role_ids': ['roleID1', 'roleID2'] +} +``` + +> [!TIP] +> See the full example in the samples directory: [scoped_token_generation_example.py](samples/service_account/scoped_token_generation_example.py) +> See [docs.skyflow.com](https://docs.skyflow.com) for more details on authentication, access control, and governance for Skyflow. + +#### Generate bearer tokens with `ctx` for context-aware authorization + +Embed context values into a bearer token during generation so you can reference those values in your policies. This enables more flexible access controls, such as tracking end-user identity when making API calls using service accounts, and facilitates using signed data tokens during detokenization. + +Generate bearer tokens containing context information using a service account with the `context_id` identifier. Context information is represented as a JWT claim in a Skyflow-generated bearer token. Tokens generated from such service accounts include a `context_identifier` claim, are valid for 60 minutes, and can be used to make API calls to the Data and Management APIs, depending on the service account's permissions. + +The `ctx` parameter accepts either a **string** or a **dict**: + +**String context** — use when your policy references a single context value: + +```python +options = {'ctx': 'user_12345'} +token, _ = generate_bearer_token(filepath, options) +``` + +**Dict context** — use when your policy needs multiple context values for conditional data access. Each key in the dict maps to a Skyflow CEL policy variable under `request.context.*`: + +```python +options = { + 'ctx': { + 'role': 'admin', + 'department': 'finance', + 'user_id': 'user_12345', + } +} +token, _ = generate_bearer_token(filepath, options) +``` + +With the dict above, your Skyflow policies can reference `request.context.role`, `request.context.department`, and `request.context.user_id` to make conditional access decisions. + +Dict keys must contain only alphanumeric characters and underscores (`[a-zA-Z0-9_]`). Invalid keys will raise a `SkyflowError`. + +> [!TIP] +> See the full example in the samples directory: [token_generation_with_context_example.py](samples/service_account/token_generation_with_context_example.py) +> See Skyflow's [context-aware authorization](https://docs.skyflow.com) and [conditional data access](https://docs.skyflow.com) docs for policy variable syntax like `request.context.*`. + +#### Generate signed data tokens: `generate_signed_data_tokens(filepath, options)` + +Digitally sign data tokens with a service account's private key to add an extra layer of protection. Skyflow generates data tokens when sensitive data is inserted into the vault. Detokenize signed tokens only by providing the signed data token along with a bearer token generated from the service account's credentials. The service account must have the necessary permissions and context to successfully detokenize the signed data tokens. + +The `ctx` parameter on signed data tokens also accepts either a **string** or a **dict**, using the same format as bearer tokens: + +```python +# String context +options = { + 'ctx': 'user_12345', + 'data_tokens': ['dataToken1', 'dataToken2'], + 'time_to_live': 90, +} + +# Dict context +options = { + 'ctx': { + 'role': 'analyst', + 'department': 'research', + }, + 'data_tokens': ['dataToken1', 'dataToken2'], + 'time_to_live': 90, +} +``` + +> [!TIP] +> See the full example in the samples directory: [signed_token_generation_example.py](samples/service_account/signed_token_generation_example.py) +> See [docs.skyflow.com](https://docs.skyflow.com) for more details on authentication, access control, and governance for Skyflow. + +## Logging + +The SDK provides logging using Python's inbuilt `logging` library. By default the logging level of the SDK is set to `LogLevel.ERROR`. This can be changed by using `set_log_level(log_level)` as shown below: + +Currently, the following five log levels are supported: + +- `DEBUG`: +When `LogLevel.DEBUG` is passed, logs at all levels will be printed (DEBUG, INFO, WARN, ERROR). +- `INFO`: +When `LogLevel.INFO` is passed, INFO logs for every event that occurs during SDK flow execution will be printed, along with WARN and ERROR logs. +- `WARN`: +When `LogLevel.WARN` is passed, only WARN and ERROR logs will be printed. +- `ERROR`: +When `LogLevel.ERROR` is passed, only ERROR logs will be printed. +- `OFF`: +`LogLevel.OFF` can be used to turn off all logging from the Skyflow Python SDK. + +**Note:** The ranking of logging levels is as follows: `DEBUG` < `INFO` < `WARN` < `ERROR` < `OFF`. + +### Example: Setting LogLevel to INFO + +```python +from skyflow import Skyflow, LogLevel, Env + +# Define vault configuration +vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': {'api_key': ''} +} + +skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.INFO) # Recommended to use LogLevel.ERROR in production + .build() +) +``` + +## Using the client in production + +**Build the client once and reuse it.** `Skyflow.builder()...build()` returns a long-lived client that lazily creates and caches an HTTP client and bearer token per vault. Construct it once at startup (for example, as a module-level singleton or a dependency-injected instance) and reuse it across requests. Rebuilding the client on every request discards these caches and forces unnecessary token regeneration. + +```python +# At application startup +skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() +) + +# Reuse `skyflow_client` for the lifetime of the process +``` + +**Bearer token refresh is automatic.** When you authenticate with a service-account credentials file/string (or API key), the SDK caches the generated bearer token and regenerates it automatically once it expires. You don't need to manage token lifecycle yourself for the common case. (For the rare expire-mid-request case, see [Bearer token expiration edge cases](#bearer-token-expiration-edge-cases).) + +**Configuration mutation is not concurrency-safe.** Methods that change client configuration at runtime — `add_vault_config`, `update_vault_config`, `remove_vault_config`, the `*_connection_config` methods, and `update_skyflow_credentials` — mutate shared client state without locking. Perform configuration changes during setup, not concurrently with in-flight requests from other threads. Once configured, reusing the built client to issue operations is the intended usage pattern. + +**Timeouts and retries.** The SDK does not currently expose request timeout or automatic-retry configuration. If you need strict timeout or retry guarantees, wrap your SDK calls with your own timeout/retry logic at the application layer. + +## Error handling + +### Catching `SkyflowError` instances + +Wrap your calls to the Skyflow SDK in try/except blocks as a best practice. Use the `SkyflowError` class to identify errors coming from Skyflow versus general request/response errors. + +```python +from skyflow.error import SkyflowError + +try: + # ...call the Skyflow SDK + pass +except SkyflowError as error: + # Handle Skyflow specific errors + print("Skyflow Specific Error:", { + "code": error.http_code, + "message": error.message, + "details": error.details, + }) +except Exception as error: + # Handle generic errors + print("Unexpected Error:", error) +``` + +### Bearer token expiration edge cases + +When using bearer tokens for authentication and API requests, a token may expire after verification but before the actual API call completes. This causes the request to fail unexpectedly. An error from this edge case looks like this: + +```txt +message: Authentication failed. Bearer token is expired. Use a valid bearer token. See https://docs.skyflow.com/api-authentication/ +``` + +If you encounter this kind of error, retry the request. During the retry the SDK detects that the previous bearer token has expired and generates a new one for the current and subsequent requests. + +> [!TIP] +> See the full example in the samples directory: [bearer_token_expiry_example.py](samples/service_account/bearer_token_expiry_example.py) +> See [docs.skyflow.com](https://docs.skyflow.com) for more details on authentication, access control, and governance for Skyflow. + +## Troubleshooting + +Most first-run problems come from configuration mismatches. Every error raised by the SDK is a `SkyflowError` exposing `http_code`, `message`, and `details` — inspect these first (see [Error handling](#error-handling)). + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| `pip install skyflow` fails / `RuntimeError: skyflow requires Python 3.9+` | Python older than 3.9 | Use Python 3.9 or above. | +| Connection/DNS failures, or 404 on every call | Wrong `cluster_id` | `cluster_id` is the first segment of your vault URL: `https://{cluster_id}.vault.skyflowapis.com`. | +| Requests hit the wrong host / unexpected auth failures | Wrong `env` | Match `env` to where your vault runs (`Env.PROD`, `Env.SANDBOX`, `Env.DEV`, `Env.STAGE`). | +| `401 Unauthorized` | Invalid or expired credentials | Verify your API key / service-account credentials. Regenerate if needed. | +| `403 Forbidden` | Service account lacks permission for the operation | Grant the service account a role with the required permissions, or use a [scoped token](#generate-bearer-tokens-scoped-to-certain-roles) with the right role. | +| `404` referencing a table or column | Table/column doesn't exist or name mismatch | Confirm the table and column names match your vault schema exactly (case-sensitive). | +| Vault not found / 404 with a valid `cluster_id` | Wrong `vault_id` | Copy `vault_id` from the vault's details page in Skyflow Studio. | +| `Authentication failed. Bearer token is expired.` | Token expired between verification and the API call | Retry the request; the SDK regenerates the token. See [Bearer token expiration edge cases](#bearer-token-expiration-edge-cases). | +| Unexpected credential is used | Multiple credentials provided | Only one credential type is used at a time; the last one added takes precedence. Provide exactly one. | +| `RequestMethod.PATCH` raises `AttributeError` | `PATCH` is not a supported connection method | Use `GET`, `POST`, `PUT`, or `DELETE` (see [`RequestMethod`](../docs/api_reference.md#requestmethod)). | + +If you're stuck, set `set_log_level(LogLevel.DEBUG)` during development for detailed SDK logs (see [Logging](#logging)). + +## Security + +### Reporting a Vulnerability + +If you discover a potential security issue in this project, reach out to us at [security@skyflow.com](mailto:security@skyflow.com). + +Don't create public GitHub issues or Pull Requests, as malicious actors could potentially view them. diff --git a/v2/requirements.txt b/skyvault/requirements.txt similarity index 100% rename from v2/requirements.txt rename to skyvault/requirements.txt diff --git a/samples/README.md b/skyvault/samples/README.md similarity index 100% rename from samples/README.md rename to skyvault/samples/README.md diff --git a/samples/detect_api/deidentify_file.py b/skyvault/samples/detect_api/deidentify_file.py similarity index 100% rename from samples/detect_api/deidentify_file.py rename to skyvault/samples/detect_api/deidentify_file.py diff --git a/samples/detect_api/deidentify_file_async.py b/skyvault/samples/detect_api/deidentify_file_async.py similarity index 100% rename from samples/detect_api/deidentify_file_async.py rename to skyvault/samples/detect_api/deidentify_file_async.py diff --git a/samples/detect_api/deidentify_text.py b/skyvault/samples/detect_api/deidentify_text.py similarity index 100% rename from samples/detect_api/deidentify_text.py rename to skyvault/samples/detect_api/deidentify_text.py diff --git a/samples/detect_api/get_detect_run.py b/skyvault/samples/detect_api/get_detect_run.py similarity index 100% rename from samples/detect_api/get_detect_run.py rename to skyvault/samples/detect_api/get_detect_run.py diff --git a/samples/detect_api/reidentify_text.py b/skyvault/samples/detect_api/reidentify_text.py similarity index 100% rename from samples/detect_api/reidentify_text.py rename to skyvault/samples/detect_api/reidentify_text.py diff --git a/samples/service_account/bearer_token_expiry_example.py b/skyvault/samples/service_account/bearer_token_expiry_example.py similarity index 100% rename from samples/service_account/bearer_token_expiry_example.py rename to skyvault/samples/service_account/bearer_token_expiry_example.py diff --git a/samples/service_account/scoped_token_generation_example.py b/skyvault/samples/service_account/scoped_token_generation_example.py similarity index 100% rename from samples/service_account/scoped_token_generation_example.py rename to skyvault/samples/service_account/scoped_token_generation_example.py diff --git a/samples/service_account/signed_token_generation_example.py b/skyvault/samples/service_account/signed_token_generation_example.py similarity index 100% rename from samples/service_account/signed_token_generation_example.py rename to skyvault/samples/service_account/signed_token_generation_example.py diff --git a/samples/service_account/token_generation_example.py b/skyvault/samples/service_account/token_generation_example.py similarity index 100% rename from samples/service_account/token_generation_example.py rename to skyvault/samples/service_account/token_generation_example.py diff --git a/samples/service_account/token_generation_with_context_example.py b/skyvault/samples/service_account/token_generation_with_context_example.py similarity index 100% rename from samples/service_account/token_generation_with_context_example.py rename to skyvault/samples/service_account/token_generation_with_context_example.py diff --git a/samples/vault_api/client_operations.py b/skyvault/samples/vault_api/client_operations.py similarity index 100% rename from samples/vault_api/client_operations.py rename to skyvault/samples/vault_api/client_operations.py diff --git a/samples/vault_api/credentials_options.py b/skyvault/samples/vault_api/credentials_options.py similarity index 100% rename from samples/vault_api/credentials_options.py rename to skyvault/samples/vault_api/credentials_options.py diff --git a/samples/vault_api/delete_records.py b/skyvault/samples/vault_api/delete_records.py similarity index 100% rename from samples/vault_api/delete_records.py rename to skyvault/samples/vault_api/delete_records.py diff --git a/samples/vault_api/detokenize_records.py b/skyvault/samples/vault_api/detokenize_records.py similarity index 100% rename from samples/vault_api/detokenize_records.py rename to skyvault/samples/vault_api/detokenize_records.py diff --git a/samples/vault_api/get_column_values.py b/skyvault/samples/vault_api/get_column_values.py similarity index 100% rename from samples/vault_api/get_column_values.py rename to skyvault/samples/vault_api/get_column_values.py diff --git a/samples/vault_api/get_records.py b/skyvault/samples/vault_api/get_records.py similarity index 100% rename from samples/vault_api/get_records.py rename to skyvault/samples/vault_api/get_records.py diff --git a/samples/vault_api/insert_byot.py b/skyvault/samples/vault_api/insert_byot.py similarity index 100% rename from samples/vault_api/insert_byot.py rename to skyvault/samples/vault_api/insert_byot.py diff --git a/samples/vault_api/insert_records.py b/skyvault/samples/vault_api/insert_records.py similarity index 100% rename from samples/vault_api/insert_records.py rename to skyvault/samples/vault_api/insert_records.py diff --git a/samples/vault_api/invoke_connection.py b/skyvault/samples/vault_api/invoke_connection.py similarity index 100% rename from samples/vault_api/invoke_connection.py rename to skyvault/samples/vault_api/invoke_connection.py diff --git a/samples/vault_api/query_records.py b/skyvault/samples/vault_api/query_records.py similarity index 100% rename from samples/vault_api/query_records.py rename to skyvault/samples/vault_api/query_records.py diff --git a/samples/vault_api/tokenize_records.py b/skyvault/samples/vault_api/tokenize_records.py similarity index 100% rename from samples/vault_api/tokenize_records.py rename to skyvault/samples/vault_api/tokenize_records.py diff --git a/samples/vault_api/update_record.py b/skyvault/samples/vault_api/update_record.py similarity index 100% rename from samples/vault_api/update_record.py rename to skyvault/samples/vault_api/update_record.py diff --git a/samples/vault_api/upload_file.py b/skyvault/samples/vault_api/upload_file.py similarity index 100% rename from samples/vault_api/upload_file.py rename to skyvault/samples/vault_api/upload_file.py diff --git a/v2/setup.py b/skyvault/setup.py similarity index 97% rename from v2/setup.py rename to skyvault/setup.py index 4e7f78bc..5fa9fe84 100644 --- a/v2/setup.py +++ b/skyvault/setup.py @@ -17,7 +17,7 @@ REPO_ROOT = os.path.dirname(HERE) COMMON_SRC = os.path.join(REPO_ROOT, 'common') -with open(os.path.join(REPO_ROOT, 'README.md'), 'r', encoding='utf-8') as f: +with open(os.path.join(HERE, 'README.md'), 'r', encoding='utf-8') as f: long_description = f.read() # Anything under common/ that must never ride along into a built wheel. diff --git a/v2/skyflow/__init__.py b/skyvault/skyflow/__init__.py similarity index 100% rename from v2/skyflow/__init__.py rename to skyvault/skyflow/__init__.py diff --git a/v2/skyflow/client/__init__.py b/skyvault/skyflow/client/__init__.py similarity index 100% rename from v2/skyflow/client/__init__.py rename to skyvault/skyflow/client/__init__.py diff --git a/v2/skyflow/client/skyflow.py b/skyvault/skyflow/client/skyflow.py similarity index 100% rename from v2/skyflow/client/skyflow.py rename to skyvault/skyflow/client/skyflow.py diff --git a/v2/skyflow/error/__init__.py b/skyvault/skyflow/error/__init__.py similarity index 100% rename from v2/skyflow/error/__init__.py rename to skyvault/skyflow/error/__init__.py diff --git a/v2/skyflow/generated/__init__.py b/skyvault/skyflow/generated/__init__.py similarity index 100% rename from v2/skyflow/generated/__init__.py rename to skyvault/skyflow/generated/__init__.py diff --git a/v2/skyflow/generated/rest/__init__.py b/skyvault/skyflow/generated/rest/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/__init__.py rename to skyvault/skyflow/generated/rest/__init__.py diff --git a/v2/skyflow/generated/rest/audit/__init__.py b/skyvault/skyflow/generated/rest/audit/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/audit/__init__.py rename to skyvault/skyflow/generated/rest/audit/__init__.py diff --git a/v2/skyflow/generated/rest/audit/client.py b/skyvault/skyflow/generated/rest/audit/client.py similarity index 100% rename from v2/skyflow/generated/rest/audit/client.py rename to skyvault/skyflow/generated/rest/audit/client.py diff --git a/v2/skyflow/generated/rest/audit/raw_client.py b/skyvault/skyflow/generated/rest/audit/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/audit/raw_client.py rename to skyvault/skyflow/generated/rest/audit/raw_client.py diff --git a/v2/skyflow/generated/rest/audit/types/__init__.py b/skyvault/skyflow/generated/rest/audit/types/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/audit/types/__init__.py rename to skyvault/skyflow/generated/rest/audit/types/__init__.py diff --git a/v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_action_type.py b/skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_action_type.py similarity index 100% rename from v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_action_type.py rename to skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_action_type.py diff --git a/v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_access_type.py b/skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_access_type.py similarity index 100% rename from v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_access_type.py rename to skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_access_type.py diff --git a/v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_actor_type.py b/skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_actor_type.py similarity index 100% rename from v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_actor_type.py rename to skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_actor_type.py diff --git a/v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_auth_mode.py b/skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_auth_mode.py similarity index 100% rename from v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_auth_mode.py rename to skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_context_auth_mode.py diff --git a/v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_resource_type.py b/skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_resource_type.py similarity index 100% rename from v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_resource_type.py rename to skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_filter_ops_resource_type.py diff --git a/v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_sort_ops_order_by.py b/skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_sort_ops_order_by.py similarity index 100% rename from v2/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_sort_ops_order_by.py rename to skyvault/skyflow/generated/rest/audit/types/audit_service_list_audit_events_request_sort_ops_order_by.py diff --git a/v2/skyflow/generated/rest/bin_lookup/__init__.py b/skyvault/skyflow/generated/rest/authentication/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/bin_lookup/__init__.py rename to skyvault/skyflow/generated/rest/authentication/__init__.py diff --git a/v2/skyflow/generated/rest/authentication/client.py b/skyvault/skyflow/generated/rest/authentication/client.py similarity index 100% rename from v2/skyflow/generated/rest/authentication/client.py rename to skyvault/skyflow/generated/rest/authentication/client.py diff --git a/v2/skyflow/generated/rest/authentication/raw_client.py b/skyvault/skyflow/generated/rest/authentication/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/authentication/raw_client.py rename to skyvault/skyflow/generated/rest/authentication/raw_client.py diff --git a/v2/skyflow/generated/rest/guardrails/__init__.py b/skyvault/skyflow/generated/rest/bin_lookup/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/guardrails/__init__.py rename to skyvault/skyflow/generated/rest/bin_lookup/__init__.py diff --git a/v2/skyflow/generated/rest/bin_lookup/client.py b/skyvault/skyflow/generated/rest/bin_lookup/client.py similarity index 100% rename from v2/skyflow/generated/rest/bin_lookup/client.py rename to skyvault/skyflow/generated/rest/bin_lookup/client.py diff --git a/v2/skyflow/generated/rest/bin_lookup/raw_client.py b/skyvault/skyflow/generated/rest/bin_lookup/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/bin_lookup/raw_client.py rename to skyvault/skyflow/generated/rest/bin_lookup/raw_client.py diff --git a/v2/skyflow/generated/rest/client.py b/skyvault/skyflow/generated/rest/client.py similarity index 100% rename from v2/skyflow/generated/rest/client.py rename to skyvault/skyflow/generated/rest/client.py diff --git a/v2/skyflow/generated/rest/core/__init__.py b/skyvault/skyflow/generated/rest/core/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/core/__init__.py rename to skyvault/skyflow/generated/rest/core/__init__.py diff --git a/v2/skyflow/generated/rest/core/api_error.py b/skyvault/skyflow/generated/rest/core/api_error.py similarity index 100% rename from v2/skyflow/generated/rest/core/api_error.py rename to skyvault/skyflow/generated/rest/core/api_error.py diff --git a/v2/skyflow/generated/rest/core/client_wrapper.py b/skyvault/skyflow/generated/rest/core/client_wrapper.py similarity index 100% rename from v2/skyflow/generated/rest/core/client_wrapper.py rename to skyvault/skyflow/generated/rest/core/client_wrapper.py diff --git a/v2/skyflow/generated/rest/core/datetime_utils.py b/skyvault/skyflow/generated/rest/core/datetime_utils.py similarity index 100% rename from v2/skyflow/generated/rest/core/datetime_utils.py rename to skyvault/skyflow/generated/rest/core/datetime_utils.py diff --git a/v2/skyflow/generated/rest/core/file.py b/skyvault/skyflow/generated/rest/core/file.py similarity index 100% rename from v2/skyflow/generated/rest/core/file.py rename to skyvault/skyflow/generated/rest/core/file.py diff --git a/v2/skyflow/generated/rest/core/force_multipart.py b/skyvault/skyflow/generated/rest/core/force_multipart.py similarity index 100% rename from v2/skyflow/generated/rest/core/force_multipart.py rename to skyvault/skyflow/generated/rest/core/force_multipart.py diff --git a/v2/skyflow/generated/rest/core/http_client.py b/skyvault/skyflow/generated/rest/core/http_client.py similarity index 100% rename from v2/skyflow/generated/rest/core/http_client.py rename to skyvault/skyflow/generated/rest/core/http_client.py diff --git a/v2/skyflow/generated/rest/core/http_response.py b/skyvault/skyflow/generated/rest/core/http_response.py similarity index 100% rename from v2/skyflow/generated/rest/core/http_response.py rename to skyvault/skyflow/generated/rest/core/http_response.py diff --git a/v2/skyflow/generated/rest/core/jsonable_encoder.py b/skyvault/skyflow/generated/rest/core/jsonable_encoder.py similarity index 100% rename from v2/skyflow/generated/rest/core/jsonable_encoder.py rename to skyvault/skyflow/generated/rest/core/jsonable_encoder.py diff --git a/v2/skyflow/generated/rest/core/pydantic_utilities.py b/skyvault/skyflow/generated/rest/core/pydantic_utilities.py similarity index 100% rename from v2/skyflow/generated/rest/core/pydantic_utilities.py rename to skyvault/skyflow/generated/rest/core/pydantic_utilities.py diff --git a/v2/skyflow/generated/rest/core/query_encoder.py b/skyvault/skyflow/generated/rest/core/query_encoder.py similarity index 100% rename from v2/skyflow/generated/rest/core/query_encoder.py rename to skyvault/skyflow/generated/rest/core/query_encoder.py diff --git a/v2/skyflow/generated/rest/core/remove_none_from_dict.py b/skyvault/skyflow/generated/rest/core/remove_none_from_dict.py similarity index 100% rename from v2/skyflow/generated/rest/core/remove_none_from_dict.py rename to skyvault/skyflow/generated/rest/core/remove_none_from_dict.py diff --git a/v2/skyflow/generated/rest/core/request_options.py b/skyvault/skyflow/generated/rest/core/request_options.py similarity index 100% rename from v2/skyflow/generated/rest/core/request_options.py rename to skyvault/skyflow/generated/rest/core/request_options.py diff --git a/v2/skyflow/generated/rest/core/serialization.py b/skyvault/skyflow/generated/rest/core/serialization.py similarity index 100% rename from v2/skyflow/generated/rest/core/serialization.py rename to skyvault/skyflow/generated/rest/core/serialization.py diff --git a/v2/skyflow/generated/rest/environment.py b/skyvault/skyflow/generated/rest/environment.py similarity index 100% rename from v2/skyflow/generated/rest/environment.py rename to skyvault/skyflow/generated/rest/environment.py diff --git a/v2/skyflow/generated/rest/errors/__init__.py b/skyvault/skyflow/generated/rest/errors/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/errors/__init__.py rename to skyvault/skyflow/generated/rest/errors/__init__.py diff --git a/v2/skyflow/generated/rest/errors/bad_request_error.py b/skyvault/skyflow/generated/rest/errors/bad_request_error.py similarity index 100% rename from v2/skyflow/generated/rest/errors/bad_request_error.py rename to skyvault/skyflow/generated/rest/errors/bad_request_error.py diff --git a/skyvault/skyflow/generated/rest/errors/internal_server_error.py b/skyvault/skyflow/generated/rest/errors/internal_server_error.py new file mode 100644 index 00000000..d7a796c6 --- /dev/null +++ b/skyvault/skyflow/generated/rest/errors/internal_server_error.py @@ -0,0 +1,11 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.api_error import ApiError +from ..types.error_response import ErrorResponse + + +class InternalServerError(ApiError): + def __init__(self, body: ErrorResponse, headers: typing.Optional[typing.Dict[str, str]] = None): + super().__init__(status_code=500, headers=headers, body=body) diff --git a/v2/skyflow/generated/rest/errors/not_found_error.py b/skyvault/skyflow/generated/rest/errors/not_found_error.py similarity index 100% rename from v2/skyflow/generated/rest/errors/not_found_error.py rename to skyvault/skyflow/generated/rest/errors/not_found_error.py diff --git a/v2/skyflow/generated/rest/errors/unauthorized_error.py b/skyvault/skyflow/generated/rest/errors/unauthorized_error.py similarity index 100% rename from v2/skyflow/generated/rest/errors/unauthorized_error.py rename to skyvault/skyflow/generated/rest/errors/unauthorized_error.py diff --git a/v2/skyflow/generated/rest/files/__init__.py b/skyvault/skyflow/generated/rest/files/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/files/__init__.py rename to skyvault/skyflow/generated/rest/files/__init__.py diff --git a/v2/skyflow/generated/rest/files/client.py b/skyvault/skyflow/generated/rest/files/client.py similarity index 100% rename from v2/skyflow/generated/rest/files/client.py rename to skyvault/skyflow/generated/rest/files/client.py diff --git a/v2/skyflow/generated/rest/files/raw_client.py b/skyvault/skyflow/generated/rest/files/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/files/raw_client.py rename to skyvault/skyflow/generated/rest/files/raw_client.py diff --git a/v2/skyflow/generated/rest/files/types/__init__.py b/skyvault/skyflow/generated/rest/files/types/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/__init__.py rename to skyvault/skyflow/generated/rest/files/types/__init__.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_output_transcription.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_output_transcription.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_output_transcription.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_audio_request_deidentify_audio_output_transcription.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_document_pdf_request_deidentify_pdf_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_document_pdf_request_deidentify_pdf_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_document_pdf_request_deidentify_pdf_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_document_pdf_request_deidentify_pdf_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_masking_method.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_masking_method.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_masking_method.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_image_request_deidentify_image_masking_method.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_document_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_document_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_document_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_document_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_presentation_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_presentation_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_presentation_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_presentation_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_spreadsheet_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_spreadsheet_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_spreadsheet_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_spreadsheet_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_structured_text_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_structured_text_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_structured_text_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_structured_text_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_text_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_text_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_text_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_request_deidentify_text_entity_types_item.py diff --git a/v2/skyflow/generated/rest/files/types/deidentify_file_request_entity_types_item.py b/skyvault/skyflow/generated/rest/files/types/deidentify_file_request_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/files/types/deidentify_file_request_entity_types_item.py rename to skyvault/skyflow/generated/rest/files/types/deidentify_file_request_entity_types_item.py diff --git a/v2/skyflow/generated/rest/query/__init__.py b/skyvault/skyflow/generated/rest/guardrails/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/query/__init__.py rename to skyvault/skyflow/generated/rest/guardrails/__init__.py diff --git a/v2/skyflow/generated/rest/guardrails/client.py b/skyvault/skyflow/generated/rest/guardrails/client.py similarity index 100% rename from v2/skyflow/generated/rest/guardrails/client.py rename to skyvault/skyflow/generated/rest/guardrails/client.py diff --git a/v2/skyflow/generated/rest/guardrails/raw_client.py b/skyvault/skyflow/generated/rest/guardrails/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/guardrails/raw_client.py rename to skyvault/skyflow/generated/rest/guardrails/raw_client.py diff --git a/v2/skyflow/generated/rest/py.typed b/skyvault/skyflow/generated/rest/py.typed similarity index 100% rename from v2/skyflow/generated/rest/py.typed rename to skyvault/skyflow/generated/rest/py.typed diff --git a/v2/skyflow/generated/rest/tokens/__init__.py b/skyvault/skyflow/generated/rest/query/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/tokens/__init__.py rename to skyvault/skyflow/generated/rest/query/__init__.py diff --git a/v2/skyflow/generated/rest/query/client.py b/skyvault/skyflow/generated/rest/query/client.py similarity index 100% rename from v2/skyflow/generated/rest/query/client.py rename to skyvault/skyflow/generated/rest/query/client.py diff --git a/v2/skyflow/generated/rest/query/raw_client.py b/skyvault/skyflow/generated/rest/query/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/query/raw_client.py rename to skyvault/skyflow/generated/rest/query/raw_client.py diff --git a/v2/skyflow/generated/rest/records/__init__.py b/skyvault/skyflow/generated/rest/records/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/records/__init__.py rename to skyvault/skyflow/generated/rest/records/__init__.py diff --git a/v2/skyflow/generated/rest/records/client.py b/skyvault/skyflow/generated/rest/records/client.py similarity index 100% rename from v2/skyflow/generated/rest/records/client.py rename to skyvault/skyflow/generated/rest/records/client.py diff --git a/v2/skyflow/generated/rest/records/raw_client.py b/skyvault/skyflow/generated/rest/records/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/records/raw_client.py rename to skyvault/skyflow/generated/rest/records/raw_client.py diff --git a/v2/skyflow/generated/rest/records/types/__init__.py b/skyvault/skyflow/generated/rest/records/types/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/records/types/__init__.py rename to skyvault/skyflow/generated/rest/records/types/__init__.py diff --git a/v2/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_order_by.py b/skyvault/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_order_by.py similarity index 100% rename from v2/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_order_by.py rename to skyvault/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_order_by.py diff --git a/v2/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_redaction.py b/skyvault/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_redaction.py similarity index 100% rename from v2/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_redaction.py rename to skyvault/skyflow/generated/rest/records/types/record_service_bulk_get_record_request_redaction.py diff --git a/v2/skyflow/generated/rest/records/types/record_service_get_record_request_redaction.py b/skyvault/skyflow/generated/rest/records/types/record_service_get_record_request_redaction.py similarity index 100% rename from v2/skyflow/generated/rest/records/types/record_service_get_record_request_redaction.py rename to skyvault/skyflow/generated/rest/records/types/record_service_get_record_request_redaction.py diff --git a/v2/skyflow/generated/rest/strings/__init__.py b/skyvault/skyflow/generated/rest/strings/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/strings/__init__.py rename to skyvault/skyflow/generated/rest/strings/__init__.py diff --git a/v2/skyflow/generated/rest/strings/client.py b/skyvault/skyflow/generated/rest/strings/client.py similarity index 100% rename from v2/skyflow/generated/rest/strings/client.py rename to skyvault/skyflow/generated/rest/strings/client.py diff --git a/v2/skyflow/generated/rest/strings/raw_client.py b/skyvault/skyflow/generated/rest/strings/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/strings/raw_client.py rename to skyvault/skyflow/generated/rest/strings/raw_client.py diff --git a/v2/skyflow/generated/rest/strings/types/__init__.py b/skyvault/skyflow/generated/rest/strings/types/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/strings/types/__init__.py rename to skyvault/skyflow/generated/rest/strings/types/__init__.py diff --git a/v2/skyflow/generated/rest/strings/types/deidentify_string_request_entity_types_item.py b/skyvault/skyflow/generated/rest/strings/types/deidentify_string_request_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/strings/types/deidentify_string_request_entity_types_item.py rename to skyvault/skyflow/generated/rest/strings/types/deidentify_string_request_entity_types_item.py diff --git a/skyvault/skyflow/generated/rest/tokens/__init__.py b/skyvault/skyflow/generated/rest/tokens/__init__.py new file mode 100644 index 00000000..5cde0202 --- /dev/null +++ b/skyvault/skyflow/generated/rest/tokens/__init__.py @@ -0,0 +1,4 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + diff --git a/v2/skyflow/generated/rest/tokens/client.py b/skyvault/skyflow/generated/rest/tokens/client.py similarity index 100% rename from v2/skyflow/generated/rest/tokens/client.py rename to skyvault/skyflow/generated/rest/tokens/client.py diff --git a/v2/skyflow/generated/rest/tokens/raw_client.py b/skyvault/skyflow/generated/rest/tokens/raw_client.py similarity index 100% rename from v2/skyflow/generated/rest/tokens/raw_client.py rename to skyvault/skyflow/generated/rest/tokens/raw_client.py diff --git a/v2/skyflow/generated/rest/types/__init__.py b/skyvault/skyflow/generated/rest/types/__init__.py similarity index 100% rename from v2/skyflow/generated/rest/types/__init__.py rename to skyvault/skyflow/generated/rest/types/__init__.py diff --git a/v2/skyflow/generated/rest/types/audit_event_audit_resource_type.py b/skyvault/skyflow/generated/rest/types/audit_event_audit_resource_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/audit_event_audit_resource_type.py rename to skyvault/skyflow/generated/rest/types/audit_event_audit_resource_type.py diff --git a/v2/skyflow/generated/rest/types/audit_event_context.py b/skyvault/skyflow/generated/rest/types/audit_event_context.py similarity index 100% rename from v2/skyflow/generated/rest/types/audit_event_context.py rename to skyvault/skyflow/generated/rest/types/audit_event_context.py diff --git a/v2/skyflow/generated/rest/types/audit_event_data.py b/skyvault/skyflow/generated/rest/types/audit_event_data.py similarity index 100% rename from v2/skyflow/generated/rest/types/audit_event_data.py rename to skyvault/skyflow/generated/rest/types/audit_event_data.py diff --git a/v2/skyflow/generated/rest/types/audit_event_http_info.py b/skyvault/skyflow/generated/rest/types/audit_event_http_info.py similarity index 100% rename from v2/skyflow/generated/rest/types/audit_event_http_info.py rename to skyvault/skyflow/generated/rest/types/audit_event_http_info.py diff --git a/v2/skyflow/generated/rest/types/batch_record_method.py b/skyvault/skyflow/generated/rest/types/batch_record_method.py similarity index 100% rename from v2/skyflow/generated/rest/types/batch_record_method.py rename to skyvault/skyflow/generated/rest/types/batch_record_method.py diff --git a/v2/skyflow/generated/rest/types/context_access_type.py b/skyvault/skyflow/generated/rest/types/context_access_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/context_access_type.py rename to skyvault/skyflow/generated/rest/types/context_access_type.py diff --git a/v2/skyflow/generated/rest/types/context_auth_mode.py b/skyvault/skyflow/generated/rest/types/context_auth_mode.py similarity index 100% rename from v2/skyflow/generated/rest/types/context_auth_mode.py rename to skyvault/skyflow/generated/rest/types/context_auth_mode.py diff --git a/v2/skyflow/generated/rest/types/deidentified_file_output.py b/skyvault/skyflow/generated/rest/types/deidentified_file_output.py similarity index 100% rename from v2/skyflow/generated/rest/types/deidentified_file_output.py rename to skyvault/skyflow/generated/rest/types/deidentified_file_output.py diff --git a/v2/skyflow/generated/rest/types/deidentified_file_output_processed_file_extension.py b/skyvault/skyflow/generated/rest/types/deidentified_file_output_processed_file_extension.py similarity index 100% rename from v2/skyflow/generated/rest/types/deidentified_file_output_processed_file_extension.py rename to skyvault/skyflow/generated/rest/types/deidentified_file_output_processed_file_extension.py diff --git a/v2/skyflow/generated/rest/types/deidentified_file_output_processed_file_type.py b/skyvault/skyflow/generated/rest/types/deidentified_file_output_processed_file_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/deidentified_file_output_processed_file_type.py rename to skyvault/skyflow/generated/rest/types/deidentified_file_output_processed_file_type.py diff --git a/v2/skyflow/generated/rest/types/deidentify_file_response.py b/skyvault/skyflow/generated/rest/types/deidentify_file_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/deidentify_file_response.py rename to skyvault/skyflow/generated/rest/types/deidentify_file_response.py diff --git a/v2/skyflow/generated/rest/types/deidentify_string_response.py b/skyvault/skyflow/generated/rest/types/deidentify_string_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/deidentify_string_response.py rename to skyvault/skyflow/generated/rest/types/deidentify_string_response.py diff --git a/v2/skyflow/generated/rest/types/detect_guardrails_response.py b/skyvault/skyflow/generated/rest/types/detect_guardrails_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/detect_guardrails_response.py rename to skyvault/skyflow/generated/rest/types/detect_guardrails_response.py diff --git a/v2/skyflow/generated/rest/types/detect_guardrails_response_validation.py b/skyvault/skyflow/generated/rest/types/detect_guardrails_response_validation.py similarity index 100% rename from v2/skyflow/generated/rest/types/detect_guardrails_response_validation.py rename to skyvault/skyflow/generated/rest/types/detect_guardrails_response_validation.py diff --git a/v2/skyflow/generated/rest/types/detect_runs_response.py b/skyvault/skyflow/generated/rest/types/detect_runs_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/detect_runs_response.py rename to skyvault/skyflow/generated/rest/types/detect_runs_response.py diff --git a/v2/skyflow/generated/rest/types/detect_runs_response_output_type.py b/skyvault/skyflow/generated/rest/types/detect_runs_response_output_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/detect_runs_response_output_type.py rename to skyvault/skyflow/generated/rest/types/detect_runs_response_output_type.py diff --git a/v2/skyflow/generated/rest/types/detect_runs_response_status.py b/skyvault/skyflow/generated/rest/types/detect_runs_response_status.py similarity index 100% rename from v2/skyflow/generated/rest/types/detect_runs_response_status.py rename to skyvault/skyflow/generated/rest/types/detect_runs_response_status.py diff --git a/v2/skyflow/generated/rest/types/detokenize_record_response_value_type.py b/skyvault/skyflow/generated/rest/types/detokenize_record_response_value_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/detokenize_record_response_value_type.py rename to skyvault/skyflow/generated/rest/types/detokenize_record_response_value_type.py diff --git a/skyvault/skyflow/generated/rest/types/error_response.py b/skyvault/skyflow/generated/rest/types/error_response.py new file mode 100644 index 00000000..7c0491bb --- /dev/null +++ b/skyvault/skyflow/generated/rest/types/error_response.py @@ -0,0 +1,20 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .error_response_error import ErrorResponseError + + +class ErrorResponse(UniversalBaseModel): + error: ErrorResponseError + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_delete_token_response.py b/skyvault/skyflow/generated/rest/types/error_response_error.py similarity index 58% rename from flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_delete_token_response.py rename to skyvault/skyflow/generated/rest/types/error_response_error.py index 9129dbff..efe080d3 100644 --- a/flowvault/skyflow_flowvault/generated/rest/types/v_1_flow_delete_token_response.py +++ b/skyvault/skyflow/generated/rest/types/error_response_error.py @@ -4,15 +4,20 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel -from .v_1_delete_token_response_object import V1DeleteTokenResponseObject +from .http_code import HttpCode -class V1FlowDeleteTokenResponse(UniversalBaseModel): - tokens: typing.Optional[typing.List[V1DeleteTokenResponseObject]] = pydantic.Field(default=None) +class ErrorResponseError(UniversalBaseModel): + grpc_code: int = pydantic.Field() """ - Tokens data for Delete + gRPC status codes. See https://grpc.io/docs/guides/status-codes. """ + http_code: HttpCode + http_status: str + message: str + details: typing.Optional[typing.List[typing.Dict[str, typing.Optional[typing.Any]]]] = None + if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 else: diff --git a/v2/skyflow/generated/rest/types/file_data.py b/skyvault/skyflow/generated/rest/types/file_data.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data.py rename to skyvault/skyflow/generated/rest/types/file_data.py diff --git a/v2/skyflow/generated/rest/types/file_data_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_data_format.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_audio.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_audio.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_audio.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_audio.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_audio_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_audio_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_audio_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_audio_data_format.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_document.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_document.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_document.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_document.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_document_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_document_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_document_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_document_data_format.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_image.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_image.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_image.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_image.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_image_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_image_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_image_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_image_data_format.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_pdf.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_pdf.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_pdf.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_pdf.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_presentation.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_presentation.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_presentation.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_presentation.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_presentation_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_presentation_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_presentation_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_presentation_data_format.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_spreadsheet.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_spreadsheet.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_spreadsheet.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_spreadsheet.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_spreadsheet_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_spreadsheet_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_spreadsheet_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_spreadsheet_data_format.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_structured_text.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_structured_text.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_structured_text.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_structured_text.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_structured_text_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_structured_text_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_structured_text_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_structured_text_data_format.py diff --git a/v2/skyflow/generated/rest/types/file_data_deidentify_text.py b/skyvault/skyflow/generated/rest/types/file_data_deidentify_text.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_deidentify_text.py rename to skyvault/skyflow/generated/rest/types/file_data_deidentify_text.py diff --git a/v2/skyflow/generated/rest/types/file_data_reidentify_file.py b/skyvault/skyflow/generated/rest/types/file_data_reidentify_file.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_reidentify_file.py rename to skyvault/skyflow/generated/rest/types/file_data_reidentify_file.py diff --git a/v2/skyflow/generated/rest/types/file_data_reidentify_file_data_format.py b/skyvault/skyflow/generated/rest/types/file_data_reidentify_file_data_format.py similarity index 100% rename from v2/skyflow/generated/rest/types/file_data_reidentify_file_data_format.py rename to skyvault/skyflow/generated/rest/types/file_data_reidentify_file_data_format.py diff --git a/v2/skyflow/generated/rest/types/format.py b/skyvault/skyflow/generated/rest/types/format.py similarity index 100% rename from v2/skyflow/generated/rest/types/format.py rename to skyvault/skyflow/generated/rest/types/format.py diff --git a/v2/skyflow/generated/rest/types/format_masked_item.py b/skyvault/skyflow/generated/rest/types/format_masked_item.py similarity index 100% rename from v2/skyflow/generated/rest/types/format_masked_item.py rename to skyvault/skyflow/generated/rest/types/format_masked_item.py diff --git a/v2/skyflow/generated/rest/types/format_plaintext_item.py b/skyvault/skyflow/generated/rest/types/format_plaintext_item.py similarity index 100% rename from v2/skyflow/generated/rest/types/format_plaintext_item.py rename to skyvault/skyflow/generated/rest/types/format_plaintext_item.py diff --git a/v2/skyflow/generated/rest/types/format_redacted_item.py b/skyvault/skyflow/generated/rest/types/format_redacted_item.py similarity index 100% rename from v2/skyflow/generated/rest/types/format_redacted_item.py rename to skyvault/skyflow/generated/rest/types/format_redacted_item.py diff --git a/v2/skyflow/generated/rest/types/googlerpc_status.py b/skyvault/skyflow/generated/rest/types/googlerpc_status.py similarity index 100% rename from v2/skyflow/generated/rest/types/googlerpc_status.py rename to skyvault/skyflow/generated/rest/types/googlerpc_status.py diff --git a/skyvault/skyflow/generated/rest/types/http_code.py b/skyvault/skyflow/generated/rest/types/http_code.py new file mode 100644 index 00000000..5fc9a3fb --- /dev/null +++ b/skyvault/skyflow/generated/rest/types/http_code.py @@ -0,0 +1,3 @@ +# This file was auto-generated by Fern from our API Definition. + +HttpCode = int diff --git a/v2/skyflow/generated/rest/types/identify_response.py b/skyvault/skyflow/generated/rest/types/identify_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/identify_response.py rename to skyvault/skyflow/generated/rest/types/identify_response.py diff --git a/v2/skyflow/generated/rest/types/locations.py b/skyvault/skyflow/generated/rest/types/locations.py similarity index 100% rename from v2/skyflow/generated/rest/types/locations.py rename to skyvault/skyflow/generated/rest/types/locations.py diff --git a/v2/skyflow/generated/rest/types/protobuf_any.py b/skyvault/skyflow/generated/rest/types/protobuf_any.py similarity index 100% rename from v2/skyflow/generated/rest/types/protobuf_any.py rename to skyvault/skyflow/generated/rest/types/protobuf_any.py diff --git a/v2/skyflow/generated/rest/types/redaction_enum_redaction.py b/skyvault/skyflow/generated/rest/types/redaction_enum_redaction.py similarity index 100% rename from v2/skyflow/generated/rest/types/redaction_enum_redaction.py rename to skyvault/skyflow/generated/rest/types/redaction_enum_redaction.py diff --git a/v2/skyflow/generated/rest/types/reidentified_file_output.py b/skyvault/skyflow/generated/rest/types/reidentified_file_output.py similarity index 100% rename from v2/skyflow/generated/rest/types/reidentified_file_output.py rename to skyvault/skyflow/generated/rest/types/reidentified_file_output.py diff --git a/v2/skyflow/generated/rest/types/reidentified_file_output_processed_file_extension.py b/skyvault/skyflow/generated/rest/types/reidentified_file_output_processed_file_extension.py similarity index 100% rename from v2/skyflow/generated/rest/types/reidentified_file_output_processed_file_extension.py rename to skyvault/skyflow/generated/rest/types/reidentified_file_output_processed_file_extension.py diff --git a/v2/skyflow/generated/rest/types/reidentify_file_response.py b/skyvault/skyflow/generated/rest/types/reidentify_file_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/reidentify_file_response.py rename to skyvault/skyflow/generated/rest/types/reidentify_file_response.py diff --git a/v2/skyflow/generated/rest/types/reidentify_file_response_output_type.py b/skyvault/skyflow/generated/rest/types/reidentify_file_response_output_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/reidentify_file_response_output_type.py rename to skyvault/skyflow/generated/rest/types/reidentify_file_response_output_type.py diff --git a/v2/skyflow/generated/rest/types/reidentify_file_response_status.py b/skyvault/skyflow/generated/rest/types/reidentify_file_response_status.py similarity index 100% rename from v2/skyflow/generated/rest/types/reidentify_file_response_status.py rename to skyvault/skyflow/generated/rest/types/reidentify_file_response_status.py diff --git a/v2/skyflow/generated/rest/types/request_action_type.py b/skyvault/skyflow/generated/rest/types/request_action_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/request_action_type.py rename to skyvault/skyflow/generated/rest/types/request_action_type.py diff --git a/v2/skyflow/generated/rest/types/resource_id.py b/skyvault/skyflow/generated/rest/types/resource_id.py similarity index 100% rename from v2/skyflow/generated/rest/types/resource_id.py rename to skyvault/skyflow/generated/rest/types/resource_id.py diff --git a/v2/skyflow/generated/rest/types/shift_dates.py b/skyvault/skyflow/generated/rest/types/shift_dates.py similarity index 100% rename from v2/skyflow/generated/rest/types/shift_dates.py rename to skyvault/skyflow/generated/rest/types/shift_dates.py diff --git a/v2/skyflow/generated/rest/types/shift_dates_entity_types_item.py b/skyvault/skyflow/generated/rest/types/shift_dates_entity_types_item.py similarity index 100% rename from v2/skyflow/generated/rest/types/shift_dates_entity_types_item.py rename to skyvault/skyflow/generated/rest/types/shift_dates_entity_types_item.py diff --git a/v2/skyflow/generated/rest/types/string_response_entities.py b/skyvault/skyflow/generated/rest/types/string_response_entities.py similarity index 100% rename from v2/skyflow/generated/rest/types/string_response_entities.py rename to skyvault/skyflow/generated/rest/types/string_response_entities.py diff --git a/v2/skyflow/generated/rest/types/token_type_mapping.py b/skyvault/skyflow/generated/rest/types/token_type_mapping.py similarity index 100% rename from v2/skyflow/generated/rest/types/token_type_mapping.py rename to skyvault/skyflow/generated/rest/types/token_type_mapping.py diff --git a/v2/skyflow/generated/rest/types/token_type_mapping_default.py b/skyvault/skyflow/generated/rest/types/token_type_mapping_default.py similarity index 100% rename from v2/skyflow/generated/rest/types/token_type_mapping_default.py rename to skyvault/skyflow/generated/rest/types/token_type_mapping_default.py diff --git a/v2/skyflow/generated/rest/types/token_type_mapping_entity_only_item.py b/skyvault/skyflow/generated/rest/types/token_type_mapping_entity_only_item.py similarity index 100% rename from v2/skyflow/generated/rest/types/token_type_mapping_entity_only_item.py rename to skyvault/skyflow/generated/rest/types/token_type_mapping_entity_only_item.py diff --git a/v2/skyflow/generated/rest/types/token_type_mapping_entity_unq_counter_item.py b/skyvault/skyflow/generated/rest/types/token_type_mapping_entity_unq_counter_item.py similarity index 100% rename from v2/skyflow/generated/rest/types/token_type_mapping_entity_unq_counter_item.py rename to skyvault/skyflow/generated/rest/types/token_type_mapping_entity_unq_counter_item.py diff --git a/v2/skyflow/generated/rest/types/token_type_mapping_vault_token_item.py b/skyvault/skyflow/generated/rest/types/token_type_mapping_vault_token_item.py similarity index 100% rename from v2/skyflow/generated/rest/types/token_type_mapping_vault_token_item.py rename to skyvault/skyflow/generated/rest/types/token_type_mapping_vault_token_item.py diff --git a/v2/skyflow/generated/rest/types/transformations.py b/skyvault/skyflow/generated/rest/types/transformations.py similarity index 100% rename from v2/skyflow/generated/rest/types/transformations.py rename to skyvault/skyflow/generated/rest/types/transformations.py diff --git a/v2/skyflow/generated/rest/types/upload_file_v_2_response.py b/skyvault/skyflow/generated/rest/types/upload_file_v_2_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/upload_file_v_2_response.py rename to skyvault/skyflow/generated/rest/types/upload_file_v_2_response.py diff --git a/v2/skyflow/generated/rest/types/uuid_.py b/skyvault/skyflow/generated/rest/types/uuid_.py similarity index 100% rename from v2/skyflow/generated/rest/types/uuid_.py rename to skyvault/skyflow/generated/rest/types/uuid_.py diff --git a/v2/skyflow/generated/rest/types/v_1_audit_after_options.py b/skyvault/skyflow/generated/rest/types/v_1_audit_after_options.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_audit_after_options.py rename to skyvault/skyflow/generated/rest/types/v_1_audit_after_options.py diff --git a/v2/skyflow/generated/rest/types/v_1_audit_event_response.py b/skyvault/skyflow/generated/rest/types/v_1_audit_event_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_audit_event_response.py rename to skyvault/skyflow/generated/rest/types/v_1_audit_event_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_audit_response.py b/skyvault/skyflow/generated/rest/types/v_1_audit_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_audit_response.py rename to skyvault/skyflow/generated/rest/types/v_1_audit_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_audit_response_event.py b/skyvault/skyflow/generated/rest/types/v_1_audit_response_event.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_audit_response_event.py rename to skyvault/skyflow/generated/rest/types/v_1_audit_response_event.py diff --git a/v2/skyflow/generated/rest/types/v_1_audit_response_event_request.py b/skyvault/skyflow/generated/rest/types/v_1_audit_response_event_request.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_audit_response_event_request.py rename to skyvault/skyflow/generated/rest/types/v_1_audit_response_event_request.py diff --git a/v2/skyflow/generated/rest/types/v_1_batch_operation_response.py b/skyvault/skyflow/generated/rest/types/v_1_batch_operation_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_batch_operation_response.py rename to skyvault/skyflow/generated/rest/types/v_1_batch_operation_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_batch_record.py b/skyvault/skyflow/generated/rest/types/v_1_batch_record.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_batch_record.py rename to skyvault/skyflow/generated/rest/types/v_1_batch_record.py diff --git a/v2/skyflow/generated/rest/types/v_1_bin_list_response.py b/skyvault/skyflow/generated/rest/types/v_1_bin_list_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_bin_list_response.py rename to skyvault/skyflow/generated/rest/types/v_1_bin_list_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_bulk_delete_record_response.py b/skyvault/skyflow/generated/rest/types/v_1_bulk_delete_record_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_bulk_delete_record_response.py rename to skyvault/skyflow/generated/rest/types/v_1_bulk_delete_record_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_bulk_get_record_response.py b/skyvault/skyflow/generated/rest/types/v_1_bulk_get_record_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_bulk_get_record_response.py rename to skyvault/skyflow/generated/rest/types/v_1_bulk_get_record_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_byot.py b/skyvault/skyflow/generated/rest/types/v_1_byot.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_byot.py rename to skyvault/skyflow/generated/rest/types/v_1_byot.py diff --git a/v2/skyflow/generated/rest/types/v_1_card.py b/skyvault/skyflow/generated/rest/types/v_1_card.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_card.py rename to skyvault/skyflow/generated/rest/types/v_1_card.py diff --git a/v2/skyflow/generated/rest/types/v_1_delete_file_response.py b/skyvault/skyflow/generated/rest/types/v_1_delete_file_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_delete_file_response.py rename to skyvault/skyflow/generated/rest/types/v_1_delete_file_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_delete_record_response.py b/skyvault/skyflow/generated/rest/types/v_1_delete_record_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_delete_record_response.py rename to skyvault/skyflow/generated/rest/types/v_1_delete_record_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_detokenize_record_request.py b/skyvault/skyflow/generated/rest/types/v_1_detokenize_record_request.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_detokenize_record_request.py rename to skyvault/skyflow/generated/rest/types/v_1_detokenize_record_request.py diff --git a/v2/skyflow/generated/rest/types/v_1_detokenize_record_response.py b/skyvault/skyflow/generated/rest/types/v_1_detokenize_record_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_detokenize_record_response.py rename to skyvault/skyflow/generated/rest/types/v_1_detokenize_record_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_detokenize_response.py b/skyvault/skyflow/generated/rest/types/v_1_detokenize_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_detokenize_response.py rename to skyvault/skyflow/generated/rest/types/v_1_detokenize_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_field_records.py b/skyvault/skyflow/generated/rest/types/v_1_field_records.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_field_records.py rename to skyvault/skyflow/generated/rest/types/v_1_field_records.py diff --git a/v2/skyflow/generated/rest/types/v_1_file_av_scan_status.py b/skyvault/skyflow/generated/rest/types/v_1_file_av_scan_status.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_file_av_scan_status.py rename to skyvault/skyflow/generated/rest/types/v_1_file_av_scan_status.py diff --git a/v2/skyflow/generated/rest/types/v_1_get_auth_token_response.py b/skyvault/skyflow/generated/rest/types/v_1_get_auth_token_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_get_auth_token_response.py rename to skyvault/skyflow/generated/rest/types/v_1_get_auth_token_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_get_file_scan_status_response.py b/skyvault/skyflow/generated/rest/types/v_1_get_file_scan_status_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_get_file_scan_status_response.py rename to skyvault/skyflow/generated/rest/types/v_1_get_file_scan_status_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_get_query_response.py b/skyvault/skyflow/generated/rest/types/v_1_get_query_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_get_query_response.py rename to skyvault/skyflow/generated/rest/types/v_1_get_query_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_insert_record_response.py b/skyvault/skyflow/generated/rest/types/v_1_insert_record_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_insert_record_response.py rename to skyvault/skyflow/generated/rest/types/v_1_insert_record_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_member_type.py b/skyvault/skyflow/generated/rest/types/v_1_member_type.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_member_type.py rename to skyvault/skyflow/generated/rest/types/v_1_member_type.py diff --git a/v2/skyflow/generated/rest/types/v_1_record_meta_properties.py b/skyvault/skyflow/generated/rest/types/v_1_record_meta_properties.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_record_meta_properties.py rename to skyvault/skyflow/generated/rest/types/v_1_record_meta_properties.py diff --git a/v2/skyflow/generated/rest/types/v_1_tokenize_record_request.py b/skyvault/skyflow/generated/rest/types/v_1_tokenize_record_request.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_tokenize_record_request.py rename to skyvault/skyflow/generated/rest/types/v_1_tokenize_record_request.py diff --git a/v2/skyflow/generated/rest/types/v_1_tokenize_record_response.py b/skyvault/skyflow/generated/rest/types/v_1_tokenize_record_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_tokenize_record_response.py rename to skyvault/skyflow/generated/rest/types/v_1_tokenize_record_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_tokenize_response.py b/skyvault/skyflow/generated/rest/types/v_1_tokenize_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_tokenize_response.py rename to skyvault/skyflow/generated/rest/types/v_1_tokenize_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_update_record_response.py b/skyvault/skyflow/generated/rest/types/v_1_update_record_response.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_update_record_response.py rename to skyvault/skyflow/generated/rest/types/v_1_update_record_response.py diff --git a/v2/skyflow/generated/rest/types/v_1_vault_field_mapping.py b/skyvault/skyflow/generated/rest/types/v_1_vault_field_mapping.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_vault_field_mapping.py rename to skyvault/skyflow/generated/rest/types/v_1_vault_field_mapping.py diff --git a/v2/skyflow/generated/rest/types/v_1_vault_schema_config.py b/skyvault/skyflow/generated/rest/types/v_1_vault_schema_config.py similarity index 100% rename from v2/skyflow/generated/rest/types/v_1_vault_schema_config.py rename to skyvault/skyflow/generated/rest/types/v_1_vault_schema_config.py diff --git a/v2/skyflow/generated/rest/types/word_character_count.py b/skyvault/skyflow/generated/rest/types/word_character_count.py similarity index 100% rename from v2/skyflow/generated/rest/types/word_character_count.py rename to skyvault/skyflow/generated/rest/types/word_character_count.py diff --git a/v2/skyflow/generated/rest/version.py b/skyvault/skyflow/generated/rest/version.py similarity index 100% rename from v2/skyflow/generated/rest/version.py rename to skyvault/skyflow/generated/rest/version.py diff --git a/v2/skyflow/py.typed b/skyvault/skyflow/py.typed similarity index 100% rename from v2/skyflow/py.typed rename to skyvault/skyflow/py.typed diff --git a/v2/skyflow/service_account/__init__.py b/skyvault/skyflow/service_account/__init__.py similarity index 100% rename from v2/skyflow/service_account/__init__.py rename to skyvault/skyflow/service_account/__init__.py diff --git a/v2/skyflow/service_account/_utils.py b/skyvault/skyflow/service_account/_utils.py similarity index 100% rename from v2/skyflow/service_account/_utils.py rename to skyvault/skyflow/service_account/_utils.py diff --git a/v2/skyflow/service_account/client/__init__.py b/skyvault/skyflow/service_account/client/__init__.py similarity index 100% rename from v2/skyflow/service_account/client/__init__.py rename to skyvault/skyflow/service_account/client/__init__.py diff --git a/v2/skyflow/service_account/client/auth_client.py b/skyvault/skyflow/service_account/client/auth_client.py similarity index 100% rename from v2/skyflow/service_account/client/auth_client.py rename to skyvault/skyflow/service_account/client/auth_client.py diff --git a/v2/skyflow/utils/__init__.py b/skyvault/skyflow/utils/__init__.py similarity index 100% rename from v2/skyflow/utils/__init__.py rename to skyvault/skyflow/utils/__init__.py diff --git a/v2/skyflow/utils/_helpers.py b/skyvault/skyflow/utils/_helpers.py similarity index 100% rename from v2/skyflow/utils/_helpers.py rename to skyvault/skyflow/utils/_helpers.py diff --git a/v2/skyflow/utils/_skyflow_messages.py b/skyvault/skyflow/utils/_skyflow_messages.py similarity index 100% rename from v2/skyflow/utils/_skyflow_messages.py rename to skyvault/skyflow/utils/_skyflow_messages.py diff --git a/v2/skyflow/utils/_utils.py b/skyvault/skyflow/utils/_utils.py similarity index 100% rename from v2/skyflow/utils/_utils.py rename to skyvault/skyflow/utils/_utils.py diff --git a/v2/skyflow/utils/_version.py b/skyvault/skyflow/utils/_version.py similarity index 100% rename from v2/skyflow/utils/_version.py rename to skyvault/skyflow/utils/_version.py diff --git a/v2/skyflow/utils/constants.py b/skyvault/skyflow/utils/constants.py similarity index 100% rename from v2/skyflow/utils/constants.py rename to skyvault/skyflow/utils/constants.py diff --git a/v2/skyflow/utils/enums/__init__.py b/skyvault/skyflow/utils/enums/__init__.py similarity index 100% rename from v2/skyflow/utils/enums/__init__.py rename to skyvault/skyflow/utils/enums/__init__.py diff --git a/v2/skyflow/utils/enums/content_types.py b/skyvault/skyflow/utils/enums/content_types.py similarity index 100% rename from v2/skyflow/utils/enums/content_types.py rename to skyvault/skyflow/utils/enums/content_types.py diff --git a/v2/skyflow/utils/enums/detect_entities.py b/skyvault/skyflow/utils/enums/detect_entities.py similarity index 100% rename from v2/skyflow/utils/enums/detect_entities.py rename to skyvault/skyflow/utils/enums/detect_entities.py diff --git a/v2/skyflow/utils/enums/detect_output_transcriptions.py b/skyvault/skyflow/utils/enums/detect_output_transcriptions.py similarity index 100% rename from v2/skyflow/utils/enums/detect_output_transcriptions.py rename to skyvault/skyflow/utils/enums/detect_output_transcriptions.py diff --git a/v2/skyflow/utils/enums/env.py b/skyvault/skyflow/utils/enums/env.py similarity index 100% rename from v2/skyflow/utils/enums/env.py rename to skyvault/skyflow/utils/enums/env.py diff --git a/v2/skyflow/utils/enums/log_level.py b/skyvault/skyflow/utils/enums/log_level.py similarity index 100% rename from v2/skyflow/utils/enums/log_level.py rename to skyvault/skyflow/utils/enums/log_level.py diff --git a/v2/skyflow/utils/enums/masking_method.py b/skyvault/skyflow/utils/enums/masking_method.py similarity index 100% rename from v2/skyflow/utils/enums/masking_method.py rename to skyvault/skyflow/utils/enums/masking_method.py diff --git a/v2/skyflow/utils/enums/redaction_type.py b/skyvault/skyflow/utils/enums/redaction_type.py similarity index 100% rename from v2/skyflow/utils/enums/redaction_type.py rename to skyvault/skyflow/utils/enums/redaction_type.py diff --git a/v2/skyflow/utils/enums/request_method.py b/skyvault/skyflow/utils/enums/request_method.py similarity index 100% rename from v2/skyflow/utils/enums/request_method.py rename to skyvault/skyflow/utils/enums/request_method.py diff --git a/v2/skyflow/utils/enums/token_mode.py b/skyvault/skyflow/utils/enums/token_mode.py similarity index 100% rename from v2/skyflow/utils/enums/token_mode.py rename to skyvault/skyflow/utils/enums/token_mode.py diff --git a/v2/skyflow/utils/enums/token_type.py b/skyvault/skyflow/utils/enums/token_type.py similarity index 100% rename from v2/skyflow/utils/enums/token_type.py rename to skyvault/skyflow/utils/enums/token_type.py diff --git a/v2/skyflow/utils/logger/__init__.py b/skyvault/skyflow/utils/logger/__init__.py similarity index 100% rename from v2/skyflow/utils/logger/__init__.py rename to skyvault/skyflow/utils/logger/__init__.py diff --git a/v2/skyflow/utils/logger/_log_helpers.py b/skyvault/skyflow/utils/logger/_log_helpers.py similarity index 100% rename from v2/skyflow/utils/logger/_log_helpers.py rename to skyvault/skyflow/utils/logger/_log_helpers.py diff --git a/v2/skyflow/utils/logger/_logger.py b/skyvault/skyflow/utils/logger/_logger.py similarity index 100% rename from v2/skyflow/utils/logger/_logger.py rename to skyvault/skyflow/utils/logger/_logger.py diff --git a/v2/skyflow/utils/validations/__init__.py b/skyvault/skyflow/utils/validations/__init__.py similarity index 100% rename from v2/skyflow/utils/validations/__init__.py rename to skyvault/skyflow/utils/validations/__init__.py diff --git a/v2/skyflow/utils/validations/_validations.py b/skyvault/skyflow/utils/validations/_validations.py similarity index 100% rename from v2/skyflow/utils/validations/_validations.py rename to skyvault/skyflow/utils/validations/_validations.py diff --git a/v2/skyflow/vault/__init__.py b/skyvault/skyflow/vault/__init__.py similarity index 100% rename from v2/skyflow/vault/__init__.py rename to skyvault/skyflow/vault/__init__.py diff --git a/v2/skyflow/vault/client/__init__.py b/skyvault/skyflow/vault/client/__init__.py similarity index 100% rename from v2/skyflow/vault/client/__init__.py rename to skyvault/skyflow/vault/client/__init__.py diff --git a/v2/skyflow/vault/client/client.py b/skyvault/skyflow/vault/client/client.py similarity index 100% rename from v2/skyflow/vault/client/client.py rename to skyvault/skyflow/vault/client/client.py diff --git a/v2/skyflow/vault/connection/__init__.py b/skyvault/skyflow/vault/connection/__init__.py similarity index 100% rename from v2/skyflow/vault/connection/__init__.py rename to skyvault/skyflow/vault/connection/__init__.py diff --git a/v2/skyflow/vault/connection/_invoke_connection_request.py b/skyvault/skyflow/vault/connection/_invoke_connection_request.py similarity index 100% rename from v2/skyflow/vault/connection/_invoke_connection_request.py rename to skyvault/skyflow/vault/connection/_invoke_connection_request.py diff --git a/v2/skyflow/vault/connection/_invoke_connection_response.py b/skyvault/skyflow/vault/connection/_invoke_connection_response.py similarity index 100% rename from v2/skyflow/vault/connection/_invoke_connection_response.py rename to skyvault/skyflow/vault/connection/_invoke_connection_response.py diff --git a/v2/skyflow/vault/controller/__init__.py b/skyvault/skyflow/vault/controller/__init__.py similarity index 100% rename from v2/skyflow/vault/controller/__init__.py rename to skyvault/skyflow/vault/controller/__init__.py diff --git a/v2/skyflow/vault/controller/_audit.py b/skyvault/skyflow/vault/controller/_audit.py similarity index 100% rename from v2/skyflow/vault/controller/_audit.py rename to skyvault/skyflow/vault/controller/_audit.py diff --git a/v2/skyflow/vault/controller/_bin_look_up.py b/skyvault/skyflow/vault/controller/_bin_look_up.py similarity index 100% rename from v2/skyflow/vault/controller/_bin_look_up.py rename to skyvault/skyflow/vault/controller/_bin_look_up.py diff --git a/v2/skyflow/vault/controller/_connections.py b/skyvault/skyflow/vault/controller/_connections.py similarity index 100% rename from v2/skyflow/vault/controller/_connections.py rename to skyvault/skyflow/vault/controller/_connections.py diff --git a/v2/skyflow/vault/controller/_detect.py b/skyvault/skyflow/vault/controller/_detect.py similarity index 100% rename from v2/skyflow/vault/controller/_detect.py rename to skyvault/skyflow/vault/controller/_detect.py diff --git a/v2/skyflow/vault/controller/_vault.py b/skyvault/skyflow/vault/controller/_vault.py similarity index 100% rename from v2/skyflow/vault/controller/_vault.py rename to skyvault/skyflow/vault/controller/_vault.py diff --git a/v2/skyflow/vault/data/__init__.py b/skyvault/skyflow/vault/data/__init__.py similarity index 100% rename from v2/skyflow/vault/data/__init__.py rename to skyvault/skyflow/vault/data/__init__.py diff --git a/v2/skyflow/vault/data/_delete_request.py b/skyvault/skyflow/vault/data/_delete_request.py similarity index 100% rename from v2/skyflow/vault/data/_delete_request.py rename to skyvault/skyflow/vault/data/_delete_request.py diff --git a/v2/skyflow/vault/data/_delete_response.py b/skyvault/skyflow/vault/data/_delete_response.py similarity index 100% rename from v2/skyflow/vault/data/_delete_response.py rename to skyvault/skyflow/vault/data/_delete_response.py diff --git a/v2/skyflow/vault/data/_file_upload_request.py b/skyvault/skyflow/vault/data/_file_upload_request.py similarity index 100% rename from v2/skyflow/vault/data/_file_upload_request.py rename to skyvault/skyflow/vault/data/_file_upload_request.py diff --git a/v2/skyflow/vault/data/_file_upload_response.py b/skyvault/skyflow/vault/data/_file_upload_response.py similarity index 100% rename from v2/skyflow/vault/data/_file_upload_response.py rename to skyvault/skyflow/vault/data/_file_upload_response.py diff --git a/v2/skyflow/vault/data/_get_request.py b/skyvault/skyflow/vault/data/_get_request.py similarity index 100% rename from v2/skyflow/vault/data/_get_request.py rename to skyvault/skyflow/vault/data/_get_request.py diff --git a/v2/skyflow/vault/data/_get_response.py b/skyvault/skyflow/vault/data/_get_response.py similarity index 100% rename from v2/skyflow/vault/data/_get_response.py rename to skyvault/skyflow/vault/data/_get_response.py diff --git a/v2/skyflow/vault/data/_insert_request.py b/skyvault/skyflow/vault/data/_insert_request.py similarity index 100% rename from v2/skyflow/vault/data/_insert_request.py rename to skyvault/skyflow/vault/data/_insert_request.py diff --git a/v2/skyflow/vault/data/_insert_response.py b/skyvault/skyflow/vault/data/_insert_response.py similarity index 100% rename from v2/skyflow/vault/data/_insert_response.py rename to skyvault/skyflow/vault/data/_insert_response.py diff --git a/v2/skyflow/vault/data/_query_request.py b/skyvault/skyflow/vault/data/_query_request.py similarity index 100% rename from v2/skyflow/vault/data/_query_request.py rename to skyvault/skyflow/vault/data/_query_request.py diff --git a/v2/skyflow/vault/data/_query_response.py b/skyvault/skyflow/vault/data/_query_response.py similarity index 100% rename from v2/skyflow/vault/data/_query_response.py rename to skyvault/skyflow/vault/data/_query_response.py diff --git a/v2/skyflow/vault/data/_update_request.py b/skyvault/skyflow/vault/data/_update_request.py similarity index 100% rename from v2/skyflow/vault/data/_update_request.py rename to skyvault/skyflow/vault/data/_update_request.py diff --git a/v2/skyflow/vault/data/_update_response.py b/skyvault/skyflow/vault/data/_update_response.py similarity index 100% rename from v2/skyflow/vault/data/_update_response.py rename to skyvault/skyflow/vault/data/_update_response.py diff --git a/v2/skyflow/vault/data/_upload_file_request.py b/skyvault/skyflow/vault/data/_upload_file_request.py similarity index 100% rename from v2/skyflow/vault/data/_upload_file_request.py rename to skyvault/skyflow/vault/data/_upload_file_request.py diff --git a/v2/skyflow/vault/detect/__init__.py b/skyvault/skyflow/vault/detect/__init__.py similarity index 100% rename from v2/skyflow/vault/detect/__init__.py rename to skyvault/skyflow/vault/detect/__init__.py diff --git a/v2/skyflow/vault/detect/_audio_bleep.py b/skyvault/skyflow/vault/detect/_audio_bleep.py similarity index 100% rename from v2/skyflow/vault/detect/_audio_bleep.py rename to skyvault/skyflow/vault/detect/_audio_bleep.py diff --git a/v2/skyflow/vault/detect/_date_transformation.py b/skyvault/skyflow/vault/detect/_date_transformation.py similarity index 100% rename from v2/skyflow/vault/detect/_date_transformation.py rename to skyvault/skyflow/vault/detect/_date_transformation.py diff --git a/v2/skyflow/vault/detect/_deidentify_file_request.py b/skyvault/skyflow/vault/detect/_deidentify_file_request.py similarity index 100% rename from v2/skyflow/vault/detect/_deidentify_file_request.py rename to skyvault/skyflow/vault/detect/_deidentify_file_request.py diff --git a/v2/skyflow/vault/detect/_deidentify_file_response.py b/skyvault/skyflow/vault/detect/_deidentify_file_response.py similarity index 100% rename from v2/skyflow/vault/detect/_deidentify_file_response.py rename to skyvault/skyflow/vault/detect/_deidentify_file_response.py diff --git a/v2/skyflow/vault/detect/_deidentify_text_request.py b/skyvault/skyflow/vault/detect/_deidentify_text_request.py similarity index 100% rename from v2/skyflow/vault/detect/_deidentify_text_request.py rename to skyvault/skyflow/vault/detect/_deidentify_text_request.py diff --git a/v2/skyflow/vault/detect/_deidentify_text_response.py b/skyvault/skyflow/vault/detect/_deidentify_text_response.py similarity index 100% rename from v2/skyflow/vault/detect/_deidentify_text_response.py rename to skyvault/skyflow/vault/detect/_deidentify_text_response.py diff --git a/v2/skyflow/vault/detect/_entity_info.py b/skyvault/skyflow/vault/detect/_entity_info.py similarity index 100% rename from v2/skyflow/vault/detect/_entity_info.py rename to skyvault/skyflow/vault/detect/_entity_info.py diff --git a/v2/skyflow/vault/detect/_file.py b/skyvault/skyflow/vault/detect/_file.py similarity index 100% rename from v2/skyflow/vault/detect/_file.py rename to skyvault/skyflow/vault/detect/_file.py diff --git a/v2/skyflow/vault/detect/_file_input.py b/skyvault/skyflow/vault/detect/_file_input.py similarity index 100% rename from v2/skyflow/vault/detect/_file_input.py rename to skyvault/skyflow/vault/detect/_file_input.py diff --git a/v2/skyflow/vault/detect/_get_detect_run_request.py b/skyvault/skyflow/vault/detect/_get_detect_run_request.py similarity index 100% rename from v2/skyflow/vault/detect/_get_detect_run_request.py rename to skyvault/skyflow/vault/detect/_get_detect_run_request.py diff --git a/v2/skyflow/vault/detect/_reidentify_text_request.py b/skyvault/skyflow/vault/detect/_reidentify_text_request.py similarity index 100% rename from v2/skyflow/vault/detect/_reidentify_text_request.py rename to skyvault/skyflow/vault/detect/_reidentify_text_request.py diff --git a/v2/skyflow/vault/detect/_reidentify_text_response.py b/skyvault/skyflow/vault/detect/_reidentify_text_response.py similarity index 100% rename from v2/skyflow/vault/detect/_reidentify_text_response.py rename to skyvault/skyflow/vault/detect/_reidentify_text_response.py diff --git a/v2/skyflow/vault/detect/_text_index.py b/skyvault/skyflow/vault/detect/_text_index.py similarity index 100% rename from v2/skyflow/vault/detect/_text_index.py rename to skyvault/skyflow/vault/detect/_text_index.py diff --git a/v2/skyflow/vault/detect/_token_format.py b/skyvault/skyflow/vault/detect/_token_format.py similarity index 100% rename from v2/skyflow/vault/detect/_token_format.py rename to skyvault/skyflow/vault/detect/_token_format.py diff --git a/v2/skyflow/vault/detect/_transformations.py b/skyvault/skyflow/vault/detect/_transformations.py similarity index 100% rename from v2/skyflow/vault/detect/_transformations.py rename to skyvault/skyflow/vault/detect/_transformations.py diff --git a/v2/skyflow/vault/tokens/__init__.py b/skyvault/skyflow/vault/tokens/__init__.py similarity index 100% rename from v2/skyflow/vault/tokens/__init__.py rename to skyvault/skyflow/vault/tokens/__init__.py diff --git a/v2/skyflow/vault/tokens/_detokenize_request.py b/skyvault/skyflow/vault/tokens/_detokenize_request.py similarity index 100% rename from v2/skyflow/vault/tokens/_detokenize_request.py rename to skyvault/skyflow/vault/tokens/_detokenize_request.py diff --git a/v2/skyflow/vault/tokens/_detokenize_response.py b/skyvault/skyflow/vault/tokens/_detokenize_response.py similarity index 100% rename from v2/skyflow/vault/tokens/_detokenize_response.py rename to skyvault/skyflow/vault/tokens/_detokenize_response.py diff --git a/v2/skyflow/vault/tokens/_tokenize_request.py b/skyvault/skyflow/vault/tokens/_tokenize_request.py similarity index 100% rename from v2/skyflow/vault/tokens/_tokenize_request.py rename to skyvault/skyflow/vault/tokens/_tokenize_request.py diff --git a/v2/skyflow/vault/tokens/_tokenize_response.py b/skyvault/skyflow/vault/tokens/_tokenize_response.py similarity index 100% rename from v2/skyflow/vault/tokens/_tokenize_response.py rename to skyvault/skyflow/vault/tokens/_tokenize_response.py diff --git a/v2/tests/__init__.py b/skyvault/tests/__init__.py similarity index 100% rename from v2/tests/__init__.py rename to skyvault/tests/__init__.py diff --git a/v2/tests/client/__init__.py b/skyvault/tests/client/__init__.py similarity index 100% rename from v2/tests/client/__init__.py rename to skyvault/tests/client/__init__.py diff --git a/v2/tests/client/test_skyflow.py b/skyvault/tests/client/test_skyflow.py similarity index 100% rename from v2/tests/client/test_skyflow.py rename to skyvault/tests/client/test_skyflow.py diff --git a/v2/tests/service_account/__init__.py b/skyvault/tests/service_account/__init__.py similarity index 100% rename from v2/tests/service_account/__init__.py rename to skyvault/tests/service_account/__init__.py diff --git a/v2/tests/service_account/invalid_creds.json b/skyvault/tests/service_account/invalid_creds.json similarity index 100% rename from v2/tests/service_account/invalid_creds.json rename to skyvault/tests/service_account/invalid_creds.json diff --git a/v2/tests/service_account/test__utils.py b/skyvault/tests/service_account/test__utils.py similarity index 100% rename from v2/tests/service_account/test__utils.py rename to skyvault/tests/service_account/test__utils.py diff --git a/v2/tests/utils/__init__.py b/skyvault/tests/utils/__init__.py similarity index 100% rename from v2/tests/utils/__init__.py rename to skyvault/tests/utils/__init__.py diff --git a/v2/tests/utils/logger/__init__.py b/skyvault/tests/utils/logger/__init__.py similarity index 100% rename from v2/tests/utils/logger/__init__.py rename to skyvault/tests/utils/logger/__init__.py diff --git a/v2/tests/utils/logger/test__log_helpers.py b/skyvault/tests/utils/logger/test__log_helpers.py similarity index 100% rename from v2/tests/utils/logger/test__log_helpers.py rename to skyvault/tests/utils/logger/test__log_helpers.py diff --git a/v2/tests/utils/logger/test__logger.py b/skyvault/tests/utils/logger/test__logger.py similarity index 100% rename from v2/tests/utils/logger/test__logger.py rename to skyvault/tests/utils/logger/test__logger.py diff --git a/v2/tests/utils/test__helpers.py b/skyvault/tests/utils/test__helpers.py similarity index 100% rename from v2/tests/utils/test__helpers.py rename to skyvault/tests/utils/test__helpers.py diff --git a/v2/tests/utils/test__utils.py b/skyvault/tests/utils/test__utils.py similarity index 100% rename from v2/tests/utils/test__utils.py rename to skyvault/tests/utils/test__utils.py diff --git a/v2/tests/utils/validations/__init__.py b/skyvault/tests/utils/validations/__init__.py similarity index 100% rename from v2/tests/utils/validations/__init__.py rename to skyvault/tests/utils/validations/__init__.py diff --git a/v2/tests/utils/validations/test__validations.py b/skyvault/tests/utils/validations/test__validations.py similarity index 100% rename from v2/tests/utils/validations/test__validations.py rename to skyvault/tests/utils/validations/test__validations.py diff --git a/v2/tests/vault/__init__.py b/skyvault/tests/vault/__init__.py similarity index 100% rename from v2/tests/vault/__init__.py rename to skyvault/tests/vault/__init__.py diff --git a/v2/tests/vault/client/__init__.py b/skyvault/tests/vault/client/__init__.py similarity index 100% rename from v2/tests/vault/client/__init__.py rename to skyvault/tests/vault/client/__init__.py diff --git a/v2/tests/vault/client/test__client.py b/skyvault/tests/vault/client/test__client.py similarity index 100% rename from v2/tests/vault/client/test__client.py rename to skyvault/tests/vault/client/test__client.py diff --git a/v2/tests/vault/connection/__init__.py b/skyvault/tests/vault/connection/__init__.py similarity index 100% rename from v2/tests/vault/connection/__init__.py rename to skyvault/tests/vault/connection/__init__.py diff --git a/v2/tests/vault/connection/test_responses.py b/skyvault/tests/vault/connection/test_responses.py similarity index 100% rename from v2/tests/vault/connection/test_responses.py rename to skyvault/tests/vault/connection/test_responses.py diff --git a/v2/tests/vault/controller/__init__.py b/skyvault/tests/vault/controller/__init__.py similarity index 100% rename from v2/tests/vault/controller/__init__.py rename to skyvault/tests/vault/controller/__init__.py diff --git a/v2/tests/vault/controller/test__audit_binlookup.py b/skyvault/tests/vault/controller/test__audit_binlookup.py similarity index 100% rename from v2/tests/vault/controller/test__audit_binlookup.py rename to skyvault/tests/vault/controller/test__audit_binlookup.py diff --git a/v2/tests/vault/controller/test__connection.py b/skyvault/tests/vault/controller/test__connection.py similarity index 100% rename from v2/tests/vault/controller/test__connection.py rename to skyvault/tests/vault/controller/test__connection.py diff --git a/v2/tests/vault/controller/test__detect.py b/skyvault/tests/vault/controller/test__detect.py similarity index 100% rename from v2/tests/vault/controller/test__detect.py rename to skyvault/tests/vault/controller/test__detect.py diff --git a/v2/tests/vault/controller/test__vault.py b/skyvault/tests/vault/controller/test__vault.py similarity index 100% rename from v2/tests/vault/controller/test__vault.py rename to skyvault/tests/vault/controller/test__vault.py diff --git a/v2/tests/vault/data/__init__.py b/skyvault/tests/vault/data/__init__.py similarity index 100% rename from v2/tests/vault/data/__init__.py rename to skyvault/tests/vault/data/__init__.py diff --git a/v2/tests/vault/data/test_responses.py b/skyvault/tests/vault/data/test_responses.py similarity index 100% rename from v2/tests/vault/data/test_responses.py rename to skyvault/tests/vault/data/test_responses.py diff --git a/v2/tests/vault/detect/__init__.py b/skyvault/tests/vault/detect/__init__.py similarity index 100% rename from v2/tests/vault/detect/__init__.py rename to skyvault/tests/vault/detect/__init__.py diff --git a/v2/tests/vault/detect/test_models.py b/skyvault/tests/vault/detect/test_models.py similarity index 100% rename from v2/tests/vault/detect/test_models.py rename to skyvault/tests/vault/detect/test_models.py diff --git a/v2/tests/vault/tokens/__init__.py b/skyvault/tests/vault/tokens/__init__.py similarity index 100% rename from v2/tests/vault/tokens/__init__.py rename to skyvault/tests/vault/tokens/__init__.py diff --git a/v2/tests/vault/tokens/test_responses.py b/skyvault/tests/vault/tokens/test_responses.py similarity index 100% rename from v2/tests/vault/tokens/test_responses.py rename to skyvault/tests/vault/tokens/test_responses.py diff --git a/tests/contract/_adapter_loader.py b/tests/contract/_adapter_loader.py index 2534a174..c23c0269 100644 --- a/tests/contract/_adapter_loader.py +++ b/tests/contract/_adapter_loader.py @@ -6,7 +6,7 @@ Usage (run once per variant, in that variant's own installed/PYTHONPATH environment -- v2's skyflow and flowvault's skyflow_flowvault can never coexist in one process): - SKYFLOW_TEST_VARIANT=v2 PYTHONPATH=.:v2 python -m unittest discover -s tests/contract -t . + SKYFLOW_TEST_VARIANT=v2 PYTHONPATH=.:skyvault python -m unittest discover -s tests/contract -t . SKYFLOW_TEST_VARIANT=v3 PYTHONPATH=.:flowvault python -m unittest discover -s tests/contract -t . """ import os @@ -20,7 +20,7 @@ else: raise RuntimeError( "SKYFLOW_TEST_VARIANT must be set to 'v2' or 'v3' before running tests/contract/ " - "(e.g. SKYFLOW_TEST_VARIANT=v2 PYTHONPATH=.:v2 python -m unittest discover -s tests/contract -t .)" + "(e.g. SKYFLOW_TEST_VARIANT=v2 PYTHONPATH=.:skyvault python -m unittest discover -s tests/contract -t .)" ) VARIANT = _VARIANT diff --git a/tests/contract/adapters/v3_adapter.py b/tests/contract/adapters/v3_adapter.py index d5a7ceb3..782cf432 100644 --- a/tests/contract/adapters/v3_adapter.py +++ b/tests/contract/adapters/v3_adapter.py @@ -5,7 +5,7 @@ from skyflow_flowvault.vault.client.client import VaultClient from skyflow_flowvault.vault.controller import VaultController -from skyflow_flowvault.vault.data import InsertRequest +from skyflow_flowvault.vault.data import InsertRequestRecord, InsertRequest def build_vault(): @@ -18,13 +18,13 @@ def build_vault(): vault_client = VaultClient(config) vault_client.initialize_client_configuration = MagicMock() # skip real credential/URL resolution insert_api = MagicMock() - vault_client.get_insert_api = MagicMock(return_value=insert_api) + vault_client.get_records_api = MagicMock(return_value=insert_api) vault = VaultController(vault_client) return vault, insert_api def build_insert_request(n): - return InsertRequest(table="contract_table", values=[dict(values={"field": f"value{i}"}) for i in range(n)]) + return InsertRequest(table_name="contract_table", records=[InsertRequestRecord(data={"field": f"value{i}"}) for i in range(n)]) def call_insert(vault, insert_api, request): @@ -35,18 +35,18 @@ def fake_insert(**kwargs): ] return SimpleNamespace(data=SimpleNamespace(records=records), headers={}) - insert_api.with_raw_response.insert.side_effect = fake_insert + insert_api.with_raw_response.insert_records.side_effect = fake_insert response = vault.insert(request) - call_count = insert_api.with_raw_response.insert.call_count + call_count = insert_api.with_raw_response.insert_records.call_count return response, call_count -# v3's InsertResponse now shares the exact same shape as v2's (inserted_fields/errors, each -# entry tagged request_index) -- kept as separate accessor functions per adapter anyway, since -# the contract module intentionally treats each variant's response as opaque. +# v3's InsertResponse is a single records list (FlowDB contract) with success/failure inline; +# kept as separate accessor functions per adapter, since the contract module intentionally +# treats each variant's response as opaque. def count_successes(response): - return len(response.inserted_fields) + return len([r for r in (response.records or []) if r.get("error") is None]) def count_errors(response): - return len(response.errors) if response.errors else 0 + return len([r for r in (response.records or []) if r.get("error") is not None])