SK-3118: FlowDB support for Python SDK (skyflow-flowvault) + module segregation

.codespellrc

@@ -3,7 +3,7 @@
 ignore-words-list = Skyflow,skyflow,skyflowapi,skyflowapis,deidentify,reidentify,detokenize,upsert,upserting,binlookup,byot,creds,fpe,devsecops,formdata,vaultid,dotenv,usecwd,runid,dateutil,Homogenous
 
 # Skip these files and folders
-skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,./skyflow/generated,setup.py
+skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,generated,setup.py
 
 # If you want to verify it is working, you can uncomment this line to see what files it checks
 # count = 

.github/workflows/contract-tests.yml

@@ -0,0 +1,116 @@
+name: Contract Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+      - skyvault-release/**
+      - flowvault-release/**
+    paths:
+      - "skyvault/**"
+      - "flowvault/**"
+      - "common/**"
+      - "ci-scripts/contract/**"
+      - ".github/workflows/contract-tests.yml"
+
+jobs:
+  contract-tests:
+    name: Contract Tests (${{ matrix.module }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - module: skyvault
+            pkg: skyflow
+          - module: flowvault
+            pkg: skyflow_flowvault
+
+    permissions:
+      contents: read
+      pull-requests: write
+
+    env:
+      GRIFFE_VERSION: "2.2.0"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Install griffe
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install "griffe==${GRIFFE_VERSION}"
+
+      - name: Verify public API surface against the committed baseline
+        run: |
+          python ci-scripts/contract/griffe_contract.py check \
+            "${{ matrix.module }}" \
+            "${{ matrix.module }}/api-report/${{ matrix.pkg }}.api.json"
+
+      - name: How to update the baseline
+        if: failure()
+        run: |
+          echo "### Public API contract drift in ${{ matrix.module }} ###"
+          echo "If this change is intentional, run:"
+          echo "    ci-scripts/contract-snapshot-update.sh ${{ matrix.module }}"
+          echo "review the api-report/${{ matrix.pkg }}.api.json diff, and commit it with your change."
+
+      - name: Detect baseline change
+        id: baseline-diff
+        if: always() && github.event.pull_request
+        run: |
+          git fetch origin "${{ github.event.pull_request.base.ref }}" --depth=1
+          BASELINE="${{ matrix.module }}/api-report/${{ matrix.pkg }}.api.json"
+          if ! git diff --quiet "origin/${{ github.event.pull_request.base.ref }}" HEAD -- "$BASELINE"; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            {
+              echo 'diff<<GRIFFE_EOF'
+              git diff "origin/${{ github.event.pull_request.base.ref }}" HEAD -- "$BASELINE" | head -300
+              echo 'GRIFFE_EOF'
+            } >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Comment contract baseline change on PR
+        if: always() && github.event.pull_request && steps.baseline-diff.outputs.changed == 'true'
+        uses: actions/github-script@v7
+        env:
+          MODULE: ${{ matrix.module }}
+          PKG: ${{ matrix.pkg }}
+          DIFF: ${{ steps.baseline-diff.outputs.diff }}
+        with:
+          script: |
+            const module = process.env.MODULE;
+            const pkg = process.env.PKG;
+            const marker = `<!-- contract-baseline-diff:${module} -->`;
+            const body = `${marker}\n## Public API contract change (\`${module}\`)\n\n`
+              + `This PR changes \`${module}/api-report/${pkg}.api.json\` (the approved public API `
+              + `contract for \`${pkg}\`). Review the surface change below:\n\n`
+              + '```diff\n' + (process.env.DIFF || '(diff too large - see the file change)') + '\n```';
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const existing = comments.find(c => c.body && c.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                comment_id: existing.id, body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: context.issue.number, body,
+              });
+            }

.github/workflows/internal-release.yml

@@ -1,23 +1,53 @@
-name: Internal Release
+name: Publish module to the JFrog Artifactory
 
 on:
   push:
+    # '**' not '*.*': Actions glob '*' does not match '/', so '*.*' let slash
+    # tags (flowvault/v1.0.0) through and fired this branch-only workflow.
     tags-ignore:
-      - '*.*'
+      - '**'
     paths-ignore:
-      - "setup.py"
-      - "*.yml"
       - "*.md"
-      - "skyflow/utils/_version.py"
-      - "samples/**"
     branches:
+      - flowvault-release/*
+      - skyvault-release/*
+      # Legacy: predates the per-module naming, still maps to skyvault.
       - release/*
 
 jobs:
+  resolve-module:
+    runs-on: ubuntu-latest
+    # Skip our own bump commit, or this loops: bump -> push -> release -> bump.
+    # PAT-authenticated pushes DO trigger workflows; GITHUB_TOKEN pushes do not.
+    # build-and-deploy needs this job, so skipping here skips the run.
+    if: ${{ !contains(github.event.head_commit.message, '[AUTOMATED]') }}
+    outputs:
+      module: ${{ steps.set-module.outputs.module }}
+    steps:
+      # Explicit match, no catch-all: defaulting once published the wrong module.
+      - name: Resolve module from branch name
+        id: set-module
+        env:
+          BRANCH: ${{ github.ref_name }}
+        run: |
+          case "$BRANCH" in
+            flowvault-release/*) MODULE="flowvault" ;;
+            skyvault-release/*)  MODULE="skyvault" ;;
+            release/*)           MODULE="skyvault" ;;
+            *)
+              echo "::error::Branch '$BRANCH' does not map to a module."
+              exit 1
+              ;;
+          esac
+          echo "Branch '$BRANCH' -> module '$MODULE'"
+          echo "module=$MODULE" >> "$GITHUB_OUTPUT"
+
   build-and-deploy:
+    needs: resolve-module
     uses: ./.github/workflows/shared-build-and-deploy.yml
     with:
       ref: ${{ github.ref_name }}
       tag: 'internal'
+      module: ${{ needs.resolve-module.outputs.module }}
     secrets: inherit
 

.github/workflows/pr-flowvault.yml

@@ -0,0 +1,84 @@
+name: PR CI Checks (flowvault)
+
+# flowvault is a folder under main, alongside skyvault - not a branch.
+# This workflow fires for PRs targeting main or a flowvault-release/* branch
+# that actually touch flowvault or its common dependency, and only builds/
+# tests those two modules. skyvault (and the full 3-module suite) is covered
+# by pr.yml, not here.
+
+on:
+  pull_request:
+    branches: [ "main", "flowvault-release/**" ]
+    paths:
+      - "flowvault/**"
+      - "common/**"
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      # flowvault depends on common as a local path dependency, so common
+      # must be built and installed first or flowvault's own build/install
+      # will fail to resolve it.
+      - name: Build and install common
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          cd common
+          python setup.py sdist bdist_wheel
+          pip install dist/*.whl
+
+      - name: Build flowvault
+        run: |
+          cd flowvault
+          python setup.py sdist bdist_wheel
+
+  test:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "credentials.json"
+          json: ${{ secrets.VALID_SKYFLOW_CREDS_TEST }}
+
+      - name: Run flowvault unit tests
+        run: |
+          python -m pip install --upgrade pip setuptools wheel coverage
+          cp credentials.json flowvault/credentials.json
+
+          # flowvault depends on common as a local path dependency.
+          cd common
+          python setup.py sdist bdist_wheel
+          pip install dist/*.whl
+          cd ..
+
+          cd flowvault
+          python setup.py sdist bdist_wheel
+          pip install dist/*.whl
+          if [ -f requirements.txt ]; then
+            pip install -r requirements.txt
+          fi
+          python -m coverage run --source=. -m unittest discover
+          coverage xml -o test-coverage.xml
+
+      - name: Codecov
+        uses: codecov/codecov-action@v2.1.0
+        with:
+          token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
+          files: flowvault/test-coverage.xml
+          flags: flowvault
+          name: codecov-skyflow-python-flowvault
+          verbose: true

.github/workflows/ci.yml → .github/workflows/pr.yml

@@ -1,4 +1,4 @@
-name: CI Checks
+name: PR CI Checks
 
 on: [pull_request]
 

.github/workflows/release.yml

@@ -1,18 +1,71 @@
-name: Public Release
+name: Public release
+
+# Triggered by publishing a GitHub Release, not a raw tag push: the Release
+# carries both facts needed here - target_commitish (the branch picked in the
+# UI; a tag records only a commit) and tag_name (module prefix + version).
+#
+# Beta and final share this workflow - 'release' events cannot be filtered by
+# tag pattern, and both behave identically downstream. Kind comes from the tag.
 
 on:
-  push:
-    tags: "*.*.*"
-    paths-ignore:
-      - "setup.py"
-      - "*.yml"
-      - "*.md"
-      - "skyflow/utils/_version.py"
+  release:
+    types: [published]
 
 jobs:
+  resolve-release:
+    runs-on: ubuntu-latest
+    outputs:
+      module: ${{ steps.parse.outputs.module }}
+      version: ${{ steps.parse.outputs.version }}
+      kind: ${{ steps.parse.outputs.kind }}
+    steps:
+      - name: Parse module, version and release kind from the tag
+        id: parse
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+          BRANCH: ${{ github.event.release.target_commitish }}
+        run: |
+          # Expected: <module>/v<semver>[-beta.N]  e.g. flowvault/v1.0.0,
+          # skyvault/v2.1.2, flowvault/v1.0.0-beta.1
+          if [[ ! "$TAG" =~ ^[a-z]+/v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]]; then
+            echo "::error::Tag '$TAG' is not <module>/v<semver>[-beta.N]." \
+                 "Examples: flowvault/v1.0.0, skyvault/v2.1.2, flowvault/v1.0.0-beta.1"
+            exit 1
+          fi
+
+          PREFIX="${TAG%%/*}"      # flowvault/v1.0.0 -> flowvault
+          VERSION="${TAG#*/}"      # flowvault/v1.0.0 -> v1.0.0
+          VERSION="${VERSION#v}"   # v1.0.0           -> 1.0.0
+
+          # Tag prefix -> module directory (both match the directory name).
+          case "$PREFIX" in
+            flowvault) MODULE="flowvault" ;;
+            skyvault)  MODULE="skyvault" ;;
+            *)
+              echo "::error::Unknown module prefix '$PREFIX' in tag '$TAG'"
+              exit 1
+              ;;
+          esac
+
+          if [[ "$VERSION" == *-beta.* ]]; then KIND="beta"; else KIND="public"; fi
+
+          if [ -z "$BRANCH" ]; then
+            echo "::error::Release has no target_commitish - cannot determine the release branch."
+            exit 1
+          fi
+
+          echo "Tag '$TAG' -> module='$MODULE' version='$VERSION' kind='$KIND' branch='$BRANCH'"
+          echo "module=$MODULE"   >> "$GITHUB_OUTPUT"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "kind=$KIND"       >> "$GITHUB_OUTPUT"
+
   build-and-deploy:
+    needs: resolve-release
     uses: ./.github/workflows/shared-build-and-deploy.yml
     with:
-      ref: main
-      tag: 'public'
+      ref: ${{ github.event.release.tag_name }}
+      tag: ${{ needs.resolve-release.outputs.kind }}
+      module: ${{ needs.resolve-release.outputs.module }}
+      version: ${{ needs.resolve-release.outputs.version }}
+      release-branch: ${{ github.event.release.target_commitish }}
     secrets: inherit

.github/workflows/semgrep.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Run Semgrep
         run: |
-          semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --sarif . > results.sarif
+          semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --exclude generated --sarif . > results.sarif
 
       - name: Upload SARIF file
         uses: github/codeql-action/upload-sarif@v3