SK-2954: common/v2/flowvault split + flowvault insert support

.codespellrc

@@ -3,7 +3,7 @@
 ignore-words-list = Skyflow,skyflow,skyflowapi,skyflowapis,deidentify,reidentify,detokenize,upsert,upserting,binlookup,byot,creds,fpe,devsecops,formdata,vaultid,dotenv,usecwd,runid,dateutil,Homogenous
 
 # Skip these files and folders
-skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,./skyflow/generated,setup.py
+skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,generated,setup.py
 
 # If you want to verify it is working, you can uncomment this line to see what files it checks
 # count = 

.github/workflows/internal-release.yml

@@ -1,23 +1,53 @@
-name: Internal Release
+name: Publish module to the JFrog Artifactory
 
 on:
   push:
+    # '**' not '*.*': Actions glob '*' does not match '/', so '*.*' let slash
+    # tags (flowvault/v1.0.0) through and fired this branch-only workflow.
     tags-ignore:
-      - '*.*'
+      - '**'
     paths-ignore:
-      - "setup.py"
-      - "*.yml"
       - "*.md"
-      - "skyflow/utils/_version.py"
-      - "samples/**"
     branches:
+      - flowvault-release/*
+      - skyvault-release/*
+      # Legacy: predates the per-module naming, still maps to skyvault.
       - release/*
 
 jobs:
+  resolve-module:
+    runs-on: ubuntu-latest
+    # Skip our own bump commit, or this loops: bump -> push -> release -> bump.
+    # PAT-authenticated pushes DO trigger workflows; GITHUB_TOKEN pushes do not.
+    # build-and-deploy needs this job, so skipping here skips the run.
+    if: ${{ !contains(github.event.head_commit.message, '[AUTOMATED]') }}
+    outputs:
+      module: ${{ steps.set-module.outputs.module }}
+    steps:
+      # Explicit match, no catch-all: defaulting once published the wrong module.
+      - name: Resolve module from branch name
+        id: set-module
+        env:
+          BRANCH: ${{ github.ref_name }}
+        run: |
+          case "$BRANCH" in
+            flowvault-release/*) MODULE="flowvault" ;;
+            skyvault-release/*)  MODULE="skyvault" ;;
+            release/*)           MODULE="skyvault" ;;
+            *)
+              echo "::error::Branch '$BRANCH' does not map to a module."
+              exit 1
+              ;;
+          esac
+          echo "Branch '$BRANCH' -> module '$MODULE'"
+          echo "module=$MODULE" >> "$GITHUB_OUTPUT"
+
   build-and-deploy:
+    needs: resolve-module
     uses: ./.github/workflows/shared-build-and-deploy.yml
     with:
       ref: ${{ github.ref_name }}
       tag: 'internal'
+      module: ${{ needs.resolve-module.outputs.module }}
     secrets: inherit
 

.github/workflows/pr-flowvault.yml

@@ -0,0 +1,84 @@
+name: PR CI Checks (flowvault)
+
+# flowvault is a folder under main, alongside skyvault - not a branch.
+# This workflow fires for PRs targeting main or a flowvault-release/* branch
+# that actually touch flowvault or its common dependency, and only builds/
+# tests those two modules. skyvault (and the full 3-module suite) is covered
+# by pr.yml, not here.
+
+on:
+  pull_request:
+    branches: [ "main", "flowvault-release/**" ]
+    paths:
+      - "flowvault/**"
+      - "common/**"
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      # flowvault depends on common as a local path dependency, so common
+      # must be built and installed first or flowvault's own build/install
+      # will fail to resolve it.
+      - name: Build and install common
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          cd common
+          python setup.py sdist bdist_wheel
+          pip install dist/*.whl
+
+      - name: Build flowvault
+        run: |
+          cd flowvault
+          python setup.py sdist bdist_wheel
+
+  test:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "credentials.json"
+          json: ${{ secrets.VALID_SKYFLOW_CREDS_TEST }}
+
+      - name: Run flowvault unit tests
+        run: |
+          python -m pip install --upgrade pip setuptools wheel coverage
+          cp credentials.json flowvault/credentials.json
+
+          # flowvault depends on common as a local path dependency.
+          cd common
+          python setup.py sdist bdist_wheel
+          pip install dist/*.whl
+          cd ..
+
+          cd flowvault
+          python setup.py sdist bdist_wheel
+          pip install dist/*.whl
+          if [ -f requirements.txt ]; then
+            pip install -r requirements.txt
+          fi
+          python -m coverage run --source=. -m unittest discover
+          coverage xml -o test-coverage.xml
+
+      - name: Codecov
+        uses: codecov/codecov-action@v2.1.0
+        with:
+          token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
+          files: flowvault/test-coverage.xml
+          flags: flowvault
+          name: codecov-skyflow-python-flowvault
+          verbose: true

.github/workflows/ci.yml → .github/workflows/pr.yml

@@ -1,4 +1,4 @@
-name: CI Checks
+name: PR CI Checks
 
 on: [pull_request]
 

.github/workflows/release.yml

@@ -1,18 +1,71 @@
-name: Public Release
+name: Public release
+
+# Triggered by publishing a GitHub Release, not a raw tag push: the Release
+# carries both facts needed here - target_commitish (the branch picked in the
+# UI; a tag records only a commit) and tag_name (module prefix + version).
+#
+# Beta and final share this workflow - 'release' events cannot be filtered by
+# tag pattern, and both behave identically downstream. Kind comes from the tag.
 
 on:
-  push:
-    tags: "*.*.*"
-    paths-ignore:
-      - "setup.py"
-      - "*.yml"
-      - "*.md"
-      - "skyflow/utils/_version.py"
+  release:
+    types: [published]
 
 jobs:
+  resolve-release:
+    runs-on: ubuntu-latest
+    outputs:
+      module: ${{ steps.parse.outputs.module }}
+      version: ${{ steps.parse.outputs.version }}
+      kind: ${{ steps.parse.outputs.kind }}
+    steps:
+      - name: Parse module, version and release kind from the tag
+        id: parse
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+          BRANCH: ${{ github.event.release.target_commitish }}
+        run: |
+          # Expected: <module>/v<semver>[-beta.N]  e.g. flowvault/v1.0.0,
+          # skyvault/v2.1.2, flowvault/v1.0.0-beta.1
+          if [[ ! "$TAG" =~ ^[a-z]+/v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]]; then
+            echo "::error::Tag '$TAG' is not <module>/v<semver>[-beta.N]." \
+                 "Examples: flowvault/v1.0.0, skyvault/v2.1.2, flowvault/v1.0.0-beta.1"
+            exit 1
+          fi
+
+          PREFIX="${TAG%%/*}"      # flowvault/v1.0.0 -> flowvault
+          VERSION="${TAG#*/}"      # flowvault/v1.0.0 -> v1.0.0
+          VERSION="${VERSION#v}"   # v1.0.0           -> 1.0.0
+
+          # Tag prefix -> module directory (both match the directory name).
+          case "$PREFIX" in
+            flowvault) MODULE="flowvault" ;;
+            skyvault)  MODULE="skyvault" ;;
+            *)
+              echo "::error::Unknown module prefix '$PREFIX' in tag '$TAG'"
+              exit 1
+              ;;
+          esac
+
+          if [[ "$VERSION" == *-beta.* ]]; then KIND="beta"; else KIND="public"; fi
+
+          if [ -z "$BRANCH" ]; then
+            echo "::error::Release has no target_commitish - cannot determine the release branch."
+            exit 1
+          fi
+
+          echo "Tag '$TAG' -> module='$MODULE' version='$VERSION' kind='$KIND' branch='$BRANCH'"
+          echo "module=$MODULE"   >> "$GITHUB_OUTPUT"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "kind=$KIND"       >> "$GITHUB_OUTPUT"
+
   build-and-deploy:
+    needs: resolve-release
     uses: ./.github/workflows/shared-build-and-deploy.yml
     with:
-      ref: main
-      tag: 'public'
+      ref: ${{ github.event.release.tag_name }}
+      tag: ${{ needs.resolve-release.outputs.kind }}
+      module: ${{ needs.resolve-release.outputs.module }}
+      version: ${{ needs.resolve-release.outputs.version }}
+      release-branch: ${{ github.event.release.target_commitish }}
     secrets: inherit