From ce8178fc08799251e53ee6ea91561d1b14b18f6a Mon Sep 17 00:00:00 2001 From: Andre Manoel Date: Mon, 10 Aug 2026 14:37:10 -0300 Subject: [PATCH 1/6] feat: add optional Slurm package Add the shared-namespace leaf package and publish it through the same-version data-designer[slurm] extra. Cover resolver, namespace, base-only isolation, and built-wheel installation behavior.\n\nCloses #852 Signed-off-by: Andre Manoel --- .github/workflows/ci.yml | 25 +++ AGENTS.md | 5 +- Makefile | 62 +++++-- packages/data-designer-slurm/README.md | 9 + packages/data-designer-slurm/pyproject.toml | 50 +++++ .../src/data_designer/slurm/__init__.py | 6 + .../data-designer-slurm/tests/test_package.py | 12 ++ packages/data-designer/pyproject.toml | 4 +- .../tests/test_dependency_audit.py | 36 +++- pyproject.toml | 1 + scripts/audit_package_dependencies.py | 41 ++-- scripts/test_slurm_package_install.py | 175 ++++++++++++++++++ uv.lock | 18 ++ 13 files changed, 417 insertions(+), 27 deletions(-) create mode 100644 packages/data-designer-slurm/README.md create mode 100644 packages/data-designer-slurm/pyproject.toml create mode 100644 packages/data-designer-slurm/src/data_designer/slurm/__init__.py create mode 100644 packages/data-designer-slurm/tests/test_package.py create mode 100644 scripts/test_slurm_package_install.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fa6044c4..0b2c09eba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,31 @@ jobs: uv run --with pytest --with pytest-asyncio --with pytest-httpx --with pytest-env \ pytest packages/data-designer/tests + test-slurm-package: + name: Test Slurm package wheels + needs: validate-dispatch + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "latest" + python-version: "3.11" + enable-cache: true + + - name: Install development dependencies + run: make install-dev + + - name: Run package tests + run: .venv/bin/pytest packages/data-designer-slurm/tests + + - name: Run built-wheel installation tests + run: make test-slurm-wheel-install + # =========================================================================== # Combined Coverage Check # Runs all tests together to verify overall coverage threshold diff --git a/AGENTS.md b/AGENTS.md index c9c3b4c43..5e36877b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,15 +7,16 @@ If you are an agent helping a user **build a dataset**, use the [`data-designer` ## The Layering Is Structural -The `data_designer` namespace is split across three installable packages that merge at runtime via PEP 420 implicit namespace packages (no top-level `__init__.py`). +The `data_designer` namespace is split across four installable packages that merge at runtime via PEP 420 implicit namespace packages (no top-level `__init__.py`). | Package | Path | Owns | |---------|------|------| | `data-designer-config` | `packages/data-designer-config/` | `data_designer.config` — column configs, model configs, sampler params, builder API, plugin system, lazy imports | | `data-designer-engine` | `packages/data-designer-engine/` | `data_designer.engine` — column generators, dataset builders, DAG execution, model facade, validators, sampling | | `data-designer` | `packages/data-designer/` | `data_designer.interface` — public `DataDesigner` class, results, errors; `data_designer.cli` — CLI entry point; `data_designer.integrations` | +| `data-designer-slurm` | `packages/data-designer-slurm/` | `data_designer.slurm` — optional Slurm batch execution | -**Dependency direction (left depends on right):** interface → engine → config. Never import against this flow. +**Dependency direction (left depends on right):** Slurm → interface → engine → config. Never import against this flow. ## Core Concepts diff --git a/Makefile b/Makefile index cf8075a08..224a9cef6 100644 --- a/Makefile +++ b/Makefile @@ -9,17 +9,20 @@ LICENSE_PYTHON_VERSION ?= 3.11 # Package directories CONFIG_PKG := packages/data-designer-config ENGINE_PKG := packages/data-designer-engine +SLURM_PKG := packages/data-designer-slurm INTERFACE_PKG := packages/data-designer # Package source and test paths CONFIG_PATHS := $(CONFIG_PKG)/src $(CONFIG_PKG)/tests ENGINE_PATHS := $(ENGINE_PKG)/src $(ENGINE_PKG)/tests +SLURM_PATHS := $(SLURM_PKG)/src $(SLURM_PKG)/tests INTERFACE_PATHS := $(INTERFACE_PKG)/src $(INTERFACE_PKG)/tests $(INTERFACE_PKG)/dev-tools ALL_PKG_PATHS := packages/ scripts/ tests_e2e/ # Test directories CONFIG_TESTS := $(CONFIG_PKG)/tests ENGINE_TESTS := $(ENGINE_PKG)/tests +SLURM_TESTS := $(SLURM_PKG)/tests INTERFACE_TESTS := $(INTERFACE_PKG)/tests define install-pre-commit-hooks @@ -50,6 +53,7 @@ help: @echo " test - Run all unit tests" @echo " coverage - Run tests with coverage report" @echo " test-e2e - Run e2e plugin tests" + @echo " test-slurm-wheel-install - Test optional Slurm package wheel installation" @echo " health-checks - Run provider health checks" @echo " test-run-tutorials - Run tutorial notebooks as e2e tests" @echo " test-run-recipes - Run recipe scripts as e2e tests" @@ -109,7 +113,7 @@ help: @echo " publish VERSION=X.Y.Z ALLOW_BRANCH=1 - Publish from non-main branch" @echo " publish VERSION=X.Y.Z FORCE_TAG=1 - Overwrite existing git tag" @echo "" - @echo "📦 Per-Package Commands (use suffix: -config, -engine, -interface):" + @echo "📦 Per-Package Commands (use suffix: -config, -engine, -slurm, -interface):" @echo " test- - Run tests for a specific package" @echo " lint- - Lint a specific package" @echo " lint-fix- - Fix lint issues in a specific package" @@ -129,7 +133,7 @@ help: install: @echo "📦 Installing DataDesigner workspace (all packages in editable mode)..." - @echo " Packages: data-designer-config → data-designer-engine → data-designer" + @echo " Packages: data-designer-config → data-designer-engine → data-designer → data-designer-slurm" uv sync --all-packages @echo "✅ Installation complete!" @echo "" @@ -137,7 +141,7 @@ install: install-dev: @echo "📦 Installing DataDesigner workspace in development mode..." - @echo " Packages: data-designer-config → data-designer-engine → data-designer" + @echo " Packages: data-designer-config → data-designer-engine → data-designer → data-designer-slurm" @echo " Groups: dev (pytest, coverage, etc.)" uv sync --all-packages --group dev $(call install-pre-commit-hooks) @@ -148,17 +152,18 @@ install-dev: @echo " packages/data-designer-config/ - Configuration layer (lightweight)" @echo " packages/data-designer-engine/ - Generation engine (heavy deps)" @echo " packages/data-designer/ - Full package with CLI" + @echo " packages/data-designer-slurm/ - Optional Slurm batch execution" @echo "" @echo "💡 Next steps:" @echo " make verify-imports - Verify all packages are working" @echo " make test - Run all tests across packages" - @echo " make test- - Run tests for specific package (config, engine, interface)" + @echo " make test- - Run tests for specific package (config, engine, slurm, interface)" @echo " make lint - Lint all code" @echo " make build - Build all package wheels" install-dev-notebooks: @echo "📦 Installing DataDesigner workspace with notebook dependencies..." - @echo " Packages: data-designer-config → data-designer-engine → data-designer" + @echo " Packages: data-designer-config → data-designer-engine → data-designer → data-designer-slurm" @echo " Groups: dev + docs + notebooks (Jupyter, jupytext, etc.)" uv sync --all-packages --group dev --group docs --group notebooks $(call install-pre-commit-hooks) @@ -168,7 +173,7 @@ install-dev-notebooks: install-dev-recipes: @echo "📦 Installing DataDesigner workspace with recipe dependencies..." - @echo " Packages: data-designer-config → data-designer-engine → data-designer" + @echo " Packages: data-designer-config → data-designer-engine → data-designer → data-designer-slurm" @echo " Groups: dev + recipes (bm25s, pymupdf, etc.)" uv sync --all-packages --group dev --group recipes $(call install-pre-commit-hooks) @@ -180,7 +185,7 @@ install-dev-recipes: # TESTING # ============================================================================== -test: test-config test-engine test-interface +test: test-config test-engine test-interface test-slurm @echo "✅ All package tests complete!" test-config: @@ -195,6 +200,14 @@ test-interface: @echo "🧪 Testing data-designer (interface)..." uv run --group dev pytest $(INTERFACE_TESTS) +test-slurm: + @echo "🧪 Testing data-designer-slurm..." + .venv/bin/pytest $(SLURM_TESTS) + +test-slurm-wheel-install: + @echo "🧪 Testing data-designer-slurm wheel installation..." + .venv/bin/python scripts/test_slurm_package_install.py + # ------------------------------------------------------------------------------ # Isolated Testing (mirrors CI behavior) # Each package is installed independently to verify dependency boundaries @@ -325,12 +338,12 @@ test-run-all-examples: test-run-tutorials test-run-recipes # CODE QUALITY - FORMATTING # ============================================================================== -format: format-config format-engine format-interface +format: format-config format-engine format-interface format-slurm @echo "📐 Formatting scripts and tests_e2e..." uv run ruff format scripts/ tests_e2e/ @echo "✅ Formatting complete!" -format-check: format-check-config format-check-engine format-check-interface +format-check: format-check-config format-check-engine format-check-interface format-check-slurm @echo "📐 Checking scripts and tests_e2e formatting..." uv run ruff format --check scripts/ tests_e2e/ @echo "✅ Formatting check complete! Run 'make format' to auto-fix issues." @@ -347,6 +360,10 @@ format-interface: @echo "📐 Formatting data-designer (interface)..." uv run ruff format $(INTERFACE_PATHS) --exclude '**/_version.py' +format-slurm: + @echo "📐 Formatting data-designer-slurm..." + .venv/bin/ruff format $(SLURM_PATHS) + format-check-config: @echo "📐 Checking data-designer-config formatting..." uv run ruff format --check $(CONFIG_PATHS) --exclude '**/_version.py' @@ -359,16 +376,20 @@ format-check-interface: @echo "📐 Checking data-designer (interface) formatting..." uv run ruff format --check $(INTERFACE_PATHS) --exclude '**/_version.py' +format-check-slurm: + @echo "📐 Checking data-designer-slurm formatting..." + .venv/bin/ruff format --check $(SLURM_PATHS) + # ============================================================================== # CODE QUALITY - LINTING # ============================================================================== -lint: lint-config lint-engine lint-interface +lint: lint-config lint-engine lint-interface lint-slurm @echo "🔍 Linting scripts and tests_e2e..." uv run ruff check --output-format=full scripts/ tests_e2e/ @echo "✅ Linting complete! Run 'make lint-fix' to auto-fix issues." -lint-fix: lint-fix-config lint-fix-engine lint-fix-interface +lint-fix: lint-fix-config lint-fix-engine lint-fix-interface lint-fix-slurm @echo "🔍 Fixing lint issues in scripts and tests_e2e..." uv run ruff check --fix scripts/ tests_e2e/ @echo "✅ Linting with autofix complete!" @@ -385,6 +406,10 @@ lint-interface: @echo "🔍 Linting data-designer (interface)..." uv run ruff check --output-format=full $(INTERFACE_PATHS) --exclude '**/_version.py' +lint-slurm: + @echo "🔍 Linting data-designer-slurm..." + .venv/bin/ruff check --output-format=full $(SLURM_PATHS) + lint-fix-config: @echo "🔍 Fixing lint issues in data-designer-config..." uv run ruff check --fix $(CONFIG_PATHS) --exclude '**/_version.py' @@ -397,6 +422,10 @@ lint-fix-interface: @echo "🔍 Fixing lint issues in data-designer (interface)..." uv run ruff check --fix $(INTERFACE_PATHS) --exclude '**/_version.py' +lint-fix-slurm: + @echo "🔍 Fixing lint issues in data-designer-slurm..." + .venv/bin/ruff check --fix $(SLURM_PATHS) + # ============================================================================== # CODE QUALITY - COMBINED CHECKS # ============================================================================== @@ -416,11 +445,14 @@ check-engine: format-check-engine lint-engine check-interface: format-check-interface lint-interface @echo "✅ Checks complete for data-designer (interface)!" +check-slurm: format-check-slurm lint-slurm + @echo "✅ Checks complete for data-designer-slurm!" + # ============================================================================== # BUILD # ============================================================================== -build: build-config build-engine build-interface +build: build-config build-engine build-interface build-slurm @echo "✅ All packages built!" build-config: @@ -435,6 +467,10 @@ build-interface: @echo "🏗️ Building data-designer (interface)..." cd $(INTERFACE_PKG) && uv build -o dist +build-slurm: + @echo "🏗️ Building data-designer-slurm..." + cd $(SLURM_PKG) && uv build -o dist + # ============================================================================== # UTILITIES # ============================================================================== @@ -444,6 +480,7 @@ verify-imports: uv run python -c "from data_designer.config.config_builder import DataDesignerConfigBuilder; print(' ✓ config')" uv run python -c "from data_designer.engine.compiler import compile_data_designer_config; print(' ✓ engine')" uv run python -c "from data_designer.interface.data_designer import DataDesigner; print(' ✓ interface')" + .venv/bin/python -c "import data_designer.slurm; print(' ✓ slurm')" @echo "✅ All imports verified!" show-versions: @@ -451,6 +488,7 @@ show-versions: @uv run python -c "from data_designer.config._version import __version__; print(f' data-designer-config: {__version__}')" 2>/dev/null || echo " data-designer-config: (not installed)" @uv run python -c "from data_designer.engine._version import __version__; print(f' data-designer-engine: {__version__}')" 2>/dev/null || echo " data-designer-engine: (not installed)" @uv run python -c "from data_designer.interface._version import __version__; print(f' data-designer: {__version__}')" 2>/dev/null || echo " data-designer: (not installed)" + @.venv/bin/python -c 'from importlib.metadata import version; print(" data-designer-slurm: " + version("data-designer-slurm"))' 2>/dev/null || echo " data-designer-slurm: (not installed)" # ============================================================================== # LICENSE CHECKS diff --git a/packages/data-designer-slurm/README.md b/packages/data-designer-slurm/README.md new file mode 100644 index 000000000..274293c5a --- /dev/null +++ b/packages/data-designer-slurm/README.md @@ -0,0 +1,9 @@ +# data-designer-slurm + +Optional Slurm batch execution support for Data Designer. + +Install it through the Data Designer extra: + +```bash +pip install "data-designer[slurm]" +``` diff --git a/packages/data-designer-slurm/pyproject.toml b/packages/data-designer-slurm/pyproject.toml new file mode 100644 index 000000000..8d197e6f0 --- /dev/null +++ b/packages/data-designer-slurm/pyproject.toml @@ -0,0 +1,50 @@ +[project] +name = "data-designer-slurm" +dynamic = ["version", "dependencies"] +description = "Slurm batch execution for Data Designer" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" + +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] + +[build-system] +requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"] +build-backend = "hatchling.build" + +[tool.hatch.version] +source = "uv-dynamic-versioning" + +[tool.uv-dynamic-versioning] +vcs = "git" +style = "pep440" +bump = true + +[tool.hatch.metadata.hooks.uv-dynamic-versioning] +dependencies = ["data-designer=={{ version }}"] + +[tool.hatch.build.targets.wheel] +packages = ["src/data_designer"] + +[tool.ruff] +extend = "../../pyproject.toml" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +package = true + +[tool.uv.sources] +data-designer = { workspace = true } diff --git a/packages/data-designer-slurm/src/data_designer/slurm/__init__.py b/packages/data-designer-slurm/src/data_designer/slurm/__init__.py new file mode 100644 index 000000000..30443170a --- /dev/null +++ b/packages/data-designer-slurm/src/data_designer/slurm/__init__.py @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Slurm batch execution for Data Designer.""" + +from __future__ import annotations diff --git a/packages/data-designer-slurm/tests/test_package.py b/packages/data-designer-slurm/tests/test_package.py new file mode 100644 index 000000000..6428ecd43 --- /dev/null +++ b/packages/data-designer-slurm/tests/test_package.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import data_designer +import data_designer.slurm + + +def test_slurm_uses_shared_namespace() -> None: + assert data_designer.__file__ is None + assert data_designer.slurm.__name__ == "data_designer.slurm" diff --git a/packages/data-designer/pyproject.toml b/packages/data-designer/pyproject.toml index 9cd9b23d6..0f29804c5 100644 --- a/packages/data-designer/pyproject.toml +++ b/packages/data-designer/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "data-designer" -dynamic = ["version", "dependencies"] +dynamic = ["version", "dependencies", "optional-dependencies"] description = "General framework for synthetic data generation" readme = "README.md" requires-python = ">=3.10" @@ -56,6 +56,7 @@ dependencies = [ "rich>=13.7.1,<15", "typer>=0.12.0,<1", ] +optional-dependencies = { slurm = ["data-designer-slurm=={{ version }}"] } [tool.hatch.build.targets.wheel] packages = ["src/data_designer"] @@ -74,3 +75,4 @@ package = true [tool.uv.sources] data-designer-config = { workspace = true } data-designer-engine = { workspace = true } +data-designer-slurm = { workspace = true } diff --git a/packages/data-designer/tests/test_dependency_audit.py b/packages/data-designer/tests/test_dependency_audit.py index 509433f94..0b1961b8e 100644 --- a/packages/data-designer/tests/test_dependency_audit.py +++ b/packages/data-designer/tests/test_dependency_audit.py @@ -45,10 +45,11 @@ def audit( workspace: Path, distributions: dict[str, list[str]], requirements: dict[str, list[str]] | None = None, + selected_extras: dict[str, set[str]] | None = None, ) -> dict: module = DEPENDENCY_AUDIT assert isinstance(module, ModuleType) - return module.audit_repository(workspace, distributions, requirements) + return module.audit_repository(workspace, distributions, requirements, selected_extras) def test_marks_transitively_guaranteed_gap_low(workspace: Path) -> None: @@ -144,3 +145,36 @@ def test_applies_module_distribution_override(tmp_path: Path) -> None: result = audit(tmp_path, {}) assert result["packages"][0]["missing"][0]["dependency"] == "pyyaml" + + +def test_includes_selected_dynamic_optional_dependencies(tmp_path: Path) -> None: + base = tmp_path / "packages" / "data-designer" + leaf = tmp_path / "packages" / "data-designer-slurm" + (base / "src").mkdir(parents=True) + (leaf / "src").mkdir(parents=True) + (base / "pyproject.toml").write_text( + """ +[project] +name = "data-designer" +dynamic = ["optional-dependencies"] + +[tool.hatch.metadata.hooks.uv-dynamic-versioning] +optional-dependencies = { slurm = ["data-designer-slurm=={{ version }}"] } +""".lstrip() + ) + (leaf / "pyproject.toml").write_text( + """ +[project] +name = "data-designer-slurm" +dynamic = ["dependencies"] + +[tool.hatch.metadata.hooks.uv-dynamic-versioning] +dependencies = ["data-designer=={{ version }}"] +""".lstrip() + ) + + result = audit(tmp_path, {}, selected_extras={"data-designer": {"slurm"}}) + + packages = {package["package"]: package for package in result["packages"]} + assert packages["data-designer"]["declared"] == ["data-designer-slurm"] + assert packages["data-designer-slurm"]["declared"] == ["data-designer"] diff --git a/pyproject.toml b/pyproject.toml index cba354e04..14fd4de23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ build-backend = "hatchling.build" members = [ "packages/data-designer-config", "packages/data-designer-engine", + "packages/data-designer-slurm", "packages/data-designer", ] diff --git a/scripts/audit_package_dependencies.py b/scripts/audit_package_dependencies.py index f044646d6..1a6dca0e8 100644 --- a/scripts/audit_package_dependencies.py +++ b/scripts/audit_package_dependencies.py @@ -66,22 +66,29 @@ def requirement_dependencies( return dict(dependencies) -def declared_dependencies(pyproject_path: Path) -> dict[str, set[str]]: +def declared_dependencies(pyproject_path: Path, selected_extras: set[str] | None = None) -> dict[str, set[str]]: with pyproject_path.open("rb") as file: config = tomllib.load(file) - project_dependencies = config.get("project", {}).get("dependencies", []) - dynamic_dependencies = ( - config.get("tool", {}) - .get("hatch", {}) - .get("metadata", {}) - .get("hooks", {}) - .get("uv-dynamic-versioning", {}) - .get("dependencies", []) + project = config.get("project", {}) + metadata_hook = ( + config.get("tool", {}).get("hatch", {}).get("metadata", {}).get("hooks", {}).get("uv-dynamic-versioning", {}) ) + project_dependencies = project.get("dependencies", []) + dynamic_dependencies = metadata_hook.get("dependencies", []) dependencies = requirement_dependencies(project_dependencies) for name, extras in requirement_dependencies(dynamic_dependencies, allow_version_template=True).items(): dependencies.setdefault(name, set()).update(extras) + + for extra in selected_extras or set(): + optional_dependencies = project.get("optional-dependencies", {}).get(extra, []) + dynamic_optional_dependencies = metadata_hook.get("optional-dependencies", {}).get(extra, []) + for name, extras in requirement_dependencies(optional_dependencies).items(): + dependencies.setdefault(name, set()).update(extras) + for name, extras in requirement_dependencies( + dynamic_optional_dependencies, allow_version_template=True + ).items(): + dependencies.setdefault(name, set()).update(extras) return dependencies @@ -134,6 +141,7 @@ def audit_repository( repository_root: Path, module_distributions: dict[str, list[str]] | None = None, requirement_map: dict[str, list[str]] | None = None, + selected_extras_by_project: dict[str, set[str]] | None = None, ) -> dict[str, Any]: repository_root = repository_root.resolve() package_dirs = sorted(path.parent for path in repository_root.glob("packages/*/pyproject.toml")) @@ -144,7 +152,10 @@ def audit_repository( project_name = normalize_name(tomllib.load(file)["project"]["name"]) projects[project_name] = { "path": str(package_dir.relative_to(repository_root)), - "declarations": declared_dependencies(package_dir / "pyproject.toml"), + "declarations": declared_dependencies( + package_dir / "pyproject.toml", + (selected_extras_by_project or {}).get(project_name), + ), "imports": imported_modules(package_dir / "src", repository_root), } @@ -243,9 +254,17 @@ def main() -> None: parser = argparse.ArgumentParser(description="Inventory package import/dependency gaps") parser.add_argument("--root", type=Path, default=Path.cwd()) parser.add_argument("--output", type=Path) + parser.add_argument("--extra", action="append", default=[], metavar="PACKAGE:EXTRA") args = parser.parse_args() - result = audit_repository(args.root) + selected_extras_by_project: dict[str, set[str]] = defaultdict(set) + for value in args.extra: + package, separator, extra = value.partition(":") + if not separator or not package or not extra: + parser.error("--extra must use PACKAGE:EXTRA format") + selected_extras_by_project[normalize_name(package)].add(extra) + + result = audit_repository(args.root, selected_extras_by_project=dict(selected_extras_by_project)) payload = json.dumps(result, indent=2) + "\n" if args.output: args.output.write_text(payload) diff --git a/scripts/test_slurm_package_install.py b/scripts/test_slurm_package_install.py new file mode 100644 index 000000000..06ca72cf9 --- /dev/null +++ b/scripts/test_slurm_package_install.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from email.message import Message +from email.parser import BytesParser +from pathlib import Path +from zipfile import ZipFile + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +REPOSITORY_ROOT = Path(__file__).parents[1] +PACKAGE_PATHS = ( + "packages/data-designer-config", + "packages/data-designer-engine", + "packages/data-designer", + "packages/data-designer-slurm", +) + + +def run(command: list[str], *, cwd: Path, check: bool = True) -> subprocess.CompletedProcess[str]: + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + environment.pop("VIRTUAL_ENV", None) + result = subprocess.run(command, cwd=cwd, env=environment, capture_output=True, text=True, check=False) + if check and result.returncode: + raise RuntimeError(result.stdout + result.stderr) + return result + + +def wheel_metadata(path: Path) -> Message: + with ZipFile(path) as wheel: + metadata_path = next(name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")) + return BytesParser().parsebytes(wheel.read(metadata_path)) + + +def build_wheels(uv: str, wheel_directory: Path) -> dict[str, Path]: + for package_path in PACKAGE_PATHS: + run( + [uv, "build", "--wheel", "--out-dir", str(wheel_directory), package_path], + cwd=REPOSITORY_ROOT, + ) + + wheels = {} + for wheel_path in wheel_directory.glob("*.whl"): + name = canonicalize_name(wheel_metadata(wheel_path)["Name"]) + wheels[name] = wheel_path + return wheels + + +def requirement(metadata: Message, dependency_name: str) -> Requirement: + requirements = [Requirement(value) for value in metadata.get_all("Requires-Dist", [])] + return next(item for item in requirements if canonicalize_name(item.name) == dependency_name) + + +def python_path(environment_path: Path) -> Path: + if os.name == "nt": + return environment_path / "Scripts" / "python.exe" + return environment_path / "bin" / "python" + + +def create_environment(uv: str, environment_path: Path, *, cwd: Path) -> Path: + run([uv, "venv", "--python", sys.executable, str(environment_path)], cwd=cwd) + return python_path(environment_path) + + +def install(uv: str, python: Path, wheel_directory: Path, package: str, *, cwd: Path) -> None: + run( + [ + uv, + "pip", + "install", + "--python", + str(python), + "--prerelease=allow", + "--find-links", + str(wheel_directory), + package, + ], + cwd=cwd, + ) + + +def verify_install(python: Path, version: str, *, slurm: bool, cwd: Path) -> None: + statement = f""" +from importlib.metadata import version +from importlib.util import find_spec + +import data_designer +import data_designer.config +import data_designer.engine +import data_designer.interface + +assert data_designer.__file__ is None +assert version("data-designer") == {version!r} +assert (find_spec("data_designer.slurm") is not None) is {slurm!r} +""" + if slurm: + statement += f'\nimport data_designer.slurm\nassert version("data-designer-slurm") == {version!r}\n' + run([str(python), "-c", statement], cwd=cwd) + + +def main() -> None: + uv = shutil.which("uv") + if uv is None: + raise RuntimeError("uv is required") + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + wheel_directory = root / "wheels" + wheel_directory.mkdir() + wheels = build_wheels(uv, wheel_directory) + + base_wheel = wheels["data-designer"] + leaf_wheel = wheels["data-designer-slurm"] + base_metadata = wheel_metadata(base_wheel) + leaf_metadata = wheel_metadata(leaf_wheel) + version = base_metadata["Version"] + assert leaf_metadata["Version"] == version + + base_leaf_requirement = requirement(base_metadata, "data-designer-slurm") + leaf_base_requirement = requirement(leaf_metadata, "data-designer") + assert str(base_leaf_requirement.specifier) == f"=={version}" + assert str(leaf_base_requirement.specifier) == f"=={version}" + assert base_leaf_requirement.marker is not None + assert base_leaf_requirement.marker.evaluate({"extra": "slurm"}) + assert not base_leaf_requirement.marker.evaluate({"extra": ""}) + assert "slurm" in base_metadata.get_all("Provides-Extra", []) + with ZipFile(leaf_wheel) as wheel: + assert "data_designer/__init__.py" not in wheel.namelist() + + base_python = create_environment(uv, root / "base", cwd=root) + install(uv, base_python, wheel_directory, f"data-designer=={version}", cwd=root) + verify_install(base_python, version, slurm=False, cwd=root) + + extra_python = create_environment(uv, root / "extra", cwd=root) + install(uv, extra_python, wheel_directory, f"data-designer[slurm]=={version}", cwd=root) + verify_install(extra_python, version, slurm=True, cwd=root) + + leaf_python = create_environment(uv, root / "leaf", cwd=root) + install(uv, leaf_python, wheel_directory, f"data-designer-slurm=={version}", cwd=root) + verify_install(leaf_python, version, slurm=True, cwd=root) + + leaf_only_directory = root / "leaf-only" + leaf_only_directory.mkdir() + shutil.copy2(leaf_wheel, leaf_only_directory) + missing_python = create_environment(uv, root / "missing", cwd=root) + result = run( + [ + uv, + "pip", + "install", + "--python", + str(missing_python), + "--no-index", + "--find-links", + str(leaf_only_directory), + str(leaf_wheel), + ], + cwd=root, + check=False, + ) + assert result.returncode != 0 + assert "data-designer" in (result.stdout + result.stderr) + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index cc0e032a1..6643fb177 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,7 @@ members = [ "data-designer", "data-designer-config", "data-designer-engine", + "data-designer-slurm", "data-designer-workspace", ] @@ -825,10 +826,16 @@ dependencies = [ { name = "typer" }, ] +[package.optional-dependencies] +slurm = [ + { name = "data-designer-slurm" }, +] + [package.metadata] requires-dist = [ { name = "data-designer-config", editable = "packages/data-designer-config" }, { name = "data-designer-engine", editable = "packages/data-designer-engine" }, + { name = "data-designer-slurm", marker = "extra == 'slurm'", editable = "packages/data-designer-slurm" }, { name = "huggingface-hub", specifier = ">=1.0.1,<2" }, { name = "opentelemetry-api", specifier = ">=1.43,<1.44" }, { name = "opentelemetry-exporter-prometheus", specifier = ">=0.64b0,<0.65" }, @@ -843,6 +850,7 @@ requires-dist = [ { name = "rich", specifier = ">=13.7.1,<15" }, { name = "typer", specifier = ">=0.12.0,<1" }, ] +provides-extras = ["slurm"] [[package]] name = "data-designer-config" @@ -958,6 +966,16 @@ requires-dist = [ { name = "wcwidth", specifier = ">=0.2.13,<1" }, ] +[[package]] +name = "data-designer-slurm" +source = { editable = "packages/data-designer-slurm" } +dependencies = [ + { name = "data-designer" }, +] + +[package.metadata] +requires-dist = [{ name = "data-designer", editable = "packages/data-designer" }] + [[package]] name = "data-designer-workspace" version = "0.0.0" From 7463983ed5b4494ae75730537a3422c83a2fed26 Mon Sep 17 00:00:00 2001 From: Andre Manoel Date: Mon, 10 Aug 2026 14:52:38 -0300 Subject: [PATCH 2/6] fix: address Slurm package review findings Signed-off-by: Andre Manoel --- .github/workflows/agentic-ci-daily.yml | 1 + .github/workflows/ci.yml | 6 ++++-- AGENTS.md | 4 ++-- Makefile | 13 +++++++------ scripts/test_slurm_package_install.py | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/agentic-ci-daily.yml b/.github/workflows/agentic-ci-daily.yml index 5362199bc..ba5c4c033 100644 --- a/.github/workflows/agentic-ci-daily.yml +++ b/.github/workflows/agentic-ci-daily.yml @@ -135,6 +135,7 @@ jobs: if: matrix.suite == 'dependencies' run: | .venv/bin/python scripts/audit_package_dependencies.py \ + --extra data-designer:slurm \ --output /tmp/dependency-inventory.json jq '.packages[] | {package, missing, unresolved_modules}' \ /tmp/dependency-inventory.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b2c09eba..d8e209837 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,7 +306,7 @@ jobs: test-summary: name: Test (Python ${{ matrix.python-version }} on ${{ matrix.os }}) runs-on: ubuntu-latest - needs: [validate-dispatch, test-config, test-engine, test-interface] + needs: [validate-dispatch, test-config, test-engine, test-interface, test-slurm-package] if: always() strategy: matrix: @@ -319,12 +319,14 @@ jobs: if [[ "${{ needs.validate-dispatch.result }}" != "success" ]] || \ [[ "${{ needs.test-config.result }}" != "success" ]] || \ [[ "${{ needs.test-engine.result }}" != "success" ]] || \ - [[ "${{ needs.test-interface.result }}" != "success" ]]; then + [[ "${{ needs.test-interface.result }}" != "success" ]] || \ + [[ "${{ needs.test-slurm-package.result }}" != "success" ]]; then echo "One or more test jobs failed" echo "validate-dispatch: ${{ needs.validate-dispatch.result }}" echo "test-config: ${{ needs.test-config.result }}" echo "test-engine: ${{ needs.test-engine.result }}" echo "test-interface: ${{ needs.test-interface.result }}" + echo "test-slurm-package: ${{ needs.test-slurm-package.result }}" exit 1 fi echo "All test jobs passed successfully" diff --git a/AGENTS.md b/AGENTS.md index 5e36877b2..bac52c0db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ The `data_designer` namespace is split across four installable packages that mer | `data-designer` | `packages/data-designer/` | `data_designer.interface` — public `DataDesigner` class, results, errors; `data_designer.cli` — CLI entry point; `data_designer.integrations` | | `data-designer-slurm` | `packages/data-designer-slurm/` | `data_designer.slurm` — optional Slurm batch execution | -**Dependency direction (left depends on right):** Slurm → interface → engine → config. Never import against this flow. +**Import direction (left imports right):** Slurm → interface → engine → config. The `data-designer[slurm]` extra creates a packaging-only reverse edge; no code may import against this flow. ## Core Concepts @@ -35,7 +35,7 @@ The `data_designer` namespace is split across four installable packages that mer ## Structural Invariants -- **Import direction** — interface → engine → config (left depends on right). No reverse imports. +- **Import direction** — Slurm → interface → engine → config (left imports right). No reverse imports. - **Fast imports** — heavy third-party libraries are lazy-loaded via `data_designer.lazy_heavy_imports`. See [STYLEGUIDE.md](STYLEGUIDE.md) for the pattern. - **No relative imports** — absolute imports only, enforced by ruff rule `TID`. - **Typed code** — all functions, methods, and class attributes require type annotations. Modern syntax: `list[str]`, `str | None`. diff --git a/Makefile b/Makefile index 224a9cef6..39ef6bafe 100644 --- a/Makefile +++ b/Makefile @@ -786,6 +786,7 @@ clean-dist: rm -rf $(CONFIG_PKG)/dist rm -rf $(ENGINE_PKG)/dist rm -rf $(INTERFACE_PKG)/dist + rm -rf $(SLURM_PKG)/dist rm -f packages/*/src/data_designer/*/_version.py @echo "✅ Dist directories cleaned!" @@ -804,20 +805,20 @@ clean-test-coverage: # ============================================================================== .PHONY: bench-cli-startup bench-cli-startup-verbose \ - build build-config build-engine build-interface \ - check-all check-all-fix check-config check-engine check-interface \ + build build-config build-engine build-interface build-slurm \ + check-all check-all-fix check-config check-engine check-interface check-slurm \ check-dependency-licenses check-fern-docs check-fern-docs-locally check-fern-links check-fern-published-docs check-fern-release-version check-fern-theme-access check-license-headers \ clean clean-dist clean-notebooks clean-pycache clean-test-coverage \ convert-execute-notebooks \ coverage coverage-config coverage-engine coverage-interface \ - format format-check format-check-config format-check-engine format-check-interface \ - format-config format-engine format-interface \ + format format-check format-check-config format-check-engine format-check-interface format-check-slurm \ + format-config format-engine format-interface format-slurm \ generate-colab-notebooks generate-fern-notebooks generate-fern-notebooks-with-outputs help \ install install-dev install-dev-notebooks install-dev-recipes install-docs-deps \ - lint lint-config lint-engine lint-fix lint-fix-config lint-fix-engine lint-fix-interface lint-interface \ + lint lint-config lint-engine lint-fix lint-fix-config lint-fix-engine lint-fix-interface lint-fix-slurm lint-interface lint-slurm \ perf-import perf-import-runtime prepare-fern-docs prepare-fern-release publish serve-fern-docs-dev serve-fern-docs-local-theme serve-fern-docs-locally show-versions \ health-checks \ test test-config test-config-isolated test-e2e test-engine test-engine-isolated \ - test-interface test-interface-isolated test-isolated \ + test-interface test-interface-isolated test-isolated test-slurm test-slurm-wheel-install \ test-run-all-examples test-run-recipes test-run-tutorials \ update-license-headers verify-imports diff --git a/scripts/test_slurm_package_install.py b/scripts/test_slurm_package_install.py index 06ca72cf9..c42bdf477 100644 --- a/scripts/test_slurm_package_install.py +++ b/scripts/test_slurm_package_install.py @@ -168,7 +168,7 @@ def main() -> None: check=False, ) assert result.returncode != 0 - assert "data-designer" in (result.stdout + result.stderr) + assert f"data-designer=={version}" in (result.stdout + result.stderr) if __name__ == "__main__": From b8e2ffc982bea7c24ed60fe25cd69b63f0d338c9 Mon Sep 17 00:00:00 2001 From: Andre Manoel Date: Tue, 11 Aug 2026 11:16:48 -0300 Subject: [PATCH 3/6] fix: publish Slurm package Signed-off-by: Andre Manoel --- scripts/publish.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/publish.sh b/scripts/publish.sh index 70e29fd8b..28889713a 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -4,7 +4,7 @@ # # Publish script for DataDesigner -# Publishes all three subpackages to PyPI with the same version. +# Publishes all four subpackages to PyPI with the same version. # # Usage: # ./scripts/publish.sh 0.3.9rc1 # Full publish @@ -32,6 +32,7 @@ PACKAGE_DIRS=( "packages/data-designer-config" "packages/data-designer-engine" "packages/data-designer" + "packages/data-designer-slurm" ) PYPIRC_FILE="$HOME/.pypirc" @@ -544,6 +545,7 @@ main() { echo " https://test.pypi.org/project/data-designer-config/$VERSION/" echo " https://test.pypi.org/project/data-designer-engine/$VERSION/" echo " https://test.pypi.org/project/data-designer/$VERSION/" + echo " https://test.pypi.org/project/data-designer-slurm/$VERSION/" echo "" echo "Test installation with:" echo " pip install --index-url $TEST_PYPI_URL data-designer==$VERSION" @@ -564,6 +566,7 @@ main() { echo " https://pypi.org/project/data-designer-config/$VERSION/" echo " https://pypi.org/project/data-designer-engine/$VERSION/" echo " https://pypi.org/project/data-designer/$VERSION/" + echo " https://pypi.org/project/data-designer-slurm/$VERSION/" fi } From 9290cd129b0c3730c679ae402ad39635c22eae31 Mon Sep 17 00:00:00 2001 From: Andre Manoel Date: Thu, 13 Aug 2026 10:40:04 -0300 Subject: [PATCH 4/6] fix: scope workspace dependency extras Signed-off-by: Andre Manoel --- .agents/recipes/dependencies/recipe.md | 11 +-- .agents/recipes/structure/recipe.md | 13 ++-- .agents/recipes/test-health/recipe.md | 2 +- DEVELOPMENT.md | 5 +- Makefile | 4 +- architecture/overview.md | 8 ++- .../tests/test_dependency_audit.py | 71 +++++++++++++++++++ scripts/audit_package_dependencies.py | 21 ++++-- 8 files changed, 110 insertions(+), 25 deletions(-) diff --git a/.agents/recipes/dependencies/recipe.md b/.agents/recipes/dependencies/recipe.md index 9e0aebf10..f96bae48e 100644 --- a/.agents/recipes/dependencies/recipe.md +++ b/.agents/recipes/dependencies/recipe.md @@ -11,7 +11,7 @@ permissions: # Dependency Audit -Audit the dependency graph across all three packages. Write findings to +Audit the dependency graph across all four packages. Write findings to `/tmp/audit-{{suite}}.md`. Dependabot handles version bump PRs. This recipe focuses on what Dependabot @@ -39,11 +39,12 @@ Read the `pyproject.toml` for each package: cat packages/data-designer-config/pyproject.toml cat packages/data-designer-engine/pyproject.toml cat packages/data-designer/pyproject.toml +cat packages/data-designer-slurm/pyproject.toml ``` -Note: engine and interface packages use `uv-dynamic-versioning` to inject -dependencies. Check both static declarations and the dynamic versioning -config. +Note: engine, interface, and Slurm packages use `uv-dynamic-versioning` to +inject dependencies. Check both static declarations and the dynamic +versioning config, including selected optional dependencies. ### 2. Direct dependency declaration gaps @@ -79,7 +80,7 @@ intentionally deferred but still need to be declared as dependencies. Check that shared dependencies use consistent version constraints: ```bash -# Extract dependency specs from all three pyproject.toml files +# Extract dependency specs from all four pyproject.toml files grep -E "^\s+\"[a-zA-Z]" packages/*/pyproject.toml ``` diff --git a/.agents/recipes/structure/recipe.md b/.agents/recipes/structure/recipe.md index 1960974f5..513e4b878 100644 --- a/.agents/recipes/structure/recipe.md +++ b/.agents/recipes/structure/recipe.md @@ -19,21 +19,24 @@ to `/tmp/audit-{{suite}}.md`. Before starting, read the authoritative sources for the structural rules: 1. **`AGENTS.md`** (repo root) - "The Layering Is Structural" section defines - the three packages, their ownership, and the dependency direction rule. + the four packages, their ownership, and the dependency direction rule. 2. **`architecture/overview.md`** - system architecture diagram, package layout, and the "no reverse imports" rule. The canonical rules: ``` -data-designer (interface) -> data-designer-engine -> data-designer-config +data-designer-slurm -> data-designer (interface) -> data-designer-engine -> data-designer-config ``` - `packages/data-designer-config/` must NOT import from `data_designer.engine`, - `data_designer.interface`, or `data_designer.cli` + `data_designer.interface`, `data_designer.cli`, or `data_designer.slurm` - `packages/data-designer-engine/` must NOT import from - `data_designer.interface` or `data_designer.cli` -- `packages/data-designer/` CAN import from both engine and config + `data_designer.interface`, `data_designer.cli`, or `data_designer.slurm` +- `packages/data-designer/` CAN import from engine and config but must NOT + import from `data_designer.slurm` +- `packages/data-designer-slurm/` uses public interface/config APIs and must + NOT import engine internals **What CI already enforces**: ruff rule `TID` catches relative imports. The CI test matrix runs config, engine, and interface tests in isolation (separate diff --git a/.agents/recipes/test-health/recipe.md b/.agents/recipes/test-health/recipe.md index b9253033a..cac26fddb 100644 --- a/.agents/recipes/test-health/recipe.md +++ b/.agents/recipes/test-health/recipe.md @@ -306,7 +306,7 @@ Write the report to `/tmp/audit-{{suite}}.md`: | Check | Status | Detail | |-------|--------|--------| -| Package imports | OK/FAIL | All three packages import cleanly | +| Package imports | OK/FAIL | All four packages import cleanly | | Import timing | OK/FAIL | X.XXs (budget: 3s) | | Registry completeness | OK/WARN | Column types resolve to config classes | diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 09a33f9b5..ddf02600b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -105,16 +105,17 @@ uv run ruff format --check # Check formatting ### Running Tests -`make test` runs all three package test suites in sequence (config, engine, interface). When iterating on a single package, run its tests directly: +`make test` runs all four package test suites in sequence (config, engine, interface, Slurm). When iterating on a single package, run its tests directly: ```bash -# Run all tests (config + engine + interface) +# Run all tests (config + engine + interface + Slurm) make test # Run a single package's tests make test-config # data-designer-config make test-engine # data-designer-engine make test-interface # data-designer (interface) +make test-slurm # data-designer-slurm # Run a specific test file uv run pytest tests/config/test_sampler_constraints.py diff --git a/Makefile b/Makefile index 39ef6bafe..4368aadd3 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ help: @echo "═════════════════════════════════════════════════════════════" @echo "" @echo "📦 Installation (uv workspace - all packages in editable mode):" - @echo " install - Install all packages (config → engine → interface)" + @echo " install - Install all packages (config → engine → interface → slurm)" @echo " install-dev - Install all packages + dev tools (pytest, etc.)" @echo " install-dev-notebooks - Install all packages + dev + docs + notebook tools" @echo " install-dev-recipes - Install all packages + dev + recipe dependencies" @@ -60,7 +60,7 @@ help: @echo " test-run-all-examples - Run all tutorials and recipes as e2e tests" @echo "" @echo "🔬 Isolated Testing (mirrors CI - uses temp venv):" - @echo " test-isolated - Run all isolated tests (config → engine → interface)" + @echo " test-isolated - Run core package isolated tests (config → engine → interface)" @echo " test-config-isolated - Test config with ONLY config installed" @echo " test-engine-isolated - Test engine with ONLY engine+config installed" @echo " test-interface-isolated - Test interface with full package installed" diff --git a/architecture/overview.md b/architecture/overview.md index 35c53e591..946e5b162 100644 --- a/architecture/overview.md +++ b/architecture/overview.md @@ -1,11 +1,13 @@ # System Architecture -DataDesigner is split across three installable packages that merge at runtime into a single `data_designer` namespace via PEP 420 implicit namespace packages (no top-level `__init__.py`). +DataDesigner is split across four installable packages that merge at runtime into a single `data_designer` namespace via PEP 420 implicit namespace packages (no top-level `__init__.py`). ## Overview ``` ┌─────────────────────────────────────────────────────────┐ +│ data-designer-slurm (optional Slurm execution) │ +├─────────────────────────────────────────────────────────┤ │ data-designer (interface + CLI + integrations) │ │ DataDesigner class, CLI commands, HuggingFace Hub │ ├─────────────────────────────────────────────────────────┤ @@ -19,7 +21,7 @@ DataDesigner is split across three installable packages that merge at runtime in └─────────────────────────────────────────────────────────┘ ``` -**Dependency direction:** interface → engine → config. No reverse imports. +**Import direction:** Slurm → interface → engine → config. The `data-designer[slurm]` extra creates a packaging-only reverse edge from the interface distribution to the optional leaf; no base package imports `data_designer.slurm`. Users declare what their data should look like through config objects (columns, types, relationships, validation rules). The engine compiles those configs into an execution plan and generates the dataset. The interface package provides the public `DataDesigner` class and CLI that wire everything together. @@ -50,7 +52,7 @@ Users declare what their data should look like through config objects (columns, ## Design Decisions -- **PEP 420 namespace packages** allow the three packages to be installed independently while sharing the `data_designer` namespace. This enables lighter installs (e.g., config-only for validation tooling) without import conflicts. +- **PEP 420 namespace packages** allow the four packages to be installed independently while sharing the `data_designer` namespace. This enables lighter installs (e.g., config-only for validation tooling) and an optional Slurm leaf without import conflicts. - **Lazy imports throughout** — `__getattr__`-based lazy loading in `data_designer.config` and `data_designer.interface`, plus `lazy_heavy_imports` for numpy/pandas, keep startup fast. - **Async-only execution** gives `DatasetBuilder` one scheduling path with row-group parallelism and DAG-aware dispatch behind the public interface. - **`TaskRegistry` subclasses: one instance per class** — `TaskRegistry.__new__` (`registry/base.py`) ensures a single instance of each concrete registry (column generators, profilers, processors). **`ModelRegistry`** and **`MCPRegistry`** are ordinary classes, constructed per run with injected dependencies. **`PluginRegistry`** (`plugins/registry.py`) uses `__new__` so entry points are discovered once per process. diff --git a/packages/data-designer/tests/test_dependency_audit.py b/packages/data-designer/tests/test_dependency_audit.py index 0b1961b8e..8393a281b 100644 --- a/packages/data-designer/tests/test_dependency_audit.py +++ b/packages/data-designer/tests/test_dependency_audit.py @@ -178,3 +178,74 @@ def test_includes_selected_dynamic_optional_dependencies(tmp_path: Path) -> None packages = {package["package"]: package for package in result["packages"]} assert packages["data-designer"]["declared"] == ["data-designer-slurm"] assert packages["data-designer-slurm"]["declared"] == ["data-designer"] + + +@pytest.mark.parametrize( + ("consumer_dependency", "expected_guaranteed_by"), + [("base", []), ("base[leaf]", ["base"])], +) +def test_scopes_selected_workspace_extra_to_dependency_edge( + tmp_path: Path, + consumer_dependency: str, + expected_guaranteed_by: list[str], +) -> None: + base = tmp_path / "packages" / "base" + leaf = tmp_path / "packages" / "leaf" + consumer = tmp_path / "packages" / "consumer" + for package in (base, leaf, consumer): + (package / "src").mkdir(parents=True) + (base / "pyproject.toml").write_text( + """ +[project] +name = "base" +dynamic = ["optional-dependencies"] + +[tool.hatch.metadata.hooks.uv-dynamic-versioning] +optional-dependencies = { leaf = ["leaf=={{ version }}"] } +""".lstrip() + ) + (leaf / "pyproject.toml").write_text('[project]\nname = "leaf"\ndependencies = []\n') + (consumer / "pyproject.toml").write_text( + f'[project]\nname = "consumer"\ndependencies = ["{consumer_dependency}"]\n' + ) + (consumer / "src" / "consumer.py").write_text("import leaf_module\n") + + result = audit( + tmp_path, + {"leaf_module": ["leaf"]}, + selected_extras={"base": {"leaf"}}, + ) + + consumer_result = next(package for package in result["packages"] if package["package"] == "consumer") + assert consumer_result["missing"][0]["guaranteed_by"] == expected_guaranteed_by + assert consumer_result["missing"][0]["severity"] == ("low" if expected_guaranteed_by else "high") + + +def test_does_not_follow_workspace_cycle_back_through_audited_project(tmp_path: Path) -> None: + base = tmp_path / "packages" / "base" + leaf = tmp_path / "packages" / "leaf" + for package in (base, leaf): + (package / "src").mkdir(parents=True) + (base / "pyproject.toml").write_text( + """ +[project] +name = "base" +dependencies = ["support"] +dynamic = ["optional-dependencies"] + +[tool.hatch.metadata.hooks.uv-dynamic-versioning] +optional-dependencies = { leaf = ["leaf=={{ version }}"] } +""".lstrip() + ) + (leaf / "pyproject.toml").write_text('[project]\nname = "leaf"\ndependencies = ["base"]\n') + (base / "src" / "base.py").write_text("import transitive_module\n") + + result = audit( + tmp_path, + {"transitive_module": ["transitive"]}, + {"support": ["transitive"]}, + selected_extras={"base": {"leaf"}}, + ) + + base_result = next(package for package in result["packages"] if package["package"] == "base") + assert base_result["missing"][0]["guaranteed_by"] == ["support"] diff --git a/scripts/audit_package_dependencies.py b/scripts/audit_package_dependencies.py index 1a6dca0e8..e4eac95b7 100644 --- a/scripts/audit_package_dependencies.py +++ b/scripts/audit_package_dependencies.py @@ -170,16 +170,20 @@ def audit_repository( requirement_cache: dict[tuple[str, frozenset[str]], dict[str, set[str]]] = {} def requirements_for(distribution_name: str, selected_extras: set[str]) -> dict[str, set[str]]: - if distribution_name in projects: - return projects[distribution_name]["declarations"] - if requirement_map is not None: - return requirement_dependencies(requirement_map.get(distribution_name, []), selected_extras) cache_key = (distribution_name, frozenset(selected_extras)) if cache_key not in requirement_cache: - requirement_cache[cache_key] = installed_requirements(distribution_name, selected_extras) + if distribution_name in projects: + package_path = repository_root / projects[distribution_name]["path"] / "pyproject.toml" + requirement_cache[cache_key] = declared_dependencies(package_path, selected_extras) + elif requirement_map is not None: + requirement_cache[cache_key] = requirement_dependencies( + requirement_map.get(distribution_name, []), selected_extras + ) + else: + requirement_cache[cache_key] = installed_requirements(distribution_name, selected_extras) return requirement_cache[cache_key] - def dependency_closure(distribution_name: str, selected_extras: set[str]) -> set[str]: + def dependency_closure(distribution_name: str, selected_extras: set[str], project_name: str) -> set[str]: closure = set() pending = [(distribution_name, selected_extras)] visited: set[tuple[str, frozenset[str]]] = set() @@ -189,6 +193,8 @@ def dependency_closure(distribution_name: str, selected_extras: set[str]) -> set if current_key in visited: continue visited.add(current_key) + if current == project_name: + continue for dependency, dependency_extras in requirements_for(current, extras).items(): closure.add(dependency) pending.append((dependency, dependency_extras)) @@ -199,7 +205,8 @@ def dependency_closure(distribution_name: str, selected_extras: set[str]) -> set declarations = project["declarations"] declared = set(declarations) dependency_closures = { - dependency: dependency_closure(dependency, extras) for dependency, extras in declarations.items() + dependency: dependency_closure(dependency, extras, project_name) + for dependency, extras in declarations.items() } resolved_imports: dict[str, dict[str, set[str]]] = defaultdict(lambda: {"modules": set(), "files": set()}) unresolved = [] From b428d5f20a355b9d0bc79292ad07e9fa240f5337 Mon Sep 17 00:00:00 2001 From: Andre Manoel Date: Mon, 17 Aug 2026 11:51:02 -0300 Subject: [PATCH 5/6] fix: publish Slurm before base package Upload the Slurm distribution before data-designer so the exact-version extra is resolvable as soon as the base package is published. Signed-off-by: Andre Manoel --- packages/data-designer-slurm/tests/test_package.py | 10 ++++++++++ scripts/publish.sh | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/data-designer-slurm/tests/test_package.py b/packages/data-designer-slurm/tests/test_package.py index 6428ecd43..b9ed63874 100644 --- a/packages/data-designer-slurm/tests/test_package.py +++ b/packages/data-designer-slurm/tests/test_package.py @@ -3,10 +3,20 @@ from __future__ import annotations +from pathlib import Path + import data_designer import data_designer.slurm +REPO_ROOT = Path(__file__).resolve().parents[3] + def test_slurm_uses_shared_namespace() -> None: assert data_designer.__file__ is None assert data_designer.slurm.__name__ == "data_designer.slurm" + + +def test_slurm_is_published_before_base_extra() -> None: + publish_script = (REPO_ROOT / "scripts" / "publish.sh").read_text() + + assert publish_script.index('"packages/data-designer-slurm"') < publish_script.index('"packages/data-designer"') diff --git a/scripts/publish.sh b/scripts/publish.sh index 28889713a..aec30b0c0 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -31,8 +31,9 @@ NC='\033[0m' # No Color PACKAGE_DIRS=( "packages/data-designer-config" "packages/data-designer-engine" - "packages/data-designer" + # Publish the provider before the package that advertises its exact-version extra. "packages/data-designer-slurm" + "packages/data-designer" ) PYPIRC_FILE="$HOME/.pypirc" From 13636aadfb7a263b8cc3cc347675c0e30a6a08a4 Mon Sep 17 00:00:00 2001 From: Andre Manoel Date: Mon, 17 Aug 2026 17:56:48 -0300 Subject: [PATCH 6/6] docs: generalize Slurm package wording Signed-off-by: Andre Manoel --- AGENTS.md | 2 +- Makefile | 2 +- packages/data-designer-slurm/README.md | 2 +- packages/data-designer-slurm/pyproject.toml | 2 +- .../data-designer-slurm/src/data_designer/slurm/__init__.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bac52c0db..274407f46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ The `data_designer` namespace is split across four installable packages that mer | `data-designer-config` | `packages/data-designer-config/` | `data_designer.config` — column configs, model configs, sampler params, builder API, plugin system, lazy imports | | `data-designer-engine` | `packages/data-designer-engine/` | `data_designer.engine` — column generators, dataset builders, DAG execution, model facade, validators, sampling | | `data-designer` | `packages/data-designer/` | `data_designer.interface` — public `DataDesigner` class, results, errors; `data_designer.cli` — CLI entry point; `data_designer.integrations` | -| `data-designer-slurm` | `packages/data-designer-slurm/` | `data_designer.slurm` — optional Slurm batch execution | +| `data-designer-slurm` | `packages/data-designer-slurm/` | `data_designer.slurm` — optional Slurm execution | **Import direction (left imports right):** Slurm → interface → engine → config. The `data-designer[slurm]` extra creates a packaging-only reverse edge; no code may import against this flow. diff --git a/Makefile b/Makefile index 4368aadd3..7cebae70d 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ install-dev: @echo " packages/data-designer-config/ - Configuration layer (lightweight)" @echo " packages/data-designer-engine/ - Generation engine (heavy deps)" @echo " packages/data-designer/ - Full package with CLI" - @echo " packages/data-designer-slurm/ - Optional Slurm batch execution" + @echo " packages/data-designer-slurm/ - Optional Slurm execution" @echo "" @echo "💡 Next steps:" @echo " make verify-imports - Verify all packages are working" diff --git a/packages/data-designer-slurm/README.md b/packages/data-designer-slurm/README.md index 274293c5a..04d29ccbd 100644 --- a/packages/data-designer-slurm/README.md +++ b/packages/data-designer-slurm/README.md @@ -1,6 +1,6 @@ # data-designer-slurm -Optional Slurm batch execution support for Data Designer. +Optional Slurm execution support for Data Designer. Install it through the Data Designer extra: diff --git a/packages/data-designer-slurm/pyproject.toml b/packages/data-designer-slurm/pyproject.toml index 8d197e6f0..17d2906df 100644 --- a/packages/data-designer-slurm/pyproject.toml +++ b/packages/data-designer-slurm/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "data-designer-slurm" dynamic = ["version", "dependencies"] -description = "Slurm batch execution for Data Designer" +description = "Slurm execution for Data Designer" readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" diff --git a/packages/data-designer-slurm/src/data_designer/slurm/__init__.py b/packages/data-designer-slurm/src/data_designer/slurm/__init__.py index 30443170a..0c471e0c7 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/__init__.py +++ b/packages/data-designer-slurm/src/data_designer/slurm/__init__.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Slurm batch execution for Data Designer.""" +"""Slurm execution for Data Designer.""" from __future__ import annotations