From 2286e75517c7a41f528d3e5b531ff4d21e46ae0a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 8 Sep 2026 18:59:04 -0500 Subject: [PATCH 1/4] Fix unified pip UCX base tags --- .devcontainer/cuda12.9-pip/devcontainer.json | 2 +- .devcontainer/cuda13.3-pip/devcontainer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/cuda12.9-pip/devcontainer.json b/.devcontainer/cuda12.9-pip/devcontainer.json index 62019e1e..a00382cf 100644 --- a/.devcontainer/cuda12.9-pip/devcontainer.json +++ b/.devcontainer/cuda12.9-pip/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "12.9", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:26.10-cpp-cuda12.9-ucx1.19.0-openmpi5.0.10" + "BASE": "rapidsai/devcontainers:26.10-cpp-cuda12.9-ucx1.21.0-openmpi5.0.10" }, "cacheFrom": [ "ghcr.io/rapidsai/devcontainers/devcontainer:26.10-cuda12.9-pip" diff --git a/.devcontainer/cuda13.3-pip/devcontainer.json b/.devcontainer/cuda13.3-pip/devcontainer.json index 08794677..4008d343 100644 --- a/.devcontainer/cuda13.3-pip/devcontainer.json +++ b/.devcontainer/cuda13.3-pip/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "13.3", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:26.10-cpp-cuda13.3-ucx1.19.0-openmpi5.0.10" + "BASE": "rapidsai/devcontainers:26.10-cpp-cuda13.3-ucx1.21.0-openmpi5.0.10" }, "cacheFrom": [ "ghcr.io/rapidsai/devcontainers/devcontainer:26.10-cuda13.3-pip" From 5b72ce42771b998234506f325c8548a2f8d80124 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 8 Sep 2026 22:58:37 -0500 Subject: [PATCH 2/4] Keep pip OpenMPI from shadowing system UCX The PyPI OpenMPI wheel bundles UCX 1.20 libraries. Unified pip containers now use system UCX 1.21, so installing that wheel creates a mixed UCX runtime and leaves libucp with unresolved 1.21 symbols during C++ links.\n\nExclude and uninstall the wheel whenever system UCX is preferred. The base image already supplies OpenMPI linked against that same system UCX, preserving one internally consistent MPI/UCX stack even for reused virtual environments. --- .../rapids-build-utils/bin/make-pip-env.sh | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index c0b6a716..d1b90087 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -43,11 +43,24 @@ make_pip_env() { local -r new_env_path="$(realpath -m "/tmp/${env_file_name}")"; local -r old_env_path="$(realpath -m "${HOME}/.local/share/venvs/${env_file_name}")"; + local -a excluded_packages=(ninja); + + # The PyPI OpenMPI wheel bundles its own UCX libraries. Do not install it when the + # devcontainer is configured to use system UCX, because loading those bundled + # libraries alongside system UCX can produce an ABI-incompatible mix. The base + # image already provides an OpenMPI build linked against its system UCX. + if [ "${RAPIDS_LIBUCX_PREFER_SYSTEM_LIBRARY:-false}" = true ]; then + excluded_packages+=(openmpi); + fi + # Create the python env without ninja. # ninja -$(ulimit -n) fails with `ninja: FATAL: pipe: Too many open files`. # This appears to have been fixed 13 years ago (https://github.com/ninja-build/ninja/issues/233), # so that fix needs to be integrated into the kitware pip ninja builds. - rapids-make-pip-dependencies --exclude <(echo ninja) "${OPTS[@]}" > "${new_env_path}"; + rapids-make-pip-dependencies \ + --exclude <(printf '%s\n' "${excluded_packages[@]}") \ + "${OPTS[@]}" \ + > "${new_env_path}"; if test -f "${new_env_path}"; then @@ -63,7 +76,6 @@ make_pip_env() { . "${HOME}/.local/share/venvs/${env_name}/bin/activate"; python -m pip install -U pip; python -m pip install "${pre[@]}" -U -r "${new_env_path}"; - python -m pip uninstall -y ninja >/dev/null 2>&1; # If the venv does exist but it's different from the generated one, # print the diff between the envs and update it elif ! diff -BNqw "${old_env_path}" "${new_env_path}" >/dev/null 2>&1; then @@ -82,9 +94,13 @@ make_pip_env() { . "${HOME}/.local/share/venvs/${env_name}/bin/activate"; python -m pip install -U pip; python -m pip install "${pre[@]}" -U -r "${new_env_path}"; - python -m pip uninstall -y ninja >/dev/null 2>&1; fi + # Keep excluded packages out of reused venvs as well as newly-created ones. + # shellcheck disable=SC1090 + . "${HOME}/.local/share/venvs/${env_name}/bin/activate"; + python -m pip uninstall -y "${excluded_packages[@]}" >/dev/null 2>&1; + cp -a "${new_env_path}" "${old_env_path}"; fi } From 93801e3961a6def53014fb1df8b67c5916f3d754 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 8 Sep 2026 23:24:11 -0500 Subject: [PATCH 3/4] Pair conda tileiras with its CUDA toolchain The CUDA feature exports an incomplete profiler-only system toolkit for conda devcontainers. Conda's tileiras then honors that CUDA_HOME instead of using the matching compiler libraries installed in the environment, causing every TileIR version probe and cuVS kernel export to fail.\n\nAllow the CUDA profile to accept an explicit package-managed toolkit root, and select the rapids conda environment only for the CUDA 13.3 conda configuration that installs tileiras. Other CUDA and package-manager combinations retain the system default. --- .devcontainer/cuda13.3-conda/devcontainer.json | 5 ++++- features/src/cuda/.bashrc | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.devcontainer/cuda13.3-conda/devcontainer.json b/.devcontainer/cuda13.3-conda/devcontainer.json index 60668e23..2572a58f 100644 --- a/.devcontainer/cuda13.3-conda/devcontainer.json +++ b/.devcontainer/cuda13.3-conda/devcontainer.json @@ -18,7 +18,10 @@ "--ulimit", "nofile=500000" ], - "containerEnv": {"PYTHON_VERSION": "3.13"}, + "containerEnv": { + "PYTHON_VERSION": "3.13", + "RAPIDS_CUDA_HOME": "/home/coder/.conda/envs/rapids" + }, "hostRequirements": {"gpu": "optional"}, "features": { "./features/src/cuda": { diff --git a/features/src/cuda/.bashrc b/features/src/cuda/.bashrc index 23e18057..667acd4d 100644 --- a/features/src/cuda/.bashrc +++ b/features/src/cuda/.bashrc @@ -1,6 +1,9 @@ export NVARCH="${NVARCH}"; -export CUDA_HOME="${CUDA_HOME}"; -export CUDA_PATH="${CUDA_HOME}"; +# A package-managed CUDA toolkit can override the system toolkit installed by +# this feature. This keeps tools such as conda's tileiras compiler paired with +# the CUDA compiler libraries from the same environment. +export CUDA_HOME="${RAPIDS_CUDA_HOME:-${CUDA_HOME}}"; +export CUDA_PATH="${RAPIDS_CUDA_HOME:-${CUDA_HOME}}"; export CUDA_VERSION="${CUDA_VERSION}"; export CUDA_VERSION_MAJOR="${CUDA_VERSION_MAJOR}"; export CUDA_VERSION_MINOR="${CUDA_VERSION_MINOR}"; From cfd93912b2b980071f7189f0a25ac8e7018a9e69 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 9 Sep 2026 08:40:04 -0500 Subject: [PATCH 4/4] Remove devcontainer CUDA_HOME override cuTile dependencies should be declared by the project that needs them, while tileiras comes from the system CUDA Toolkit selected for that build. Redirecting CUDA_HOME for the entire conda devcontainer hides that dependency boundary and changes unrelated CUDA consumers, so restore the feature's normal system-toolkit environment. --- .devcontainer/cuda13.3-conda/devcontainer.json | 5 +---- features/src/cuda/.bashrc | 7 ++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.devcontainer/cuda13.3-conda/devcontainer.json b/.devcontainer/cuda13.3-conda/devcontainer.json index 2572a58f..60668e23 100644 --- a/.devcontainer/cuda13.3-conda/devcontainer.json +++ b/.devcontainer/cuda13.3-conda/devcontainer.json @@ -18,10 +18,7 @@ "--ulimit", "nofile=500000" ], - "containerEnv": { - "PYTHON_VERSION": "3.13", - "RAPIDS_CUDA_HOME": "/home/coder/.conda/envs/rapids" - }, + "containerEnv": {"PYTHON_VERSION": "3.13"}, "hostRequirements": {"gpu": "optional"}, "features": { "./features/src/cuda": { diff --git a/features/src/cuda/.bashrc b/features/src/cuda/.bashrc index 667acd4d..23e18057 100644 --- a/features/src/cuda/.bashrc +++ b/features/src/cuda/.bashrc @@ -1,9 +1,6 @@ export NVARCH="${NVARCH}"; -# A package-managed CUDA toolkit can override the system toolkit installed by -# this feature. This keeps tools such as conda's tileiras compiler paired with -# the CUDA compiler libraries from the same environment. -export CUDA_HOME="${RAPIDS_CUDA_HOME:-${CUDA_HOME}}"; -export CUDA_PATH="${RAPIDS_CUDA_HOME:-${CUDA_HOME}}"; +export CUDA_HOME="${CUDA_HOME}"; +export CUDA_PATH="${CUDA_HOME}"; export CUDA_VERSION="${CUDA_VERSION}"; export CUDA_VERSION_MAJOR="${CUDA_VERSION_MAJOR}"; export CUDA_VERSION_MINOR="${CUDA_VERSION_MINOR}";