From 71b665b1ac2a7e93ab2f513ca1e14b73ced9a801 Mon Sep 17 00:00:00 2001 From: Andrew Cheng Date: Thu, 2 Jul 2026 16:18:37 +0000 Subject: [PATCH] Add added_between/removed_between filters for content diffs across arbitrary repository versions This PR adds two range filters to the content viewset, added_between and removed_between, that compute the net content diff between two arbitrary (non-adjacent) repository versions, rather than only the single-step diff against the immediate predecessor that repository_version_added and repository_version_removed provide. Each filter takes exactly two comma-separated Repository Version (or Repository) HREF/PRNs in 'base,target' order. added_between=base,target returns the content present in target but not in base; removed_between=base,target returns the content present in base but not in target. The two are symmetric: added_between=a,b is equivalent to removed_between=b,a. Benchmarks on a synthetic 200k-unit system with 1000-unit diffs between repo versions show this approach is ~5-15x faster than running an equivalent complex q query, with the query going from 1486 ms -> 99 ms for a 50k/50k repo version pair and 2403 ms -> 479 ms for a 199k/199k repo version pair. Planning time also decreases significantly and stays ~0.2 ms. repository_version_added and repository_version_removed are unchanged and continue to compute the single-step diff against the immediate predecessor, preserving backward compatibility. Closes #7831 Assisted by: Claude Opus 4.8 --- CHANGES/7831.feature | 5 + pulpcore/app/models/repository.py | 25 ++++- pulpcore/app/viewsets/content.py | 12 +++ pulpcore/app/viewsets/custom_filters.py | 86 ++++++++++++++++- .../api/using_plugin/test_repo_versions.py | 92 +++++++++++++++++++ 5 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 CHANGES/7831.feature diff --git a/CHANGES/7831.feature b/CHANGES/7831.feature new file mode 100644 index 00000000000..82c18216388 --- /dev/null +++ b/CHANGES/7831.feature @@ -0,0 +1,5 @@ +Added optional `added_between` and `removed_between` filters to the content list endpoints. Each +takes two repository versions (by HREF/PRN) as `base,target` and returns the net set of content +added or removed going from the base version to the target version, allowing diffs between two +arbitrary (possibly non-adjacent) repository versions instead of only the single-step difference +against the immediate predecessor. diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py index f18ca998f73..cb54c9b2649 100644 --- a/pulpcore/app/models/repository.py +++ b/pulpcore/app/models/repository.py @@ -1028,13 +1028,24 @@ def get_content(self, content_qs=None): if content_qs is None: content_qs = Content.objects - # Try to not even attempt to evaluate the content_ids on the python side. - content_ids_subquery = ( + return content_qs.filter(pk__in=self.content_ids_subquery()) + + def content_ids_subquery(self): + """ + Return this version's ``content_ids`` as a database-side ``unnest`` subquery. + + Using a subquery keeps the content unit UUIDs inside PostgreSQL instead of loading the + whole array into Python and passing each UUID as a bound query parameter. This avoids the + per-query parameter limit and the memory/serialization cost for large repository versions. + + Returns: + django.db.models.QuerySet: A values queryset yielding the content unit UUIDs. + """ + return ( RepositoryVersion.objects.filter(pk=self.pk) .annotate(cids=Func(F("content_ids"), function="unnest")) .values_list("cids", flat=True) ) - return content_qs.filter(pk__in=content_ids_subquery) @property def content(self): @@ -1148,7 +1159,9 @@ def added(self, base_version=None): if not base_version: return Content.objects.filter(version_memberships__version_added=self) - return Content.objects.filter(pk__in=self.content).exclude(pk__in=base_version.content) + return Content.objects.filter(pk__in=self.content_ids_subquery()).exclude( + pk__in=base_version.content_ids_subquery() + ) def removed(self, base_version=None): """ @@ -1161,7 +1174,9 @@ def removed(self, base_version=None): if not base_version: return Content.objects.filter(version_memberships__version_removed=self) - return Content.objects.filter(pk__in=base_version.content).exclude(pk__in=self.content) + return Content.objects.filter(pk__in=base_version.content_ids_subquery()).exclude( + pk__in=self.content_ids_subquery() + ) def contains(self, content): """ diff --git a/pulpcore/app/viewsets/content.py b/pulpcore/app/viewsets/content.py index a23ad0e13dd..56a786c399b 100644 --- a/pulpcore/app/viewsets/content.py +++ b/pulpcore/app/viewsets/content.py @@ -23,6 +23,8 @@ ContentAddedRepositoryVersionFilter, ContentRemovedRepositoryVersionFilter, ContentRepositoryVersionFilter, + RepositoryVersionAddedBetweenFilter, + RepositoryVersionRemovedBetweenFilter, ) @@ -125,6 +127,14 @@ class ContentFilter(BaseFilterSet): Return Content which was added in this repository version. repository_version_removed: Return Content which was removed from this repository version. + added_between: + Given two Repository Version HREFs/PRNs as 'base,target', return the net set of + Content added going from the base version to the target version (present in target + but not in base). Diffs two arbitrary (possibly non-adjacent) versions. + removed_between: + Given two Repository Version HREFs/PRNs as 'base,target', return the net set of + Content removed going from the base version to the target version (present in base + but not in target). Diffs two arbitrary (possibly non-adjacent) versions. orphaned_for: Return Content which has been orphaned for a given number of minutes; -1 uses ORPHAN_PROTECTION_TIME value. @@ -135,6 +145,8 @@ class ContentFilter(BaseFilterSet): repository_version = ContentRepositoryVersionFilter() repository_version_added = ContentAddedRepositoryVersionFilter() repository_version_removed = ContentRemovedRepositoryVersionFilter() + added_between = RepositoryVersionAddedBetweenFilter() + removed_between = RepositoryVersionRemovedBetweenFilter() orphaned_for = OrphanedFilter( help_text="Minutes Content has been orphaned for. -1 uses ORPHAN_PROTECTION_TIME." ) diff --git a/pulpcore/app/viewsets/custom_filters.py b/pulpcore/app/viewsets/custom_filters.py index 72db8022db7..7480ee63a40 100644 --- a/pulpcore/app/viewsets/custom_filters.py +++ b/pulpcore/app/viewsets/custom_filters.py @@ -9,7 +9,7 @@ from django.conf import settings from django.db.models import Q -from django_filters import BaseInFilter, CharFilter, Filter +from django_filters import BaseInFilter, BaseRangeFilter, CharFilter, Filter from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers @@ -163,6 +163,90 @@ def filter(self, qs, value): raise NotImplementedError() +class RepositoryVersionBetweenFilter(BaseRangeFilter, RepoVersionHrefPrnFilter): + """ + Base class for range filters that diff content between two arbitrary repository versions. + + The filter accepts exactly two comma-separated Repository Version (or Repository) HREF/PRN + values in ``base,target`` order: the first is the base version and the second is the target + version. Subclasses implement :meth:`diff` to return the appropriate net set of content. + """ + + def diff(self, base_version, target_version): + """ + Return the content that differs between ``base_version`` and ``target_version``. + + Args: + base_version (pulpcore.app.models.RepositoryVersion): The base repository version. + target_version (pulpcore.app.models.RepositoryVersion): The target repository version. + + Returns: + django.db.models.query.QuerySet: A Content queryset with the diff. + """ + raise NotImplementedError() + + def filter(self, qs, value): + """ + Args: + qs (django.db.models.query.QuerySet): The Content Queryset + value (list): The ``[base, target]`` RepositoryVersion href/prn pair to diff between. + """ + if not value: + return qs + + base_version = self.get_repository_version(value[0]) + target_version = self.get_repository_version(value[1]) + return qs.filter(pk__in=self.diff(base_version, target_version)) + + +class RepositoryVersionAddedBetweenFilter(RepositoryVersionBetweenFilter): + """ + Filter Content to the net set added going from a base repository version to a target version. + + Given ``base,target``, returns the content present in ``target`` but not in ``base`` -- the + net difference across two arbitrary (possibly non-adjacent) repository versions, rather than + the single-step difference computed by ``repository_version_added``. + """ + + def __init__(self, *args, **kwargs): + kwargs.setdefault( + "help_text", + _( + "Two Repository Versions referenced by HREF/PRN as 'base,target'. Returns the " + "content added going from the base version to the target version (present in " + "target but not in base)." + ), + ) + super().__init__(*args, **kwargs) + + def diff(self, base_version, target_version): + return target_version.added(base_version=base_version) + + +class RepositoryVersionRemovedBetweenFilter(RepositoryVersionBetweenFilter): + """ + Filter Content to the net set removed going from a base repository version to a target version. + + Given ``base,target``, returns the content present in ``base`` but not in ``target`` -- the + net difference across two arbitrary (possibly non-adjacent) repository versions, rather than + the single-step difference computed by ``repository_version_removed``. + """ + + def __init__(self, *args, **kwargs): + kwargs.setdefault( + "help_text", + _( + "Two Repository Versions referenced by HREF/PRN as 'base,target'. Returns the " + "content removed going from the base version to the target version (present in " + "base but not in target)." + ), + ) + super().__init__(*args, **kwargs) + + def diff(self, base_version, target_version): + return target_version.removed(base_version=base_version) + + class RepositoryVersionFilter(RepoVersionHrefPrnFilter): """ Filter by RepositoryVersion href/prn. diff --git a/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py b/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py index c8b482024c1..e273bc2e332 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py +++ b/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py @@ -176,6 +176,98 @@ def test_add_remove_content( assert latest_version.content_summary.removed == {} +@pytest.fixture +def file_repo_three_versions( + file_bindings, + file_repository_factory, + file_9_contents, + monitor_task, +): + """Build a repo with three versions whose diffs span more than one step. + + Using the content units "A" through "E", the versions end up as:: + + v1: add A, B, C -> present {A, B, C} + v2: add D, remove A -> present {B, C, D} + v3: add E, remove B -> present {C, D, E} + + So a diff between the non-adjacent versions v1 and v3 differs from the single-step diff at v3. + """ + repo = file_repository_factory() + contents = file_9_contents + + def modify(add=(), remove=()): + body = { + "add_content_units": [contents[name].pulp_href for name in add], + "remove_content_units": [contents[name].pulp_href for name in remove], + } + monitor_task(file_bindings.RepositoriesFileApi.modify(repo.pulp_href, body).task) + return file_bindings.RepositoriesFileApi.read(repo.pulp_href).latest_version_href + + v1 = modify(add=["A", "B", "C"]) + v2 = modify(add=["D"], remove=["A"]) + v3 = modify(add=["E"], remove=["B"]) + return repo, v1, v2, v3 + + +def _relative_paths(list_response): + return {content.relative_path for content in list_response.results} + + +@pytest.mark.parallel +def test_added_between_filter(file_bindings, file_repo_three_versions): + """``added_between`` returns the net content added between two arbitrary versions.""" + _, v1, v2, v3 = file_repo_three_versions + + # Single-step diff against the immediate predecessor (v2 -> v3) for reference. + single_step = file_bindings.ContentFilesApi.list(repository_version_added=v3) + assert _relative_paths(single_step) == {"E"} + + # added_between=base,target => content in target (v3) but not in base (v1). + # {C, D, E} - {A, B, C} => {D, E}. + net = file_bindings.ContentFilesApi.list(added_between=[v1, v3]) + assert _relative_paths(net) == {"D", "E"} + + +@pytest.mark.parallel +def test_removed_between_filter(file_bindings, file_repo_three_versions): + """``removed_between`` returns the net content removed between two arbitrary versions.""" + _, v1, v2, v3 = file_repo_three_versions + + # Single-step diff against the immediate predecessor (v2 -> v3) for reference. + single_step = file_bindings.ContentFilesApi.list(repository_version_removed=v3) + assert _relative_paths(single_step) == {"B"} + + # removed_between=base,target => content in base (v1) but not in target (v3). + # {A, B, C} - {C, D, E} => {A, B}. + net = file_bindings.ContentFilesApi.list(removed_between=[v1, v3]) + assert _relative_paths(net) == {"A", "B"} + + +@pytest.mark.parallel +def test_added_removed_between_are_symmetric(file_bindings, file_repo_three_versions): + """``added_between=a,b`` yields the same set as ``removed_between=b,a`` (order reversed).""" + _, v1, v2, v3 = file_repo_three_versions + + added = file_bindings.ContentFilesApi.list(added_between=[v1, v3]) + removed_reversed = file_bindings.ContentFilesApi.list(removed_between=[v3, v1]) + assert _relative_paths(added) == _relative_paths(removed_reversed) == {"D", "E"} + + +@pytest.mark.parallel +def test_between_filter_requires_two_versions(file_bindings, file_repo_three_versions): + """``added_between`` / ``removed_between`` reject anything other than exactly two versions.""" + _, v1, v2, v3 = file_repo_three_versions + + with pytest.raises(file_bindings.ApiException) as ctx: + file_bindings.ContentFilesApi.list(added_between=[v1]) + assert ctx.value.status == 400 + + with pytest.raises(file_bindings.ApiException) as ctx: + file_bindings.ContentFilesApi.list(removed_between=[v1, v2, v3]) + assert ctx.value.status == 400 + + @pytest.mark.parallel def test_add_remove_repo_version( file_bindings,