diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 50be266df0..7751eab34d 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,22 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +369,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1860,16 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..052fcff12e 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,22 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +105,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +153,16 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..5d42fea272 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,22 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +212,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +945,16 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..7c43b13bc0 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,22 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +193,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +274,16 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 1ae9da3083..c940eeac56 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", + "diffable-rdf>=0.4.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..8e28488d55 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -42,6 +42,7 @@ import pyoxigraph as ox import rdflib +from diffable_rdf import wl_relabel_quads from rdflib.compare import to_canonical_graph @@ -287,6 +288,7 @@ def _is_safe_prefix_iri(iri: str) -> bool: def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. @@ -300,6 +302,13 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Derive blank-node labels from each node's own + neighbourhood instead of RDFC-1.0's global ``c14nN`` numbering, so that + editing one part of a schema does not renumber unrelated blank nodes. + Output is deterministic and isomorphic either way; only the choice of + label changes. Off by default because enabling it relabels existing + output. Has no effect on the rdflib fallback path (non-standard RDF), + which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -330,6 +339,23 @@ def canonicalize_rdf_graph( RDFCanonicalizationWarning, stacklevel=2, ) + if diff_stable: + # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, + # and this path exists precisely because pyoxigraph refused the + # graph, so there are none to refine. The fallback is deterministic + # but not diff-stable: say so rather than returning output that + # silently ignores the argument. ``shaclgen --include-annotations`` + # reaches this path, because an annotation tag without a ``:`` + # becomes a literal predicate. + warnings.warn( + "diff_stable=True was requested but this graph took the rdflib fallback, " + "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " + "deterministic but NOT diff-stable: an unrelated edit may still renumber " + "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " + "predicates) to get diff-stable labels.", + RDFCanonicalizationWarning, + stacklevel=2, + ) return _deterministic_fallback_serialize(graph, output_format) dataset = ox.Dataset() @@ -339,13 +365,26 @@ def canonicalize_rdf_graph( # 3. Canonicalize blank node labels with RDFC-1.0. dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) + quads = list(dataset) + + # 3b. Optionally re-label blank nodes with locality-sensitive hashes. + # RDFC-1.0 guarantees that identical graphs get identical labels, but it + # does not guarantee that *similar* graphs get similar labels: the labels + # are assigned by a global ordering, so inserting one blank node can + # renumber every label after it and turn a one-line semantic change into a + # whole-file diff. Weisfeiler-Lehman labels are derived only from each + # node's local neighbourhood, so unrelated regions keep their labels. + # Output stays deterministic and isomorphic either way; this only changes + # which label each blank node receives. + if diff_stable: + quads = wl_relabel_quads(quads) + # 4. Sort triples for deterministic ordering. # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset # iteration order is not sorted and varies across processes (verified # empirically against pyoxigraph 0.5.8). The explicit string-key sort # is load-bearing for byte-identical output across runs; see # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) sorted_triples = sorted( (ox.Triple(q.subject, q.predicate, q.object) for q in quads), key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..1bc07583af 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -507,3 +507,163 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + import difflib + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_is_opt_in(): + """The default must keep producing exactly the output it produced before.""" + graph = _make_graph_with_bnodes() + assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) + + +def test_diff_stable_preserves_semantics(): + """Relabelling blank nodes must not change what the graph means.""" + graph = _make_graph_with_bnodes() + + plain = rdflib.Graph() + plain.parse(data=canonicalize_rdf_graph(graph), format="turtle") + stable = rdflib.Graph() + stable.parse(data=canonicalize_rdf_graph(graph, diff_stable=True), format="turtle") + + assert rdflib.compare.isomorphic(plain, stable) + + +def test_diff_stable_is_deterministic(): + """Diff stability must not cost determinism, which is the stronger property.""" + graph = _make_graph_with_bnodes() + outputs = {canonicalize_rdf_graph(graph, diff_stable=True) for _ in range(5)} + assert len(outputs) == 1 + + +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): + """Inserting one subject must not relabel the blank nodes of the others. + + RDFC-1.0 numbers blank nodes ``c14nN`` in a global order, so a subject + sorting before the others shifts every subsequent label and rewrites + most of the file. This is the entire reason the option exists, so the + assertion is on the *ratio*, not on an absolute line count that would + be brittle across rdflib versions. + """ + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +def _generator_cases(): + """The four generators that serialize RDF, with the args that make them do so.""" + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.rdfgen import RDFGenerator + from linkml.generators.shaclgen import ShaclGenerator + from linkml.generators.shexgen import ShExGenerator + + return [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + # rdfgen and shexgen resolve JSON-LD contexts (linkml types, shex.jsonld); + # the `network` marker serves those from local stubs. See tests/conftest.py. + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + # ShExGenerator only emits RDF in this format; its default is ShExC text, + # where blank-node labelling does not apply. + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ] + + +@pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) +def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): + """Every RDF generator must actually apply the option, not merely accept it. + + Asserting only that the attribute exists would pass even if a generator + forgot to pass it down to :func:`canonicalize_rdf_graph`. Instead this + checks the observable consequence: RDFC-1.0 names blank nodes ``c14nN``, + while diff-stable labels are neighbourhood hashes, so a generator that + honours the flag emits no ``c14nN`` label at all. + """ + assert generator.diff_stable is False, f"{generator.__name__} must default to off" + + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8", newline="\n") + + plain = generator(str(schema), **kwargs).serialize() + stable = generator(str(schema), diff_stable=True, **kwargs).serialize() + + # Guards the test itself: if the fixture stopped producing blank nodes the + # assertion below would hold vacuously. + assert re.search(r"c14n\d+", plain), f"{generator.__name__} output has no blank nodes to relabel" + assert not re.search(r"c14n\d+", stable), ( + f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " + "the flag is probably not threaded into canonicalize_rdf_graph()" + ) + + +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): + """A request the fallback cannot honour must be reported, not ignored. + + ``wl_relabel_quads`` consumes canonical pyoxigraph quads, and the rdflib + fallback exists precisely because pyoxigraph refused the graph. Returning + the same bytes for ``diff_stable=True`` and ``diff_stable=False`` without + saying so lets a caller believe the output is diff-stable when it is not. + """ + graph = _make_graph_with_bnodes() + # A relative IRI is non-standard RDF, so pyoxigraph rejects the graph and + # canonicalize_rdf_graph degrades to rdflib -- the same path that + # ``shaclgen --include-annotations`` takes via its literal predicates. + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + + with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + + # The warning is the contract: the bytes really are identical, which is + # exactly why staying silent would be misleading. + assert stable == plain diff --git a/uv.lock b/uv.lock index d8cddaa9e1..3c867304c8 100644 --- a/uv.lock +++ b/uv.lock @@ -1085,6 +1085,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2618,6 +2631,7 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, + { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2650,6 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" },