test: add schema-driven round-trip and fixture tests

.github/workflows/run_tests.yml

@@ -15,6 +15,10 @@ on:
     branches: [ "main" ]
   # Allows for manually running this workflow from the Actions tab
   workflow_dispatch:
+  # Weekly full-model run, so the exhaustive round-trip sweep is exercised
+  # even in weeks without a push to main
+  schedule:
+    - cron: '0 3 * * 1'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -61,19 +65,28 @@ jobs:
           sarif_file: docs/reports/code_issues.sarif
 
       # Run all tests in the project.
+      # The round-trip test covers every openMINDS type when
+      # OPENMINDS_TEST_ALL_TYPES is set, which takes several minutes. Pull
+      # requests run a representative sample instead. The full sweep runs
+      # on push to main, which is where the openMINDS pipeline lands
+      # regenerated type classes, and on the weekly schedule.
       - name: Run tests
         if: always()
         uses: matlab-actions/run-command@v3
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENMINDS_TEST_ALL_TYPES: ${{ (github.event_name == 'pull_request') && '0' || '1' }}
         with:
           command: |
             doCreateBadge = "${{ matrix.MATLABVersion }}" == "${{ env.LatestMATLABVersion }}";
             addpath(genpath("tools")); testToolbox("CreateBadge", doCreateBadge)
 
-      # Commit updated SVG badges for the issues and tests (if changed)
+      # Commit updated SVG badges for the issues and tests (if changed).
+      # Push events only, so badge commits land on main and nowhere else:
+      # committing them on a pull_request event rewrites the PR head branch
+      # and breaks any stack built on top of it.
       - name: Commit svg badges if updated
-        if: matrix.MATLABVersion == env.LatestMATLABVersion
+        if: matrix.MATLABVersion == env.LatestMATLABVersion && github.event_name == 'push'
         continue-on-error: true
         run: |
           git config user.name "${{ github.workflow }} by ${{ github.actor }}"

.gitignore

@@ -47,4 +47,6 @@ codegen/
 .MATLABDriveTag
 
 # Python cache
-__pycache__
+__pycache__
+# Generated by the tutorial livescripts during test runs
+code/livescripts/example_metadata.jsonld

tools/tests/+ommtest/+helper/buildFixtureCollection.m

@@ -0,0 +1,56 @@
+function collection = buildFixtureCollection()
+%buildFixtureCollection Canonical instance graph used by the fixture tests
+%
+%   collection = ommtest.helper.buildFixtureCollection() returns a
+%   collection covering the structural features that serialization has to
+%   get right: a scalar string, a string list, a linked instance, an
+%   embedded instance, a controlled instance reference, a date, and a
+%   number.
+%
+%   Every instance is given an explicit identifier so the serialized
+%   document is byte-for-byte reproducible. Without that, blank node
+%   identifiers are random and no golden file could be compared.
+%
+%   This function is the single definition of the fixture content. The
+%   golden files are generated from it by
+%   ommtest.helper.regenerateFixtures, so refreshing fixtures after a model
+%   version bump is one command plus a review of the diff.
+%
+%   Output Arguments:
+%     collection - An openminds.Collection holding the fixture graph.
+%
+%   See also ommtest.helper.regenerateFixtures
+
+    baseIRI = "https://openminds.om-i.org/instances/matlabTestFixture/";
+
+    contactInformation = openminds.core.ContactInformation( ...
+        'id', baseIRI + "contact-001");
+    contactInformation.email = "ada@example.org";
+
+    person = openminds.core.Person('id', baseIRI + "person-001");
+    person.givenName = "Ada";
+    person.familyName = "Lovelace";
+    person.alternateName = ["A. Lovelace", "Ada L."];
+    person.contactInformation = contactInformation;
+
+    quantitativeValue = openminds.core.QuantitativeValue();
+    quantitativeValue.value = 42;
+    quantitativeValue.unit = ommtest.helper.controlledInstance( ...
+        "openminds.controlledterms.UnitOfMeasurement", "day");
+
+    specimenAge = openminds.core.SpecimenAge();
+    specimenAge.age = quantitativeValue;
+    specimenAge.reference = ommtest.helper.controlledInstance( ...
+        "openminds.controlledterms.AgeReference", "birth");
+
+    subjectState = openminds.core.SubjectState('id', baseIRI + "subjectState-001");
+    subjectState.age = specimenAge;
+
+    subject = openminds.core.Subject('id', baseIRI + "subject-001");
+    subject.lookupLabel = "fixtureSubject";
+    subject.species = ommtest.helper.controlledInstance( ...
+        "openminds.controlledterms.Species", "Homo sapiens");
+    subject.studiedState = subjectState;
+
+    collection = openminds.Collection(person, subject);
+end

tools/tests/+ommtest/+helper/controlledInstance.m

@@ -0,0 +1,36 @@
+function instance = controlledInstance(className, instanceName)
+%controlledInstance Create a controlled instance by name, whichever API it uses
+%
+%   instance = ommtest.helper.controlledInstance(className, instanceName)
+%   returns the named controlled instance of the given openMINDS type.
+%
+%   openMINDS exposes two disjoint mechanisms for controlled instances and
+%   there is no common entry point. Subclasses of
+%   openminds.abstract.ControlledTerm take the instance name directly in
+%   their constructor, while types using the
+%   openminds.internal.mixin.HasControlledInstance mixin require the static
+%   fromName method and reject a string constructor argument. This function
+%   dispatches on the superclass so callers do not have to know which
+%   mechanism a given type uses.
+%
+%   Input Arguments:
+%     className    - Full MATLAB class name of the controlled type.
+%     instanceName - Name of the controlled instance, e.g. "Homo sapiens".
+%
+%   Output Arguments:
+%     instance - The requested controlled instance.
+
+    arguments
+        className (1,1) string
+        instanceName (1,1) string
+    end
+
+    usesMixin = any(ismember(superclasses(className), ...
+        {'openminds.internal.mixin.HasControlledInstance'}));
+
+    if usesMixin
+        instance = feval(sprintf("%s.fromName", className), instanceName);
+    else
+        instance = feval(className, instanceName);
+    end
+end

tools/tests/+ommtest/+helper/fixtureNamespaceTag.m

@@ -0,0 +1,21 @@
+function tag = fixtureNamespaceTag()
+%fixtureNamespaceTag Short tag naming the active openMINDS namespace
+%
+%   tag = ommtest.helper.fixtureNamespaceTag() returns "omi" when the
+%   active model uses the https://openminds.om-i.org namespace and
+%   "ebrains" when it uses https://openminds.ebrains.eu.
+%
+%   Fixtures are named by namespace rather than by version number because
+%   the namespace is what actually appears in the serialized document.
+
+    baseIRI = openminds.constant.BaseURI();
+
+    if startsWith(baseIRI, "https://openminds.om-i.org")
+        tag = "omi";
+    elseif startsWith(baseIRI, "https://openminds.ebrains.eu")
+        tag = "ebrains";
+    else
+        error('ommtest:fixtureNamespaceTag:UnknownNamespace', ...
+            'No fixture tag defined for base IRI "%s".', baseIRI)
+    end
+end

tools/tests/+ommtest/+helper/fixturePath.m

@@ -0,0 +1,13 @@
+function folderPath = fixturePath()
+%fixturePath Folder holding the golden JSON-LD fixtures
+%
+%   folderPath = ommtest.helper.fixturePath() returns the absolute path of
+%   the fixtures folder, resolved relative to this file so it does not
+%   depend on the current working folder.
+
+    thisFile = mfilename('fullpath');
+    helperFolder = fileparts(thisFile);            % +helper
+    packageFolder = fileparts(helperFolder);       % +ommtest
+    testsFolder = fileparts(packageFolder);        % tools/tests
+    folderPath = string(fullfile(testsFolder, 'fixtures'));
+end

tools/tests/+ommtest/+helper/knownRoundTripGap.m

@@ -0,0 +1,64 @@
+function reason = knownRoundTripGap(typeName)
+%knownRoundTripGap Reason a type is not expected to survive a JSON-LD round trip
+%
+%   reason = ommtest.helper.knownRoundTripGap(typeName) returns a string
+%   explaining why the given openMINDS type currently fails to round trip
+%   through JSON-LD, or an empty string if the type is expected to succeed.
+%
+%   Every entry here is a defect in the library, not in the test. This list
+%   is expected to shrink. When a fix lands, the corresponding entry must
+%   be removed so the round-trip test starts guarding the fixed behaviour.
+%
+%   Input Arguments:
+%     typeName - Short name of an openMINDS type, e.g. "Person".
+%
+%   Output Arguments:
+%     reason - Explanation of the gap, or "" if the type should round trip.
+%
+%   See also ommtest.helper.synthesizeInstance
+
+    arguments
+        typeName (1,1) string
+    end
+
+    reason = "";
+
+    if isControlledTermType(typeName)
+        reason = "Controlled terms defined by the user lose every property " + ...
+            "on reload. ControlledTermBase/initializeControlledTerm discards " + ...
+            "the decoded struct and passes only the identifier to " + ...
+            "deserializeFromName, which finds no matching controlled instance " + ...
+            "and returns an empty object.";
+        return
+    end
+
+    if ismember(typeName, residualGapTypes())
+        reason = "Multi-valued properties linking to controlled instances " + ...
+            "lose all but the first entry on reload.";
+    end
+end
+
+function tf = isControlledTermType(typeName)
+    className = openminds.enum.Types(typeName).ClassName;
+    tf = any(ismember(superclasses(className), {'openminds.abstract.ControlledTerm'}));
+end
+
+function typeNames = residualGapTypes()
+% Types that fail for reasons other than the controlled term defect.
+%
+%   Unlike the controlled term case there is no clean structural predicate
+%   for these, so they are listed explicitly. Determined by sweeping every
+%   type through save and load; see the round-trip test for the procedure.
+
+    typeNames = [ ...
+        "Accessibility", "AtlasAnnotation", "ChemicalSubstance", ...
+        "ContentType", "CustomAnnotation", "DataAnalysis", "DataCopy", ...
+        "DatasetVersion", "Dependency", "File", "FileBundle", ...
+        "FilePathPattern", "GenericComputation", "LocalFile", ...
+        "ModelValidation", "Optimization", "ParcellationTerminologyVersion", ...
+        "QuantitativeRelationAssessment", "Setup", "Simulation", ...
+        "SoftwareVersion", "SubjectGroup", "SubjectGroupState", ...
+        "SubjectState", "TissueSample", "TissueSampleCollection", ...
+        "TissueSampleCollectionState", "TissueSampleState", ...
+        "ValidationTest", "Visualization"];
+end

tools/tests/+ommtest/+helper/regenerateFixtures.m

@@ -0,0 +1,42 @@
+function outputPath = regenerateFixtures(options)
+%regenerateFixtures Write the golden JSON-LD fixture for the active model
+%
+%   ommtest.helper.regenerateFixtures() writes the golden fixture file for
+%   the currently active openMINDS model version, from the graph defined by
+%   ommtest.helper.buildFixtureCollection.
+%
+%   Run this after a model version bump, then review the diff. A change in
+%   the golden file is a change in the serialized output of the library and
+%   should be understood before it is committed.
+%
+%   Name-Value Arguments:
+%     FixtureFolder - Folder to write to. Defaults to the fixtures folder
+%                     next to the tests.
+%
+%   Output Arguments:
+%     outputPath - Path of the file that was written.
+%
+%   See also ommtest.helper.buildFixtureCollection, ommtest.helper.fixturePath
+
+    arguments
+        options.FixtureFolder (1,1) string = ommtest.helper.fixturePath()
+    end
+
+    if ~isfolder(options.FixtureFolder)
+        mkdir(options.FixtureFolder)
+    end
+
+    collection = ommtest.helper.buildFixtureCollection();
+    outputPath = fullfile(options.FixtureFolder, currentFixtureName());
+    collection.save(outputPath);
+
+    fprintf('Wrote fixture: %s\n', outputPath);
+
+    if ~nargout
+        clear outputPath
+    end
+end
+
+function fileName = currentFixtureName()
+    fileName = "collection_" + ommtest.helper.fixtureNamespaceTag() + ".jsonld";
+end

tools/tests/+ommtest/+helper/roundTripTypeSelection.m

@@ -0,0 +1,37 @@
+function typeNames = roundTripTypeSelection()
+%roundTripTypeSelection Types to exercise in the JSON-LD round-trip test
+%
+%   typeNames = ommtest.helper.roundTripTypeSelection() returns a cell
+%   array of openMINDS type names to run the round-trip test against.
+%
+%   Round tripping every type takes several minutes, which is too slow for
+%   a per-commit test run. By default this returns an evenly spaced sample
+%   of the model, which gives fast regression signal on every commit. Set
+%   the environment variable OPENMINDS_TEST_ALL_TYPES to "1" to return
+%   every type instead, for scheduled runs and for the schema rebuild
+%   pipeline.
+%
+%   The sample is a fixed stride through the type list rather than a
+%   curated set, so it needs no maintenance as the model changes and it
+%   still spans the breadth of the model.
+%
+%   Output Arguments:
+%     typeNames - Cell array of type names, for use as a TestParameter.
+
+    allTypeNames = string(cellstr(enumeration('openminds.enum.Types')));
+    allTypeNames(allTypeNames == "None") = [];
+    allTypeNames = sort(allTypeNames);
+
+    if isFullSweepRequested()
+        typeNames = cellstr(allTypeNames);
+        return
+    end
+
+    numSampled = 30;
+    stride = max(1, floor(numel(allTypeNames) / numSampled));
+    typeNames = cellstr(allTypeNames(1:stride:end));
+end
+
+function tf = isFullSweepRequested()
+    tf = strcmp(getenv('OPENMINDS_TEST_ALL_TYPES'), '1');
+end