From d1428f67f84b743b89fa3163297cb4b5278d5e00 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 19 Aug 2026 09:46:34 +0200 Subject: [PATCH] add(rules): mapping of resource types for research committees # Conflicts: # cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py --- .../transform/models/research_committee.py | 3 +- .../rdm/records/transform/transform.py | 7 + .../transform/xml_processing/rules/base.py | 11 +- .../xml_processing/rules/research.py | 32 +- .../rules/research_committee.py | 423 ++++++++++++++++++ cds_migrator_kit/rdm/streams.yaml | 6 +- setup.cfg | 5 + tests/cds-rdm/test_publications_rules.py | 104 +++++ 8 files changed, 576 insertions(+), 15 deletions(-) create mode 100644 cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py diff --git a/cds_migrator_kit/rdm/records/transform/models/research_committee.py b/cds_migrator_kit/rdm/records/transform/models/research_committee.py index 57bbfc75..bb27821f 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/models/research_committee.py @@ -95,11 +95,10 @@ class ResearchCommitteeModel(CdsOverdo): _default_fields = { "custom_fields": {}, - "resource_type": {"id": "publication-other"}, } research_comm_model = ResearchCommitteeModel( bases=(rdm_base_publication_model,), - entry_point_group="cds_migrator_kit.migrator.rdm.rules.publication", + entry_point_group="cds_migrator_kit.migrator.rules.research_committee", ) diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index 6b91988e..dcb6d622 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -422,6 +422,11 @@ def creators(json, key="creators"): return _creators def _resource_type(entry): + # `_resource_type_rank` is bookkeeping for the 980__/697C_ + # resource_type rule and research_committee.py's report-number + # detection (see research.py:resource_type) - drop it before it + # reaches the final record. + entry.pop("_resource_type_rank", None) try: return entry["resource_type"] except KeyError: @@ -617,6 +622,8 @@ def field_departments(record_json, custom_fields_dict): if result and result not in custom_fields_dict["cern:departments"]: custom_fields_dict["cern:departments"].append(result) elif not result: + if department.lower() == "cern?": + continue subj = json_output["metadata"].get("subjects", []) subj.append({"subject": department}) json_output["metadata"]["subjects"] = subj diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index e2e4b511..229cf29c 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -551,8 +551,7 @@ def corporate_author(self, key, value): departments = self.get("custom_fields", {}).get("cern:departments", []) if department and department not in departments: departments.append(department) - self["custom_fields"]["cern:departments"] = departments - raise IgnoreKey("contributors") + self["custom_fields"]["cern:departments"] = departments if "b" in value: unit = value.get("b") if unit: @@ -877,6 +876,14 @@ def related_identifiers_787(self, key, value): "relation_type": {"id": "references"}, "resource_type": {"id": "publication-report"}, }, + "addendum to": { + "relation_type": {"id": "issupplementto"}, + "resource_type": {"id": "publication-report"}, + }, + "complemented by": { + "relation_type": {"id": "issuplementedby"}, + "resource_type": {"id": "publication-report"}, + }, "preprint": { "relation_type": {"id": "references"}, "resource_type": {"id": "publication-preprint"}, diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index 6d851c38..6c5cd36f 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -798,20 +798,36 @@ def resource_type(self, key, value): subjects = self.get("subjects") subjects.append({"subject": value_a if value_a else value_b}) self["subjects"] = subjects - raise IgnoreKey("resource_type") - raise UnexpectedValue( - "Unknown resource type (Publications)", value=best_value, field=key - ) - - if current: - current_key = next((k for k, v in mapping.items() if v == current), None) - current_rank = priority.get(current_key, float("inf")) + # This function is invoked once per repeated 980__/697C_ occurrence + # (see CdsOverdo.do), with `self["resource_type"]` accumulating the + # best match across calls. An unmapped value here doesn't mean the + # record has no resource type - another occurrence, processed later, + # may still resolve to a known one, so don't abort the whole record. + # If no occurrence ever resolves, the missing resource_type is + # caught downstream (see `_resource_type` in transform.py). + raise IgnoreKey("resource_type") + # `current` may be `resource_type`'s value from an earlier 980__/697C_ + # occurrence on this same record, but it may also be a default seeded by + # the model's `_default_fields` (e.g. ResearchCommitteeModel sets + # resource_type to publication-other upfront) before this rule ever ran. + # Reverse-looking up `current` in `mapping` can't tell those apart - a + # seeded default happens to equal "alephdraft"'s mapped value, so it + # would get treated as an already-decided, low-priority match and block + # any later occurrence whose value isn't in `priority` (rank stays + # `inf`, and `inf < inf` is False). Track the rank of our own previous + # decision explicitly instead, so a seeded default never poisons this + # comparison. + current_rank = self.get("_resource_type_rank") + + if current and current_rank is not None: if rank < current_rank: + self["_resource_type_rank"] = rank return mapping[best_value] else: raise IgnoreKey("resource_type") else: + self["_resource_type_rank"] = rank return mapping[best_value] diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py new file mode 100644 index 00000000..fe988843 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py @@ -0,0 +1,423 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM research committee rules.""" + +import re + +from cds_migrator_kit.transform.xml_processing.quality.decorators import ( + for_each_value, +) +from cds_migrator_kit.transform.xml_processing.quality.parsers import StringValue + +from ...models.research_committee import research_comm_model as model +from .base import report_number as _report_number_rule +from .base import series_information as _series_information_rule +from .base import title as _base_title +from .research import imprint as _imprint_rule + +# Unwrapped base rules (strip @for_each_value): CdsOverdo.do() already +# invokes rules once per repeated-tag occurrence (one 037__/088__/250__/ +# 490__ at a time), so re-wrapping the already-wrapped rule here would +# double-apply its bookkeeping. See research.py's own `_raw_licenses` for +# the same trick. `title` isn't decorated with @for_each_value, so it's +# used as-is. +_base_report_number = _report_number_rule.__wrapped__ +_base_series_information = _series_information_rule.__wrapped__ +_base_imprint = _imprint_rule.__wrapped__ + +# Committees whose report numbers follow the `--` +# convention, e.g. "SPSC-I-170", "SPSLC-P-282" (see +# https://cds.cern.ch/record/493774). Keep in sync with the `committees` +# mapping in xml_processing/rules/research.py:resource_type. +REPORT_NUMBER_COMMITTEES = { + "DRDC", + "EEC", + "EMC", + "ISC", + "ISRC", + "ISTC", + "LEPC", + "NPRC", + "NSC", + "PSC", + "PSCC", + "SCC", + "SPSC", + "SPSLC", + "TCC", +} + +# segment -> resource_type, shared by every committee in +# REPORT_NUMBER_COMMITTEES unless overridden per-committee below. Most type +# segments are a single hyphen-separated token (e.g. "I" in "SPSC-I-170"), +# but some are two tokens (e.g. "Status-report" in +# "DRDC-Status-report-RD-30", https://cds.cern.ch/record/291072) - those are +# keyed with a "-" joining both tokens and checked first, see +# `_apply_committee_report_number`. +_DEFAULT_REPORT_TYPES = { + "M": {"id": "publication-meetingminutes"}, + "N": {"id": "publication-meetingminutes"}, + "S": {"id": "publication-meetingminutes"}, + "I": {"id": "publication-letter"}, + "A": {"id": "publication-meetingagenda"}, + "G": {"id": "publication-other"}, + "P": {"id": "publication-proposal"}, + "T": {"id": "publication-technicalnote"}, + "TDR": {"id": "publication-report"}, + "UG": {"id": "publication-report"}, + "SR": {"id": "publication-report"}, + "RD": {"id": "publication-report"}, + "STATUS-REPORT": {"id": "publication-report"}, + "MEMO": {"id": "publication-memorandum"}, + "INTERNAL-REPORT": {"id": "publication-report"}, +} + +# Longest type-token-sequence first, so "STATUS-REPORT" (2 tokens) is tried +# before a hypothetical single-token match on just "STATUS". +_TYPE_TOKEN_LENGTHS = sorted( + {len(t.split("-")) for t in _DEFAULT_REPORT_TYPES}, reverse=True +) + +# Per-committee overrides of `_DEFAULT_REPORT_TYPES`. "R" has no default +# mapping - it's only defined for SPSC. +_COMMITTEE_TYPE_OVERRIDES = { + "SPSC": { + "M": {"id": "publication-memorandum"}, + "R": {"id": "publication-report"}, + }, +} + +# Extra subject tagged onto the record for specific (committee, type) pairs. +# "*" matches any committee. +_TYPE_SUBJECTS = { + ("SPSC", "R"): "recommendation", + ("*", "UG"): "collection:upgrade cost group", +} + + +# 250__$a (edition), 490__$a (series) and 245__ (title/subtitle) sometimes +# spell out the document type as free text instead of, or in addition to, +# a -- report number, e.g. "Memorandum", +# "Proposal", "Letter of Intent", "Addendum" (https://cds.cern.ch/record/1005022 +# has 250__$a "Addendum") - see `_apply_series_resource_type`, +# `_apply_edition_resource_type` and `_apply_title_resource_type`. +_SERIES_RESOURCE_TYPES = { + "memorandum": {"id": "publication-memorandum"}, + "proposal": {"id": "publication-proposal"}, + "proposals": {"id": "publication-proposal"}, + "letter of intent": {"id": "publication-letter"}, + "letter": {"id": "publication-letter"}, + "letter of intention": {"id": "publication-letter"}, + "minutes": {"id": "publication-meetingminutes"}, + "agenda": {"id": "publication-meetingagenda"}, + "report": {"id": "publication-report"}, + "status report": {"id": "publication-report"}, + "progress report": {"id": "publication-report"}, + "addendum": {"id": "publication-other"}, + "decisions": {"id": "publication-meetingminutes"}, + "commentaires": {"id": "publication-peerreview"}, + "comments": {"id": "publication-peerreview"}, + "proposition": {"id": "publication-proposal"}, + "rapport": {"id": "publication-report"}, + "technical note": {"id": "publication-technicalnote"}, + "note": {"id": "publication-technicalnote"}, + "decision taken at the meeting": {"id": "publication-meetingminutes"} +} + + +# Priority tiers for the different ways a research-committee record's +# resource_type can be derived, most to least reliable/specific: +# -- report number (structured, unambiguous) > a +# document type spelled out in 250__$a edition or 490__$a series (both +# structured fields holding a short, free text value - equally reliable, +# so they share a tier) > a document type mentioned anywhere in the 245__ +# title (free text, least reliable). Lower = higher priority, matching the +# rank semantics `research.py:resource_type` already uses for the generic +# 980__/697C_ rule - see `_set_resource_type_if_higher_priority`. +_RANK_REPORT_NUMBER = float("-inf") +_RANK_SERIES = -2 +_RANK_TITLE = -1 + + +def _set_resource_type_if_higher_priority(self, resource_type, rank): + """Set `resource_type` (+ its `_resource_type_rank`), but only if + `rank` outranks (is strictly lower than) whatever has already been + decided for this record so far - by an earlier field in tag order, or + by a higher-priority rule matching the same field. + + All three research-committee-specific resource_type sources (report + number, series, title) and the generic 980__/697C_ rule + (research.py:resource_type) read/write the same `_resource_type_rank`, + so whichever ends up with the lowest rank wins regardless of which + field it came from or when it was processed. + """ + current_rank = self.get("_resource_type_rank") + if current_rank is not None and rank >= current_rank: + return + self["resource_type"] = resource_type + self["_resource_type_rank"] = rank + + +def _committee_report_type(committee, type_code): + """Resolve resource_type + optional subject for a report number type code.""" + overrides = _COMMITTEE_TYPE_OVERRIDES.get(committee, {}) + resource_type = overrides.get(type_code) or _DEFAULT_REPORT_TYPES.get(type_code) + if not resource_type: + return None, None + subject = _TYPE_SUBJECTS.get((committee, type_code)) or _TYPE_SUBJECTS.get( + ("*", type_code) + ) + return resource_type, subject + + +def _apply_committee_report_number(self, identifier): + """Detect a `--` report number (e.g. + "SPSC-I-170") and derive the record's resource_type - and, for some + types, an extra subject - from the type code. + + is usually one token ("I"), but can be two ("Status-report" in + "DRDC-Status-report-RD-30" - the remaining "RD-30" is just part of the + number, not the type). We don't know up front how many tokens the type + takes, so we try the longest known type first (see + `_TYPE_TOKEN_LENGTHS`) and fall back to shorter ones. + + This writes `resource_type` directly (rather than through a scratch key + resolved later) so it's the single source of truth from the moment + 037__/088__ is processed - see `_set_resource_type_if_higher_priority` + and `_RANK_REPORT_NUMBER` for how it's kept from being clobbered by the + other resource_type sources that run later (490__, 245__, and the + generic 980__ rule in research.py:resource_type). + """ + if not identifier or identifier.count("-") < 2: + return + + parts = identifier.split("-") + committee = parts[0].upper() + rest = [p.upper() for p in parts[1:]] + + if committee not in REPORT_NUMBER_COMMITTEES: + return + + resource_type = subject = None + for token_len in _TYPE_TOKEN_LENGTHS: + if len(rest) <= token_len: + # The type can't consume the whole identifier - at least one + # token must be left over for the number. + continue + type_code = "-".join(rest[:token_len]) + resource_type, subject = _committee_report_type(committee, type_code) + if resource_type: + break + + if not resource_type: + return + + _set_resource_type_if_higher_priority(self, resource_type, _RANK_REPORT_NUMBER) + if subject: + subjects = self.get("subjects", []) + new_subject = {"subject": subject} + if new_subject not in subjects: + subjects.append(new_subject) + self["subjects"] = subjects + + +# Type tokens from `_DEFAULT_REPORT_TYPES` safe to match anywhere in a +# report number, not just right after a known committee prefix - e.g. +# "CERN-NP-MEMO-7840" (CERN's generic CERN--- report +# numbering, not tied to a research committee at all, so +# `_apply_committee_report_number`'s REPORT_NUMBER_COMMITTEES gate doesn't +# apply). Deliberately excludes the single-letter codes (M, N, S, I, A, G, +# P, T): unanchored from a committee prefix, a bare one-letter token is far +# more likely to be a coincidental match (e.g. some unrelated department or +# series code) than an actual type marker - the longer tokens here don't +# have that problem. +_UNANCHORED_REPORT_TYPES = { + token: resource_type + for token, resource_type in _DEFAULT_REPORT_TYPES.items() + if len(token) > 1 +} +_UNANCHORED_TOKEN_LENGTHS = sorted( + {len(t.split("-")) for t in _UNANCHORED_REPORT_TYPES}, reverse=True +) + + +def _apply_generic_report_number_type(self, identifier): + """Detect one of `_UNANCHORED_REPORT_TYPES`'s tokens anywhere in a + report number and derive the record's resource_type from it - see the + comment above `_UNANCHORED_REPORT_TYPES` for why this is safe without a + committee gate, unlike `_apply_committee_report_number`. + + Same rank as a committee report number (`_RANK_REPORT_NUMBER`): both + are derived from a structured identifier rather than free text. + """ + if not identifier: + return + + tokens = identifier.upper().split("-") + for start in range(len(tokens)): + for token_len in _UNANCHORED_TOKEN_LENGTHS: + end = start + token_len + if end > len(tokens): + continue + candidate = "-".join(tokens[start:end]) + resource_type = _UNANCHORED_REPORT_TYPES.get(candidate) + if resource_type: + _set_resource_type_if_higher_priority( + self, resource_type, _RANK_REPORT_NUMBER + ) + return + + +def _apply_series_resource_type(self, value_a): + """Detect a document type spelled out in 490__$a (see + `_SERIES_RESOURCE_TYPES`) and derive the record's resource_type from it. + + See `_set_resource_type_if_higher_priority` and `_RANK_SERIES` for how + this is weighed against the other resource_type sources. + """ + resource_type = _SERIES_RESOURCE_TYPES.get(value_a.strip().lower()) + if not resource_type: + return + + _set_resource_type_if_higher_priority(self, resource_type, _RANK_SERIES) + + +def _free_text_resource_type(text): + """Return the resource_type for text containing one of + `_SERIES_RESOURCE_TYPES`'s phrases anywhere in it (e.g. "Draft minutes + of the third meeting of the EEC ...", + https://cds.cern.ch/record/1015008, or "Addendum 1"), matched at a word + boundary so e.g. "Reported" doesn't match "report" - or None if it + doesn't. + """ + text_lower = text.strip().lower() + # Longest phrase first, so "letter of intent" is tried before a + # hypothetical single-word phrase it contains. + for phrase in sorted(_SERIES_RESOURCE_TYPES, key=len, reverse=True): + if re.search(rf"\b{re.escape(phrase)}\b", text_lower): + return _SERIES_RESOURCE_TYPES[phrase] + return None + + +def _apply_edition_resource_type(self, value_a): + """Detect a document type spelled out in 250__$a (edition statement, + e.g. "Addendum", "Addendum 1" - see `_free_text_resource_type`) and + derive the record's resource_type from it. Matched the same way as the + title (anywhere, word boundary) rather than 490__ series' exact match, + since 250__ values are often followed by a number ("Addendum 2"). + + See `_set_resource_type_if_higher_priority` and `_RANK_SERIES` for how + this is weighed against the other resource_type sources - 250__ shares + a priority tier with 490__ series, see the comment above + `_SERIES_RESOURCE_TYPES`. + """ + resource_type = _free_text_resource_type(value_a) + if not resource_type: + return + + _set_resource_type_if_higher_priority(self, resource_type, _RANK_SERIES) + + +def _apply_title_resource_type(self, title_value): + """Detect a document type mentioned anywhere in the 245__ title (see + `_free_text_resource_type`) and derive the record's resource_type from + it. + + Titles are free text and the least reliable of the three + research-committee-specific resource_type sources, so this only wins + over an already-decided value if that value came from an even weaker + source (i.e. nothing at all) - see `_set_resource_type_if_higher_priority` + and `_RANK_TITLE`. In particular, a later 490__ series match (more + reliable, see `_apply_series_resource_type`) can still override a + title-derived guess even though 245__ is processed first (CdsOverdo.do + visits fields in tag order). + """ + resource_type = _free_text_resource_type(title_value) + if not resource_type: + return + + _set_resource_type_if_higher_priority(self, resource_type, _RANK_TITLE) + + +@model.over("related_identifiers", "(^037__)|(^088__)", override_tag=True) +@for_each_value +def report_number(self, key, value): + """Translates report_number fields for research committee records. + + In addition to the generic report_number handling (see + base.report_number), detects research-committee report numbers like + "SPSC-I-170" and derives the record's resource_type from the type code + segment (see `_apply_committee_report_number`), as well as CERN's + generic CERN--- report numbering, e.g. + "CERN-NP-MEMO-7840" (see `_apply_generic_report_number_type`). + + This re-registers (rather than adding a second, independent rule for + 037__/088__) because dojson dispatches a single rule per MARC tag - see + CdsOverdo.do. Delegating every call to the original `report_number` + (including letting its `IgnoreKey` propagate untouched) preserves 100% + of its existing identifiers/related_identifiers behaviour; only + research-committee records (routed to this model) are affected, every + other collection keeps using the unmodified rule. + """ + identifier = StringValue(value.get("a", "")).parse() + _apply_committee_report_number(self, identifier) + _apply_generic_report_number_type(self, identifier) + + return _base_report_number(self, key, value) + + +@model.over("additional_descriptions", "^490__", override_tag=True) +@for_each_value +def series_information(self, key, value): + """Translates series information for research committee records. + + In addition to the generic series_information handling (see + base.series_information), detects a document type spelled out in + 490__$a, e.g. "Memorandum" (see `_apply_series_resource_type`). + + Re-registers rather than adding a second, independent rule for 490__ - + see `report_number`'s docstring for why. + """ + _apply_series_resource_type(self, value.get("a", "")) + + return _base_series_information(self, key, value) + + +@model.over("title", "^245__", override_tag=True) +def title(self, key, value): + """Translates title for research committee records. + + In addition to the generic title handling (see base.title), detects a + document type mentioned anywhere in the title or subtitle, e.g. "Draft + minutes of the third meeting of the EEC ..." (see + `_apply_title_resource_type`). + + Re-registers rather than adding a second, independent rule for 245__ - + see `report_number`'s docstring for why. + """ + _apply_title_resource_type(self, value.get("a", "")) + _apply_title_resource_type(self, value.get("b", "")) + + return _base_title(self, key, value) + + +@model.over("imprint_info", "(^250__)", override_tag=True) +@for_each_value +def imprint(self, key, value): + """Translates edition statement for research committee records. + + In addition to the generic imprint/edition handling (see + research.imprint), detects a document type spelled out in 250__$a, + e.g. "Addendum" (see `_apply_edition_resource_type`). + + Re-registers rather than adding a second, independent rule for 250__ - + see `report_number`'s docstring for why. + """ + _apply_edition_resource_type(self, value.get("a", "")) + + return _base_imprint(self, key, value) diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index bd80235b..41d60462 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -140,10 +140,10 @@ records: communities_ids: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "28bf99b9-1e72-405a-b95a-828019831def" - scps_comm: - data_dir: cds_migrator_kit/rdm/data/committees/scps + scp_adv: + data_dir: cds_migrator_kit/rdm/data/committees/sc_sdv extract: - dirpath: cds_migrator_kit/rdm/data/committees/scps + dirpath: cds_migrator_kit/rdm/data/committees/sc_adv transform: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users diff --git a/setup.cfg b/setup.cfg index 6ddede48..2ee4daa2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -209,6 +209,11 @@ cds_migrator_kit.migrator.rules.technical_support = base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research technical_support = cds_migrator_kit.rdm.records.transform.xml_processing.rules.technical_support +cds_migrator_kit.migrator.rules.research_committee = + base = cds_migrator_kit.transform.xml_processing.rules.base + base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base + publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research + research_committee = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research_committee cds_migrator_kit.migrator.rules.people = people = cds_migrator_kit.rdm.users.transform.xml_processing.rules.people invenio_pidstore.minters = diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index 7fcf4dd7..86c648d8 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -21,6 +21,7 @@ journal, meeting, oa_level_from_license, + resource_type, udc, ) @@ -1216,3 +1217,106 @@ def test_funding_model_not_overwritten_by_second_tag(self): record, "540__", [{"f": "SCOAP3"}, {"f": "Collective"}] ) assert self._cf(record)["cern:oa_funding_model"] == {"id": "scoap3"} + + +class TestResourceType: + """Test resource_type function from publications.py (980__/697C_ rule). + + 980__/697C_ is a repeated MARC field: dojson invokes this rule once per + occurrence (see CdsOverdo.do), accumulating the best match into + record["resource_type"] across calls. An unmapped value on one + occurrence must not stop later occurrences from being considered. + """ + + def _apply(self, record, key, value): + """Simulate one dojson `do()` step for a repeated-field occurrence.""" + try: + record["resource_type"] = resource_type(record, key, value) + except IgnoreKey: + pass + + def test_record_290988_note_resolved_after_unmapped_occurrence(self): + """https://cds.cern.ch/record/290988 has three 980__ occurrences: + SCICOMMPUBLDRDC (a committee, not a resource type), INTNOTEATLASPUBL + (unmapped), and NOTE (maps to publication-technicalnote). The + unmapped middle occurrence must not prevent NOTE from being applied. + """ + record = {"custom_fields": {}} + + self._apply(record, "980__", {"a": "SCICOMMPUBLDRDC"}) + self._apply(record, "980__", {"a": "INTNOTEATLASPUBL"}) + self._apply(record, "980__", {"a": "NOTE"}) + + assert record["resource_type"] == {"id": "publication-technicalnote"} + assert record["custom_fields"]["cern:committees"] == [{"id": "DRDC"}] + + def test_unmapped_value_ignored_not_raised(self): + """An unmapped 980__a is skipped, not raised - it no longer aborts + processing of the record's remaining fields.""" + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + resource_type(record, "980__", {"a": "INTNOTEATLASPUBL"}) + assert "resource_type" not in record + + def test_unmapped_only_occurrence_still_ignored_not_raised(self): + """Even as the only occurrence, an unmapped value is skipped. Records + left without any resource_type are caught downstream by the + required-field check (see `_resource_type` in transform.py).""" + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + resource_type(record, "980__", {"a": "totally-unknown-type"}) + assert "resource_type" not in record + + def test_committee_sets_custom_field_not_resource_type(self): + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + resource_type(record, "980__", {"a": "SCICOMMPUBLSPSC"}) + assert record["custom_fields"]["cern:committees"] == [{"id": "SPSC"}] + assert "resource_type" not in record + + def test_ignored_type_dropped_silently(self): + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + resource_type(record, "980__", {"a": "aleph_misc"}) + assert "resource_type" not in record + assert "cern:committees" not in record["custom_fields"] + + def test_experiment_marker_sets_custom_field(self): + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + resource_type(record, "980__", {"a": "LHCBCERNTALK"}) + assert record["custom_fields"]["cern:experiments"] == ["LHCb"] + assert "resource_type" not in record + + def test_higher_priority_value_replaces_note(self): + record = {"custom_fields": {}} + self._apply(record, "980__", {"a": "NOTE"}) + + result = resource_type(record, "980__", {"a": "ARTICLE"}) + assert result == {"id": "publication-article"} + + def test_lower_priority_value_ignored(self): + record = {"custom_fields": {}} + self._apply(record, "980__", {"a": "conferencepaper"}) + + with pytest.raises(IgnoreKey): + resource_type(record, "980__", {"a": "note"}) + assert record["resource_type"] == {"id": "publication-conferencepaper"} + + def test_seeded_default_resource_type_does_not_block_real_value(self): + """A `resource_type` value present without a matching + `_resource_type_rank` (e.g. seeded by a model's `_default_fields`, + or set directly by another rule - see + xml_processing/rules/research_committee.py) must not be mistaken + for an already-decided match and block a real, unranked mapped + value like "report".""" + record = {"custom_fields": {}, "resource_type": {"id": "publication-other"}} + result = resource_type(record, "980__", {"a": "REPORT"}) + assert result == {"id": "publication-report"} + + def test_697C_lexi_adds_subject_instead_of_resource_type(self): + record = {"custom_fields": {}, "subjects": []} + with pytest.raises(IgnoreKey): + resource_type(record, "697C_", {"a": "lexisomething"}) + assert record["subjects"] == [{"subject": "lexisomething"}] + assert "resource_type" not in record