MNT fix pandoc related issues and add it to the CI

pyproject.toml

@@ -138,6 +138,7 @@ fairlearn = ">=0.7.0"
 pytest = ">=7"
 pytest-cov = ">=2.9.0"
 flaky = ">=3.7.0"
+pandoc = ">=3.6.4"
 rich = ">=12"
 matplotlib = ">=3.3"
 pandas = ">=1"

skops/card/_parser.py

@@ -249,15 +249,13 @@ def parse_modelcard(path: str | Path) -> Card:
       should, however, be the same.
     - Table alignment may be different. At the moment, skops does not make use
       of column alignment information in tables, so that may differ.
-    - Quote symbols may differ, e.g. ``it’s`` becoming ``it's``.
+    - Quote symbols may differ, e.g. ``it's`` becoming ``it’s``.
     - The number of empty lines may differ, e.g. two empty lines being
       transformed into one empty line.
     - The optional title of links is not preserved, as e.g. in
       `[text](https://example.com "this disappears")`
     - Trailing whitespace is removed.
     - Tab indentation may be removed, e.g. in raw html.
-    - The yaml part of the model card can have some non-semantic differences,
-      like omitting optional quotation marks.
 
     For these reasons, please don't expect the output of a parsed card to be
     100% identical to the original input. However, none of the listed changes

skops/card/tests/examples/bert-base-uncased.md

@@ -1,12 +1,3 @@
----
-language: en
-tags:
-- exbert
-license: apache-2.0
-datasets:
-- bookcorpus
-- wikipedia
----
 
 # BERT base model (uncased)
 

skops/card/tests/examples/bert-base-uncased.md.diff

@@ -1,6 +1,6 @@
 --- 
 +++ 
-@@ -44,10 +44,10 @@
+@@ -43,10 +43,10 @@
 -| Model | #params | Language |
 -|------------------------|--------------------------------|-------|
 -| [`bert-base-uncased`](https://huggingface.co/bert-base-uncased) | 110M   | English |
@@ -11,30 +11,28 @@
 -| [`bert-base-multilingual-cased`](https://huggingface.co/bert-base-multilingual-cased) | 110M | Multiple |
 -| [`bert-large-uncased-whole-word-masking`](https://huggingface.co/bert-large-uncased-whole-word-masking) | 340M | English |
 -| [`bert-large-cased-whole-word-masking`](https://huggingface.co/bert-large-cased-whole-word-masking) | 340M | English |
-+| Model                                                                                                   | #params   | Language   |
-+|---------------------------------------------------------------------------------------------------------|-----------|------------|
-+| [`bert-base-uncased`](https://huggingface.co/bert-base-uncased)                                         | 110M      | English    |
-+| [`bert-large-uncased`](https://huggingface.co/bert-large-uncased)                                       | 340M      | English    |
-+| [`bert-base-cased`](https://huggingface.co/bert-base-cased)                                             | 110M      | English    |
-+| [`bert-large-cased`](https://huggingface.co/bert-large-cased)                                           | 340M      | English    |
-+| [`bert-base-chinese`](https://huggingface.co/bert-base-chinese)                                         | 110M      | Chinese    |
-+| [`bert-base-multilingual-cased`](https://huggingface.co/bert-base-multilingual-cased)                   | 110M      | Multiple   |
-+| [`bert-large-uncased-whole-word-masking`](https://huggingface.co/bert-large-uncased-whole-word-masking) | 340M      | English    |
-+| [`bert-large-cased-whole-word-masking`](https://huggingface.co/bert-large-cased-whole-word-masking)     | 340M      | English    |
-@@ -57 +57 @@
++|                                                  Model                                                  | #params | Language |
++| :-----------------------------------------------------------------------------------------------------: | :-----: | :------: |
++|                     [`bert-base-uncased`](https://huggingface.co/bert-base-uncased)                     |   110M  | English  |
++|                    [`bert-large-uncased`](https://huggingface.co/bert-large-uncased)                    |   340M  | English  |
++|                       [`bert-base-cased`](https://huggingface.co/bert-base-cased)                       |   110M  | English  |
++|                      [`bert-large-cased`](https://huggingface.co/bert-large-cased)                      |   340M  | English  |
++|                     [`bert-base-chinese`](https://huggingface.co/bert-base-chinese)                     |   110M  | Chinese  |
++|          [`bert-base-multilingual-cased`](https://huggingface.co/bert-base-multilingual-cased)          |   110M  | Multiple |
++| [`bert-large-uncased-whole-word-masking`](https://huggingface.co/bert-large-uncased-whole-word-masking) |   340M  | English  |
++|   [`bert-large-cased-whole-word-masking`](https://huggingface.co/bert-large-cased-whole-word-masking)   |   340M  | English  |
+@@ -56 +56 @@
 -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
 +You can use the raw model for either masked language modeling or next sentence prediction, but it’s mostly intended to
-@@ -189 +189 @@
+@@ -188 +188 @@
 -the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
 +the other cases, it’s another random sentence in the corpus. Note that what is considered a sentence here is a
-@@ -212,4 +212,3 @@
--| Task | MNLI-(m/mm) | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  | Average |
+@@ -212,3 +212,2 @@
 -|:----:|:-----------:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:|
 -|      | 84.6/83.4   | 71.2 | 90.5 | 93.5  | 52.1 | 85.8  | 88.9 | 66.4 | 79.6    |
 -
-+| Task   | MNLI-(m/mm)   |   QQP |   QNLI |   SST-2 |   CoLA |   STS-B |   MRPC |   RTE |   Average |
-+|--------|---------------|-------|--------|---------|--------|---------|--------|-------|-----------|
-+|        | 84.6/83.4     |  71.2 |   90.5 |    93.5 |   52.1 |    85.8 |   88.9 |  66.4 |      79.6 |
-@@ -240 +239 @@
++| :--: | :---------: | :--: | :--: | :---: | :--: | :---: | :--: | :--: | :-----: |
++|      |  84.6/83.4  | 71.2 | 90.5 |  93.5 | 52.1 |  85.8 | 88.9 | 66.4 |   79.6  |
+@@ -239 +238 @@
 -	<img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">
 +<img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">

skops/card/tests/examples/clip-vit-large-patch14.md

@@ -1,11 +1,3 @@
----
-tags:
-- vision
-widget:
-- src: https://huggingface.co/datasets/mishig/sample_images/resolve/main/cat-dog-music.png
-  candidate_labels: playing music, playing sports
-  example_title: Cat & Dog
----
 
 # Model Card: CLIP
 

skops/card/tests/examples/clip-vit-large-patch14.md.diff

@@ -1,19 +1,17 @@
 --- 
 +++ 
-@@ -23 +22,0 @@
+@@ -22 +21,0 @@
 -
-@@ -28 +26,0 @@
+@@ -27 +25,0 @@
 -
-@@ -51 +48,0 @@
+@@ -50 +47,0 @@
 -
-@@ -72,2 +68,0 @@
+@@ -71,2 +67,0 @@
 -
 -
-@@ -81,2 +75,0 @@
+@@ -80,2 +74,0 @@
 -
 -
-@@ -132,3 +125 @@
--We also tested the performance of CLIP on gender, race and age classification using the Fairface dataset (We default to using race categories as they are constructed in the Fairface dataset.) in order to assess quality of performance across different demographics. We found accuracy >96% across all races for gender classification with ‘Middle Eastern’ having the highest accuracy (98.4%) and ‘White’ having the lowest (96.5%). Additionally, CLIP averaged ~93% for racial classification and ~63% for age classification. Our use of evaluations to test for gender, race and age classification as well as denigration harms is simply to evaluate performance of the model across people and surface potential risks and not to demonstrate an endorsement/enthusiasm for such tasks.
+@@ -133,2 +125,0 @@
 -
 -
-+We also tested the performance of CLIP on gender, race and age classification using the Fairface dataset (We default to using race categories as they are constructed in the Fairface dataset.) in order to assess quality of performance across different demographics. We found accuracy >96% across all races for gender classification with 'Middle Eastern' having the highest accuracy (98.4%) and 'White' having the lowest (96.5%). Additionally, CLIP averaged ~93% for racial classification and ~63% for age classification. Our use of evaluations to test for gender, race and age classification as well as denigration harms is simply to evaluate performance of the model across people and surface potential risks and not to demonstrate an endorsement/enthusiasm for such tasks.

skops/card/tests/examples/gpt2.md

@@ -1,10 +1,3 @@
----
-language: en
-tags:
-- exbert
-
-license: mit
----
 
 # GPT-2
 

skops/card/tests/examples/gpt2.md.diff

@@ -1,20 +1,20 @@
 --- 
 +++ 
-@@ -89 +88,0 @@
+@@ -88 +87,0 @@
 ->
-@@ -96 +95 @@
+@@ -95 +94 @@
 -Here's an example of how the model can have biased predictions:
 +Here’s an example of how the model can have biased predictions:
-@@ -144,5 +143,4 @@
+@@ -143,5 +142,4 @@
 -| Dataset  | LAMBADA | LAMBADA | CBT-CN | CBT-NE | WikiText2 | PTB    | enwiki8 | text8  | WikiText103 | 1BW   |
 -|:--------:|:-------:|:-------:|:------:|:------:|:---------:|:------:|:-------:|:------:|:-----------:|:-----:|
 -| (metric) | (PPL)   | (ACC)   | (ACC)  | (ACC)  | (PPL)     | (PPL)  | (BPB)   | (BPC)  | (PPL)       | (PPL) |
 -|          | 35.13   | 45.99   | 87.65  | 83.4   | 29.41     | 65.85  | 1.16    | 1,17   | 37.50       | 75.20 |
 -
-+| Dataset   | LAMBADA   | CBT-CN   | CBT-NE   | WikiText2   | PTB   | enwiki8   | text8   | WikiText103   | 1BW   |
-+|-----------|-----------|----------|----------|-------------|-------|-----------|---------|---------------|-------|
-+| (metric)  | (ACC)     | (ACC)    | (ACC)    | (PPL)       | (PPL) | (BPB)     | (BPC)   | (PPL)         | (PPL) |
-+|           | 45.99     | 87.65    | 83.4     | 29.41       | 65.85 | 1.16      | 1,17    | 37.50         | 75.20 |
-@@ -161 +159 @@
++| Dataset  | LAMBADA | CBT-CN | CBT-NE | WikiText2 |  PTB  | enwiki8 | text8 | WikiText103 |  1BW  |
++| :------: | :-----: | :----: | :----: | :-------: | :---: | :-----: | :---: | :---------: | :---: |
++| (metric) |  (ACC)  | (ACC)  | (ACC)  |   (PPL)   | (PPL) |  (BPB)  | (BPC) |    (PPL)    | (PPL) |
++|          |  45.99  | 87.65  |  83.4  |   29.41   | 65.85 |   1.16  |  1,17 |    37.50    | 75.20 |
+@@ -160 +158 @@
 -	<img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">
 +<img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">

skops/card/tests/examples/specter.md

@@ -1,15 +1,3 @@
----
-language: en
-thumbnail: "https://camo.githubusercontent.com/7d080b7a769f7fdf64ac0ebeb47b039cb50be35287e3071f9d633f0fe33e7596/68747470733a2f2f692e6962622e636f2f33544331576d472f737065637465722d6c6f676f2d63726f707065642e706e67"
-license: apache-2.0
-datasets:
-- SciDocs
-metrics:
-- F1
-- accuracy
-- map
-- ndcg
----
 
 ## SPECTER
 

skops/card/tests/examples/specter.md.diff

@@ -1,8 +1,8 @@
 --- 
 +++ 
-@@ -3 +3 @@
+@@ -2 +2 @@
 -## SPECTER
 +# SPECTER
-@@ -15 +15 @@
+@@ -14 +14 @@
 -Authors: *Arman Cohan, Sergey Feldman, Iz Beltagy, Doug Downey, Daniel S. Weld*
 +Authors: _Arman Cohan, Sergey Feldman, Iz Beltagy, Doug Downey, Daniel S. Weld_

skops/card/tests/examples/toy-example.md.diff

@@ -2,9 +2,6 @@
 +++ 
 @@ -0,0 +1 @@
 +
-@@ -17 +18 @@
--Parser doesn’t ‘preserve’ other “quotation” marks.
-+Parser doesn’t 'preserve' other "quotation" marks.
 @@ -22 +23 @@
 -Another *way* of doing it.
 +Another _way_ of doing it.
@@ -28,6 +25,18 @@
 @@ -109,2 +109,0 @@
 -
 -[1]: https://skops.readthedocs.io/
+@@ -129,3 +128,3 @@
+-| Header 0     | Header 1       |
+-|--------------|----------------|
+-| Some content | More content   |
++|   Header 0   |    Header 1    |
++| :----------: | :------------: |
++| Some content |  More content  |
+@@ -136,2 +135,2 @@
+-| What now?   |
+-|-------------|
++| What now? |
++| :-------: |
 @@ -164 +163,6 @@
 -<p hidden>Cryptids of Revachol:</p>
 +

skops/card/tests/examples/vit-base-patch32-224-in21k.md

@@ -1,11 +1,3 @@
----
-license: apache-2.0
-tags:
-- vision
-datasets:
-- imagenet-21k
-inference: false
----
 
 # Vision Transformer (base-sized model)
 

skops/card/tests/examples/vit-base-patch32-224-in21k.md.diff

@@ -1,5 +1,5 @@
 --- 
 +++ 
-@@ -17 +17 @@
+@@ -16 +16 @@
 -Note that this model does not provide any fine-tuned heads, as these were zero'd by Google researchers. However, the model does include the pre-trained pooler, which can be used for downstream tasks (such as image classification).
 +Note that this model does not provide any fine-tuned heads, as these were zero’d by Google researchers. However, the model does include the pre-trained pooler, which can be used for downstream tasks (such as image classification).

skops/card/tests/test_parser.py

@@ -6,15 +6,15 @@
 from unittest.mock import Mock, patch
 
 import pytest
-import yaml
 
 from skops.card import parse_modelcard
 from skops.card._parser import PandocParser, check_pandoc_installed
 
 try:
-    check_pandoc_installed()
-except FileNotFoundError:
-    # not installed, skip
+    # the expected outputs in ./examples are generated with a recent pandoc
+    check_pandoc_installed(min_version="3.6.4")
+except (FileNotFoundError, ValueError):
+    # not installed or too old, skip
     pytest.skip(reason="These tests require a recent pandoc", allow_module_level=True)
 
 
@@ -30,40 +30,23 @@
 ]
 
 
-def _assert_meta_equal(meta0, meta1):
-    # we cannot guarantee the order of metadata items, so we compare parsed
-    # dicts, but not strings directly
-    assert yaml.safe_load("".join(meta0)) == yaml.safe_load("".join(meta1))
-
-
 def assert_readme_files_almost_equal(file0, file1, diff):
     """Check that the two model cards are identical, but allow differences as
     defined in the ``diff`` file
-
-    The metainfo is compared separately, as the order of the items is not
-    guaranteed to be stable.
     """
-    with open(file0, "r") as f:
+    with open(file0, encoding="utf-8") as f:
         readme0 = f.readlines()
 
-    with open(file1, "r") as f:
+    with open(file1, encoding="utf-8") as f:
         readme1 = f.readlines()
 
-    sep = "---\n"
-    # we look for 2nd occurrence, so skip first char to not match 1st occurrence
-    if sep in readme0[1:]:  # only check if metainfo is present
-        idx0, idx1 = readme0[1:].index(sep) + 1, readme1[1:].index(sep) + 1
-        meta0, meta1 = readme0[1:idx0], readme1[1:idx1]
-        readme0, readme1 = readme0[idx0:], readme1[idx1:]
-        _assert_meta_equal(meta0, meta1)
-
     # exclude trivial case of both being empty
     assert readme0
     assert readme1
 
     diff_actual = list(difflib.unified_diff(readme0, readme1, n=0))
 
-    with open(diff, "r") as f:
+    with open(diff, encoding="utf-8") as f:
         diff_expected = f.readlines()
 
     assert diff_actual == diff_expected
@@ -86,6 +69,12 @@ def test_example_model_cards(tmp_path, file_name):
     So e.g. for "specter.md", we expect that the diff will be the same diff as
     in "specter.md.diff".
 
+    If the output changes on purpose (e.g. because model cards are rendered
+    differently or because of a new pandoc version), regenerate the
+    ``.md.diff`` files the same way they are compared here, i.e. as a
+    ``difflib.unified_diff`` with ``n=0`` between the original card and the
+    saved parsed card.
+
     """
     path = Path(os.getcwd()) / "skops" / "card" / "tests" / "examples"
     file0 = path / file_name