Add full equivalence between different types of linebreaks

tests/wikitext/test_sections.py

@@ -1,4 +1,4 @@
-from pytest import mark, warns
+from pytest import warns
 
 from wikitextparser import WikiText, parse
 
@@ -13,8 +13,6 @@ def test_blank_lead():
     assert '== s ==\nc\n' == wt.sections[1].string
 
 
-# Todo: Parser should also work with windows line endings.
-@mark.xfail
 def test_multiline_with_carriage_return():
     s = 'text\r\n= s =\r\n{|\r\n| a \r\n|}\r\ntext'
     p = parse(s)

wikitextparser/_argument.py

@@ -2,16 +2,16 @@
 
 from typing import MutableSequence
 
-from regex import DOTALL, MULTILINE, Match
+from regex import DOTALL, Match
 
 from ._spans import TypeToSpans
 from ._wikitext import SECTION_HEADING, SubWikiText, rc
 
 ARG_SHADOW_FULLMATCH = rc(
     rb'[|:](?<pre_eq>(?:[^=]*+(?:'
     + SECTION_HEADING
-    + rb'\n)?+)*+)(?:\Z|(?<eq>=)(?<post_eq>.*+))',
-    MULTILINE | DOTALL,
+    + rb'\R)?+)*+)(?:\Z|(?<eq>=)(?<post_eq>.*+))',
+    DOTALL,
 ).fullmatch
 
 

wikitextparser/_cell.py

@@ -24,7 +24,7 @@
         |
         (?P<attrs>
             (?:
-                [^|\n]
+                [^|\r\n]
                 (?!
                     # attrs end with `|`; or `!!` if sep is `!`
                     (?P=sep){2}
@@ -48,9 +48,9 @@
         (?P=sep){2}|
         \|!!|
         # start of the next newline-cell
-        \n\s*+[!|]|
+        \R\s*+[!|]|
         # end of cell-string
-        $
+        \Z
     )
     """,
     VERBOSE,
@@ -83,7 +83,7 @@
                 (?:
                     # inline header attrs end with `|` (above) or `!!` (below)
                     (?!!{2})
-                    [^|\n]
+                    [^|\r\n]
                 )*+
             )
             # attrs-data separator
@@ -96,13 +96,13 @@
     (?P<data>.*?)
     (?=
         # start of the next newline-cell
-        \n\s*+[!|]|
+        \R\s*+[!|]|
         # start of the next inline-cell
         \|\||
         !!|
         \|!!|
         # end of cell-string
-        $
+        \Z
     )
     """,
     VERBOSE | DOTALL,
@@ -119,7 +119,7 @@
         (?!\|)
         |
         (?P<attrs>
-            [^|\n]*? # non-_header attrs end with a `|`
+            [^|\r\n]*? # non-_header attrs end with a `|`
         )
         # attribute-data separator
         \|
@@ -132,8 +132,8 @@
         [^|]*?
         (?=
             \|\|| # start of the next inline-cell
-            \n\s*+[!|]| # start of the next newline-cell
-            $ # end of cell-string
+            \R\s*+[!|]| # start of the next newline-cell
+            \Z # end of cell-string
         )
     )
     """,
@@ -188,7 +188,7 @@ def _match(self) -> Match[bytes]:
         if cache_string == string:
             return cache_match  # type: ignore
         shadow = self._shadow
-        if shadow[0] == 10:  # ord('\n')
+        if shadow[0] == 10 or shadow[0] == 13:  # ord('\n'), ord('\r')
             m: Match[bytes] = NEWLINE_CELL_MATCH(shadow)  # type: ignore
             self._header = m['sep'] == 33  # ord('!')
         elif self._header:
@@ -260,7 +260,7 @@ def set_attr(self, attr_name: str, attr_value: str) -> None:
             return
         # There is no attributes span in this cell. Create one.
         fmt = ' {}="{}" |' if attr_value else ' {} |'
-        if shadow[0] == 10:  # ord('\n')
+        if shadow[0] == 10 or shadow[0] == 13:  # ord('\n'), ord('\r')
             self.insert(
                 cell_match.start('sep') + 1, fmt.format(attr_name, attr_value)
             )

wikitextparser/_comment_bold_italic.py

@@ -2,22 +2,22 @@
 
 from typing import MutableSequence
 
-from regex import DOTALL, MULTILINE, Match
+from regex import DOTALL, Match
 
 from ._spans import TypeToSpans
 from ._wikitext import SubWikiText, rc
 
 COMMENT_PATTERN = r'<!--[\s\S]*?(?>-->|\Z)'
-COMMA_COMMENT = "'(?>" + COMMENT_PATTERN + ')*+'
-COMMENT_COMMA = '(?>' + COMMENT_PATTERN + ")*+'"
+COMMA_COMMENT = r"'(?>" + COMMENT_PATTERN + r')*+'
+COMMENT_COMMA = r'(?>' + COMMENT_PATTERN + r")*+'"
 BOLD_FULLMATCH = rc(
-    COMMA_COMMENT * 2 + "'(.*?)(?>'" + COMMENT_COMMA * 2 + '|$)',
-    MULTILINE | DOTALL,
+    COMMA_COMMENT * 2 + r"'(.*?)(?>'" + COMMENT_COMMA * 2 + r'|(?=\R|\Z))',
+    DOTALL,
 ).fullmatch
 ITALIC_FULLMATCH = rc(
-    COMMA_COMMENT + "'(.*?)(?>'" + COMMENT_COMMA + '|$)', DOTALL
+    COMMA_COMMENT + r"'(.*?)(?>'" + COMMENT_COMMA + r'|\Z)', DOTALL
 ).fullmatch
-ITALIC_NOEND_FULLMATCH = rc(COMMA_COMMENT + "'(.*)", DOTALL).fullmatch
+ITALIC_NOEND_FULLMATCH = rc(COMMA_COMMENT + r"'(.*)", DOTALL).fullmatch
 
 
 class Comment(SubWikiText):

wikitextparser/_section.py

@@ -4,7 +4,7 @@
 
 from ._wikitext import SubWikiText, rc
 
-HEADER_MATCH = rc(rb'\0*+(={1,6})([^\n]+?)\1[ \t\0]*+(\n|\Z)').match
+HEADER_MATCH = rc(rb'\0*+(={1,6})([^\r\n]+?)\1[ \t\0]*+(\R|\Z)').match
 
 
 class Section(SubWikiText):

wikitextparser/_spans.py

@@ -20,7 +20,7 @@
 rc = partial(rc, cache_pattern=False)
 # According to https://www.mediawiki.org/wiki/Manual:$wgLegalTitleChars
 # illegal title characters are: r'[]{}|#<>[\u0000-\u0020]'
-VALID_TITLE_CHARS = rb'[^\|\{\}\[\2\]\3<>\n]*+'
+VALID_TITLE_CHARS = rb'[^\|\{\}\[\2\]\3<>\r\n]*+'
 # Parser functions
 # According to https://www.mediawiki.org/wiki/Help:Magic_words
 # See also:
@@ -49,7 +49,7 @@
     + rb'\}\})'
 ).finditer
 # External links
-INVALID_URL_CHARS = rb' \t\n"<>\[\]'
+INVALID_URL_CHARS = rb' \t\r\n"<>\[\]'
 VALID_URL_CHARS = rb'[^' + INVALID_URL_CHARS + rb']++'
 # See more info on literal IPv6 see:
 # https://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers
@@ -67,7 +67,7 @@
 # Wikilinks
 # https://www.mediawiki.org/wiki/Help:Links#Internal_links
 WIKILINK_PARAM_FINDITER = rc(
-    rb'(?<!(?>^|[^\[\0])(?:(?>\[\0*+){2})*+\[\0*+)'  # != 2N + 1
+    rb'(?<!(?>\A|[^\[\0])(?:(?>\[\0*+){2})*+\[\0*+)'  # != 2N + 1
     rb'\[\0*\['
     rb'(?![\ \0]*+' + BARE_EXTERNAL_LINK + rb')' + VALID_TITLE_CHARS + rb'(?>'
     rb'\|'
@@ -98,11 +98,11 @@
     REVERSE,
 ).finditer
 image_pattern_search = rc(
-    rb'^\[\[[ \t]*+' + regex_pattern(FILE_NAMESACE) + rb'[ \t]*+:',
+    rb'\A\[\[[ \t]*+' + regex_pattern(FILE_NAMESACE) + rb'[ \t]*+:',
     IGNORECASE,
 ).search
 MARKUP = b''.maketrans(b"=|[]'{}", b'\1_\2\3___')
-BRACES_PIPE_NEWLINE = b''.maketrans(b'|{}\n', b'____')
+BRACES_PIPE_NEWLINE = b''.maketrans(b'|{}\r\n', b'_____')
 BRACKETS = b''.maketrans(b'[]', b'__')
 
 PARSABLE_TAG_EXTENSION_NAME = regex_pattern(_parsable_tag_extensions)
@@ -147,7 +147,7 @@
 # Tags:
 # https://infra.spec.whatwg.org/#ascii-whitespace
 # \0 was added as a special case for wikitextparser
-SPACE_CHARS = rb' \t\n\u000C\r\0'  # \s - \v
+SPACE_CHARS = rb' \t\r\n\u000C\0'  # \s - \v
 # http://stackoverflow.com/a/93029/2705757
 # chrs = (chr(i) for i in range(sys.maxunicode))
 # control_chars = ''.join(c for c in chrs if unicodedata.category(c) == 'Cc')

wikitextparser/_table.py

@@ -23,20 +23,20 @@
         {\|
         (?:
             (?:
-                (?!\n\s*+\|)
+                (?!\R\s*+\|)
                 [\s\S]
             )*?
         )
         # Start of caption line
-        \n\s*+\|\+
+        \R\s*+\|\+
     )
     # Optional caption attrs
     (?:
-        (?P<attrs>[^\n|]*+)
+        (?P<attrs>[^\r\n|]*+)
         \|(?!\|)
     )?
     (?P<caption>.*?)
-    (?:\n[\|\!]|\|\|)
+    (?:\R[\|\!]|\|\|)
     """,
     DOTALL | VERBOSE,
 ).match
@@ -49,7 +49,9 @@
 # Captions are optional and only one should be placed between table-start
 # and the first row. Others captions are not part of the table and will
 # be ignored.
-FIRST_NON_CAPTION_LINE = rc(rb'\n[\t \0]*+(\|(?!\+)|!)|\Z').search
+FIRST_NON_CAPTION_LINE = rc(rb'\R[\t \0]*+(\|(?!\+)|!)|\Z').search
+
+FIRST_LINEBREAK = rc(rb'\R').search
 
 
 def head_int(value):
@@ -91,13 +93,13 @@ def _match_table(self) -> list[list[Any]]:
         """Return match_table."""
         table_shadow = self._table_shadow
         # Remove table-start and table-end marks.
-        pos = table_shadow.find(10)  # ord('\n')
+        pos = FIRST_LINEBREAK(table_shadow).span()[0]
         lsp = _lstrip_increase(table_shadow, pos)
         # Remove everything until the first row
         try:
             # while condition may raise IndexError of table is empty
             while table_shadow[lsp] not in b'!|':
-                nlp = table_shadow.find(10, lsp)  # ord('\n')
+                nlp = FIRST_LINEBREAK(table_shadow, lsp).span()[0]
                 pos = nlp
                 lsp = _lstrip_increase(table_shadow, pos)
         except IndexError:
@@ -362,9 +364,9 @@ def caption(self, newcaption: str) -> None:
             )
             return
         # There is no caption. Create one.
-        h, s, t = shadow.partition(b'\n')
+        m = FIRST_LINEBREAK(shadow)
         # Insert caption after the first one.
-        self.insert(len(h + s), '|+' + newcaption + '\n')
+        self.insert(m.span()[1], '|+' + newcaption + m[0].decode())
 
     @property
     def _attrs_match(self) -> Any:
@@ -373,7 +375,7 @@ def _attrs_match(self) -> Any:
         if cache_string == string:
             return cache_match
         shadow = self._shadow
-        attrs_match = ATTRS_MATCH(shadow, 2, shadow.find(10))  # ord('\n')
+        attrs_match = ATTRS_MATCH(shadow, 2, FIRST_LINEBREAK(shadow).span()[0])
         self._attrs_match_cache = attrs_match, string
         return attrs_match
 
@@ -390,10 +392,10 @@ def caption_attrs(self) -> str | None:
     @caption_attrs.setter
     def caption_attrs(self, attrs: str) -> None:
         shadow = self._shadow
-        h, s, t = shadow.partition(b'\n')
+        p = FIRST_LINEBREAK(shadow)
         m = CAPTION_MATCH(shadow)
         if not m:  # There is no caption-line
-            self.insert(len(h + s), '|+' + attrs + '|\n')
+            self.insert(p.span()[1], '|+' + attrs + '|' + p[0].decode())
         else:  # Caption and attrs or Caption but no attrs
             end = m.end('attrs')
             if end != -1:
@@ -574,7 +576,8 @@ def _row_separator_increase(shadow: bytearray, pos: int) -> int:
     lsp = _lstrip_increase(shadow, ncl)
     while shadow[lsp : lsp + 2] == b'|-':  # type: ignore
         # We are on a row separator line.
-        pos = shadow.find(10, lsp + 2)  # ord('\n')
+        m = FIRST_LINEBREAK(shadow, lsp + 2)
+        pos = m.span()[0]
         pos = FIRST_NON_CAPTION_LINE(shadow, pos).start()  # type: ignore
         lsp = _lstrip_increase(shadow, pos)
     return pos

wikitextparser/_template.py

@@ -13,7 +13,7 @@
 
 TL_NAME_ARGS_FULLMATCH = rc(rb'[^|}]*+(?#name)(?<arg>\|[^|]*+)*+').fullmatch
 STARTING_WS_MATCH = rc(r'\s*+').match
-ENDING_WS_MATCH = rc(r'(?>\n[ \t]*)*+', REVERSE).match
+ENDING_WS_MATCH = rc(r'(?>\R[ \t]*)*+', REVERSE).match
 SPACE_AFTER_SEARCH = rc(r'\s*+(?=\|)').search
 
 T = TypeVar('T')

wikitextparser/_wikilist.py

@@ -3,29 +3,29 @@
 from operator import attrgetter
 from typing import Iterable, MutableSequence
 
-from regex import MULTILINE, Match, escape, fullmatch
+from regex import Match, escape, fullmatch
 
 from ._spans import TypeToSpans
 from ._wikitext import EXTERNAL_LINK_FINDITER, SubWikiText
 
 # See includes/parser/BlockLevelPass.php for how MW parses list blocks.
-SUBLIST_PATTERN = rb'(?>^' rb'(?&pattern)' rb'[:;#*].*+' rb'(?>\n|\Z)' rb')*+'
+SUBLIST_PATTERN = rb'(?>(?<=\R|\A)' rb'(?&pattern)' rb'[:;#*].*+' rb'(?>\R|\Z)' rb')*+'
 SUBLIST_WITH_SECOND_PATTERN = (
-    rb'[*#;:].*+(?>\n|\Z)' rb'(?>' rb'(?&pattern)[*#;:].*+(?>\n|\Z)' rb')*+'
+    rb'[*#;:].*+(?>\R|\Z)' rb'(?>' rb'(?&pattern)[*#;:].*+(?>\R|\Z)' rb')*+'
 )
 LIST_PATTERN_FORMAT = (
-    rb'(?<fullitem>^'
+    rb'(?<fullitem>(?<=\R|\A)'
     rb'(?<pattern>{pattern})'
     rb'(?>'
     rb'(?(?<=;\s*+)'
     # mark inline definition as an item
-    rb'(?<item>[^:\n]*+)(?<fullitem>:(?<item>.*+))?+'
-    rb'(?>\n|\Z)' + SUBLIST_PATTERN + rb'|'
+    rb'(?<item>[^:\r\n]*+)(?<fullitem>:(?<item>.*+))?+'
+    rb'(?>\R|\Z)' + SUBLIST_PATTERN + rb'|'
     # non-definition
     rb'(?>'
     rb'(?<item>)'
     + SUBLIST_WITH_SECOND_PATTERN
-    + rb'|(?<item>.*+)(?>\n|\Z)'
+    + rb'|(?<item>.*+)(?>\R|\Z)'
     + SUBLIST_PATTERN
     + rb')'
     rb')'
@@ -58,7 +58,6 @@ def __init__(
                         b'{pattern}', pattern.encode(), 1
                     ),
                     self._list_shadow,
-                    MULTILINE,
                 ),
                 self.string,
             )
@@ -84,7 +83,6 @@ def _match(self) -> Match[bytes]:
                 b'{pattern}', self.pattern.encode(), 1
             ),
             self._list_shadow,
-            MULTILINE,
         )
         self._match_cache = cache_match, string
         return cache_match  # type: ignore