diff --git a/tests/wikitext/test_sections.py b/tests/wikitext/test_sections.py index 5fb4b4a..c7e3538 100644 --- a/tests/wikitext/test_sections.py +++ b/tests/wikitext/test_sections.py @@ -1,4 +1,4 @@ -from pytest import mark, warns +from pytest import warns from wikitextparser import WikiText, parse @@ -13,8 +13,6 @@ def test_blank_lead(): assert '== s ==\nc\n' == wt.sections[1].string -# Todo: Parser should also work with windows line endings. -@mark.xfail def test_multiline_with_carriage_return(): s = 'text\r\n= s =\r\n{|\r\n| a \r\n|}\r\ntext' p = parse(s) diff --git a/wikitextparser/_argument.py b/wikitextparser/_argument.py index e6a68bc..87b2356 100644 --- a/wikitextparser/_argument.py +++ b/wikitextparser/_argument.py @@ -2,7 +2,7 @@ from typing import MutableSequence -from regex import DOTALL, MULTILINE, Match +from regex import DOTALL, Match from ._spans import TypeToSpans from ._wikitext import SECTION_HEADING, SubWikiText, rc @@ -10,8 +10,8 @@ ARG_SHADOW_FULLMATCH = rc( rb'[|:](?(?:[^=]*+(?:' + SECTION_HEADING - + rb'\n)?+)*+)(?:\Z|(?=)(?.*+))', - MULTILINE | DOTALL, + + rb'\R)?+)*+)(?:\Z|(?=)(?.*+))', + DOTALL, ).fullmatch diff --git a/wikitextparser/_cell.py b/wikitextparser/_cell.py index 5b9eff3..60d7137 100644 --- a/wikitextparser/_cell.py +++ b/wikitextparser/_cell.py @@ -24,7 +24,7 @@ | (?P (?: - [^|\n] + [^|\r\n] (?! # attrs end with `|`; or `!!` if sep is `!` (?P=sep){2} @@ -48,9 +48,9 @@ (?P=sep){2}| \|!!| # start of the next newline-cell - \n\s*+[!|]| + \R\s*+[!|]| # end of cell-string - $ + \Z ) """, VERBOSE, @@ -83,7 +83,7 @@ (?: # inline header attrs end with `|` (above) or `!!` (below) (?!!{2}) - [^|\n] + [^|\r\n] )*+ ) # attrs-data separator @@ -96,13 +96,13 @@ (?P.*?) (?= # start of the next newline-cell - \n\s*+[!|]| + \R\s*+[!|]| # start of the next inline-cell \|\|| !!| \|!!| # end of cell-string - $ + \Z ) """, VERBOSE | DOTALL, @@ -119,7 +119,7 @@ (?!\|) | (?P - [^|\n]*? # non-_header attrs end with a `|` + [^|\r\n]*? # non-_header attrs end with a `|` ) # attribute-data separator \| @@ -132,8 +132,8 @@ [^|]*? (?= \|\|| # start of the next inline-cell - \n\s*+[!|]| # start of the next newline-cell - $ # end of cell-string + \R\s*+[!|]| # start of the next newline-cell + \Z # end of cell-string ) ) """, @@ -188,7 +188,7 @@ def _match(self) -> Match[bytes]: if cache_string == string: return cache_match # type: ignore shadow = self._shadow - if shadow[0] == 10: # ord('\n') + if shadow[0] == 10 or shadow[0] == 13: # ord('\n'), ord('\r') m: Match[bytes] = NEWLINE_CELL_MATCH(shadow) # type: ignore self._header = m['sep'] == 33 # ord('!') elif self._header: @@ -260,7 +260,7 @@ def set_attr(self, attr_name: str, attr_value: str) -> None: return # There is no attributes span in this cell. Create one. fmt = ' {}="{}" |' if attr_value else ' {} |' - if shadow[0] == 10: # ord('\n') + if shadow[0] == 10 or shadow[0] == 13: # ord('\n'), ord('\r') self.insert( cell_match.start('sep') + 1, fmt.format(attr_name, attr_value) ) diff --git a/wikitextparser/_comment_bold_italic.py b/wikitextparser/_comment_bold_italic.py index cd1deb9..2205d83 100644 --- a/wikitextparser/_comment_bold_italic.py +++ b/wikitextparser/_comment_bold_italic.py @@ -2,22 +2,22 @@ from typing import MutableSequence -from regex import DOTALL, MULTILINE, Match +from regex import DOTALL, Match from ._spans import TypeToSpans from ._wikitext import SubWikiText, rc COMMENT_PATTERN = r'|\Z)' -COMMA_COMMENT = "'(?>" + COMMENT_PATTERN + ')*+' -COMMENT_COMMA = '(?>' + COMMENT_PATTERN + ")*+'" +COMMA_COMMENT = r"'(?>" + COMMENT_PATTERN + r')*+' +COMMENT_COMMA = r'(?>' + COMMENT_PATTERN + r")*+'" BOLD_FULLMATCH = rc( - COMMA_COMMENT * 2 + "'(.*?)(?>'" + COMMENT_COMMA * 2 + '|$)', - MULTILINE | DOTALL, + COMMA_COMMENT * 2 + r"'(.*?)(?>'" + COMMENT_COMMA * 2 + r'|(?=\R|\Z))', + DOTALL, ).fullmatch ITALIC_FULLMATCH = rc( - COMMA_COMMENT + "'(.*?)(?>'" + COMMENT_COMMA + '|$)', DOTALL + COMMA_COMMENT + r"'(.*?)(?>'" + COMMENT_COMMA + r'|\Z)', DOTALL ).fullmatch -ITALIC_NOEND_FULLMATCH = rc(COMMA_COMMENT + "'(.*)", DOTALL).fullmatch +ITALIC_NOEND_FULLMATCH = rc(COMMA_COMMENT + r"'(.*)", DOTALL).fullmatch class Comment(SubWikiText): diff --git a/wikitextparser/_section.py b/wikitextparser/_section.py index 49313f0..1b7e0b8 100644 --- a/wikitextparser/_section.py +++ b/wikitextparser/_section.py @@ -4,7 +4,7 @@ from ._wikitext import SubWikiText, rc -HEADER_MATCH = rc(rb'\0*+(={1,6})([^\n]+?)\1[ \t\0]*+(\n|\Z)').match +HEADER_MATCH = rc(rb'\0*+(={1,6})([^\r\n]+?)\1[ \t\0]*+(\R|\Z)').match class Section(SubWikiText): diff --git a/wikitextparser/_spans.py b/wikitextparser/_spans.py index 14911c9..cf07743 100644 --- a/wikitextparser/_spans.py +++ b/wikitextparser/_spans.py @@ -20,7 +20,7 @@ rc = partial(rc, cache_pattern=False) # According to https://www.mediawiki.org/wiki/Manual:$wgLegalTitleChars # illegal title characters are: r'[]{}|#<>[\u0000-\u0020]' -VALID_TITLE_CHARS = rb'[^\|\{\}\[\2\]\3<>\n]*+' +VALID_TITLE_CHARS = rb'[^\|\{\}\[\2\]\3<>\r\n]*+' # Parser functions # According to https://www.mediawiki.org/wiki/Help:Magic_words # See also: @@ -49,7 +49,7 @@ + rb'\}\})' ).finditer # External links -INVALID_URL_CHARS = rb' \t\n"<>\[\]' +INVALID_URL_CHARS = rb' \t\r\n"<>\[\]' VALID_URL_CHARS = rb'[^' + INVALID_URL_CHARS + rb']++' # See more info on literal IPv6 see: # https://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers @@ -67,7 +67,7 @@ # Wikilinks # https://www.mediawiki.org/wiki/Help:Links#Internal_links WIKILINK_PARAM_FINDITER = rc( - rb'(?^|[^\[\0])(?:(?>\[\0*+){2})*+\[\0*+)' # != 2N + 1 + rb'(?\A|[^\[\0])(?:(?>\[\0*+){2})*+\[\0*+)' # != 2N + 1 rb'\[\0*\[' rb'(?![\ \0]*+' + BARE_EXTERNAL_LINK + rb')' + VALID_TITLE_CHARS + rb'(?>' rb'\|' @@ -98,11 +98,11 @@ REVERSE, ).finditer image_pattern_search = rc( - rb'^\[\[[ \t]*+' + regex_pattern(FILE_NAMESACE) + rb'[ \t]*+:', + rb'\A\[\[[ \t]*+' + regex_pattern(FILE_NAMESACE) + rb'[ \t]*+:', IGNORECASE, ).search MARKUP = b''.maketrans(b"=|[]'{}", b'\1_\2\3___') -BRACES_PIPE_NEWLINE = b''.maketrans(b'|{}\n', b'____') +BRACES_PIPE_NEWLINE = b''.maketrans(b'|{}\r\n', b'_____') BRACKETS = b''.maketrans(b'[]', b'__') PARSABLE_TAG_EXTENSION_NAME = regex_pattern(_parsable_tag_extensions) @@ -147,7 +147,7 @@ # Tags: # https://infra.spec.whatwg.org/#ascii-whitespace # \0 was added as a special case for wikitextparser -SPACE_CHARS = rb' \t\n\u000C\r\0' # \s - \v +SPACE_CHARS = rb' \t\r\n\u000C\0' # \s - \v # http://stackoverflow.com/a/93029/2705757 # chrs = (chr(i) for i in range(sys.maxunicode)) # control_chars = ''.join(c for c in chrs if unicodedata.category(c) == 'Cc') diff --git a/wikitextparser/_table.py b/wikitextparser/_table.py index da963be..845ad20 100644 --- a/wikitextparser/_table.py +++ b/wikitextparser/_table.py @@ -23,20 +23,20 @@ {\| (?: (?: - (?!\n\s*+\|) + (?!\R\s*+\|) [\s\S] )*? ) # Start of caption line - \n\s*+\|\+ + \R\s*+\|\+ ) # Optional caption attrs (?: - (?P[^\n|]*+) + (?P[^\r\n|]*+) \|(?!\|) )? (?P.*?) - (?:\n[\|\!]|\|\|) + (?:\R[\|\!]|\|\|) """, DOTALL | VERBOSE, ).match @@ -49,7 +49,9 @@ # Captions are optional and only one should be placed between table-start # and the first row. Others captions are not part of the table and will # be ignored. -FIRST_NON_CAPTION_LINE = rc(rb'\n[\t \0]*+(\|(?!\+)|!)|\Z').search +FIRST_NON_CAPTION_LINE = rc(rb'\R[\t \0]*+(\|(?!\+)|!)|\Z').search + +FIRST_LINEBREAK = rc(rb'\R').search def head_int(value): @@ -91,13 +93,13 @@ def _match_table(self) -> list[list[Any]]: """Return match_table.""" table_shadow = self._table_shadow # Remove table-start and table-end marks. - pos = table_shadow.find(10) # ord('\n') + pos = FIRST_LINEBREAK(table_shadow).span()[0] lsp = _lstrip_increase(table_shadow, pos) # Remove everything until the first row try: # while condition may raise IndexError of table is empty while table_shadow[lsp] not in b'!|': - nlp = table_shadow.find(10, lsp) # ord('\n') + nlp = FIRST_LINEBREAK(table_shadow, lsp).span()[0] pos = nlp lsp = _lstrip_increase(table_shadow, pos) except IndexError: @@ -362,9 +364,9 @@ def caption(self, newcaption: str) -> None: ) return # There is no caption. Create one. - h, s, t = shadow.partition(b'\n') + m = FIRST_LINEBREAK(shadow) # Insert caption after the first one. - self.insert(len(h + s), '|+' + newcaption + '\n') + self.insert(m.span()[1], '|+' + newcaption + m[0].decode()) @property def _attrs_match(self) -> Any: @@ -373,7 +375,7 @@ def _attrs_match(self) -> Any: if cache_string == string: return cache_match shadow = self._shadow - attrs_match = ATTRS_MATCH(shadow, 2, shadow.find(10)) # ord('\n') + attrs_match = ATTRS_MATCH(shadow, 2, FIRST_LINEBREAK(shadow).span()[0]) self._attrs_match_cache = attrs_match, string return attrs_match @@ -390,10 +392,10 @@ def caption_attrs(self) -> str | None: @caption_attrs.setter def caption_attrs(self, attrs: str) -> None: shadow = self._shadow - h, s, t = shadow.partition(b'\n') + p = FIRST_LINEBREAK(shadow) m = CAPTION_MATCH(shadow) if not m: # There is no caption-line - self.insert(len(h + s), '|+' + attrs + '|\n') + self.insert(p.span()[1], '|+' + attrs + '|' + p[0].decode()) else: # Caption and attrs or Caption but no attrs end = m.end('attrs') if end != -1: @@ -574,7 +576,8 @@ def _row_separator_increase(shadow: bytearray, pos: int) -> int: lsp = _lstrip_increase(shadow, ncl) while shadow[lsp : lsp + 2] == b'|-': # type: ignore # We are on a row separator line. - pos = shadow.find(10, lsp + 2) # ord('\n') + m = FIRST_LINEBREAK(shadow, lsp + 2) + pos = m.span()[0] pos = FIRST_NON_CAPTION_LINE(shadow, pos).start() # type: ignore lsp = _lstrip_increase(shadow, pos) return pos diff --git a/wikitextparser/_template.py b/wikitextparser/_template.py index c450c34..3fda09d 100644 --- a/wikitextparser/_template.py +++ b/wikitextparser/_template.py @@ -13,7 +13,7 @@ TL_NAME_ARGS_FULLMATCH = rc(rb'[^|}]*+(?#name)(?\|[^|]*+)*+').fullmatch STARTING_WS_MATCH = rc(r'\s*+').match -ENDING_WS_MATCH = rc(r'(?>\n[ \t]*)*+', REVERSE).match +ENDING_WS_MATCH = rc(r'(?>\R[ \t]*)*+', REVERSE).match SPACE_AFTER_SEARCH = rc(r'\s*+(?=\|)').search T = TypeVar('T') diff --git a/wikitextparser/_wikilist.py b/wikitextparser/_wikilist.py index bca10c6..6ad6a05 100644 --- a/wikitextparser/_wikilist.py +++ b/wikitextparser/_wikilist.py @@ -3,29 +3,29 @@ from operator import attrgetter from typing import Iterable, MutableSequence -from regex import MULTILINE, Match, escape, fullmatch +from regex import Match, escape, fullmatch from ._spans import TypeToSpans from ._wikitext import EXTERNAL_LINK_FINDITER, SubWikiText # See includes/parser/BlockLevelPass.php for how MW parses list blocks. -SUBLIST_PATTERN = rb'(?>^' rb'(?&pattern)' rb'[:;#*].*+' rb'(?>\n|\Z)' rb')*+' +SUBLIST_PATTERN = rb'(?>(?<=\R|\A)' rb'(?&pattern)' rb'[:;#*].*+' rb'(?>\R|\Z)' rb')*+' SUBLIST_WITH_SECOND_PATTERN = ( - rb'[*#;:].*+(?>\n|\Z)' rb'(?>' rb'(?&pattern)[*#;:].*+(?>\n|\Z)' rb')*+' + rb'[*#;:].*+(?>\R|\Z)' rb'(?>' rb'(?&pattern)[*#;:].*+(?>\R|\Z)' rb')*+' ) LIST_PATTERN_FORMAT = ( - rb'(?^' + rb'(?(?<=\R|\A)' rb'(?{pattern})' rb'(?>' rb'(?(?<=;\s*+)' # mark inline definition as an item - rb'(?[^:\n]*+)(?:(?.*+))?+' - rb'(?>\n|\Z)' + SUBLIST_PATTERN + rb'|' + rb'(?[^:\r\n]*+)(?:(?.*+))?+' + rb'(?>\R|\Z)' + SUBLIST_PATTERN + rb'|' # non-definition rb'(?>' rb'(?)' + SUBLIST_WITH_SECOND_PATTERN - + rb'|(?.*+)(?>\n|\Z)' + + rb'|(?.*+)(?>\R|\Z)' + SUBLIST_PATTERN + rb')' rb')' @@ -58,7 +58,6 @@ def __init__( b'{pattern}', pattern.encode(), 1 ), self._list_shadow, - MULTILINE, ), self.string, ) @@ -84,7 +83,6 @@ def _match(self) -> Match[bytes]: b'{pattern}', self.pattern.encode(), 1 ), self._list_shadow, - MULTILINE, ) self._match_cache = cache_match, string return cache_match # type: ignore diff --git a/wikitextparser/_wikitext.py b/wikitextparser/_wikitext.py index 50f1e49..a7f152f 100644 --- a/wikitextparser/_wikitext.py +++ b/wikitextparser/_wikitext.py @@ -15,7 +15,6 @@ from regex import ( DOTALL, IGNORECASE, - MULTILINE, VERBOSE, Match, finditer, @@ -67,8 +66,8 @@ ).sub # Sections -SECTION_HEADING = rb'^\0*+(?={1,6})[^\r\n]+?(?P=equals)[ \t\0]*+\r?+$' -SUB_SECTION = rb'(?:^\0*+(?P=equals)=[^\r\n]+?(?P=equals)=[ \t\0]*+\r?+$.*?)*' +SECTION_HEADING = rb'(?<=\R|\A)\0*+(?={1,6})[^\r\n]+?(?P=equals)[ \t\0]*+(?=\R|\Z)' +SUB_SECTION = rb'(?:(?<=\R|\A)\0*+(?P=equals)=[^\r\n]+?(?P=equals)=[ \t\0]*+(?=\R|\Z).*?)*' LEAD_SECTION = rb'(?
(?).*?)' SECTIONS_FULLMATCH = rc( LEAD_SECTION @@ -76,7 +75,7 @@ + SECTION_HEADING + rb'.*?' # heading # section content rb')*', - DOTALL | MULTILINE | VERBOSE, + DOTALL | VERBOSE, ).fullmatch SECTIONS_TOP_LEVELS_ONLY = rc( LEAD_SECTION @@ -85,7 +84,7 @@ + rb'.*?' + SUB_SECTION + rb')*', - DOTALL | MULTILINE | VERBOSE, + DOTALL | VERBOSE, ).fullmatch # Tables @@ -93,20 +92,20 @@ rb""" # Table-start # Always starts on a new line with optional leading spaces or indentation. - (?<=^[ :\0]*+) + (?<=\R[ :\0]*+|\A[ :\0]*+) {\| # Table contents (?: # Any character, as long as it is not indicating another table-start - (?!^\ *+\{\|). + (?!\R\ *+\{\|). )*? # Table-end - \r?\n\s*+ + \R\s*+ (?> \|} | \Z ) """, - DOTALL | MULTILINE | VERBOSE, + DOTALL | VERBOSE, ).finditer -substitute_apostrophes = rc(rb"('\0*+){2,}+(?=[^']|$)", MULTILINE).sub +substitute_apostrophes = rc(rb"('\0*+){2,}+(?=[^']|\R|\Z)").sub BOLD_FINDITER = rc( rb""" @@ -115,9 +114,9 @@ # content (\0*+[^'\r\n]++.*?) # end token - (?:'\0*+'\0*+'|$) + (?:'\0*+'\0*+'|(?=\R|\Z)) """, - MULTILINE | VERBOSE, + VERBOSE, ).finditer ITALIC_FINDITER = rc( @@ -127,9 +126,9 @@ # content (\0*+[^'\r\n]++.*?) # end token - (?:'\0*+'|$) + (?:'\0*+'|(?=\R|\Z)) """, - MULTILINE | VERBOSE, + VERBOSE, ).finditer # Types which are detected by parse_to_spans @@ -1458,7 +1457,6 @@ def get_lists( for m in finditer( LIST_PATTERN_FORMAT.replace(b'{pattern}', ptrn.encode(), 1), shadow, - MULTILINE, ): ms, me = m.span() s, e = ss + ms, ss + me