From 5da229925c135ce661c7050cf0bb392858dc9d24 Mon Sep 17 00:00:00 2001 From: tacc-tacc Date: Fri, 4 Sep 2026 03:35:05 -0300 Subject: [PATCH 1/2] fixed template and parser function behavior --- docs/index.rst | 2 +- tests/test_parser_function.py | 77 ++++++++++++- tests/test_template.py | 4 +- wikitextparser/_argument.py | 108 +++++++++++++++-- wikitextparser/_parser_function.py | 178 ++++++++++++++++------------- wikitextparser/_template.py | 51 ++++++--- wikitextparser/_wikitext.py | 35 ++---- 7 files changed, 315 insertions(+), 140 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b9bc7aab..b317222d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,7 +41,7 @@ SubWikiTextWithAttrs SubWikiTextWithArgs -------------------- -.. autoclass:: wikitextparser._parser_function.SubWikiTextWithArgs +.. autoclass:: wikitextparser._argument.SubWikiTextWithArgs :members: :special-members: :show-inheritance: diff --git a/tests/test_parser_function.py b/tests/test_parser_function.py index 85d2705b..48449ee5 100644 --- a/tests/test_parser_function.py +++ b/tests/test_parser_function.py @@ -38,6 +38,15 @@ def test_set_name(): assert '{{#if: test | true | false }}' == pf.string +def test_normal_name(): + assert '#u ' == ParserFunction('{{ #u :a}}').normal_name() + assert '#u ' == ParserFunction('{{ #U :a}}').normal_name() + assert '#a_b' == ParserFunction('{{#a_b:}}').normal_name() + assert '#t#a' == ParserFunction('{{#t#a:a}}').normal_name() + assert '#a___b' == ParserFunction('{{#A___B:}}').normal_name() + assert '#t' == ParserFunction('{{\n #T:}}').normal_name() + + def test_pipes_inside_params_or_templates(): pf = ParserFunction('{{ #if: test | {{ text | aaa }} }}') assert [] == pf.parameters @@ -77,10 +86,70 @@ def test_tag_containing_pipe(): assert len(ParserFunction('{{text|abc}}').arguments) == 1 -@mark.skip( - reason='ParserFunction arguments currently inherit Template "=" semantics' -) def test_equal_in_if_expression(): pf = ParserFunction('{{#if: 2==2 | yes | no }}') pf.arguments[0].value = '3' - assert pf.string == '{{#if: 3 | yes | no }}' + assert pf.string == '{{#if:3| yes | no }}' + + +def test_has_arg(): + has_arg = ParserFunction('{{#pf:a|b=c}}').has_arg + assert has_arg('1') is True + assert has_arg('1', 'a') is True + assert has_arg('b') is False + assert has_arg('b', 'c') is False + assert has_arg('2') is True + assert has_arg('2', 'b=c') is True + assert has_arg('c') is False + assert has_arg('b', 'd') is False + + +def test_get_arg(): + get_arg = ParserFunction('{{#pf:a|b=c}}').get_arg + assert ':a' == get_arg('1').string # type: ignore + assert get_arg('c') is None + + +def test_name_contains_a_param_with_default(): + t = ParserFunction('{{#pf {{{p1|d1}}} : {{{p2|d2}}} }}') + assert '#pf {{{p1|d1}}} ' == t.name + assert ': {{{p2|d2}}} ' == t.arguments[0].string + t.name = 'g' + assert 'g' == t.name + + +def test_set_arg(): + t = ParserFunction('{{#pf}}') + t.set_arg('1', 'b') + assert '{{#pf:b}}' == t.string + t = ParserFunction('{{#pf:a}}') + t.set_arg('1', 'b') + assert '{{#pf:b}}' == t.string + t = ParserFunction('{{#pf:a|b}}') + t.set_arg('2', 'c') + assert '{{#pf:a|c}}' == t.string + t = ParserFunction('{{#pf:a|b}}') + t.set_arg('4', 'c') + assert '{{#pf:a|b}}' == t.string + t = ParserFunction('{{#pf:a|b}}') + t.set_arg('xd', 'c') + assert '{{#pf:a|b}}' == t.string + + +def test_del_arg(): + t = ParserFunction('{{#pf:a}}') + t.del_arg('1') + assert '{{#pf}}' == t.string + t = ParserFunction('{{#pf:a|b}}') + t.del_arg('2') + assert '{{#pf:a}}' == t.string + + +def test_lists(): + l1, l2 = ParserFunction('{{#pf:*a\n*b|*c\n*d}}').get_lists() + assert l1.items == ['a', 'b'] + assert l2.items == ['c', 'd'] + assert ParserFunction('{{#pf:;https://a.b :d}}').get_lists('[;:]')[0].items == [ + 'https://a.b ', + 'd', + ] diff --git a/tests/test_template.py b/tests/test_template.py index 49cd7dc6..06a56b43 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -290,11 +290,13 @@ def test_multi_set_positional_args(): assert '{{t|p|q}}' == t.string -@mark.xfail def test_invalid_position(): t = Template('{{t}}') t.set_arg('2', 'a', positional=True) assert '{{t|2=a}}' == t.string + t = Template('{{t}}') + t.set_arg('v', 'a', positional=True) + assert '{{t|v=a}}' == t.string def test_force_new_to_positional_when_old_is_keyword(): diff --git a/wikitextparser/_argument.py b/wikitextparser/_argument.py index f3bfd58a..74fb6ce1 100644 --- a/wikitextparser/_argument.py +++ b/wikitextparser/_argument.py @@ -1,10 +1,12 @@ from __future__ import annotations -from collections.abc import MutableSequence +from bisect import insort +from collections.abc import Iterable, MutableSequence from regex import DOTALL, Match from ._spans import TypeToSpans +from ._wikilist import WikiList from ._wikitext import SECTION_HEADING, SubWikiText, rc ARG_SHADOW_FULLMATCH = rc( @@ -24,7 +26,7 @@ class Argument(SubWikiText): See https://www.mediawiki.org/wiki/Help:Templates for more information. """ - __slots__ = '_parent', '_shadow_match_cache' + __slots__ = '_ignore_equals', '_parent', '_shadow_match_cache' def __init__( self, @@ -35,6 +37,7 @@ def __init__( _parent: SubWikiTextWithArgs | None = None, ): super().__init__(string, _type_to_spans, _span, _type) + self._ignore_equals = _parent._ignore_equals if _parent != None else False self._parent = _parent or self self._shadow_match_cache = None, None @@ -60,7 +63,7 @@ def name(self) -> str: """ ss = self._span_data[0] shadow_match = self._shadow_match - if shadow_match['eq']: + if not self._ignore_equals and shadow_match['eq']: s, e = shadow_match.span('pre_eq') return self._lststr[0][ss + s : ss + e] # positional argument @@ -79,7 +82,7 @@ def name(self) -> str: @name.setter def name(self, newname: str) -> None: - if self._shadow_match['eq']: + if not self._ignore_equals and self._shadow_match['eq']: self[1 : 1 + len(self._shadow_match['pre_eq'])] = newname else: self.insert(1, newname + '=') @@ -93,12 +96,12 @@ def positional(self) -> bool: Raise ValueError on trying to convert positional to keyword argument. """ - return not self._shadow_match['eq'] + return self._ignore_equals or not self._shadow_match['eq'] @positional.setter def positional(self, to_positional: bool) -> None: shadow_match = self._shadow_match - if shadow_match['eq']: + if not self._ignore_equals and shadow_match['eq']: # Keyword argument if to_positional: del self[1 : shadow_match.end('eq')] @@ -125,14 +128,14 @@ def value(self) -> str: Assign a new value to self. """ shadow_match = self._shadow_match - if shadow_match['eq']: + if not self._ignore_equals and shadow_match['eq']: return self(shadow_match.start('post_eq'), None) return self(1, None) @value.setter def value(self, newvalue: str) -> None: shadow_match = self._shadow_match - if shadow_match['eq']: + if not self._ignore_equals and shadow_match['eq']: self[shadow_match.start('post_eq') :] = newvalue else: self[1:] = newvalue @@ -140,7 +143,7 @@ def value(self, newvalue: str) -> None: @property def _lists_shadow_ss(self): shadow_match = self._shadow_match - if shadow_match['eq']: + if not self._ignore_equals and shadow_match['eq']: post_eq = shadow_match['post_eq'] ls_post_eq = post_eq.lstrip() return ( @@ -153,5 +156,88 @@ def _lists_shadow_ss(self): return bytearray(shadow_match[0][1:]), self._span_data[0] + 1 -if __name__ == '__main__': - from wikitextparser._parser_function import SubWikiTextWithArgs +class SubWikiTextWithArgs(SubWikiText): + """Define common attributes for `Template` and `ParserFunction`.""" + + __slots__ = () + + _name_args_matcher = NotImplemented + _first_arg_sep = 0 + _ignore_equals = False + + @property + def _content_span(self) -> tuple[int, int]: + return 2, -2 + + @property + def nesting_level(self) -> int: + """Return the nesting level of self. + + The minimum nesting_level is 0. Being part of any Template or + ParserFunction increases the level by one. + """ + return self._nesting_level(('Template', 'ParserFunction')) + + @property + def arguments(self) -> list[Argument]: + """Parse template content. Create self.name and self.arguments.""" + shadow = self._shadow + split_spans = self._name_args_matcher(shadow, 2, -2).spans('arg') + if not split_spans: + return [] + arguments = [] + arguments_append = arguments.append + type_to_spans = self._type_to_spans + ss, se, _, _ = span = self._span_data + type_ = id(span) + lststr = self._lststr + arg_spans = type_to_spans.setdefault(type_, []) + span_tuple_to_span_get = {(s[0], s[1]): s for s in arg_spans}.get + for arg_self_start, arg_self_end in split_spans: + # todo: add byte array + s, e, _, _ = arg_span = [ + ss + arg_self_start, + ss + arg_self_end, + None, + None, + ] + old_span = span_tuple_to_span_get((s, e)) + if old_span is None: + insort(arg_spans, arg_span) + else: + arg_span = old_span + arg = Argument(lststr, type_to_spans, arg_span, type_, self) + arg._span_data[3] = shadow[arg_self_start:arg_self_end] + arguments_append(arg) + return arguments + + def get_lists( + self, pattern: str | Iterable[str] = (r'\#', r'\*', '[:;]') + ) -> list[WikiList]: + """Return the lists in all arguments. + + For performance reasons it is usually preferred to get a specific + Argument and use the `get_lists` method of that argument instead. + """ + return [ + lst + for arg in self.arguments + for lst in arg.get_lists(pattern) + if lst + ] + + @property + def name(self) -> str: + """Template's name (includes whitespace). + + getter: Return the name. + setter: Set a new name. + """ + sep = self._shadow.find(self._first_arg_sep) + if sep == -1: + return self(2, -2) + return self(2, sep) + + @name.setter + def name(self, newname: str) -> None: + self[2 : 2 + len(self.name)] = newname diff --git a/wikitextparser/_parser_function.py b/wikitextparser/_parser_function.py index e11be736..a066f664 100644 --- a/wikitextparser/_parser_function.py +++ b/wikitextparser/_parser_function.py @@ -1,109 +1,125 @@ from __future__ import annotations -from bisect import insort from collections.abc import Iterable -from ._argument import Argument -from ._wikilist import WikiList -from ._wikitext import SubWikiText, rc +from ._argument import Argument, SubWikiTextWithArgs +from ._comment_bold_italic import COMMENT_PATTERN +from ._wikitext import WS, rc + +COMMENT_SUB = rc(COMMENT_PATTERN).sub PF_NAME_ARGS_FULLMATCH = rc( rb'[^:|}]*+(?#name)' rb'(?:[^|]*+)?+(?\|[^|]*+)*+' ).fullmatch -class SubWikiTextWithArgs(SubWikiText): - """Define common attributes for `Template` and `ParserFunction`.""" +class ParserFunction(SubWikiTextWithArgs): + """Convert strings to ParserFunction objects. + The string should start with {{ and end with }}. + """ __slots__ = () - _name_args_matcher = NotImplemented - _first_arg_sep = 0 + _name_args_matcher = PF_NAME_ARGS_FULLMATCH + _first_arg_sep = 58 + _ignore_equals = True - @property - def _content_span(self) -> tuple[int, int]: - return 2, -2 - @property - def nesting_level(self) -> int: - """Return the nesting level of self. + def normal_name(self) -> str: + """Return normal form of self.name. - The minimum nesting_level is 0. Being part of any Template or - ParserFunction increases the level by one. + - Remove comments. + - Lowercase """ - return self._nesting_level(('Template', 'ParserFunction')) - - @property - def arguments(self) -> list[Argument]: - """Parse template content. Create self.name and self.arguments.""" - shadow = self._shadow - split_spans = self._name_args_matcher(shadow, 2, -2).spans('arg') - if not split_spans: - return [] - arguments = [] - arguments_append = arguments.append - type_to_spans = self._type_to_spans - ss, se, _, _ = span = self._span_data - type_ = id(span) - lststr = self._lststr - arg_spans = type_to_spans.setdefault(type_, []) - span_tuple_to_span_get = {(s[0], s[1]): s for s in arg_spans}.get - for arg_self_start, arg_self_end in split_spans: - # todo: add byte array - s, e, _, _ = arg_span = [ - ss + arg_self_start, - ss + arg_self_end, - None, - None, - ] - old_span = span_tuple_to_span_get((s, e)) - if old_span is None: - insort(arg_spans, arg_span) - else: - arg_span = old_span - arg = Argument(lststr, type_to_spans, arg_span, type_, self) - arg._span_data[3] = shadow[arg_self_start:arg_self_end] - arguments_append(arg) - return arguments - - def get_lists( - self, pattern: str | Iterable[str] = (r'\#', r'\*', '[:;]') - ) -> list[WikiList]: - """Return the lists in all arguments. - - For performance reasons it is usually preferred to get a specific - Argument and use the `get_lists` method of that argument instead. + return COMMENT_SUB('', self.name).lstrip(WS).lower() + + def set_arg( + self, + name: str | None, + value: str, + ) -> None: + """Set the value for `name` argument. Add it if it doesn't exist. """ - return [ - lst - for arg in self.arguments - for lst in arg.get_lists(pattern) - if lst - ] + args = (*reversed(self.arguments),) + if name is not None: + # Invalid + if not is_positive_integer(name): + return - @property - def name(self) -> str: - """Template's name (includes whitespace). + # Updating an existing argument. + arg = get_arg(name, args) + if arg: + arg.positional = True + arg.value = value + return - getter: Return the name. - setter: Set a new name. - """ - sep = self._shadow.find(self._first_arg_sep) - if sep == -1: - return self(2, -2) - return self(2, sep) + last_idx = get_last_idx_positional_args(args) - @name.setter - def name(self, newname: str) -> None: - self[2 : 2 + len(self.name)] = newname + # Invalid, as it would need to fill the pf with empty arguments + if name and (last_idx != int(name) - 1): + return + # Adding a new argument + addstring = (':' if last_idx == 0 else '|') + value + self.insert(-2, addstring) -class ParserFunction(SubWikiTextWithArgs): - __slots__ = () + def get_arg(self, name: str) -> Argument | None: + """Return the last argument with the given name. - _name_args_matcher = PF_NAME_ARGS_FULLMATCH - _first_arg_sep = 58 + Return None if no argument with that name is found. + """ + return get_arg(name, reversed(self.arguments)) + + def has_arg(self, name: str, value: str | None = None) -> bool: + """Return true if the is an arg named `name`. + + Also check equality of values if `value` is provided. + + Note: If you just need to get an argument and you want to LBYL, it's + better to get_arg directly and then check if the returned value + is None. + """ + for arg in reversed(self.arguments): + if arg.name.strip(WS) == name.strip(WS): + if value: + return arg.value == value + return True + return False + + def del_arg(self, name: str) -> None: + """Delete all arguments with the given then.""" + for arg in reversed(self.arguments): + if arg.name.strip(WS) == name.strip(WS): + del arg[:] @property def parser_functions(self) -> list[ParserFunction]: return super().parser_functions[1:] + + +def is_positive_integer(x): + try: + return int(x) > 0 + except ValueError: + return False + +def get_arg(name: str, args: Iterable[Argument]) -> Argument | None: + """Return the first argument in the args that has the given name. + + Return None if no such argument is found. + + As the computation of self.arguments is a little costly, this + function was created so that other methods that have already computed + the arguments use it instead of calling self.get_arg directly. + """ + for arg in args: + if arg.name.strip(WS) == name.strip(WS): + return arg + return None + +def get_last_idx_positional_args(args: Iterable[Argument]) -> int: + idx = 0 + for arg in args: + if arg.positional: + idx += 1 + return idx diff --git a/wikitextparser/_template.py b/wikitextparser/_template.py index 42e56b89..6d7ac3f2 100644 --- a/wikitextparser/_template.py +++ b/wikitextparser/_template.py @@ -5,9 +5,8 @@ from regex import REVERSE -from ._argument import Argument +from ._argument import Argument, SubWikiTextWithArgs from ._comment_bold_italic import COMMENT_PATTERN -from ._parser_function import SubWikiTextWithArgs from ._wikitext import WS, rc COMMENT_SUB = rc(COMMENT_PATTERN).sub @@ -30,6 +29,7 @@ class Template(SubWikiTextWithArgs): _name_args_matcher = TL_NAME_ARGS_FULLMATCH _first_arg_sep = 124 + _ignore_equals = False @property def _content_span(self) -> tuple[int, int]: @@ -172,7 +172,7 @@ def rm_dup_args_safe(self, tag: str | None = None) -> None: def set_arg( self, - name: str, + name: str | None, value: str, positional: bool | None = None, before: str | None = None, @@ -191,20 +191,28 @@ def set_arg( If it's None, do what seems more appropriate. """ args = (*reversed(self.arguments),) - arg = get_arg(name, args) - # Updating an existing argument. - if arg: - if positional: - arg.positional = positional - if preserve_spacing: - val = arg.value - arg.value = val.replace(val.strip(WS), value, 1) - else: - arg.value = value - return + if name is not None: + arg = get_arg(name, args) + # Updating an existing argument. + if arg: + if positional == True: + arg.positional = True + # if positional == False but arg.positional == true, then + # positional.setter of SubWikiText will raise an exception + if preserve_spacing: + val = arg.value + arg.value = val.replace(val.strip(WS), value, 1) + else: + arg.value = value + return # Adding a new argument - if not name and positional is None: + if not name: positional = True + else: + if positional and not is_positive_integer(name): + positional = False + if positional and get_last_idx_positional_args(args) != int(name) - 1: + positional = False # Calculate the whitespace needed before arg-name and after arg-value. if not positional and preserve_spacing and args: before_names = [] @@ -232,6 +240,7 @@ def set_arg( # Ignore preserve_spacing for positional args. addstring = '|' + value else: + assert(name) # To keep the compiler happy if preserve_spacing: addstring = ( '|' @@ -309,6 +318,11 @@ def del_arg(self, name: str) -> None: def templates(self) -> list[Template]: return super().templates[1:] +def is_positive_integer(x): + try: + return int(x) > 0 + except ValueError: + return False def mode(list_: list[T]) -> T: """Return the most common item in the list. @@ -341,3 +355,10 @@ def get_arg(name: str, args: Iterable[Argument]) -> Argument | None: if arg.name.strip(WS) == name.strip(WS): return arg return None + +def get_last_idx_positional_args(args: Iterable[Argument]) -> int: + idx = 0 + for arg in args: + if arg.positional: + idx += 1 + return idx diff --git a/wikitextparser/_wikitext.py b/wikitextparser/_wikitext.py index 6099ca6d..2572333b 100644 --- a/wikitextparser/_wikitext.py +++ b/wikitextparser/_wikitext.py @@ -903,40 +903,21 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: if len(args) == 1: arg = args[0] # the first arg is both the first and last argument - if arg.positional: - arg.value = ( - newline_indent + arg.value.strip(ws) + short_indent - ) - else: - # Note that we don't add spaces before and after the - # '=' in parser functions because it could be part of - # an ordinary string. - arg.name = newline_indent + arg.name.lstrip(ws) - arg.value = arg.value.rstrip(ws) + short_indent + arg.value = ( + newline_indent + arg.value.strip(ws) + short_indent + ) continue # Special formatting for the first argument arg = args[0] - if arg.positional: - arg.value = ( - newline_indent + arg.value.strip(ws) + newline_indent - ) - else: - arg.name = newline_indent + arg.name.lstrip(ws) - arg.value = arg.value.rstrip(ws) + newline_indent + arg.value = ( + newline_indent + arg.value.strip(ws) + newline_indent + ) # Formatting the middle arguments for arg in args[1:-1]: - if arg.positional: - arg.value = ' ' + arg.value.strip(ws) + newline_indent - else: - arg.name = ' ' + arg.name.lstrip(ws) - arg.value = arg.value.rstrip(ws) + newline_indent + arg.value = ' ' + arg.value.strip(ws) + newline_indent # Special formatting for the last argument arg = args[-1] - if arg.positional: - arg.value = ' ' + arg.value.strip(ws) + short_indent - else: - arg.name = ' ' + arg.name.lstrip(ws) - arg.value = arg.value.rstrip(ws) + short_indent + arg.value = ' ' + arg.value.strip(ws) + short_indent return parsed.string From 4b7aef389d93434fde4b287e1173b2e6eec58fff Mon Sep 17 00:00:00 2001 From: tacc-tacc Date: Sat, 5 Sep 2026 21:04:06 -0300 Subject: [PATCH 2/2] reworked structure to permit the use of ignore_equals as function parameter --- tests/test_argument.py | 72 ++--- tests/test_cell.py | 2 +- tests/test_parser_function.py | 65 +++-- tests/test_spans.py | 30 +-- tests/test_template.py | 16 +- tests/wikitext/test_wikitext.py | 8 +- wikitextparser/_argument.py | 412 ++++++++++++++++++++++++----- wikitextparser/_parser_function.py | 113 +++----- wikitextparser/_template.py | 266 ++----------------- wikitextparser/_wikitext.py | 58 ++-- 10 files changed, 537 insertions(+), 505 deletions(-) diff --git a/tests/test_argument.py b/tests/test_argument.py index fc0680fe..43d0ac14 100644 --- a/tests/test_argument.py +++ b/tests/test_argument.py @@ -1,45 +1,47 @@ -from pytest import raises - from wikitextparser import Argument, Template, parse def test_basics(): a = Argument('| a = b ') - assert ' a ' == a.name - assert ' b ' == a.value - assert not a.positional + assert ' a ' == a.get_name(False) + assert ' b ' == a.get_value(False) + assert not a.is_positional(False) assert repr(a) == "Argument('| a = b ')" def test_anonymous_parameter(): a = Argument('| a ') - assert '1' == a.name - assert ' a ' == a.value + assert '1' == a.get_name(False) + assert ' a ' == a.get_value(False) def test_set_name(): a = Argument('| a = b ') - a.name = ' c ' + a.set_name(' c ', False) assert '| c = b ' == a.string + a.set_name(' c ', True) + assert '| c = c = b ' == a.string def test_set_name_at_subspan_boundary(): a = Argument('|{{ a }}={{ b }}') - a.name = ' c ' + a.set_name(' c ', False) assert '| c ={{ b }}' == a.string - assert '{{ b }}' == a.value + assert '{{ b }}' == a.get_value(False) def test_set_name_for_positional_args(): a = Argument('| b ') - a.name = a.name + a.set_name(a.get_name(False), False) assert '|1= b ' == a.string def test_value_setter(): a = Argument('| a = b ') - a.value = ' c ' + a.set_value(' c ', ignore_equals=False) assert '| a = c ' == a.string + a.set_value(' c ', ignore_equals=True) + assert '| c ' == a.string def test_removing_last_arg_should_not_effect_the_others(): @@ -51,42 +53,40 @@ def test_removing_last_arg_should_not_effect_the_others(): def test_nowikied_arg(): a = Argument('|1=3') - assert a.positional is True - assert '1' == a.name - assert '1=3' == a.value + assert a.is_positional(False) is True + assert '1' == a.get_name(False) + assert '1=3' == a.get_value(False) def test_value_after_convertion_of_positional_to_keywordk(): a = Argument("""|{{{a|{{{b}}}}}}""") - a.name = ' 1 ' - assert '{{{a|{{{b}}}}}}' == a.value + a.set_name(' 1 ', False) + assert '{{{a|{{{b}}}}}}' == a.get_value(False) def test_name_of_positionals(): assert ['1', '2', '3'] == [ - a.name for a in parse('{{t|a|b|c}}').templates[0].arguments + a.get_name(False) for a in parse('{{t|a|b|c}}').templates[0].arguments ] def test_dont_confuse_subspan_equal_with_keyword_arg_equal(): p = parse('{{text| {{text|1=first}} | b }}') a0, a1 = p.templates[0].arguments - assert ' {{text|1=first}} ' == a0.value - assert '1' == a0.name - assert ' b ' == a1.value - assert '2' == a1.name + assert ' {{text|1=first}} ' == a0.get_value(False) + assert '1' == a0.get_name(False) + assert ' b ' == a1.get_value(False) + assert '2' == a1.get_name(False) def test_setting_positionality(): a = Argument('|1=v') - a.positional = False + a.make_positional(True) assert '|1=v' == a.string - a.positional = True + a.make_positional(False) assert '|v' == a.string - a.positional = True + a.make_positional(False) assert '|v' == a.string - with raises(ValueError): - a.positional = False def test_parser_functions_at_the_end(): @@ -96,12 +96,12 @@ def test_parser_functions_at_the_end(): def test_section_not_keyword_arg(): a = Argument('|1=foo\n== section ==\nbar') - assert (a.name, a.value) == ('1', 'foo\n== section ==\nbar') + assert (a.get_name(False), a.get_value(False)) == ('1', 'foo\n== section ==\nbar') a = Argument('|\n==t==\nx') - assert (a.name, a.value) == ('1', '\n==t==\nx') + assert (a.get_name(False), a.get_value(False)) == ('1', '\n==t==\nx') # Following cases is not treated as a section headings a = Argument('|==1==\n') - assert (a.name, a.value) == ('', '=1==\n') + assert (a.get_name(False), a.get_value(False)) == ('', '=1==\n') # Todo: Prevents forming a template! # a = Argument('|\n==1==') # assert @@ -112,9 +112,9 @@ def test_argument_name_not_external_link(): # MediaWiki parses template parameters before external links, # so it goes with the named parameter in both cases. a = Argument('|[http://example.com?foo=bar]') - assert (a.name, a.value) == ('[http://example.com?foo', 'bar]') + assert (a.get_name(False), a.get_value(False)) == ('[http://example.com?foo', 'bar]') a = Argument('|http://example.com?foo=bar') - assert (a.name, a.value) == ('http://example.com?foo', 'bar') + assert (a.get_name(False), a.get_value(False)) == ('http://example.com?foo', 'bar') def test_lists(): @@ -128,13 +128,13 @@ def test_lists(): def test_equal_sign_in_val(): a, c = Template('{{t|a==b|c}}').arguments - assert a.value == '=b' - assert c.name == '1' + assert a.get_value(False) == '=b' + assert c.get_name(False) == '1' def test_tag_with_equal_sign(): - assert Argument('|aR').name == '1' + assert Argument('|aR').get_name(False) == '1' def test_section_heading_with_carriage_return_in_name(): - assert Argument('|a\r== heading ==\rb=c').name == 'a\r== heading ==\rb' + assert Argument('|a\r== heading ==\rb=c').get_name(False) == 'a\r== heading ==\rb' diff --git a/tests/test_cell.py b/tests/test_cell.py index 581379c0..2cb1f2be 100644 --- a/tests/test_cell.py +++ b/tests/test_cell.py @@ -81,7 +81,7 @@ def test_update_match_from_shadow(): assert c is not None assert c.value == '{{text|s}}' t = c.templates[0] - t.arguments[0].value = 't' + t.arguments[0].set_value('t', False) assert c.value == '{{text|t}}' diff --git a/tests/test_parser_function.py b/tests/test_parser_function.py index 48449ee5..2e226a4b 100644 --- a/tests/test_parser_function.py +++ b/tests/test_parser_function.py @@ -1,4 +1,4 @@ -from pytest import mark +from pytest import mark, raises from wikitextparser import ParserFunction, WikiText @@ -28,8 +28,8 @@ def test_name_and_args(): assert ' #if' == f.name args = f.arguments assert [': test ', '| true ', '| false '] == [a.string for a in args] - assert args[0].name == '1' - assert args[2].name == '3' + assert args[0].get_name(True) == '1' + assert args[2].get_name(True) == '3' def test_set_name(): @@ -77,8 +77,8 @@ def test_parser_function_alias_without_hash_sign(): def test_argument_with_existing_span(): """Test when the span is already in type_to_spans.""" pf = WikiText('{{formatnum:text}}').parser_functions[0] - assert pf.arguments[0].value == 'text' - assert pf.arguments[0].value == 'text' + assert pf.arguments[0].get_value(True) == 'text' + assert pf.arguments[0].get_value(True) == 'text' assert pf.string == '{{formatnum:text}}' @@ -88,26 +88,26 @@ def test_tag_containing_pipe(): def test_equal_in_if_expression(): pf = ParserFunction('{{#if: 2==2 | yes | no }}') - pf.arguments[0].value = '3' + pf.arguments[0].set_value('3', ignore_equals=True) assert pf.string == '{{#if:3| yes | no }}' def test_has_arg(): has_arg = ParserFunction('{{#pf:a|b=c}}').has_arg - assert has_arg('1') is True - assert has_arg('1', 'a') is True - assert has_arg('b') is False - assert has_arg('b', 'c') is False - assert has_arg('2') is True - assert has_arg('2', 'b=c') is True - assert has_arg('c') is False - assert has_arg('b', 'd') is False + assert has_arg('1', ignore_equals=True) is True + assert has_arg('1', 'a', ignore_equals=True) is True + assert has_arg('b', ignore_equals=True) is False + assert has_arg('b', 'c', ignore_equals=True) is False + assert has_arg('2', ignore_equals=True) is True + assert has_arg('2', 'b=c', ignore_equals=True) is True + assert has_arg('c', ignore_equals=True) is False + assert has_arg('b', 'd', ignore_equals=True) is False def test_get_arg(): get_arg = ParserFunction('{{#pf:a|b=c}}').get_arg - assert ':a' == get_arg('1').string # type: ignore - assert get_arg('c') is None + assert ':a' == get_arg('1', ignore_equals=True).string # type: ignore + assert get_arg('c', ignore_equals=True) is None def test_name_contains_a_param_with_default(): @@ -120,28 +120,37 @@ def test_name_contains_a_param_with_default(): def test_set_arg(): t = ParserFunction('{{#pf}}') - t.set_arg('1', 'b') + t.set_arg('1', 'b', ignore_equals=True) assert '{{#pf:b}}' == t.string t = ParserFunction('{{#pf:a}}') - t.set_arg('1', 'b') + t.set_arg('1', 'b', ignore_equals=True) assert '{{#pf:b}}' == t.string t = ParserFunction('{{#pf:a|b}}') - t.set_arg('2', 'c') + t.set_arg('2', 'c', ignore_equals=True) assert '{{#pf:a|c}}' == t.string + with raises(ValueError): + t = ParserFunction('{{#pf:a|b}}') + t.set_arg('4', 'c', ignore_equals=True) + with raises(ValueError): + t = ParserFunction('{{#pf:a|b}}') + t.set_arg('xd', 'c', ignore_equals=True) t = ParserFunction('{{#pf:a|b}}') - t.set_arg('4', 'c') - assert '{{#pf:a|b}}' == t.string + t.set_arg('4', 'c', ignore_equals=False) + assert '{{#pf:a|b|4=c}}' == t.string t = ParserFunction('{{#pf:a|b}}') - t.set_arg('xd', 'c') - assert '{{#pf:a|b}}' == t.string + t.set_arg('xd', 'c', ignore_equals=False) + assert '{{#pf:a|b|xd=c}}' == t.string + with raises(ValueError): + t = ParserFunction('{{#pf:a|b}}') + t.set_arg('2', 'c', False, ignore_equals=False) def test_del_arg(): t = ParserFunction('{{#pf:a}}') - t.del_arg('1') + t.del_arg('1', ignore_equals=True) assert '{{#pf}}' == t.string t = ParserFunction('{{#pf:a|b}}') - t.del_arg('2') + t.del_arg('2', ignore_equals=True) assert '{{#pf:a}}' == t.string @@ -153,3 +162,9 @@ def test_lists(): 'https://a.b ', 'd', ] + + +def test_get_last_positional_index(): + t = ParserFunction('{{#pf:a|b|c=d}}') + assert t.get_last_positional_index(ignore_equals=False) == 2 + assert t.get_last_positional_index(ignore_equals=True) == 3 diff --git a/tests/test_spans.py b/tests/test_spans.py index 56e30ca6..aa9c77fa 100644 --- a/tests/test_spans.py +++ b/tests/test_spans.py @@ -208,29 +208,29 @@ def test_keyword_and_positional_args_removal(): t1, t2 = wt.templates t1_args = t1.arguments t2_args = t2.arguments - assert '1' == t1_args[2].name - assert 'kw2' == t1_args[3].name - assert '2' == t1_args[4].name - assert '1' == t2_args[0].name - assert '2' == t2_args[1].name - assert '1' == t2_args[2].name + assert '1' == t1_args[2].get_name(False) + assert 'kw2' == t1_args[3].get_name(False) + assert '2' == t1_args[4].get_name(False) + assert '1' == t2_args[0].get_name(False) + assert '2' == t2_args[1].get_name(False) + assert '1' == t2_args[2].get_name(False) del t1_args[0][:] t1_args = t1.arguments t2_args = t2.arguments - assert '1' == t1_args[0].name - assert 'kw2' == t1_args[2].name + assert '1' == t1_args[0].get_name(False) + assert 'kw2' == t1_args[2].get_name(False) assert '|pa2' == t1_args[3].string - assert '1' == t2_args[0].name - assert '2' == t2_args[1].name - assert '1' == t2_args[2].name + assert '1' == t2_args[0].get_name(False) + assert '2' == t2_args[1].get_name(False) + assert '1' == t2_args[2].get_name(False) del t1_args[1][:] t1_args = t1.arguments t2_args = t2.arguments assert 'text{{t1|1=|kw2=a|pa2}}{{t2|a|1|1=}}text' == wt.string - assert 'pa2' == t1_args[2].value - assert '1' == t1_args[2].name - assert 'a' == t2_args[0].value - assert '1' == t2_args[0].name + assert 'pa2' == t1_args[2].get_value(False) + assert '1' == t1_args[2].get_name(False) + assert 'a' == t2_args[0].get_value(False) + assert '1' == t2_args[0].get_name(False) def test_parser_function_regex(): diff --git a/tests/test_template.py b/tests/test_template.py index 06a56b43..d4ae7e36 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -1,4 +1,4 @@ -from pytest import mark +from pytest import mark, raises from wikitextparser import Template @@ -70,7 +70,7 @@ def test_normal_name(): def test_keyword_and_positional_args(): - assert '1' == Template('{{t|kw=a|1=|pa|kw2=a|pa2}}').arguments[2].name + assert '1' == Template('{{t|kw=a|1=|pa|kw2=a|pa2}}').arguments[2].get_name(False) def test_rm_first_of_dup_args(): @@ -235,6 +235,9 @@ def test_set_arg(): t = Template('{{t\n | p1 = v1\n | p22 = v2\n}}') t.set_arg('z', 'z', preserve_spacing=True) assert '{{t\n | p1 = v1\n | p22 = v2\n | z = z\n}}' == t.string + with raises(ValueError): + t = Template('{{t|a|b|c}}') + t.set_arg('3', 'z', False) @mark.parametrize('newline', ['\n', '\r', '\r\n']) @@ -307,8 +310,8 @@ def test_force_new_to_positional_when_old_is_keyword(): def test_nowiki_makes_equal_ineffective(): a = Template('{{text|1=g}}').arguments[0] - assert a.value == '1=g' - assert a.name == '1' + assert a.get_value(False) == '1=g' + assert a.get_name(False) == '1' def test_not_name_and_positional_is_none(): @@ -343,3 +346,8 @@ def test_preserve_spacing_left_and_right(): def test_invalid_normal_name(): # 105 assert '' == Template('{{template:}}').normal_name(capitalize=True) + + +def test_get_last_positional_index(): + t = Template('{{t|a|b|c=d}}') + assert t.get_last_positional_index() == 2 diff --git a/tests/wikitext/test_wikitext.py b/tests/wikitext/test_wikitext.py index fe75d1eb..0fe2d5fc 100644 --- a/tests/wikitext/test_wikitext.py +++ b/tests/wikitext/test_wikitext.py @@ -93,8 +93,8 @@ def test_overwriting_template_args(): assert '|c' == c.string t.string = '{{t|0|a|b|c}}' assert '' == c.string - assert '0' == t.get_arg('1').value # type: ignore - assert 'c' == t.get_arg('4').value # type: ignore + assert '0' == t.get_arg('1').get_value(False) # type: ignore + assert 'c' == t.get_arg('4').get_value(False) # type: ignore def test_delitem(): @@ -217,12 +217,12 @@ def test_overwriting_or_extending_selfspan_will_cause_data_loss(): wt = WikiText('{{t|{{#if:a|b|c}}}}') a = wt.templates[0].arguments[0] pf = wt.parser_functions[0] - a.value += '' + a.set_value(a.get_value(False) + '', False) assert '|{{#if:a|b|c}}' == a.string # Note that the old parser function is overwritten assert '' == pf.string pf = a.parser_functions[0] - a.value = 'a' + a.set_value('a', False) assert '' == pf.string diff --git a/wikitextparser/_argument.py b/wikitextparser/_argument.py index 74fb6ce1..b7aeec53 100644 --- a/wikitextparser/_argument.py +++ b/wikitextparser/_argument.py @@ -1,13 +1,14 @@ from __future__ import annotations from bisect import insort -from collections.abc import Iterable, MutableSequence +from collections.abc import Callable, Iterable, MutableSequence +from typing import ClassVar, TypeVar -from regex import DOTALL, Match +from regex import DOTALL, REVERSE, Match from ._spans import TypeToSpans from ._wikilist import WikiList -from ._wikitext import SECTION_HEADING, SubWikiText, rc +from ._wikitext import SECTION_HEADING, WS, SubWikiText, rc ARG_SHADOW_FULLMATCH = rc( rb'[|:](?(?:[^=]*+(?:' @@ -15,6 +16,11 @@ + rb'\R)?+)*+)(?:\Z|(?=)(?.*+))', DOTALL, ).fullmatch +STARTING_WS_MATCH = rc(r'\s*+').match +ENDING_WS_MATCH = rc(r'(?>\R[ \t]*)*+', REVERSE).match +SPACE_AFTER_SEARCH = rc(r'\s*+(?=\|)').search + +T = TypeVar('T') class Argument(SubWikiText): @@ -26,7 +32,7 @@ class Argument(SubWikiText): See https://www.mediawiki.org/wiki/Help:Templates for more information. """ - __slots__ = '_ignore_equals', '_parent', '_shadow_match_cache' + __slots__ = '_parent', '_shadow_match_cache' def __init__( self, @@ -37,7 +43,6 @@ def __init__( _parent: SubWikiTextWithArgs | None = None, ): super().__init__(string, _type_to_spans, _span, _type) - self._ignore_equals = _parent._ignore_equals if _parent != None else False self._parent = _parent or self self._shadow_match_cache = None, None @@ -54,8 +59,7 @@ def _shadow_match(self) -> Match[bytes]: self._shadow_match_cache = shadow_match, self_string return shadow_match # type: ignore - @property - def name(self) -> str: + def get_name(self, ignore_equals: bool) -> str: """Argument's name. getter: return the position as a string, for positional arguments. @@ -63,7 +67,7 @@ def name(self) -> str: """ ss = self._span_data[0] shadow_match = self._shadow_match - if not self._ignore_equals and shadow_match['eq']: + if not ignore_equals and shadow_match['eq']: s, e = shadow_match.span('pre_eq') return self._lststr[0][ss + s : ss + e] # positional argument @@ -80,15 +84,13 @@ def name(self) -> str: position += 1 return str(position) - @name.setter - def name(self, newname: str) -> None: - if not self._ignore_equals and self._shadow_match['eq']: + def set_name(self, newname: str, ignore_equals: bool) -> None: + if not ignore_equals and self._shadow_match['eq']: self[1 : 1 + len(self._shadow_match['pre_eq'])] = newname else: self.insert(1, newname + '=') - @property - def positional(self) -> bool: + def is_positional(self, ignore_equals: bool) -> bool: """True if self is positional, False if keyword. setter: @@ -96,29 +98,14 @@ def positional(self) -> bool: Raise ValueError on trying to convert positional to keyword argument. """ - return self._ignore_equals or not self._shadow_match['eq'] + return ignore_equals or not self._shadow_match['eq'] - @positional.setter - def positional(self, to_positional: bool) -> None: + def make_positional(self, ignore_equals: bool) -> None: shadow_match = self._shadow_match - if not self._ignore_equals and shadow_match['eq']: - # Keyword argument - if to_positional: - del self[1 : shadow_match.end('eq')] - else: - return - if to_positional: - # Positional argument. to_positional is True. - return - # Positional argument. to_positional is False. - raise ValueError( - 'Converting positional argument to keyword argument is not ' - 'possible without knowing the new name. ' - 'You can use `self.name = somename` instead.' - ) + if not ignore_equals and shadow_match['eq']: + del self[1 : shadow_match.end('eq')] - @property - def value(self) -> str: + def get_value(self, ignore_equals: bool) -> str: """Value of self. Support both keyword or positional arguments. @@ -128,14 +115,13 @@ def value(self) -> str: Assign a new value to self. """ shadow_match = self._shadow_match - if not self._ignore_equals and shadow_match['eq']: + if not ignore_equals and shadow_match['eq']: return self(shadow_match.start('post_eq'), None) return self(1, None) - @value.setter - def value(self, newvalue: str) -> None: + def set_value(self, newvalue: str, ignore_equals: bool) -> None: shadow_match = self._shadow_match - if not self._ignore_equals and shadow_match['eq']: + if not ignore_equals and shadow_match['eq']: self[shadow_match.start('post_eq') :] = newvalue else: self[1:] = newvalue @@ -143,7 +129,7 @@ def value(self, newvalue: str) -> None: @property def _lists_shadow_ss(self): shadow_match = self._shadow_match - if not self._ignore_equals and shadow_match['eq']: + if shadow_match['eq']: post_eq = shadow_match['post_eq'] ls_post_eq = post_eq.lstrip() return ( @@ -155,15 +141,24 @@ def _lists_shadow_ss(self): ) return bytearray(shadow_match[0][1:]), self._span_data[0] + 1 - class SubWikiTextWithArgs(SubWikiText): """Define common attributes for `Template` and `ParserFunction`.""" - __slots__ = () + __slots__ = ('_arguments_cache', '_first_arg_sep', '_name_args_matcher', '_shadow_match_cache') - _name_args_matcher = NotImplemented - _first_arg_sep = 0 - _ignore_equals = False + _name_args_matcher: ClassVar[Callable] + _first_arg_sep: ClassVar[int] + + def __init__( + self, + string: str | MutableSequence[str], + _type_to_spans: TypeToSpans | None = None, + _span: list | None = None, + _type: str | int | None = None, + ) -> None: + self._arguments_cache = tuple[Argument]() + self._shadow_match_cache = None, None + super().__init__(string, _type_to_spans, _span, _type) @property def _content_span(self) -> tuple[int, int]: @@ -179,37 +174,46 @@ def nesting_level(self) -> int: return self._nesting_level(('Template', 'ParserFunction')) @property - def arguments(self) -> list[Argument]: + def arguments(self) -> tuple[Argument]: """Parse template content. Create self.name and self.arguments.""" + cached_shadow_match, cache_string = self._shadow_match_cache + self_string = str(self) + if cache_string == self_string: + return self._arguments_cache + shadow = self._shadow - split_spans = self._name_args_matcher(shadow, 2, -2).spans('arg') - if not split_spans: - return [] + shadow_match = self._name_args_matcher(shadow, 2, -2) + split_spans = shadow_match.spans('arg') arguments = [] - arguments_append = arguments.append - type_to_spans = self._type_to_spans - ss, se, _, _ = span = self._span_data - type_ = id(span) - lststr = self._lststr - arg_spans = type_to_spans.setdefault(type_, []) - span_tuple_to_span_get = {(s[0], s[1]): s for s in arg_spans}.get - for arg_self_start, arg_self_end in split_spans: - # todo: add byte array - s, e, _, _ = arg_span = [ - ss + arg_self_start, - ss + arg_self_end, - None, - None, - ] - old_span = span_tuple_to_span_get((s, e)) - if old_span is None: - insort(arg_spans, arg_span) - else: - arg_span = old_span - arg = Argument(lststr, type_to_spans, arg_span, type_, self) - arg._span_data[3] = shadow[arg_self_start:arg_self_end] - arguments_append(arg) - return arguments + + if split_spans: + arguments_append = arguments.append + type_to_spans = self._type_to_spans + ss, se, _, _ = span = self._span_data + type_ = id(span) + lststr = self._lststr + arg_spans = type_to_spans.setdefault(type_, []) + span_tuple_to_span_get = {(s[0], s[1]): s for s in arg_spans}.get + for arg_self_start, arg_self_end in split_spans: + # todo: add byte array + s, e, _, _ = arg_span = [ + ss + arg_self_start, + ss + arg_self_end, + None, + None, + ] + old_span = span_tuple_to_span_get((s, e)) + if old_span is None: + insort(arg_spans, arg_span) + else: + arg_span = old_span + arg = Argument(lststr, type_to_spans, arg_span, type_, self) + arg._span_data[3] = shadow[arg_self_start:arg_self_end] + arguments_append(arg) + + self._shadow_match_cache = shadow_match, self_string + self._arguments_cache = tuple(arguments) + return self._arguments_cache def get_lists( self, pattern: str | Iterable[str] = (r'\#', r'\*', '[:;]') @@ -241,3 +245,269 @@ def name(self) -> str: @name.setter def name(self, newname: str) -> None: self[2 : 2 + len(self.name)] = newname + + def rm_first_of_dup_args(self) -> None: + """Eliminate duplicate arguments by removing the first occurrences. + + Remove the first occurrences of duplicate arguments, regardless of + their value. Result of the rendered wikitext should remain the same. + Warning: Some meaningful data may be removed from wikitext. + + Also see `rm_dup_args_safe` function. + """ + names = set() + for a in reversed(self.arguments): + name = a.get_name(False).strip(WS) + if name in names: + del a[: len(a.string)] + else: + names.add(name) + + def rm_dup_args_safe(self, tag: str | None = None) -> None: + """Remove duplicate arguments in a safe manner. + + Remove the duplicate arguments only in the following situations: + 1. Both arguments have the same name AND value. (Remove one of + them.) + 2. Arguments have the same name and one of them is empty. (Remove + the empty one.) + + Warning: Although this is considered to be safe and no meaningful data + is removed from wikitext, but the result of the rendered wikitext + may actually change if the second arg is empty and removed but + the first had had a value. + + If `tag` is defined, it should be a string that will be appended to + the value of the remaining duplicate arguments. + + Also see `rm_first_of_dup_args` function. + """ + name_to_lastarg_vals: dict[str, tuple[Argument, list[str]]] = {} + # Removing positional args affects their name. By reversing the list + # we avoid encountering those kind of args. + for arg in reversed(self.arguments): + name = arg.get_name(False).strip(WS) + if arg.is_positional(False): + # Value of keyword arguments is automatically stripped by MW. + val = arg.get_value(False) + else: + # But it's not OK to strip whitespace in positional arguments. + val = arg.get_value(False).strip(WS) + if name in name_to_lastarg_vals: + # This is a duplicate argument. + if not val: + # This duplicate argument is empty. It's safe to remove it. + del arg[0 : len(arg.string)] + else: + # Try to remove any of the detected duplicates of this + # that are empty or their value equals to this one. + lastarg, dup_vals = name_to_lastarg_vals[name] + if val in dup_vals: + del arg[0 : len(arg.string)] + elif '' in dup_vals: + # This happens only if the last occurrence of name has + # been an empty string; other empty values will + # be removed as they are seen. + # In other words index of the empty argument in + # dup_vals is always 0. + del lastarg[0 : len(lastarg.string)] + dup_vals.pop(0) + else: + # It was not possible to remove any of the duplicates. + dup_vals.append(val) + if tag: + arg.set_value(arg.get_value(False) + tag, False) + else: + name_to_lastarg_vals[name] = (arg, [val]) + + def _get_last_positional_index(self, *, ignore_equals: bool) -> int: + idx = 0 + for arg in self.arguments: + if arg.is_positional(ignore_equals): + idx += 1 + return idx + + def _get_arg(self, name: str, *, ignore_equals: bool) -> Argument | None: + """Return the last argument with the given name. + + Return None if no argument with that name is found. + """ + for arg in reversed(self.arguments): + if arg.get_name(ignore_equals).strip(WS) == name.strip(WS): + return arg + return None + + def _has_arg(self, name: str, value: str | None, *, ignore_equals: bool) -> bool: + """Return true if there is an arg named `name`. + + Also check equality of values if `value` is provided. + + Note: If you just need to get an argument and you want to LBYL, it's + better to get_arg directly and then check if the returned value + is None. + """ + for arg in reversed(self.arguments): + if arg.get_name(ignore_equals).strip(WS) == name.strip(WS): + if value: + if arg.is_positional(ignore_equals): + return arg.get_value(ignore_equals) == value + return arg.get_value(ignore_equals).strip(WS) == value.strip(WS) + return True + return False + + def _set_arg( + self, + name: str | None, + value: str, + positional: bool | None, + before: str | None, + after: str | None, + preserve_spacing: bool | None, + *, + ignore_equals: bool, + ) -> None: + """Set the value for `name` argument. Add it if it doesn't exist. + + - Use `positional`, `before` and `after` keyword arguments only when + adding a new argument. + - If `before` is given, ignore `after`. + - If neither `before` nor `after` are given and it's needed to add a + new argument, then append the new argument to the end. + - If `positional` is True, try to add the given value as a positional + argument. Ignore `preserve_spacing` if positional is True. + If it's None, do what seems more appropriate. + """ + + if name is not None: + arg = self._get_arg(name, ignore_equals=ignore_equals) + # Updating an existing argument. + if arg: + if positional == True: + arg.make_positional(ignore_equals) + if positional == False and arg.is_positional(ignore_equals): + raise ValueError( + 'Converting positional argument to keyword argument is not ' + 'possible without knowing the new name. ' + 'You can use `self.set_name` instead.' + ) + if preserve_spacing: + val = arg.get_value(ignore_equals) + arg.set_value(val.replace(val.strip(WS), value, 1), ignore_equals) + else: + arg.set_value(value, ignore_equals) + return + # Adding a new argument + if not name: + positional = True + else: + if not is_positive_integer(name) or self._get_last_positional_index(ignore_equals=ignore_equals) != int(name) - 1: + positional = False + + if ignore_equals == True: + if positional == None: + positional = True + if positional == False: + raise ValueError( + 'positional = False is not supported for ignore_equals = True' + ) + + # Calculate the whitespace needed before arg-name and after arg-value. + if not positional and preserve_spacing and len(self.arguments) > 0: + before_names = [] + name_lengths = [] + before_values = [] + after_values = [] + for arg in reversed(self.arguments): + aname = arg.get_name(ignore_equals) + name_len = len(aname) + name_lengths.append(name_len) + before_names.append(STARTING_WS_MATCH(aname)[0]) # type: ignore + arg_value = arg.get_value(ignore_equals) + before_values.append(STARTING_WS_MATCH(arg_value)[0]) # type: ignore + after_values.append(ENDING_WS_MATCH(arg_value)[0]) # type: ignore + pre_name_ws_mode = mode(before_names) + name_length_mode = mode(name_lengths) + post_value_ws_mode = mode( + [SPACE_AFTER_SEARCH(self.string)[0], *after_values[1:]] # type: ignore + ) + pre_value_ws_mode = mode(before_values) + else: + preserve_spacing = False + # Calculate the string that needs to be added to the Template. + addsep = chr(self._first_arg_sep) if len(self.arguments) == 0 else '|' + if positional: + # Ignore preserve_spacing for positional args. + addstring = addsep + value + else: + assert(name) # To keep the compiler happy + if preserve_spacing: + addstring = ( + addsep + + (pre_name_ws_mode + name.strip(WS)).ljust( # type: ignore + name_length_mode # type: ignore + ) + + '=' + + pre_value_ws_mode # type: ignore + + value + + post_value_ws_mode # type: ignore + ) + else: + addstring = addsep + name + '=' + value + # Place the addstring in the right position. + if before: + arg = self._get_arg(before, ignore_equals=ignore_equals) + arg.insert(0, addstring) # type: ignore + elif after: + arg = self._get_arg(after, ignore_equals=ignore_equals) + arg.insert(len(arg.string), addstring) # type: ignore + else: + if len(self.arguments) > 0 and not positional: + arg = self.arguments[-1] + arg_string = arg.string + if preserve_spacing: + # Insert after the last argument. + # The addstring needs to be recalculated because we don't + # want to change the the whitespace before final braces. + # noinspection PyUnboundLocalVariable + arg[0 : len(arg_string)] = ( + arg.string.rstrip(WS) + + post_value_ws_mode # type: ignore + + addstring.rstrip(WS) + + after_values[0] # type: ignore + ) + else: + arg.insert(len(arg_string), addstring) + else: + # The template has no arguments or the new arg is + # positional AND is to be added at the end of the template. + self.insert(-2, addstring) + + def _del_arg(self, name: str, ignore_equals: bool) -> None: + """Delete all arguments with the given then.""" + for arg in reversed(self.arguments): + if arg.get_name(ignore_equals).strip(WS) == name.strip(WS): + del arg[:] + + +def is_positive_integer(x): + try: + return int(x) > 0 + except ValueError: + return False + +def mode(list_: list[T]) -> T: + """Return the most common item in the list. + + Return the first one if there are more than one most common items. + + Example: + + >>> mode([1,1,2,2,]) + 1 + >>> mode([1,2,2]) + 2 + >>> mode([]) + ... + ValueError: max() arg is an empty sequence + """ + return max(set(list_), key=list_.count) diff --git a/wikitextparser/_parser_function.py b/wikitextparser/_parser_function.py index a066f664..8cd3dd98 100644 --- a/wikitextparser/_parser_function.py +++ b/wikitextparser/_parser_function.py @@ -1,7 +1,5 @@ from __future__ import annotations -from collections.abc import Iterable - from ._argument import Argument, SubWikiTextWithArgs from ._comment_bold_italic import COMMENT_PATTERN from ._wikitext import WS, rc @@ -22,56 +20,28 @@ class ParserFunction(SubWikiTextWithArgs): _name_args_matcher = PF_NAME_ARGS_FULLMATCH _first_arg_sep = 58 - _ignore_equals = True def normal_name(self) -> str: """Return normal form of self.name. - Remove comments. - - Lowercase + - Lowercase. """ return COMMENT_SUB('', self.name).lstrip(WS).lower() - def set_arg( - self, - name: str | None, - value: str, - ) -> None: - """Set the value for `name` argument. Add it if it doesn't exist. - """ - args = (*reversed(self.arguments),) - if name is not None: - # Invalid - if not is_positive_integer(name): - return - - # Updating an existing argument. - arg = get_arg(name, args) - if arg: - arg.positional = True - arg.value = value - return - - last_idx = get_last_idx_positional_args(args) - - # Invalid, as it would need to fill the pf with empty arguments - if name and (last_idx != int(name) - 1): - return - - # Adding a new argument - addstring = (':' if last_idx == 0 else '|') + value - self.insert(-2, addstring) - - def get_arg(self, name: str) -> Argument | None: + def get_last_positional_index(self, ignore_equals: bool) -> int: + return super()._get_last_positional_index(ignore_equals=ignore_equals) + + def get_arg(self, name: str, *, ignore_equals: bool) -> Argument | None: """Return the last argument with the given name. Return None if no argument with that name is found. """ - return get_arg(name, reversed(self.arguments)) + return super()._get_arg(name, ignore_equals=ignore_equals) - def has_arg(self, name: str, value: str | None = None) -> bool: - """Return true if the is an arg named `name`. + def has_arg(self, name: str, value: str | None = None, *, ignore_equals: bool) -> bool: + """Return true if there is an arg named `name`. Also check equality of values if `value` is provided. @@ -79,47 +49,36 @@ def has_arg(self, name: str, value: str | None = None) -> bool: better to get_arg directly and then check if the returned value is None. """ - for arg in reversed(self.arguments): - if arg.name.strip(WS) == name.strip(WS): - if value: - return arg.value == value - return True - return False - - def del_arg(self, name: str) -> None: + return super()._has_arg(name, value, ignore_equals=ignore_equals) + + def set_arg( + self, + name: str | None, + value: str, + positional: bool | None = None, + before: str | None = None, + after: str | None = None, + preserve_spacing: bool = False, + *, + ignore_equals: bool + ) -> None: + """Set the value for `name` argument. Add it if it doesn't exist. + + - Use `positional`, `before` and `after` keyword arguments only when + adding a new argument. + - If `before` is given, ignore `after`. + - If neither `before` nor `after` are given and it's needed to add a + new argument, then append the new argument to the end. + - If `positional` is True, try to add the given value as a positional + argument. Ignore `preserve_spacing` if positional is True. + If it's None, do what seems more appropriate. + """ + super()._set_arg(name, value, positional, before, after, preserve_spacing, ignore_equals=ignore_equals) + + def del_arg(self, name: str, *, ignore_equals: bool) -> None: """Delete all arguments with the given then.""" - for arg in reversed(self.arguments): - if arg.name.strip(WS) == name.strip(WS): - del arg[:] + super()._del_arg(name, ignore_equals=ignore_equals) @property def parser_functions(self) -> list[ParserFunction]: return super().parser_functions[1:] - - -def is_positive_integer(x): - try: - return int(x) > 0 - except ValueError: - return False - -def get_arg(name: str, args: Iterable[Argument]) -> Argument | None: - """Return the first argument in the args that has the given name. - - Return None if no such argument is found. - - As the computation of self.arguments is a little costly, this - function was created so that other methods that have already computed - the arguments use it instead of calling self.get_arg directly. - """ - for arg in args: - if arg.name.strip(WS) == name.strip(WS): - return arg - return None - -def get_last_idx_positional_args(args: Iterable[Argument]) -> int: - idx = 0 - for arg in args: - if arg.positional: - idx += 1 - return idx diff --git a/wikitextparser/_template.py b/wikitextparser/_template.py index 6d7ac3f2..45f289c1 100644 --- a/wikitextparser/_template.py +++ b/wikitextparser/_template.py @@ -1,10 +1,7 @@ from __future__ import annotations -from collections.abc import Iterable from typing import TypeVar -from regex import REVERSE - from ._argument import Argument, SubWikiTextWithArgs from ._comment_bold_italic import COMMENT_PATTERN from ._wikitext import WS, rc @@ -12,9 +9,6 @@ COMMENT_SUB = rc(COMMENT_PATTERN).sub TL_NAME_ARGS_FULLMATCH = rc(rb'[^|}]*+(?#name)(?\|[^|]*+)*+').fullmatch -STARTING_WS_MATCH = rc(r'\s*+').match -ENDING_WS_MATCH = rc(r'(?>\R[ \t]*)*+', REVERSE).match -SPACE_AFTER_SEARCH = rc(r'\s*+(?=\|)').search T = TypeVar('T') @@ -29,11 +23,7 @@ class Template(SubWikiTextWithArgs): _name_args_matcher = TL_NAME_ARGS_FULLMATCH _first_arg_sep = 124 - _ignore_equals = False - @property - def _content_span(self) -> tuple[int, int]: - return 2, -2 def normal_name( self, @@ -96,79 +86,26 @@ def normal_name( name, sep, tail = name.partition('#') return ' '.join(name.split()) - def rm_first_of_dup_args(self) -> None: - """Eliminate duplicate arguments by removing the first occurrences. + def get_last_positional_index(self) -> int: + return super()._get_last_positional_index(ignore_equals=False) - Remove the first occurrences of duplicate arguments, regardless of - their value. Result of the rendered wikitext should remain the same. - Warning: Some meaningful data may be removed from wikitext. + def get_arg(self, name: str) -> Argument | None: + """Return the last argument with the given name. - Also see `rm_dup_args_safe` function. + Return None if no argument with that name is found. """ - names = set() - for a in reversed(self.arguments): - name = a.name.strip(WS) - if name in names: - del a[: len(a.string)] - else: - names.add(name) + return super()._get_arg(name, ignore_equals=False) - def rm_dup_args_safe(self, tag: str | None = None) -> None: - """Remove duplicate arguments in a safe manner. - - Remove the duplicate arguments only in the following situations: - 1. Both arguments have the same name AND value. (Remove one of - them.) - 2. Arguments have the same name and one of them is empty. (Remove - the empty one.) - - Warning: Although this is considered to be safe and no meaningful data - is removed from wikitext, but the result of the rendered wikitext - may actually change if the second arg is empty and removed but - the first had had a value. + def has_arg(self, name: str, value: str | None = None) -> bool: + """Return true if there is an arg named `name`. - If `tag` is defined, it should be a string that will be appended to - the value of the remaining duplicate arguments. + Also check equality of values if `value` is provided. - Also see `rm_first_of_dup_args` function. + Note: If you just need to get an argument and you want to LBYL, it's + better to get_arg directly and then check if the returned value + is None. """ - name_to_lastarg_vals: dict[str, tuple[Argument, list[str]]] = {} - # Removing positional args affects their name. By reversing the list - # we avoid encountering those kind of args. - for arg in reversed(self.arguments): - name = arg.name.strip(WS) - if arg.positional: - # Value of keyword arguments is automatically stripped by MW. - val = arg.value - else: - # But it's not OK to strip whitespace in positional arguments. - val = arg.value.strip(WS) - if name in name_to_lastarg_vals: - # This is a duplicate argument. - if not val: - # This duplicate argument is empty. It's safe to remove it. - del arg[0 : len(arg.string)] - else: - # Try to remove any of the detected duplicates of this - # that are empty or their value equals to this one. - lastarg, dup_vals = name_to_lastarg_vals[name] - if val in dup_vals: - del arg[0 : len(arg.string)] - elif '' in dup_vals: - # This happens only if the last occurrence of name has - # been an empty string; other empty values will - # be removed as they are seen. - # In other words index of the empty argument in - # dup_vals is always 0. - del lastarg[0 : len(lastarg.string)] - dup_vals.pop(0) - else: - # It was not possible to remove any of the duplicates. - dup_vals.append(val) - if tag: - arg.value += tag - else: - name_to_lastarg_vals[name] = (arg, [val]) + return super()._has_arg(name, value, ignore_equals=False) def set_arg( self, @@ -177,188 +114,25 @@ def set_arg( positional: bool | None = None, before: str | None = None, after: str | None = None, - preserve_spacing=False, + preserve_spacing: bool = False, ) -> None: """Set the value for `name` argument. Add it if it doesn't exist. - Use `positional`, `before` and `after` keyword arguments only when - adding a new argument. + adding a new argument. - If `before` is given, ignore `after`. - If neither `before` nor `after` are given and it's needed to add a - new argument, then append the new argument to the end. + new argument, then append the new argument to the end. - If `positional` is True, try to add the given value as a positional - argument. Ignore `preserve_spacing` if positional is True. - If it's None, do what seems more appropriate. - """ - args = (*reversed(self.arguments),) - if name is not None: - arg = get_arg(name, args) - # Updating an existing argument. - if arg: - if positional == True: - arg.positional = True - # if positional == False but arg.positional == true, then - # positional.setter of SubWikiText will raise an exception - if preserve_spacing: - val = arg.value - arg.value = val.replace(val.strip(WS), value, 1) - else: - arg.value = value - return - # Adding a new argument - if not name: - positional = True - else: - if positional and not is_positive_integer(name): - positional = False - if positional and get_last_idx_positional_args(args) != int(name) - 1: - positional = False - # Calculate the whitespace needed before arg-name and after arg-value. - if not positional and preserve_spacing and args: - before_names = [] - name_lengths = [] - before_values = [] - after_values = [] - for arg in args: - aname = arg.name - name_len = len(aname) - name_lengths.append(name_len) - before_names.append(STARTING_WS_MATCH(aname)[0]) # type: ignore - arg_value = arg.value - before_values.append(STARTING_WS_MATCH(arg_value)[0]) # type: ignore - after_values.append(ENDING_WS_MATCH(arg_value)[0]) # type: ignore - pre_name_ws_mode = mode(before_names) - name_length_mode = mode(name_lengths) - post_value_ws_mode = mode( - [SPACE_AFTER_SEARCH(self.string)[0], *after_values[1:]] # type: ignore - ) - pre_value_ws_mode = mode(before_values) - else: - preserve_spacing = False - # Calculate the string that needs to be added to the Template. - if positional: - # Ignore preserve_spacing for positional args. - addstring = '|' + value - else: - assert(name) # To keep the compiler happy - if preserve_spacing: - addstring = ( - '|' - + (pre_name_ws_mode + name.strip(WS)).ljust( # type: ignore - name_length_mode # type: ignore - ) - + '=' - + pre_value_ws_mode # type: ignore - + value - + post_value_ws_mode # type: ignore - ) - else: - addstring = '|' + name + '=' + value - # Place the addstring in the right position. - if before: - arg = get_arg(before, args) - arg.insert(0, addstring) # type: ignore - elif after: - arg = get_arg(after, args) - arg.insert(len(arg.string), addstring) # type: ignore - else: - if args and not positional: - arg = args[0] - arg_string = arg.string - if preserve_spacing: - # Insert after the last argument. - # The addstring needs to be recalculated because we don't - # want to change the the whitespace before final braces. - # noinspection PyUnboundLocalVariable - arg[0 : len(arg_string)] = ( - arg.string.rstrip(WS) - + post_value_ws_mode # type: ignore - + addstring.rstrip(WS) - + after_values[0] # type: ignore - ) - else: - arg.insert(len(arg_string), addstring) - else: - # The template has no arguments or the new arg is - # positional AND is to be added at the end of the template. - self.insert(-2, addstring) - - def get_arg(self, name: str) -> Argument | None: - """Return the last argument with the given name. - - Return None if no argument with that name is found. + argument. Ignore `preserve_spacing` if positional is True. + If it's None, do what seems more appropriate. """ - return get_arg(name, reversed(self.arguments)) - - def has_arg(self, name: str, value: str | None = None) -> bool: - """Return true if the is an arg named `name`. - - Also check equality of values if `value` is provided. - - Note: If you just need to get an argument and you want to LBYL, it's - better to get_arg directly and then check if the returned value - is None. - """ - for arg in reversed(self.arguments): - if arg.name.strip(WS) == name.strip(WS): - if value: - if arg.positional: - return arg.value == value - return arg.value.strip(WS) == value.strip(WS) - return True - return False + super()._set_arg(name, value, positional, before, after, preserve_spacing, ignore_equals=False) def del_arg(self, name: str) -> None: """Delete all arguments with the given then.""" - for arg in reversed(self.arguments): - if arg.name.strip(WS) == name.strip(WS): - del arg[:] + super()._del_arg(name, ignore_equals=False) @property def templates(self) -> list[Template]: return super().templates[1:] - -def is_positive_integer(x): - try: - return int(x) > 0 - except ValueError: - return False - -def mode(list_: list[T]) -> T: - """Return the most common item in the list. - - Return the first one if there are more than one most common items. - - Example: - - >>> mode([1,1,2,2,]) - 1 - >>> mode([1,2,2]) - 2 - >>> mode([]) - ... - ValueError: max() arg is an empty sequence - """ - return max(set(list_), key=list_.count) - - -def get_arg(name: str, args: Iterable[Argument]) -> Argument | None: - """Return the first argument in the args that has the given name. - - Return None if no such argument is found. - - As the computation of self.arguments is a little costly, this - function was created so that other methods that have already computed - the arguments use it instead of calling self.get_arg directly. - """ - for arg in args: - if arg.name.strip(WS) == name.strip(WS): - return arg - return None - -def get_last_idx_positional_args(args: Iterable[Argument]) -> int: - idx = 0 - for arg in args: - if arg.positional: - idx += 1 - return idx diff --git a/wikitextparser/_wikitext.py b/wikitextparser/_wikitext.py index 2572333b..237cc821 100644 --- a/wikitextparser/_wikitext.py +++ b/wikitextparser/_wikitext.py @@ -784,7 +784,7 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: if stripped_tl_name[0] == '{' else stripped_tl_name ) - args = template.arguments + args = list(template.arguments) if not args: continue if ':' in stripped_tl_name: @@ -793,8 +793,8 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: else: not_a_parser_function = True # Required for alignment - arg_stripped_names = [a.name.strip(ws) for a in args] - arg_positionalities = [a.positional for a in args] + arg_stripped_names = [a.get_name(False).strip(ws) for a in args] + arg_positionalities = [a.is_positional(False) for a in args] arg_name_lengths = [ wcswidth(n.replace('لا', '?')) if not p else 0 for n, p in zip(arg_stripped_names, arg_positionalities) @@ -811,22 +811,24 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: # Special formatting for the last argument. last_arg = args.pop() last_is_positional = arg_positionalities.pop() - last_value = last_arg.value + last_value = last_arg.get_value(False) last_stripped_value = last_value.strip(ws) if last_is_positional and last_value != last_stripped_value: stop_conversion = True if not last_value.endswith('\n' + indent * (level - 1)): - last_arg.value = last_value + last_comment_indent + last_arg.set_value(last_value + last_comment_indent, False) elif not_a_parser_function: stop_conversion = False - last_arg.name = ( + last_arg.set_name( ' ' + arg_stripped_names.pop() + ' ' - + ' ' * (max_name_len - arg_name_lengths.pop()) + + ' ' * (max_name_len - arg_name_lengths.pop()), + False ) - last_arg.value = ( - ' ' + last_stripped_value + '\n' + indent * (level - 1) + last_arg.set_value( + ' ' + last_stripped_value + '\n' + indent * (level - 1), + False ) elif last_is_positional: # (last_value == last_stripped_value @@ -834,16 +836,17 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: stop_conversion = True # Can't strip or adjust the position of the value # because this could be a positional argument in a template. - last_arg.value = last_value + last_comment_indent + last_arg.set_value(last_value + last_comment_indent, False) else: stop_conversion = True # This is either a parser function or a keyword # argument in a template. In both cases the name # can be lstripped and the value can be rstripped. - last_arg.name = ' ' + last_arg.name.lstrip(ws) + last_arg.set_name(' ' + last_arg.get_name(False).lstrip(ws), False) if not last_value.endswith('\n' + indent * (level - 1)): - last_arg.value = ( - last_value.rstrip(ws) + ' ' + last_comment_indent + last_arg.set_value( + last_value.rstrip(ws) + ' ' + last_comment_indent, + False ) if not args: continue @@ -854,26 +857,27 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: reversed(arg_positionalities), reversed(arg_name_lengths), ): - value = arg.value + value = arg.get_value(False) stripped_value = value.strip(ws) # Positional arguments of templates are sensitive to # whitespace. See: # https://meta.wikimedia.org/wiki/Help:Newlines_and_spaces if stop_conversion: if not value.endswith(newline_indent): - arg.value += comment_indent + arg.set_value(arg.get_value(False) + comment_indent, False) elif positional and value != stripped_value: stop_conversion = True if not value.endswith(newline_indent): - arg.value += comment_indent + arg.set_value(arg.get_value(False) + comment_indent, False) elif not_a_parser_function: - arg.name = ( + arg.set_name( ' ' + stripped_name + ' ' - + ' ' * (max_name_len - arg_name_len) + + ' ' * (max_name_len - arg_name_len), + False ) - arg.value = ' ' + stripped_value + newline_indent + arg.set_value(' ' + stripped_value + newline_indent, False) for func in reversed(parsed.parser_functions): name = func.name @@ -889,7 +893,7 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: # See: [[mw:Help:Extension:ParserFunctions#Miscellaneous]] # All args of #invoke are also whitespace-sensitive. continue - args = func.arguments + args = list(func.arguments) if not args: continue # Whitespace, including newlines, tabs, and spaces is stripped @@ -903,21 +907,23 @@ def pformat(self, indent: str = ' ', remove_comments=False) -> str: if len(args) == 1: arg = args[0] # the first arg is both the first and last argument - arg.value = ( - newline_indent + arg.value.strip(ws) + short_indent + arg.set_value( + newline_indent + arg.get_value(True).strip(ws) + short_indent, + True ) continue # Special formatting for the first argument arg = args[0] - arg.value = ( - newline_indent + arg.value.strip(ws) + newline_indent + arg.set_value( + newline_indent + arg.get_value(True).strip(ws) + newline_indent, + True ) # Formatting the middle arguments for arg in args[1:-1]: - arg.value = ' ' + arg.value.strip(ws) + newline_indent + arg.set_value(' ' + arg.get_value(True).strip(ws) + newline_indent, True) # Special formatting for the last argument arg = args[-1] - arg.value = ' ' + arg.value.strip(ws) + short_indent + arg.set_value(' ' + arg.get_value(True).strip(ws) + short_indent, True) return parsed.string