diff --git a/src/wikitextprocessor/core.py b/src/wikitextprocessor/core.py index 3ce00067..5c3e4cd9 100644 --- a/src/wikitextprocessor/core.py +++ b/src/wikitextprocessor/core.py @@ -74,6 +74,16 @@ total=True, # fields are obligatory ) +LocalizationData = TypedDict( + "LocalizationData", + { + "decimal_point": str, + "grouping_separator": str, + "grouping_method": tuple[int, ...], + }, + total=True, +) + JsonValues = Union[str, int, float, list, dict, bool, None] # Can't specify _LuaTable contents further, so no use specifying the Dict either ParentData = tuple[str, Union["_LuaTable", dict[Union[int, str], str]]] @@ -265,6 +275,8 @@ class Wtp: "suppress_special", # XXX never set to True??? "data_folder", "NAMESPACE_DATA", + "LOCALIZATION_DATA", + "LOCALIZATION_ALLOWED_REVERSABLE_NUMBER_CHARS", "LOCAL_NS_NAME_BY_ID", # Local namespace names dictionary "NS_ID_BY_LOCAL_NAME", "lang_code", @@ -322,8 +334,9 @@ def __init__( self.expand_stack: list[str] = [] # XXX: this has a confusing name self.parser_stack: list["WikiNode"] = [] self.lang_code = lang_code # dump file language code - self.data_folder = files("wikitextprocessor") / "data" / lang_code + self.init_data_folder() self.init_namespace_data() + self.init_localization_data() self.create_db() self.template_override_funcs = template_override_funcs self.beginning_of_line = False @@ -496,6 +509,39 @@ def init_namespace_data(self) -> None: for data in self.NAMESPACE_DATA.values() } + def init_localization_data(self) -> None: + loc_path = self.data_folder.joinpath("localization.json") + if not loc_path.is_file(): + # default localization + self.LOCALIZATION_DATA: LocalizationData = { + "decimal_point": ".", + "grouping_separator": ",", + "grouping_method": (3, 0), + } + self.LOCALIZATION_ALLOWED_REVERSABLE_NUMBER_CHARS: set[str] = set( + ",.0123456789" + ) + return + with loc_path.open(encoding="utf-8") as f: + self.LOCALIZATION_DATA = json.load(f) + + self.LOCALIZATION_ALLOWED_REVERSABLE_NUMBER_CHARS = set( + # XXX this could also be other numeral characters + # locale.ALT_DIGITS is buggy in our version of Python, + # check later if it's usable. + "1234567890" + + self.LOCALIZATION_DATA["decimal_point"] + + self.LOCALIZATION_DATA["grouping_separator"] + + ( + " " + if self.LOCALIZATION_DATA["grouping_separator"] == "\xa0" + else "" + ), + ) + + def init_data_folder(self) -> None: + self.data_folder = files("wikitextprocessor") / "data" / self.lang_code + def _fmt_errmsg(self, kind: str, msg: str, trace: Optional[str]) -> None: if self.quiet_output: return diff --git a/src/wikitextprocessor/data/af/localization.json b/src/wikitextprocessor/data/af/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/af/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/am/localization.json b/src/wikitextprocessor/data/am/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/am/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/as/localization.json b/src/wikitextprocessor/data/as/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/as/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/az/localization.json b/src/wikitextprocessor/data/az/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/az/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/be-x-old/localization.json b/src/wikitextprocessor/data/be-x-old/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/be-x-old/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/be/localization.json b/src/wikitextprocessor/data/be/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/be/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/bg/localization.json b/src/wikitextprocessor/data/bg/localization.json new file mode 100644 index 00000000..8544e952 --- /dev/null +++ b/src/wikitextprocessor/data/bg/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/br/localization.json b/src/wikitextprocessor/data/br/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/br/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/bs/localization.json b/src/wikitextprocessor/data/bs/localization.json new file mode 100644 index 00000000..70656132 --- /dev/null +++ b/src/wikitextprocessor/data/bs/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ca/localization.json b/src/wikitextprocessor/data/ca/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/ca/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/cs/localization.json b/src/wikitextprocessor/data/cs/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/cs/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/cy/localization.json b/src/wikitextprocessor/data/cy/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/cy/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/da/localization.json b/src/wikitextprocessor/data/da/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/da/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/de/localization.json b/src/wikitextprocessor/data/de/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/de/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/el/localization.json b/src/wikitextprocessor/data/el/localization.json new file mode 100644 index 00000000..224e9aac --- /dev/null +++ b/src/wikitextprocessor/data/el/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/en/localization.json b/src/wikitextprocessor/data/en/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/en/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/eo/localization.json b/src/wikitextprocessor/data/eo/localization.json new file mode 100644 index 00000000..70656132 --- /dev/null +++ b/src/wikitextprocessor/data/eo/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/es/localization.json b/src/wikitextprocessor/data/es/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/es/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/et/localization.json b/src/wikitextprocessor/data/et/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/et/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/eu/localization.json b/src/wikitextprocessor/data/eu/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/eu/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/fa/localization.json b/src/wikitextprocessor/data/fa/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/fa/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/fi/localization.json b/src/wikitextprocessor/data/fi/localization.json new file mode 100644 index 00000000..c8eb17b9 --- /dev/null +++ b/src/wikitextprocessor/data/fi/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/fo/localization.json b/src/wikitextprocessor/data/fo/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/fo/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/fr/localization.json b/src/wikitextprocessor/data/fr/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/fr/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ga/localization.json b/src/wikitextprocessor/data/ga/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ga/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/gd/localization.json b/src/wikitextprocessor/data/gd/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/gd/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/gl/localization.json b/src/wikitextprocessor/data/gl/localization.json new file mode 100644 index 00000000..70656132 --- /dev/null +++ b/src/wikitextprocessor/data/gl/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/gv/localization.json b/src/wikitextprocessor/data/gv/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/gv/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/he/localization.json b/src/wikitextprocessor/data/he/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/he/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/hi/localization.json b/src/wikitextprocessor/data/hi/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/hi/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/hr/localization.json b/src/wikitextprocessor/data/hr/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/hr/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/hu/localization.json b/src/wikitextprocessor/data/hu/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/hu/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/id/localization.json b/src/wikitextprocessor/data/id/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/id/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/is/localization.json b/src/wikitextprocessor/data/is/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/is/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/it/localization.json b/src/wikitextprocessor/data/it/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/it/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/iu/localization.json b/src/wikitextprocessor/data/iu/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/iu/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ja/localization.json b/src/wikitextprocessor/data/ja/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ja/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ka/localization.json b/src/wikitextprocessor/data/ka/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/ka/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/kl/localization.json b/src/wikitextprocessor/data/kl/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/kl/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/kn/localization.json b/src/wikitextprocessor/data/kn/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/kn/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ko/localization.json b/src/wikitextprocessor/data/ko/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ko/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ks/localization.json b/src/wikitextprocessor/data/ks/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/ks/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/kw/localization.json b/src/wikitextprocessor/data/kw/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/kw/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ky/localization.json b/src/wikitextprocessor/data/ky/localization.json new file mode 100644 index 00000000..c8eb17b9 --- /dev/null +++ b/src/wikitextprocessor/data/ky/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/lo/localization.json b/src/wikitextprocessor/data/lo/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/lo/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/lt/localization.json b/src/wikitextprocessor/data/lt/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/lt/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/lv/localization.json b/src/wikitextprocessor/data/lv/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/lv/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/mi/localization.json b/src/wikitextprocessor/data/mi/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/mi/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/mk/localization.json b/src/wikitextprocessor/data/mk/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/mk/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ml/localization.json b/src/wikitextprocessor/data/ml/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/ml/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/mr/localization.json b/src/wikitextprocessor/data/mr/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/mr/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ms/localization.json b/src/wikitextprocessor/data/ms/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ms/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/mt/localization.json b/src/wikitextprocessor/data/mt/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/mt/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/nl/localization.json b/src/wikitextprocessor/data/nl/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/nl/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/nn/localization.json b/src/wikitextprocessor/data/nn/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/nn/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/nso/localization.json b/src/wikitextprocessor/data/nso/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/nso/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/oc/localization.json b/src/wikitextprocessor/data/oc/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/oc/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/or/localization.json b/src/wikitextprocessor/data/or/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/or/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/pa/localization.json b/src/wikitextprocessor/data/pa/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/pa/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/pl/localization.json b/src/wikitextprocessor/data/pl/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/pl/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/pt/localization.json b/src/wikitextprocessor/data/pt/localization.json new file mode 100644 index 00000000..70656132 --- /dev/null +++ b/src/wikitextprocessor/data/pt/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ro/localization.json b/src/wikitextprocessor/data/ro/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/ro/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ru/localization.json b/src/wikitextprocessor/data/ru/localization.json new file mode 100644 index 00000000..c8eb17b9 --- /dev/null +++ b/src/wikitextprocessor/data/ru/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/rw/localization.json b/src/wikitextprocessor/data/rw/localization.json new file mode 100644 index 00000000..70656132 --- /dev/null +++ b/src/wikitextprocessor/data/rw/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/sd/localization.json b/src/wikitextprocessor/data/sd/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/sd/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/sh/localization.json b/src/wikitextprocessor/data/sh/localization.json new file mode 100644 index 00000000..70656132 --- /dev/null +++ b/src/wikitextprocessor/data/sh/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/si/localization.json b/src/wikitextprocessor/data/si/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/si/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/sk/localization.json b/src/wikitextprocessor/data/sk/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/sk/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/sl/localization.json b/src/wikitextprocessor/data/sl/localization.json new file mode 100644 index 00000000..224e9aac --- /dev/null +++ b/src/wikitextprocessor/data/sl/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/sq/localization.json b/src/wikitextprocessor/data/sq/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/sq/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/sr/localization.json b/src/wikitextprocessor/data/sr/localization.json new file mode 100644 index 00000000..70656132 --- /dev/null +++ b/src/wikitextprocessor/data/sr/localization.json @@ -0,0 +1,5 @@ +{ + "decimal_point": ",", + "grouping_method": [], + "grouping_separator": "" +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ss/localization.json b/src/wikitextprocessor/data/ss/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ss/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/st/localization.json b/src/wikitextprocessor/data/st/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/st/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/sv/localization.json b/src/wikitextprocessor/data/sv/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/sv/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ta/localization.json b/src/wikitextprocessor/data/ta/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/ta/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/te/localization.json b/src/wikitextprocessor/data/te/localization.json new file mode 100644 index 00000000..1db2947d --- /dev/null +++ b/src/wikitextprocessor/data/te/localization.json @@ -0,0 +1,9 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 2, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/tg/localization.json b/src/wikitextprocessor/data/tg/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/tg/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/th/localization.json b/src/wikitextprocessor/data/th/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/th/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/tl/localization.json b/src/wikitextprocessor/data/tl/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/tl/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/tn/localization.json b/src/wikitextprocessor/data/tn/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/tn/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/tr/localization.json b/src/wikitextprocessor/data/tr/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/tr/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ts/localization.json b/src/wikitextprocessor/data/ts/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ts/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/tt/localization.json b/src/wikitextprocessor/data/tt/localization.json new file mode 100644 index 00000000..c8eb17b9 --- /dev/null +++ b/src/wikitextprocessor/data/tt/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/uk/localization.json b/src/wikitextprocessor/data/uk/localization.json new file mode 100644 index 00000000..004579f1 --- /dev/null +++ b/src/wikitextprocessor/data/uk/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ur/localization.json b/src/wikitextprocessor/data/ur/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ur/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/uz/localization.json b/src/wikitextprocessor/data/uz/localization.json new file mode 100644 index 00000000..c8eb17b9 --- /dev/null +++ b/src/wikitextprocessor/data/uz/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": " " +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/ve/localization.json b/src/wikitextprocessor/data/ve/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/ve/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/vi/localization.json b/src/wikitextprocessor/data/vi/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/vi/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/wa/localization.json b/src/wikitextprocessor/data/wa/localization.json new file mode 100644 index 00000000..e978bf0d --- /dev/null +++ b/src/wikitextprocessor/data/wa/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ",", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "." +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/xh/localization.json b/src/wikitextprocessor/data/xh/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/xh/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/yi/localization.json b/src/wikitextprocessor/data/yi/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/yi/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/zh-classical/localization.json b/src/wikitextprocessor/data/zh-classical/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/zh-classical/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/zh-min-nan/localization.json b/src/wikitextprocessor/data/zh-min-nan/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/zh-min-nan/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/zh-yue/localization.json b/src/wikitextprocessor/data/zh-yue/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/zh-yue/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/zh/localization.json b/src/wikitextprocessor/data/zh/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/zh/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/data/zu/localization.json b/src/wikitextprocessor/data/zu/localization.json new file mode 100644 index 00000000..fd1c6b72 --- /dev/null +++ b/src/wikitextprocessor/data/zu/localization.json @@ -0,0 +1,8 @@ +{ + "decimal_point": ".", + "grouping_method": [ + 3, + 0 + ], + "grouping_separator": "," +} \ No newline at end of file diff --git a/src/wikitextprocessor/parserfns.py b/src/wikitextprocessor/parserfns.py index 796e779d..99b0a629 100644 --- a/src/wikitextprocessor/parserfns.py +++ b/src/wikitextprocessor/parserfns.py @@ -586,35 +586,120 @@ def ucfirst_fn( return capitalizeFirstOnly(t) +# XXX If we implement different numeral characters (western arabic numerals, +# bengali, etc.) this should be only used for "raw" strings and R)eversable +# strings need to allow for those other numeral characters, and if the R-flag +# is set, those numeral characters (and other things) should be allowed. +# XXX if implementing other numeral chars, check out str.maketrans() and +# translate; the mapping needs to be hardcoded or a separate config file, +# because I haven't found any sensible standard library thing for this. +ALLOWED_FORMATNUM_CHARS_NOT_R = { + ".", + ",", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", +} + + def formatnum_fn( ctx: "Wtp", fn_name: str, args: list[str], expander: Callable[[str], str] ) -> str: """Implements the formatnum parser function.""" arg0 = expander(args[0]).strip() if args else "" arg1 = expander(args[1]).strip() if len(args) >= 2 else "" + if arg1 == "R": - # Reverse formatting - # XXX this is a very simplified implementation, should handle more cases - return arg0.replace(",", "") + return _formatnum_reverse(ctx, arg0) + + if ( + arg0 == "" + # <= is a set subset comparison, for those of you who + # don't know (like me, who wrote this piece of code) + or not (set(arg0) <= ALLOWED_FORMATNUM_CHARS_NOT_R) + or arg0.count(".") > 1 + ): + return arg0 + if arg1 == "NOSEP": sep = "" else: - sep = "," - comma = "." # Really should depend on locale + sep = ctx.LOCALIZATION_DATA["grouping_separator"] + + if sep in arg0: + # separator only allowed when R)eversing + return arg0 + + decimal_point = ctx.LOCALIZATION_DATA["decimal_point"] # XXX implement support for non-english locales for digits - orig = arg0.split(".") + orig = arg0.split(".") # remember, raw input strungs "." always decimal first = orig[0] - parts = [] - first = "".join(reversed(first)) - for i in range(0, len(first), 3): - parts.append("".join(reversed(first[i : i + 3]))) - parts = [sep.join(reversed(parts))] + + parts: list[str] = [] + # Tuple with ints, usually (3, 0). + # (2, 2, 2, 3, 0) means "first do three groups of 2 and then a group of 3; + # at 0, just start repeating the previous group. + # TECHNICALLY the docs/specs say that instead of ending with a "0" this + # could end with locale. + algo = ctx.LOCALIZATION_DATA["grouping_method"] + if len(algo) > 0: + first = "".join(reversed(first)) + i = 0 + algo_i = 0 + group_size = algo[algo_i] + while i < len(first): + parts.append("".join(reversed(first[i : i + group_size]))) + i += group_size + if algo_i + 1 < len(algo): + algo_i += 1 + if algo[algo_i] > 0: + group_size = algo[algo_i] + # for i in range(0, len(first), 3): + # parts.append("".join(reversed(first[i : i + 3]))) + parts = [sep.join(reversed(parts))] + else: + parts = [first] + if len(orig) > 1: - parts.append(comma) - parts.append(".".join(orig[1:])) + parts.append(decimal_point) + parts.extend(orig[1:]) + return "".join(parts) +def _formatnum_reverse(ctx: "Wtp", arg0: str) -> str: + # Reverse formatting + # XXX this is a very simplified implementation, should handle more cases + decimal = ctx.LOCALIZATION_DATA["decimal_point"] + if ( + arg0 == "" + # <= is a set subset comparison, for those of you who + # don't know (like me, who wrote this piece of code) + or not (set(arg0) <= ctx.LOCALIZATION_ALLOWED_REVERSABLE_NUMBER_CHARS) + or arg0.count(decimal) > 1 + ): + return arg0 + + sep = ctx.LOCALIZATION_DATA["grouping_separator"] + + # XXX implement other numeral systems, like west arabic or bengali + + # Kludge for French; the locale data has non-breaking spaces as the + # separators, but it seems clear we must also allow normal spaces + if sep == "\xa0": # non-breaking space + return arg0.replace(decimal, ".").replace(sep, "").replace(" ", "") + + # Currently only doing the minimum by removing thousand separators + return arg0.replace(decimal, ".").replace(sep, "") + + def dateformat_fn( ctx: "Wtp", fn_name: str, args: list[str], expander: Callable[[str], str] ) -> str: diff --git a/tests/test_wikiprocess.py b/tests/test_wikiprocess.py index 58b5ebfa..f900ae1c 100644 --- a/tests/test_wikiprocess.py +++ b/tests/test_wikiprocess.py @@ -608,6 +608,62 @@ def test_formatnum9(self): def test_formatnum10(self): self.parserfn("{{formatnum:12345}}", "12,345") + def test_formatnum11(self): + # only numerals + . + , + self.parserfn("{{formatnum:µB}}", "µB") + + def test_formatnum12(self): + # no bad decimals + self.parserfn("{{formatnum:123.4.5}}", "123.4.5") + + def test_formatnum13(self): + # no spaces + self.parserfn("{{formatnum:123 4.5}}", "123 4.5") + + def test_formatnum13_b(self): + # ... except a the start and end + self.parserfn("{{formatnum: 1234.5 }}", "1,234.5") + + def test_formatnum14(self): + # bad commas allowed when |R|eversing + self.parserfn("{{formatnum:123,4.5|R}}", "1234.5") + + def test_formatnum15(self): + # commas in decimals allowed when |R|eversing + self.parserfn("{{formatnum:1234.5,4|R}}", "1234.54") + + def test_formatnum_fr_locale_1(self): + self.ctx.lang_code = "fr" + self.ctx.init_data_folder() + self.ctx.init_localization_data() + self.parserfn("{{formatnum:1234.54}}", "1\xa0234,54") + + def test_formatnum_fr_locale_2(self): + self.ctx.lang_code = "fr" + self.ctx.init_data_folder() + self.ctx.init_localization_data() + self.parserfn("{{formatnum:1 234,54|R}}", "1234.54") + + def test_formatnum_fr_locale_3(self): + # French locale data has non-breaking space as the numeral separator, + # which has a separate kludge so that normal space works too. + self.ctx.lang_code = "fr" + self.ctx.init_data_folder() + self.ctx.init_localization_data() + self.parserfn("{{formatnum:1\xa0234,54|R}}", "1234.54") + + def test_formatnum_hi_locale_1(self): + self.ctx.lang_code = "hi" + self.ctx.init_data_folder() + self.ctx.init_localization_data() + self.parserfn("{{formatnum:123456789.0123}}", "12,34,56,789.0123") + + def test_formatnum_hi_locale_2(self): + self.ctx.lang_code = "hi" + self.ctx.init_data_folder() + self.ctx.init_localization_data() + self.parserfn("{{formatnum:12,34,56,789.0123|R}}", "123456789.0123") + def test_dateformat1(self): self.parserfn("{{#dateformat:25 dec 2009|ymd}}", "2009 Dec 25") diff --git a/tools/dump_locale_data.py b/tools/dump_locale_data.py new file mode 100644 index 00000000..c1eec5f1 --- /dev/null +++ b/tools/dump_locale_data.py @@ -0,0 +1,105 @@ +import argparse +import json +import locale +import os +import pathlib + +from wikitextprocessor.core import LocalizationData + +parser = argparse.ArgumentParser( + description="Generate config files for locale stuff" +) +parser.add_argument( + "langdirs", + type=pathlib.Path, + help="Directory containing language configuration " + "directories, usually 'src/wikitextprocessor/data/...'." + "This is used as input to determine what language codes " + "need to have configs generated for them.", +) + +parser.add_argument( + "outputdir", + type=pathlib.Path, + help="Directory where individual directories named " + "by language code ('en/', 'de/' etc.) are outputted. " + "Recommended not to use src/wikitextprocessor/data " + "directly, but to copy-paste into that directory later.", +) +args = parser.parse_args() + + +def main() -> None: + # get the names (lang-codes) of the config directories + dirs = sorted([f.name for f in os.scandir(args.langdirs) if f.is_dir()]) + # print(dirs) + + # list of language-code -> locale code + # map. Based on your system, yet many of these locale codes fail + aliases = locale.locale_alias + + configs: dict[str, LocalizationData] = {} + + for dir_code in dirs: + main_code = dir_code.split("-")[0] + if dir_code not in aliases and main_code not in aliases: + continue + if dir_code not in aliases: + lang_code = main_code + else: + lang_code = dir_code + + alias = aliases[lang_code] + utf8ed = alias.split(".")[0] + ".UTF-8" + failed = True + plog = [] + for x in (alias, utf8ed): + try: + locale.setlocale(locale.LC_ALL, x) + failed = False + plog.append(f"{x=} succeeded") + break + except Exception as e: + plog.append(f"Failed {e=} for {x=}") + + if failed: + plog.append( + f"BOTH FAILED {dir_code=} {alias=}, {utf8ed=}; no " + "config file will be generated" + ) + print("[[" + "\n".join(plog) + "]]") + if failed: + continue + + envv = locale.localeconv() + sep: str = envv["thousands_sep"] # type: ignore + decimal: str = envv["decimal_point"] # type: ignore + grouping: tuple[int] = tuple(envv["grouping"]) # type: ignore + mon_grouping: tuple[int] = tuple(envv["mon_grouping"]) # type:ignore + + # For some reason, for hindi "grouping" and "mon_grouping" are + # different, so we will use the "non-default" one. + if grouping != mon_grouping: + if grouping == (3, 0): + grouping = mon_grouping + + configs[dir_code] = { + "decimal_point": decimal, + "grouping_method": tuple(grouping), + "grouping_separator": sep, + } + + for dirname, locdata in configs.items(): + data_folder = args.outputdir / dirname + if not data_folder.exists(): + data_folder.mkdir() + with open( + data_folder / "localization.json", + mode="w", + encoding="utf-8", + ) as f: + json.dump(locdata, f, ensure_ascii=False, indent=2) + + +if __name__ == "__main__": + main()