Meta: Rewrite GenerateEncodingIndexes in python

Author: https://github.com/ayeteadoe
Commit: 838ca8d172
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/5027
Reviewed-by: https://github.com/ADKaster ✅
Reviewed-by: https://github.com/R-Goc
Reviewed-by: https://github.com/jdahlin

Meta/generate_encoding_indexes.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024, Simon Wanner <simon@skyrising.xyz>
+# Copyright (c) 2025, ayeteadoe <ayeteadoe@gmail.com>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+
+import argparse
+import json
+
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+
+class GenerateAccessor(Enum):
+    NO = False
+    YES = True
+
+
+class GenerateInverseAccessor(Enum):
+    NO = False
+    YES = True
+
+
+@dataclass
+class LookupTable:
+    first_pointer: int
+    max_code_point: int
+    code_points: list[int]
+    generate_accessor: GenerateAccessor
+    generate_inverse_accessor: GenerateInverseAccessor
+
+
+@dataclass
+class LookupTables:
+    gb18030_ranges: list[Any]
+    indexes: dict[str, LookupTable]
+
+
+def prepare_table(
+    data: list[Any],
+    generate_accessor: GenerateAccessor = GenerateAccessor.NO,
+) -> LookupTable:
+    code_points = []
+    max_code_point = 0
+    first_pointer = 0
+
+    for entry in data:
+        if entry is None:
+            if not code_points:
+                first_pointer += 1
+            else:
+                code_points.append(0xFFFD)
+                max_code_point = max(max_code_point, code_points[-1])
+        else:
+            code_points.append(int(entry))
+            max_code_point = max(max_code_point, code_points[-1])
+
+    if generate_accessor == GenerateAccessor.YES:
+        while code_points and code_points[-1] == 0xFFFD:
+            code_points.pop()
+    else:
+        assert first_pointer == 0
+
+    return LookupTable(
+        first_pointer=first_pointer,
+        max_code_point=max_code_point,
+        code_points=code_points,
+        generate_accessor=GenerateAccessor.YES,
+        generate_inverse_accessor=GenerateInverseAccessor.YES,
+    )
+
+
+def generate_table(name: str, table: LookupTable) -> str:
+    max_u16 = (1 << 16) - 1
+    value_type = "u32" if table.max_code_point > max_u16 else "u16"
+    size = len(table.code_points)
+
+    lines = []
+
+    if table.first_pointer > 0:
+        lines.append(f"static constexpr u32 s_{name}_index_first_pointer = {table.first_pointer};")
+
+    lines.append(f"static constexpr Array<{value_type}, {size}> s_{name}_index {{")
+
+    formatted_points = []
+    for i, point in enumerate(table.code_points):
+        formatted_points.append(f"0x{point:04x}")
+        if i != len(table.code_points) - 1:
+            if i % 16 == 15:
+                formatted_points.append(",\n    ")
+            else:
+                formatted_points.append(", ")
+
+    lines.append(f"    {' '.join(formatted_points)}")
+    lines.append("};")
+
+    if table.generate_accessor:
+        lines.append(f"Optional<u32> index_{name}_code_point(u32 pointer);")
+
+    if table.generate_inverse_accessor:
+        lines.append(f"Optional<u32> code_point_{name}_index(u32 code_point);")
+
+    return "\n".join(lines)
+
+
+def generate_header_file(tables: LookupTables, output_path: Path) -> None:
+    gb18030_ranges_size = len(tables.gb18030_ranges)
+
+    content = f"""#pragma once
+
+#include <AK/Array.h>
+#include <AK/Types.h>
+
+namespace TextCodec {{
+
+struct Gb18030RangeEntry {{
+    u32 pointer;
+    u32 code_point;
+}};
+
+static constexpr Array<Gb18030RangeEntry, {gb18030_ranges_size}> s_gb18030_ranges {{ {{
+"""
+
+    for range_entry in tables.gb18030_ranges:
+        pointer = range_entry[0]
+        code_point = range_entry[1]
+        content += f"    {{ {pointer}, 0x{code_point:04x} }},\n"
+
+    content += "} };\n\n"
+
+    for name, table in tables.indexes.items():
+        content += generate_table(name, table) + "\n\n"
+
+    content += "}\n"
+
+    with open(output_path, "w") as f:
+        f.write(content)
+
+
+def generate_table_accessor(name: str, table: LookupTable) -> str:
+    if table.first_pointer > 0:
+        return f"""
+Optional<u32> index_{name}_code_point(u32 pointer)
+{{
+    if (pointer < s_{name}_index_first_pointer || pointer - s_{name}_index_first_pointer >= s_{name}_index.size())
+        return {{}};
+    auto value = s_{name}_index[pointer - s_{name}_index_first_pointer];
+    if (value == 0xfffd)
+        return {{}};
+    return value;
+}}
+"""
+    else:
+        return f"""
+Optional<u32> index_{name}_code_point(u32 pointer)
+{{
+    if (pointer >= s_{name}_index.size())
+        return {{}};
+    auto value = s_{name}_index[pointer];
+    if (value == 0xfffd)
+        return {{}};
+    return value;
+}}
+"""
+
+
+def generate_inverse_table_accessor(name: str, table: LookupTable) -> str:
+    if table.first_pointer > 0:
+        return f"""
+Optional<u32> code_point_{name}_index(u32 code_point)
+{{
+    for (u32 i = 0; i < s_{name}_index.size(); ++i) {{
+        if (s_{name}_index[i] == code_point) {{
+            return s_{name}_index_first_pointer + i;
+        }}
+    }}
+    return {{}};
+}}
+"""
+    else:
+        return f"""
+Optional<u32> code_point_{name}_index(u32 code_point)
+{{
+    for (u32 i = 0; i < s_{name}_index.size(); ++i) {{
+        if (s_{name}_index[i] == code_point) {{
+            return i;
+        }}
+    }}
+    return {{}};
+}}
+"""
+
+
+def generate_implementation_file(tables: LookupTables, output_path: Path) -> None:
+    content = """
+#include <LibTextCodec/LookupTables.h>
+
+namespace TextCodec {
+"""
+
+    for name, table in tables.indexes.items():
+        if table.generate_accessor:
+            content += generate_table_accessor(name, table)
+        if table.generate_inverse_accessor:
+            content += generate_inverse_table_accessor(name, table)
+
+    content += "\n}\n"
+
+    with open(output_path, "w") as f:
+        f.write(content)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate text codec lookup tables", add_help=False)
+    parser.add_argument("-h", required=True)
+    parser.add_argument("-c", required=True)
+    parser.add_argument("-j", required=True)
+
+    args = parser.parse_args()
+
+    with open(args.j, "r") as f:
+        data = json.load(f)
+
+    gb18030_table = prepare_table(data["gb18030"], GenerateAccessor.YES)
+
+    # FIXME: Update JSON to match GB-18030-2022 Encoding specification (https://github.com/whatwg/encoding/issues/312)
+    # NOTE: See https://commits.webkit.org/264918@main
+    gb18030_updates = {
+        7182: 0xFE10,
+        7183: 0xFE12,
+        7184: 0xFE11,
+        7185: 0xFE13,
+        7186: 0xFE14,
+        7187: 0xFE15,
+        7188: 0xFE16,
+        7201: 0xFE17,
+        7202: 0xFE18,
+        7208: 0xFE19,
+        23775: 0x9FB4,
+        23783: 0x9FB5,
+        23788: 0x9FB6,
+        23789: 0x9FB7,
+        23795: 0x9FB8,
+        23812: 0x9FB9,
+        23829: 0x9FBA,
+        23845: 0x9FBB,
+    }
+
+    for index, value in gb18030_updates.items():
+        if index < len(gb18030_table.code_points):
+            gb18030_table.code_points[index] = value
+
+    tables = LookupTables(
+        gb18030_ranges=data["gb18030-ranges"],
+        indexes={
+            "gb18030": gb18030_table,
+            "big5": prepare_table(data["big5"], GenerateAccessor.YES),
+            "jis0208": prepare_table(data["jis0208"], GenerateAccessor.YES),
+            "jis0212": prepare_table(data["jis0212"], GenerateAccessor.YES),
+            "euc_kr": prepare_table(data["euc-kr"], GenerateAccessor.YES),
+            "ibm866": prepare_table(data["ibm866"]),
+            "iso_2022_jp_katakana": prepare_table(data["iso-2022-jp-katakana"], GenerateAccessor.YES),
+            "iso_8859_2": prepare_table(data["iso-8859-2"]),
+            "iso_8859_3": prepare_table(data["iso-8859-3"]),
+            "iso_8859_4": prepare_table(data["iso-8859-4"]),
+            "iso_8859_5": prepare_table(data["iso-8859-5"]),
+            "iso_8859_6": prepare_table(data["iso-8859-6"]),
+            "iso_8859_7": prepare_table(data["iso-8859-7"]),
+            "iso_8859_8": prepare_table(data["iso-8859-8"]),
+            "iso_8859_10": prepare_table(data["iso-8859-10"]),
+            "iso_8859_13": prepare_table(data["iso-8859-13"]),
+            "iso_8859_14": prepare_table(data["iso-8859-14"]),
+            "iso_8859_15": prepare_table(data["iso-8859-15"]),
+            "iso_8859_16": prepare_table(data["iso-8859-16"]),
+            "koi8_r": prepare_table(data["koi8-r"]),
+            "koi8_u": prepare_table(data["koi8-u"]),
+            "macintosh": prepare_table(data["macintosh"]),
+            "windows_874": prepare_table(data["windows-874"]),
+            "windows_1250": prepare_table(data["windows-1250"]),
+            "windows_1251": prepare_table(data["windows-1251"]),
+            "windows_1252": prepare_table(data["windows-1252"]),
+            "windows_1253": prepare_table(data["windows-1253"]),
+            "windows_1254": prepare_table(data["windows-1254"]),
+            "windows_1255": prepare_table(data["windows-1255"]),
+            "windows_1256": prepare_table(data["windows-1256"]),
+            "windows_1257": prepare_table(data["windows-1257"]),
+            "windows_1258": prepare_table(data["windows-1258"]),
+            "x_mac_cyrillic": prepare_table(data["x-mac-cyrillic"]),
+        },
+    )
+
+    generate_header_file(tables, Path(args.h))
+    generate_implementation_file(tables, Path(args.c))
+
+
+if __name__ == "__main__":
+    main()