LibJS+LibLocale: Replace calendar weekday information with ICU

Author: https://github.com/trflynn89
Commit: 4b3e26c583
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/145

Meta/CMake/locale_data.cmake

@@ -38,9 +38,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
         message(STATUS "Skipping download of ${CLDR_ZIP_URL}, expecting the archive to have been extracted to ${CLDR_PATH}")
     endif()
 
-    set(DATE_TIME_FORMAT_DATA_HEADER DateTimeFormatData.h)
-    set(DATE_TIME_FORMAT_DATA_IMPLEMENTATION DateTimeFormatData.cpp)
-
     set(LOCALE_DATA_HEADER LocaleData.h)
     set(LOCALE_DATA_IMPLEMENTATION LocaleData.cpp)
 
@@ -53,14 +50,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
     set(RELATIVE_TIME_FORMAT_DATA_HEADER RelativeTimeFormatData.h)
     set(RELATIVE_TIME_FORMAT_DATA_IMPLEMENTATION RelativeTimeFormatData.cpp)
 
-    invoke_generator(
-        "DateTimeFormatData"
-        Lagom::GenerateDateTimeFormatData
-        "${CLDR_VERSION_FILE}"
-        "${DATE_TIME_FORMAT_DATA_HEADER}"
-        "${DATE_TIME_FORMAT_DATA_IMPLEMENTATION}"
-        arguments -r "${CLDR_CORE_PATH}"
-    )
     invoke_generator(
         "LocaleData"
         Lagom::GenerateLocaleData
@@ -95,8 +84,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
     )
 
     set(LOCALE_DATA_SOURCES
-        ${DATE_TIME_FORMAT_DATA_HEADER}
-        ${DATE_TIME_FORMAT_DATA_IMPLEMENTATION}
         ${LOCALE_DATA_HEADER}
         ${LOCALE_DATA_IMPLEMENTATION}
         ${NUMBER_FORMAT_DATA_HEADER}

Meta/Lagom/Tools/CodeGenerators/LibLocale/CMakeLists.txt

@@ -1,4 +1,3 @@
-lagom_tool(GenerateDateTimeFormatData SOURCES GenerateDateTimeFormatData.cpp LIBS LibMain LibTimeZone)
 lagom_tool(GenerateLocaleData SOURCES GenerateLocaleData.cpp LIBS LibMain)
 lagom_tool(GenerateNumberFormatData SOURCES GenerateNumberFormatData.cpp LIBS LibMain)
 lagom_tool(GeneratePluralRulesData SOURCES GeneratePluralRulesData.cpp LIBS LibMain)

Meta/Lagom/Tools/CodeGenerators/LibLocale/GenerateDateTimeFormatData.cpp

@@ -1,267 +0,0 @@
-/*
- * Copyright (c) 2021-2024, Tim Flynn <trflynn89@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include "../LibUnicode/GeneratorUtil.h" // FIXME: Move this somewhere common.
-#include <AK/AllOf.h>
-#include <AK/ByteString.h>
-#include <AK/CharacterTypes.h>
-#include <AK/Error.h>
-#include <AK/Format.h>
-#include <AK/HashMap.h>
-#include <AK/JsonObject.h>
-#include <AK/JsonParser.h>
-#include <AK/JsonValue.h>
-#include <AK/LexicalPath.h>
-#include <AK/SourceGenerator.h>
-#include <AK/StringBuilder.h>
-#include <LibCore/ArgsParser.h>
-#include <LibCore/Directory.h>
-#include <LibLocale/DateTimeFormat.h>
-
-struct CLDR {
-    HashMap<ByteString, u8> minimum_days;
-    Vector<ByteString> minimum_days_regions;
-
-    HashMap<ByteString, Locale::Weekday> first_day;
-    Vector<ByteString> first_day_regions;
-
-    HashMap<ByteString, Locale::Weekday> weekend_start;
-    Vector<ByteString> weekend_start_regions;
-
-    HashMap<ByteString, Locale::Weekday> weekend_end;
-    Vector<ByteString> weekend_end_regions;
-};
-
-static ErrorOr<void> parse_week_data(ByteString core_path, CLDR& cldr)
-{
-    // https://unicode.org/reports/tr35/tr35-dates.html#Week_Data
-    LexicalPath week_data_path(move(core_path));
-    week_data_path = week_data_path.append("supplemental"sv);
-    week_data_path = week_data_path.append("weekData.json"sv);
-
-    auto week_data = TRY(read_json_file(week_data_path.string()));
-    auto const& supplemental_object = week_data.as_object().get_object("supplemental"sv).value();
-    auto const& week_data_object = supplemental_object.get_object("weekData"sv).value();
-
-    auto parse_weekday = [](StringView day) -> Locale::Weekday {
-        if (day == "sun"sv)
-            return Locale::Weekday::Sunday;
-        if (day == "mon"sv)
-            return Locale::Weekday::Monday;
-        if (day == "tue"sv)
-            return Locale::Weekday::Tuesday;
-        if (day == "wed"sv)
-            return Locale::Weekday::Wednesday;
-        if (day == "thu"sv)
-            return Locale::Weekday::Thursday;
-        if (day == "fri"sv)
-            return Locale::Weekday::Friday;
-        if (day == "sat"sv)
-            return Locale::Weekday::Saturday;
-        VERIFY_NOT_REACHED();
-    };
-
-    auto parse_regional_weekdays = [&](auto const& region, auto const& weekday, auto& weekdays_map, auto& weekday_regions) {
-        if (region.ends_with("alt-variant"sv))
-            return;
-
-        weekdays_map.set(region, parse_weekday(weekday));
-
-        if (!weekday_regions.contains_slow(region))
-            weekday_regions.append(region);
-    };
-
-    auto const& minimum_days_object = week_data_object.get_object("minDays"sv).value();
-    auto const& first_day_object = week_data_object.get_object("firstDay"sv).value();
-    auto const& weekend_start_object = week_data_object.get_object("weekendStart"sv).value();
-    auto const& weekend_end_object = week_data_object.get_object("weekendEnd"sv).value();
-
-    minimum_days_object.for_each_member([&](auto const& region, auto const& value) {
-        auto minimum_days = value.as_string().template to_number<u8>();
-        cldr.minimum_days.set(region, *minimum_days);
-
-        if (!cldr.minimum_days_regions.contains_slow(region))
-            cldr.minimum_days_regions.append(region);
-    });
-
-    first_day_object.for_each_member([&](auto const& region, auto const& value) {
-        parse_regional_weekdays(region, value.as_string(), cldr.first_day, cldr.first_day_regions);
-    });
-    weekend_start_object.for_each_member([&](auto const& region, auto const& value) {
-        parse_regional_weekdays(region, value.as_string(), cldr.weekend_start, cldr.weekend_start_regions);
-    });
-    weekend_end_object.for_each_member([&](auto const& region, auto const& value) {
-        parse_regional_weekdays(region, value.as_string(), cldr.weekend_end, cldr.weekend_end_regions);
-    });
-
-    return {};
-}
-
-static ErrorOr<void> parse_all_locales(ByteString core_path, CLDR& cldr)
-{
-    TRY(parse_week_data(core_path, cldr));
-    return {};
-}
-
-static ByteString format_identifier(StringView owner, ByteString identifier)
-{
-    identifier = identifier.replace("-"sv, "_"sv, ReplaceMode::All);
-    identifier = identifier.replace("/"sv, "_"sv, ReplaceMode::All);
-
-    if (all_of(identifier, is_ascii_digit))
-        return ByteString::formatted("{}_{}", owner[0], identifier);
-    if (is_ascii_lower_alpha(identifier[0]))
-        return ByteString::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
-    return identifier;
-}
-
-static ErrorOr<void> generate_unicode_locale_header(Core::InputBufferedFile& file, CLDR& cldr)
-{
-    StringBuilder builder;
-    SourceGenerator generator { builder };
-
-    generator.append(R"~~~(
-#pragma once
-
-#include <AK/Types.h>
-
-namespace Locale {
-)~~~");
-
-    generate_enum(generator, format_identifier, "MinimumDaysRegion"sv, {}, cldr.minimum_days_regions);
-    generate_enum(generator, format_identifier, "FirstDayRegion"sv, {}, cldr.first_day_regions);
-    generate_enum(generator, format_identifier, "WeekendStartRegion"sv, {}, cldr.weekend_start_regions);
-    generate_enum(generator, format_identifier, "WeekendEndRegion"sv, {}, cldr.weekend_end_regions);
-
-    generator.append(R"~~~(
-}
-)~~~");
-
-    TRY(file.write_until_depleted(generator.as_string_view().bytes()));
-    return {};
-}
-
-static ErrorOr<void> generate_unicode_locale_implementation(Core::InputBufferedFile& file, CLDR& cldr)
-{
-    StringBuilder builder;
-    SourceGenerator generator { builder };
-
-    generator.append(R"~~~(
-#include <AK/Array.h>
-#include <AK/BinarySearch.h>
-#include <AK/Error.h>
-#include <AK/Optional.h>
-#include <AK/String.h>
-#include <AK/StringView.h>
-#include <LibLocale/DateTimeFormat.h>
-#include <LibLocale/DateTimeFormatData.h>
-#include <LibLocale/Locale.h>
-#include <LibLocale/LocaleData.h>
-
-namespace Locale {
-)~~~");
-
-    auto append_mapping = [&](auto const& keys, auto const& map, auto type, auto name, auto mapping_getter) {
-        generator.set("type", type);
-        generator.set("name", name);
-        generator.set("size", ByteString::number(keys.size()));
-
-        generator.append(R"~~~(
-static constexpr Array<@type@, @size@> @name@ { {)~~~");
-
-        bool first = true;
-        for (auto const& key : keys) {
-            auto const& value = map.find(key)->value;
-            auto mapping = mapping_getter(value);
-
-            generator.append(first ? " "sv : ", "sv);
-            generator.append(ByteString::number(mapping));
-            first = false;
-        }
-
-        generator.append(" } };");
-    };
-
-    append_mapping(cldr.minimum_days_regions, cldr.minimum_days, "u8"sv, "s_minimum_days"sv, [](auto minimum_days) { return minimum_days; });
-    append_mapping(cldr.first_day_regions, cldr.first_day, "u8"sv, "s_first_day"sv, [](auto first_day) { return to_underlying(first_day); });
-    append_mapping(cldr.weekend_start_regions, cldr.weekend_start, "u8"sv, "s_weekend_start"sv, [](auto weekend_start) { return to_underlying(weekend_start); });
-    append_mapping(cldr.weekend_end_regions, cldr.weekend_end, "u8"sv, "s_weekend_end"sv, [](auto weekend_end) { return to_underlying(weekend_end); });
-    generator.append("\n");
-
-    auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values, Vector<Alias> const& aliases = {}) -> ErrorOr<void> {
-        HashValueMap<ByteString> hashes;
-        TRY(hashes.try_ensure_capacity(values.size()));
-
-        for (auto const& value : values)
-            hashes.set(value.hash(), format_identifier(enum_title, value));
-        for (auto const& alias : aliases)
-            hashes.set(alias.alias.hash(), format_identifier(enum_title, alias.alias));
-
-        generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
-
-        return {};
-    };
-
-    TRY(append_from_string("MinimumDaysRegion"sv, "minimum_days_region"sv, cldr.minimum_days_regions));
-    TRY(append_from_string("FirstDayRegion"sv, "first_day_region"sv, cldr.first_day_regions));
-    TRY(append_from_string("WeekendStartRegion"sv, "weekend_start_region"sv, cldr.weekend_start_regions));
-    TRY(append_from_string("WeekendEndRegion"sv, "weekend_end_region"sv, cldr.weekend_end_regions));
-
-    auto append_regional_lookup = [&](StringView return_type, StringView lookup_type) {
-        generator.set("return_type", return_type);
-        generator.set("lookup_type", lookup_type);
-
-        generator.append(R"~~~(
-Optional<@return_type@> get_regional_@lookup_type@(StringView region)
-{
-    auto region_value = @lookup_type@_region_from_string(region);
-    if (!region_value.has_value())
-        return {};
-
-    auto region_index = to_underlying(*region_value);
-    auto @lookup_type@ = s_@lookup_type@.at(region_index);
-
-    return static_cast<@return_type@>(@lookup_type@);
-}
-)~~~");
-    };
-
-    append_regional_lookup("u8"sv, "minimum_days"sv);
-    append_regional_lookup("Weekday"sv, "first_day"sv);
-    append_regional_lookup("Weekday"sv, "weekend_start"sv);
-    append_regional_lookup("Weekday"sv, "weekend_end"sv);
-
-    generator.append(R"~~~(
-}
-)~~~");
-
-    TRY(file.write_until_depleted(generator.as_string_view().bytes()));
-    return {};
-}
-
-ErrorOr<int> serenity_main(Main::Arguments arguments)
-{
-    StringView generated_header_path;
-    StringView generated_implementation_path;
-    StringView core_path;
-
-    Core::ArgsParser args_parser;
-    args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
-    args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
-    args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
-    args_parser.parse(arguments);
-
-    auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
-    auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
-
-    CLDR cldr;
-    TRY(parse_all_locales(core_path, cldr));
-
-    TRY(generate_unicode_locale_header(*generated_header_file, cldr));
-    TRY(generate_unicode_locale_implementation(*generated_implementation_file, cldr));
-
-    return 0;
-}

Userland/Libraries/LibJS/Runtime/Intl/Locale.cpp

@@ -254,10 +254,10 @@ StringView weekday_to_string(StringView weekday)
     VERIFY_NOT_REACHED();
 }
 
-static u8 weekday_to_integer(Optional<::Locale::Weekday> weekday, ::Locale::Weekday falllback)
+static u8 weekday_to_integer(Optional<::Locale::Weekday> const& weekday, ::Locale::Weekday falllback)
 {
-    // NOTE: This fallback will be used if LibUnicode data generation is disabled. Its value should
+    // NOTE: This fallback will be used if the ICU data lookup failed. Its value should be that of the
-    //       be that of the default region ("001") in the CLDR.
+    //       default region ("001") in the CLDR.
     switch (weekday.value_or(falllback)) {
     case ::Locale::Weekday::Monday:
         return 1;
@@ -274,25 +274,18 @@ static u8 weekday_to_integer(Optional<::Locale::Weekday> weekday, ::Locale::Week
     case ::Locale::Weekday::Sunday:
         return 7;
     }
-
     VERIFY_NOT_REACHED();
 }
 
-static Vector<u8> weekend_of_locale(StringView locale)
+static Vector<u8> weekend_of_locale(ReadonlySpan<::Locale::Weekday> const& weekend_days)
 {
-    auto weekend_start = weekday_to_integer(::Locale::get_locale_weekend_start(locale), ::Locale::Weekday::Saturday);
-    auto weekend_end = weekday_to_integer(::Locale::get_locale_weekend_end(locale), ::Locale::Weekday::Sunday);
-
-    // There currently aren't any regions in the CLDR which wrap around from Sunday (7) to Monday (1).
-    // If this changes, this logic will need to be updated to handle that.
-    VERIFY(weekend_start <= weekend_end);
-
     Vector<u8> weekend;
-    weekend.ensure_capacity(weekend_end - weekend_start + 1);
+    weekend.ensure_capacity(weekend_days.size());
 
-    for (auto day = weekend_start; day <= weekend_end; ++day)
+    for (auto day : weekend_days)
-        weekend.unchecked_append(day);
+        weekend.unchecked_append(weekday_to_integer(day, day));
 
+    quick_sort(weekend);
     return weekend;
 }
 
@@ -306,10 +299,12 @@ WeekInfo week_info_of_locale(Locale const& locale_object)
     VERIFY(::Locale::parse_unicode_locale_id(locale).has_value());
 
     // 3. Let r be a record whose fields are defined by Table 3, with values based on locale.
+    auto locale_week_info = ::Locale::week_info_of_locale(locale);
+
     WeekInfo week_info {};
-    week_info.minimal_days = ::Locale::get_locale_minimum_days(locale).value_or(1);
+    week_info.minimal_days = locale_week_info.minimal_days_in_first_week;
-    week_info.first_day = weekday_to_integer(::Locale::get_locale_first_day(locale), ::Locale::Weekday::Monday);
+    week_info.first_day = weekday_to_integer(locale_week_info.first_day_of_week, ::Locale::Weekday::Monday);
-    week_info.weekend = weekend_of_locale(locale);
+    week_info.weekend = weekend_of_locale(locale_week_info.weekend_days);
 
     // 4. Let fw be loc.[[FirstDayOfWeek]].
     // 5. If fw is not undefined, then

Userland/Libraries/LibLocale/DateTimeFormat.cpp

@@ -24,6 +24,7 @@
 #include <unicode/gregocal.h>
 #include <unicode/smpdtfmt.h>
 #include <unicode/timezone.h>
+#include <unicode/ucal.h>
 
 namespace Locale {
 
@@ -530,38 +531,6 @@ static T find_regional_values_for_locale(StringView locale, GetRegionalValues&&
     return return_default_values();
 }
 
-Optional<MinimumDaysRegion> __attribute__((weak)) minimum_days_region_from_string(StringView) { return {}; }
-Optional<u8> __attribute__((weak)) get_regional_minimum_days(StringView) { return {}; }
-
-Optional<u8> get_locale_minimum_days(StringView locale)
-{
-    return find_regional_values_for_locale<Optional<u8>>(locale, get_regional_minimum_days);
-}
-
-Optional<FirstDayRegion> __attribute__((weak)) first_day_region_from_string(StringView) { return {}; }
-Optional<Weekday> __attribute__((weak)) get_regional_first_day(StringView) { return {}; }
-
-Optional<Weekday> get_locale_first_day(StringView locale)
-{
-    return find_regional_values_for_locale<Optional<Weekday>>(locale, get_regional_first_day);
-}
-
-Optional<WeekendStartRegion> __attribute__((weak)) weekend_start_region_from_string(StringView) { return {}; }
-Optional<Weekday> __attribute__((weak)) get_regional_weekend_start(StringView) { return {}; }
-
-Optional<Weekday> get_locale_weekend_start(StringView locale)
-{
-    return find_regional_values_for_locale<Optional<Weekday>>(locale, get_regional_weekend_start);
-}
-
-Optional<WeekendEndRegion> __attribute__((weak)) weekend_end_region_from_string(StringView) { return {}; }
-Optional<Weekday> __attribute__((weak)) get_regional_weekend_end(StringView) { return {}; }
-
-Optional<Weekday> get_locale_weekend_end(StringView locale)
-{
-    return find_regional_values_for_locale<Optional<Weekday>>(locale, get_regional_weekend_end);
-}
-
 // ICU does not contain a field enumeration for "literal" partitions. Define a custom field so that we may provide a
 // type for those partitions.
 static constexpr i32 LITERAL_FIELD = -1;
@@ -961,4 +930,70 @@ NonnullOwnPtr<DateTimeFormat> DateTimeFormat::create_for_pattern_options(
     return adopt_own(*new DateTimeFormatImpl(locale_data->locale(), pattern, time_zone_identifier, move(formatter)));
 }
 
+static constexpr Weekday icu_calendar_day_to_weekday(UCalendarDaysOfWeek day)
+{
+    switch (day) {
+    case UCAL_SUNDAY:
+        return Weekday::Sunday;
+    case UCAL_MONDAY:
+        return Weekday::Monday;
+    case UCAL_TUESDAY:
+        return Weekday::Tuesday;
+    case UCAL_WEDNESDAY:
+        return Weekday::Wednesday;
+    case UCAL_THURSDAY:
+        return Weekday::Thursday;
+    case UCAL_FRIDAY:
+        return Weekday::Friday;
+    case UCAL_SATURDAY:
+        return Weekday::Saturday;
+    }
+    VERIFY_NOT_REACHED();
+}
+
+WeekInfo week_info_of_locale(StringView locale)
+{
+    UErrorCode status = U_ZERO_ERROR;
+
+    auto locale_data = LocaleData::for_locale(locale);
+    if (!locale_data.has_value())
+        return {};
+
+    auto calendar = adopt_own_if_nonnull(icu::Calendar::createInstance(locale_data->locale(), status));
+    if (icu_failure(status))
+        return {};
+
+    WeekInfo week_info;
+    week_info.minimal_days_in_first_week = calendar->getMinimalDaysInFirstWeek();
+
+    if (auto day = calendar->getFirstDayOfWeek(status); icu_success(status))
+        week_info.first_day_of_week = icu_calendar_day_to_weekday(day);
+
+    auto append_if_weekend = [&](auto day) {
+        auto type = calendar->getDayOfWeekType(day, status);
+        if (icu_failure(status))
+            return;
+
+        switch (type) {
+        case UCAL_WEEKEND_ONSET:
+        case UCAL_WEEKEND_CEASE:
+        case UCAL_WEEKEND:
+            week_info.weekend_days.append(icu_calendar_day_to_weekday(day));
+            break;
+        default:
+            break;
+        }
+    };
+
+    append_if_weekend(UCAL_SUNDAY);
+    append_if_weekend(UCAL_MONDAY);
+    append_if_weekend(UCAL_TUESDAY);
+    append_if_weekend(UCAL_WEDNESDAY);
+    append_if_weekend(UCAL_THURSDAY);
+    append_if_weekend(UCAL_FRIDAY);
+    append_if_weekend(UCAL_SATURDAY);
+
+    return week_info;
+}
+
 }

Userland/Libraries/LibLocale/DateTimeFormat.h

@@ -97,22 +97,6 @@ struct CalendarPattern {
     Optional<CalendarPatternStyle> time_zone_name;
 };
 
-Optional<MinimumDaysRegion> minimum_days_region_from_string(StringView minimum_days_region);
-Optional<u8> get_regional_minimum_days(StringView region);
-Optional<u8> get_locale_minimum_days(StringView locale);
-
-Optional<FirstDayRegion> first_day_region_from_string(StringView first_day_region);
-Optional<Weekday> get_regional_first_day(StringView region);
-Optional<Weekday> get_locale_first_day(StringView locale);
-
-Optional<WeekendStartRegion> weekend_start_region_from_string(StringView weekend_start_region);
-Optional<Weekday> get_regional_weekend_start(StringView region);
-Optional<Weekday> get_locale_weekend_start(StringView locale);
-
-Optional<WeekendEndRegion> weekend_end_region_from_string(StringView weekend_end_region);
-Optional<Weekday> get_regional_weekend_end(StringView region);
-Optional<Weekday> get_locale_weekend_end(StringView locale);
-
 class DateTimeFormat {
 public:
     static NonnullOwnPtr<DateTimeFormat> create_for_date_and_time_style(
@@ -148,4 +132,11 @@ protected:
     DateTimeFormat() = default;
 };
 
+struct WeekInfo {
+    u8 minimal_days_in_first_week { 1 };
+    Optional<Weekday> first_day_of_week;
+    Vector<Weekday> weekend_days;
+};
+WeekInfo week_info_of_locale(StringView locale);
+
 }

Userland/Libraries/LibLocale/Forward.h

@@ -11,7 +11,6 @@
 namespace Locale {
 
 enum class CalendarPatternStyle : u8;
-enum class FirstDayRegion : u8;
 enum class HourCycle : u8;
 enum class Key : u8;
 enum class KeywordCalendar : u8;
@@ -21,13 +20,10 @@ enum class KeywordColNumeric : u8;
 enum class KeywordHours : u8;
 enum class KeywordNumbers : u8;
 enum class Locale : u16;
-enum class MinimumDaysRegion : u8;
 enum class NumericSymbol : u8;
 enum class PluralCategory : u8;
 enum class Style : u8;
 enum class Weekday : u8;
-enum class WeekendEndRegion : u8;
-enum class WeekendStartRegion : u8;
 
 class NumberFormat;