AK: Add a UTF-16 string with optimized short- and ASCII-string storage

This is a strictly UTF-16 string with some optimizations for ASCII.

* If created from a short UTF-8 or UTF-16 string that is also ASCII,
  then the string is stored in an inlined byte buffer.

* If created with a long UTF-8 or UTF-16 string that is also ASCII,
  then the string is stored in an outlined char buffer.

* If created with a short or long UTF-8 or UTF-16 string that is not
  ASCII, then the string is stored in an outlined char16 buffer.

We do not store short non-ASCII text in the inlined buffer to avoid
confusion with operations such as `length_in_code_units` and
`code_unit_at`. For example, "😀" would be stored as 4 UTF-8 bytes
in short string form. But we still want `length_in_code_units` to
be 2, and `code_unit_at(0)` to be 0xD83D.

AK/Utf16StringData.cpp

@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2025, Tim Flynn <trflynn89@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/TypedTransfer.h>
+#include <AK/Utf16StringData.h>
+#include <AK/Utf32View.h>
+#include <AK/Utf8View.h>
+
+#include <simdutf.h>
+
+namespace AK::Detail {
+
+// Due to internal optimizations, we have an explicit maximum string length of 2**63 - 1.
+#define VERIFY_UTF16_LENGTH(length) VERIFY(length >> Detail::UTF16_FLAG == 0);
+
+NonnullRefPtr<Utf16StringData> Utf16StringData::create_uninitialized(StorageType storage_type, size_t code_unit_length)
+{
+    auto allocation_size = storage_type == Utf16StringData::StorageType::ASCII
+        ? sizeof(Utf16StringData) + (sizeof(char) * code_unit_length)
+        : sizeof(Utf16StringData) + (sizeof(char16_t) * code_unit_length);
+
+    void* slot = malloc(allocation_size);
+    VERIFY(slot);
+
+    return adopt_ref(*new (slot) Utf16StringData(storage_type, code_unit_length));
+}
+
+template<typename ViewType>
+NonnullRefPtr<Utf16StringData> Utf16StringData::create_from_code_point_iterable(ViewType const& view)
+{
+    size_t code_unit_length = 0;
+    size_t code_point_length = 0;
+
+    for (auto code_point : view) {
+        code_unit_length += UnicodeUtils::code_unit_length_for_code_point(code_point);
+        ++code_point_length;
+    }
+
+    VERIFY_UTF16_LENGTH(code_unit_length);
+
+    auto string = create_uninitialized(StorageType::UTF16, code_unit_length);
+    string->m_length_in_code_points = code_point_length;
+
+    size_t code_unit_index = 0;
+
+    for (auto code_point : view) {
+        (void)UnicodeUtils::code_point_to_utf16(code_point, [&](auto code_unit) {
+            string->m_utf16_data[code_unit_index++] = code_unit;
+        });
+    }
+
+    return string;
+}
+
+NonnullRefPtr<Utf16StringData> Utf16StringData::from_utf8(StringView utf8_string, AllowASCIIStorage allow_ascii_storage)
+{
+    RefPtr<Utf16StringData> string;
+
+    if (allow_ascii_storage == AllowASCIIStorage::Yes && utf8_string.is_ascii()) {
+        VERIFY_UTF16_LENGTH(utf8_string.length());
+
+        string = create_uninitialized(StorageType::ASCII, utf8_string.length());
+        TypedTransfer<char>::copy(string->m_ascii_data, utf8_string.characters_without_null_termination(), utf8_string.length());
+    } else if (Utf8View view { utf8_string }; view.validate(AllowLonelySurrogates::No)) {
+        auto code_unit_length = simdutf::utf16_length_from_utf8(utf8_string.characters_without_null_termination(), utf8_string.length());
+        VERIFY_UTF16_LENGTH(code_unit_length);
+
+        string = create_uninitialized(StorageType::UTF16, code_unit_length);
+
+        auto result = simdutf::convert_utf8_to_utf16(utf8_string.characters_without_null_termination(), utf8_string.length(), string->m_utf16_data);
+        VERIFY(result == code_unit_length);
+    } else {
+        string = create_from_code_point_iterable(view);
+    }
+
+    return string.release_nonnull();
+}
+
+NonnullRefPtr<Utf16StringData> Utf16StringData::from_utf16(Utf16View const& utf16_string)
+{
+    VERIFY_UTF16_LENGTH(utf16_string.length_in_code_units());
+    RefPtr<Utf16StringData> string;
+
+    if (utf16_string.has_ascii_storage()) {
+        string = create_uninitialized(StorageType::ASCII, utf16_string.length_in_code_units());
+        TypedTransfer<char>::copy(string->m_ascii_data, utf16_string.ascii_span().data(), utf16_string.length_in_code_units());
+    } else if (utf16_string.is_ascii()) {
+        string = create_uninitialized(StorageType::ASCII, utf16_string.length_in_code_units());
+
+        auto result = simdutf::convert_utf16_to_utf8(utf16_string.utf16_span().data(), utf16_string.length_in_code_units(), string->m_ascii_data);
+        VERIFY(result == utf16_string.length_in_code_units());
+    } else {
+        string = create_uninitialized(StorageType::UTF16, utf16_string.length_in_code_units());
+        TypedTransfer<char16_t>::copy(string->m_utf16_data, utf16_string.utf16_span().data(), utf16_string.length_in_code_units());
+
+        string->m_length_in_code_points = utf16_string.m_length_in_code_points;
+    }
+
+    return string.release_nonnull();
+}
+
+NonnullRefPtr<Utf16StringData> Utf16StringData::from_utf32(Utf32View const& utf32_string)
+{
+    RefPtr<Utf16StringData> string;
+
+    auto const* utf32_data = reinterpret_cast<char32_t const*>(utf32_string.code_points());
+    auto utf32_length = utf32_string.length();
+
+    if (utf32_string.is_ascii()) {
+        VERIFY_UTF16_LENGTH(utf32_length);
+
+        string = create_uninitialized(StorageType::ASCII, utf32_length);
+
+        auto result = simdutf::convert_utf32_to_utf8(utf32_data, utf32_length, string->m_ascii_data);
+        VERIFY(result == utf32_length);
+    } else if (simdutf::validate_utf32(utf32_data, utf32_length)) {
+        auto code_unit_length = simdutf::utf16_length_from_utf32(utf32_data, utf32_length);
+        VERIFY_UTF16_LENGTH(code_unit_length);
+
+        string = create_uninitialized(StorageType::UTF16, code_unit_length);
+        string->m_length_in_code_points = utf32_length;
+
+        auto result = simdutf::convert_utf32_to_utf16(utf32_data, utf32_length, string->m_utf16_data);
+        VERIFY(result == code_unit_length);
+    } else {
+        string = create_from_code_point_iterable(utf32_string);
+    }
+
+    return string.release_nonnull();
+}
+
+size_t Utf16StringData::calculate_code_point_length() const
+{
+    ASSERT(!has_ascii_storage());
+
+    if (simdutf::validate_utf16(m_utf16_data, length_in_code_units()))
+        return simdutf::count_utf16(m_utf16_data, length_in_code_units());
+
+    size_t code_points = 0;
+    for ([[maybe_unused]] auto code_point : utf16_view())
+        ++code_points;
+    return code_points;
+}
+
+}