mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2025-04-21 03:55:24 +00:00
AK: Remove endianness override from Utf16View
Utf16View is now always in "host" endian mode. This makes it smaller and less branchy for everyone!
This commit is contained in:
Andreas Kling
Andreas Kling
parent
0e9480b944
commit
7628ddfaf7
Notes:
github-actions[bot]
2025-04-16 08:06:00 +00:00
Author: https://github.com/awesomekling Commit: https://github.com/LadybirdBrowser/ladybird/commit/7628ddfaf77 Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/4370
5 changed files with 16 additions and 159 deletions
|
|
@ -100,32 +100,11 @@ ErrorOr<String> String::from_utf16(Utf16View const& utf16)
|
|||
|
||||
String result;
|
||||
|
||||
auto utf8_length = [&]() {
|
||||
switch (utf16.endianness()) {
|
||||
case Endianness::Host:
|
||||
return simdutf::utf8_length_from_utf16(utf16.char_data(), utf16.length_in_code_units());
|
||||
case Endianness::Big:
|
||||
return simdutf::utf8_length_from_utf16be(utf16.char_data(), utf16.length_in_code_units());
|
||||
case Endianness::Little:
|
||||
return simdutf::utf8_length_from_utf16le(utf16.char_data(), utf16.length_in_code_units());
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}();
|
||||
auto utf8_length = simdutf::utf8_length_from_utf16(utf16.char_data(), utf16.length_in_code_units());
|
||||
|
||||
TRY(result.replace_with_new_string(utf8_length, [&](Bytes buffer) -> ErrorOr<void> {
|
||||
[[maybe_unused]] auto result = [&]() {
|
||||
switch (utf16.endianness()) {
|
||||
case Endianness::Host:
|
||||
return simdutf::convert_utf16_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast<char*>(buffer.data()));
|
||||
case Endianness::Big:
|
||||
return simdutf::convert_utf16be_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast<char*>(buffer.data()));
|
||||
case Endianness::Little:
|
||||
return simdutf::convert_utf16le_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast<char*>(buffer.data()));
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}();
|
||||
[[maybe_unused]] auto result = simdutf::convert_utf16_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast<char*>(buffer.data()));
|
||||
ASSERT(result == buffer.size());
|
||||
|
||||
return {};
|
||||
}));
|
||||
|
||||
|
|
|
|||
|
|
@ -259,17 +259,7 @@ ErrorOr<void> StringBuilder::try_append(Utf16View const& utf16_view)
|
|||
auto uninitialized_data_pointer = static_cast<char*>(m_buffer.end_pointer());
|
||||
|
||||
// Fast path.
|
||||
auto result = [&]() {
|
||||
switch (remaining_view.endianness()) {
|
||||
case Endianness::Host:
|
||||
return simdutf::convert_utf16_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer);
|
||||
case Endianness::Big:
|
||||
return simdutf::convert_utf16be_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer);
|
||||
case Endianness::Little:
|
||||
return simdutf::convert_utf16le_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer);
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}();
|
||||
auto result = simdutf::convert_utf16_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer);
|
||||
if (result.error == simdutf::SUCCESS) {
|
||||
auto bytes_just_written = result.count;
|
||||
m_buffer.set_size(m_buffer.size() + bytes_just_written);
|
||||
|
|
@ -281,17 +271,7 @@ ErrorOr<void> StringBuilder::try_append(Utf16View const& utf16_view)
|
|||
ASSERT(first_invalid_code_unit < remaining_view.length_in_code_units());
|
||||
|
||||
// Unfortunately, `simdutf` does not tell us how many bytes it just wrote in case of an error, so we have to calculate it ourselves.
|
||||
auto bytes_just_written = [&]() {
|
||||
switch (remaining_view.endianness()) {
|
||||
case Endianness::Host:
|
||||
return simdutf::utf8_length_from_utf16(remaining_view.char_data(), first_invalid_code_unit);
|
||||
case Endianness::Big:
|
||||
return simdutf::utf8_length_from_utf16be(remaining_view.char_data(), first_invalid_code_unit);
|
||||
case Endianness::Little:
|
||||
return simdutf::utf8_length_from_utf16le(remaining_view.char_data(), first_invalid_code_unit);
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}();
|
||||
auto bytes_just_written = simdutf::utf8_length_from_utf16(remaining_view.char_data(), first_invalid_code_unit);
|
||||
|
||||
do {
|
||||
auto code_unit = remaining_view.code_unit_at(first_invalid_code_unit++);
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ size_t Utf16View::length_in_code_points() const
|
|||
u16 Utf16View::code_unit_at(size_t index) const
|
||||
{
|
||||
VERIFY(index < length_in_code_units());
|
||||
return host_code_unit(m_code_units[index], m_endianness);
|
||||
return host_code_unit(m_code_units[index], Endianness::Host);
|
||||
}
|
||||
|
||||
u32 Utf16View::code_point_at(size_t index) const
|
||||
|
|
@ -296,31 +296,12 @@ bool Utf16View::starts_with(Utf16View const& needle) const
|
|||
|
||||
bool Utf16View::validate() const
|
||||
{
|
||||
switch (m_endianness) {
|
||||
case Endianness::Host:
|
||||
return simdutf::validate_utf16(char_data(), length_in_code_units());
|
||||
case Endianness::Big:
|
||||
return simdutf::validate_utf16be(char_data(), length_in_code_units());
|
||||
case Endianness::Little:
|
||||
return simdutf::validate_utf16le(char_data(), length_in_code_units());
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
return simdutf::validate_utf16(char_data(), length_in_code_units());
|
||||
}
|
||||
|
||||
bool Utf16View::validate(size_t& valid_code_units) const
|
||||
{
|
||||
auto result = [&]() {
|
||||
switch (m_endianness) {
|
||||
case Endianness::Host:
|
||||
return simdutf::validate_utf16_with_errors(char_data(), length_in_code_units());
|
||||
case Endianness::Big:
|
||||
return simdutf::validate_utf16be_with_errors(char_data(), length_in_code_units());
|
||||
case Endianness::Little:
|
||||
return simdutf::validate_utf16le_with_errors(char_data(), length_in_code_units());
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}();
|
||||
|
||||
auto result = simdutf::validate_utf16_with_errors(char_data(), length_in_code_units());
|
||||
valid_code_units = result.count;
|
||||
return result.error == simdutf::SUCCESS;
|
||||
}
|
||||
|
|
@ -330,16 +311,8 @@ size_t Utf16View::calculate_length_in_code_points() const
|
|||
// FIXME: simdutf's code point length method assumes valid UTF-16, whereas Utf16View uses U+FFFD as a replacement
|
||||
// for invalid code points. If we change Utf16View to only accept valid encodings as an invariant, we can
|
||||
// remove this branch.
|
||||
if (validate()) [[likely]] {
|
||||
switch (m_endianness) {
|
||||
case Endianness::Host:
|
||||
return simdutf::count_utf16(char_data(), length_in_code_units());
|
||||
case Endianness::Big:
|
||||
return simdutf::count_utf16be(char_data(), length_in_code_units());
|
||||
case Endianness::Little:
|
||||
return simdutf::count_utf16le(char_data(), length_in_code_units());
|
||||
}
|
||||
}
|
||||
if (validate()) [[likely]]
|
||||
return simdutf::count_utf16(char_data(), length_in_code_units());
|
||||
|
||||
size_t code_points = 0;
|
||||
for ([[maybe_unused]] auto code_point : *this)
|
||||
|
|
@ -397,11 +370,11 @@ u32 Utf16CodePointIterator::operator*() const
|
|||
// W2 as its 10 low-order bits.
|
||||
// 5) Add 0x10000 to U' to obtain the character value U. Terminate.
|
||||
|
||||
auto code_unit = host_code_unit(*m_ptr, m_endianness);
|
||||
auto code_unit = host_code_unit(*m_ptr, Endianness::Host);
|
||||
|
||||
if (Utf16View::is_high_surrogate(code_unit)) {
|
||||
if (m_remaining_code_units > 1) {
|
||||
auto next_code_unit = host_code_unit(*(m_ptr + 1), m_endianness);
|
||||
auto next_code_unit = host_code_unit(*(m_ptr + 1), Endianness::Host);
|
||||
|
||||
if (Utf16View::is_low_surrogate(next_code_unit))
|
||||
return Utf16View::decode_surrogate_pair(code_unit, next_code_unit);
|
||||
|
|
|
|||
|
|
@ -51,16 +51,14 @@ public:
|
|||
size_t length_in_code_units() const;
|
||||
|
||||
private:
|
||||
Utf16CodePointIterator(u16 const* ptr, size_t length, Endianness endianness)
|
||||
Utf16CodePointIterator(u16 const* ptr, size_t length)
|
||||
: m_ptr(ptr)
|
||||
, m_remaining_code_units(length)
|
||||
, m_endianness(endianness)
|
||||
{
|
||||
}
|
||||
|
||||
u16 const* m_ptr { nullptr };
|
||||
size_t m_remaining_code_units { 0 };
|
||||
Endianness m_endianness { Endianness::Host };
|
||||
};
|
||||
|
||||
class Utf16View {
|
||||
|
|
@ -74,18 +72,16 @@ public:
|
|||
Utf16View() = default;
|
||||
~Utf16View() = default;
|
||||
|
||||
explicit Utf16View(ReadonlySpan<u16> code_units, Endianness endianness = Endianness::Host)
|
||||
explicit Utf16View(ReadonlySpan<u16> code_units)
|
||||
: m_code_units(code_units)
|
||||
, m_endianness(endianness)
|
||||
{
|
||||
}
|
||||
|
||||
template<size_t Size>
|
||||
Utf16View(char16_t const (&code_units)[Size], Endianness endianness = Endianness::Host)
|
||||
Utf16View(char16_t const (&code_units)[Size])
|
||||
: m_code_units(
|
||||
reinterpret_cast<u16 const*>(&code_units[0]),
|
||||
code_units[Size - 1] == u'\0' ? Size - 1 : Size)
|
||||
, m_endianness(endianness)
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -104,10 +100,8 @@ public:
|
|||
size_t length_in_code_units() const { return m_code_units.size(); }
|
||||
size_t length_in_code_points() const;
|
||||
|
||||
Endianness endianness() const { return m_endianness; }
|
||||
|
||||
Utf16CodePointIterator begin() const { return { begin_ptr(), m_code_units.size(), m_endianness }; }
|
||||
Utf16CodePointIterator end() const { return { end_ptr(), 0, m_endianness }; }
|
||||
Utf16CodePointIterator begin() const { return { begin_ptr(), m_code_units.size() }; }
|
||||
Utf16CodePointIterator end() const { return { end_ptr(), 0 }; }
|
||||
|
||||
u16 const* data() const { return m_code_units.data(); }
|
||||
char16_t const* char_data() const { return reinterpret_cast<char16_t const*>(data()); }
|
||||
|
|
@ -142,7 +136,6 @@ private:
|
|||
|
||||
ReadonlySpan<u16> m_code_units;
|
||||
NO_UNIQUE_ADDRESS mutable Optional<size_t> m_length_in_code_points;
|
||||
Endianness m_endianness { Endianness::Host };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -357,71 +357,3 @@ TEST_CASE(starts_with)
|
|||
EXPECT(!emoji.starts_with(u"a"));
|
||||
EXPECT(!emoji.starts_with(u"🙃"));
|
||||
}
|
||||
|
||||
TEST_CASE(big_endian)
|
||||
{
|
||||
auto string = MUST(AK::utf8_to_utf16("säk😀"sv, AK::Endianness::Big));
|
||||
Utf16View view { string, AK::Endianness::Big };
|
||||
{
|
||||
EXPECT(view.validate());
|
||||
EXPECT_EQ(MUST(view.to_utf8()), "säk😀"sv);
|
||||
|
||||
EXPECT_EQ(view.length_in_code_units(), 5u);
|
||||
EXPECT_EQ(view.length_in_code_points(), 4u);
|
||||
|
||||
EXPECT_EQ(view.code_unit_at(0), 0x73u);
|
||||
EXPECT_EQ(view.code_unit_at(1), 0xe4u);
|
||||
EXPECT_EQ(view.code_unit_at(2), 0x6bu);
|
||||
EXPECT_EQ(view.code_unit_at(3), 0xd83d);
|
||||
EXPECT_EQ(view.code_unit_at(4), 0xde00u);
|
||||
|
||||
EXPECT_EQ(view.code_point_at(0), 0x73u);
|
||||
EXPECT_EQ(view.code_point_at(1), 0xe4u);
|
||||
EXPECT_EQ(view.code_point_at(2), 0x6bu);
|
||||
EXPECT_EQ(view.code_point_at(3), 0x1f600u);
|
||||
EXPECT_EQ(view.code_point_at(4), 0xde00u);
|
||||
}
|
||||
{
|
||||
Utf16Data data;
|
||||
MUST(code_point_to_utf16(data, 's', AK::Endianness::Big));
|
||||
MUST(code_point_to_utf16(data, 0xe4, AK::Endianness::Big));
|
||||
MUST(code_point_to_utf16(data, 'k', AK::Endianness::Big));
|
||||
MUST(code_point_to_utf16(data, 0x1f600, AK::Endianness::Big));
|
||||
EXPECT_EQ(data, to_array<u16>({ 0x7300, 0xe400, 0x6b00, 0x3dd8, 0x00de }));
|
||||
EXPECT_EQ(data, string);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE(little_endian)
|
||||
{
|
||||
auto string = MUST(AK::utf8_to_utf16("säk😀"sv, AK::Endianness::Little));
|
||||
Utf16View view { string, AK::Endianness::Little };
|
||||
{
|
||||
EXPECT(view.validate());
|
||||
EXPECT_EQ(MUST(view.to_utf8()), "säk😀"sv);
|
||||
|
||||
EXPECT_EQ(view.length_in_code_units(), 5u);
|
||||
EXPECT_EQ(view.length_in_code_points(), 4u);
|
||||
|
||||
EXPECT_EQ(view.code_unit_at(0), 0x73u);
|
||||
EXPECT_EQ(view.code_unit_at(1), 0xe4u);
|
||||
EXPECT_EQ(view.code_unit_at(2), 0x6bu);
|
||||
EXPECT_EQ(view.code_unit_at(3), 0xd83d);
|
||||
EXPECT_EQ(view.code_unit_at(4), 0xde00u);
|
||||
|
||||
EXPECT_EQ(view.code_point_at(0), 0x73u);
|
||||
EXPECT_EQ(view.code_point_at(1), 0xe4u);
|
||||
EXPECT_EQ(view.code_point_at(2), 0x6bu);
|
||||
EXPECT_EQ(view.code_point_at(3), 0x1f600u);
|
||||
EXPECT_EQ(view.code_point_at(4), 0xde00u);
|
||||
}
|
||||
{
|
||||
Utf16Data data;
|
||||
MUST(code_point_to_utf16(data, 's', AK::Endianness::Little));
|
||||
MUST(code_point_to_utf16(data, 0xe4, AK::Endianness::Little));
|
||||
MUST(code_point_to_utf16(data, 'k', AK::Endianness::Little));
|
||||
MUST(code_point_to_utf16(data, 0x1f600, AK::Endianness::Little));
|
||||
EXPECT_EQ(data, to_array<u16>({ 0x73, 0xe4, 0x6b, 0xd83d, 0xde00 }));
|
||||
EXPECT_EQ(data, string);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue