ladybird/AK/Utf8View.h
Timothy Flynn 9fc3e72db2 AK+Everywhere: Allow lonely UTF-16 surrogates by default
By definition, the web allows lonely surrogates by default. Let's have
our string APIs reflect this, so we don't have to pass an allow option
all over the place.
2025-07-03 09:51:56 -04:00

329 lines
11 KiB
C++

/*
* Copyright (c) 2019-2020, Sergey Bugaev <bugaevc@serenityos.org>
* Copyright (c) 2021, Max Wipfli <mail@maxwipfli.ch>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/ByteString.h>
#include <AK/Debug.h>
#include <AK/Format.h>
#include <AK/Function.h>
#include <AK/StringView.h>
#include <AK/Types.h>
#include <AK/UnicodeUtils.h>
namespace AK {
class Utf8View;
class Utf8CodePointIterator {
friend class Utf8View;
friend class ByteString;
public:
Utf8CodePointIterator() = default;
~Utf8CodePointIterator() = default;
bool operator==(Utf8CodePointIterator const&) const = default;
bool operator!=(Utf8CodePointIterator const&) const = default;
Utf8CodePointIterator& operator++();
u32 operator*() const;
// NOTE: This returns {} if the peek is at or past EOF.
Optional<u32> peek(size_t offset = 0) const;
ssize_t operator-(Utf8CodePointIterator const& other) const
{
return m_ptr - other.m_ptr;
}
u8 const* ptr() const { return m_ptr; }
// Note : These methods return the information about the underlying UTF-8 bytes.
// If the UTF-8 string encoding is not valid at the iterator's position, then the underlying bytes might be different from the
// decoded character's re-encoded bytes (which will be an `0xFFFD REPLACEMENT CHARACTER` with an UTF-8 length of three bytes).
// If your code relies on the decoded character being equivalent to the re-encoded character, use the `UTF8View::validate()`
// method on the view prior to using its iterator.
size_t underlying_code_point_length_in_bytes() const;
ReadonlyBytes underlying_code_point_bytes() const { return { m_ptr, underlying_code_point_length_in_bytes() }; }
bool done() const { return m_length == 0; }
private:
Utf8CodePointIterator(u8 const* ptr, size_t length)
: m_ptr(ptr)
, m_length(length)
{
}
u8 const* m_ptr { nullptr };
size_t m_length { 0 };
};
class Utf8View {
public:
using Iterator = Utf8CodePointIterator;
Utf8View() = default;
explicit constexpr Utf8View(StringView string)
: m_string(string)
{
}
explicit Utf8View(ByteString& string)
: m_string(string.view())
{
}
explicit Utf8View(ByteString&&) = delete;
~Utf8View() = default;
StringView as_string() const { return m_string; }
Utf8CodePointIterator begin() const { return { begin_ptr(), m_string.length() }; }
Utf8CodePointIterator end() const { return { end_ptr(), 0 }; }
Utf8CodePointIterator iterator_at_byte_offset(size_t) const;
Utf8CodePointIterator iterator_at_byte_offset_without_validation(size_t) const;
unsigned char const* bytes() const { return begin_ptr(); }
size_t byte_length() const { return m_string.length(); }
size_t code_point_offset_of(size_t code_unit_offset) const;
[[nodiscard]] size_t byte_offset_of(Utf8CodePointIterator const& it) const
{
VERIFY(it.m_ptr >= begin_ptr());
VERIFY(it.m_ptr <= end_ptr());
return it.m_ptr - begin_ptr();
}
size_t byte_offset_of(size_t code_point_offset) const;
Utf8View substring_view(size_t byte_offset, size_t byte_length) const { return Utf8View { m_string.substring_view(byte_offset, byte_length) }; }
Utf8View substring_view(size_t byte_offset) const { return substring_view(byte_offset, byte_length() - byte_offset); }
Utf8View unicode_substring_view(size_t code_point_offset, size_t code_point_length) const;
Utf8View unicode_substring_view(size_t code_point_offset) const { return unicode_substring_view(code_point_offset, length() - code_point_offset); }
bool is_empty() const { return m_string.is_empty(); }
bool is_null() const { return m_string.is_null(); }
bool starts_with(Utf8View const&) const;
bool contains(u32) const;
bool contains_any_of(ReadonlySpan<u32>) const;
Utf8View trim(Utf8View const& characters, TrimMode mode = TrimMode::Both) const;
size_t iterator_offset(Utf8CodePointIterator const& it) const
{
return byte_offset_of(it);
}
size_t length() const
{
if (!m_have_length) {
m_length = calculate_length();
m_have_length = true;
}
return m_length;
}
bool validate(AllowLonelySurrogates allow_lonely_surrogates = AllowLonelySurrogates::Yes) const
{
size_t valid_bytes = 0;
return validate(valid_bytes, allow_lonely_surrogates);
}
bool validate(size_t& valid_bytes, AllowLonelySurrogates allow_lonely_surrogates = AllowLonelySurrogates::Yes) const;
template<typename Callback>
auto for_each_split_view(Function<bool(u32)> splitter, SplitBehavior split_behavior, Callback callback) const
{
bool keep_empty = has_flag(split_behavior, SplitBehavior::KeepEmpty);
bool keep_trailing_separator = has_flag(split_behavior, SplitBehavior::KeepTrailingSeparator);
auto start_offset = 0u;
auto offset = 0u;
auto run_callback = [&]() {
auto length = offset - start_offset;
if (length == 0 && !keep_empty)
return;
auto substring = unicode_substring_view(start_offset, length);
// Reject splitter-only entries if we're not keeping empty results
if (keep_trailing_separator && !keep_empty && length == 1 && splitter(*substring.begin()))
return;
callback(substring);
};
auto iterator = begin();
while (iterator != end()) {
if (splitter(*iterator)) {
if (keep_trailing_separator)
++offset;
run_callback();
if (!keep_trailing_separator)
++offset;
start_offset = offset;
++iterator;
continue;
}
++offset;
++iterator;
}
run_callback();
}
private:
friend class Utf8CodePointIterator;
u8 const* begin_ptr() const { return reinterpret_cast<u8 const*>(m_string.characters_without_null_termination()); }
u8 const* end_ptr() const { return begin_ptr() + m_string.length(); }
size_t calculate_length() const;
struct Utf8EncodedByteData {
size_t byte_length { 0 };
u8 encoding_bits { 0 };
u8 encoding_mask { 0 };
u32 first_code_point { 0 };
u32 last_code_point { 0 };
};
static constexpr Array<Utf8EncodedByteData, 4> utf8_encoded_byte_data { {
{ 1, 0b0000'0000, 0b1000'0000, 0x0000, 0x007F },
{ 2, 0b1100'0000, 0b1110'0000, 0x0080, 0x07FF },
{ 3, 0b1110'0000, 0b1111'0000, 0x0800, 0xFFFF },
{ 4, 0b1111'0000, 0b1111'1000, 0x10000, 0x10FFFF },
} };
struct LeadingByte {
size_t byte_length { 0 };
u32 code_point_bits { 0 };
bool is_valid { false };
};
static constexpr LeadingByte decode_leading_byte(u8 byte)
{
for (auto const& data : utf8_encoded_byte_data) {
if ((byte & data.encoding_mask) != data.encoding_bits)
continue;
byte &= ~data.encoding_mask;
return { data.byte_length, byte, true };
}
return { .is_valid = false };
}
StringView m_string;
mutable size_t m_length { 0 };
mutable bool m_have_length { false };
};
template<>
struct Formatter<Utf8View> : Formatter<StringView> {
ErrorOr<void> format(FormatBuilder&, Utf8View const&);
};
inline Utf8CodePointIterator& Utf8CodePointIterator::operator++()
{
VERIFY(m_length > 0);
// OPTIMIZATION: Fast path for ASCII characters.
if (*m_ptr <= 0x7F) {
m_ptr += 1;
m_length -= 1;
return *this;
}
size_t code_point_length_in_bytes = underlying_code_point_length_in_bytes();
if (code_point_length_in_bytes > m_length) {
// We don't have enough data for the next code point. Skip one character and try again.
// The rest of the code will output replacement characters as needed for any eventual extension bytes we might encounter afterwards.
dbgln_if(UTF8_DEBUG, "Expected code point size {} is too big for the remaining length {}. Moving forward one byte.", code_point_length_in_bytes, m_length);
m_ptr += 1;
m_length -= 1;
return *this;
}
m_ptr += code_point_length_in_bytes;
m_length -= code_point_length_in_bytes;
return *this;
}
inline size_t Utf8CodePointIterator::underlying_code_point_length_in_bytes() const
{
VERIFY(m_length > 0);
auto [code_point_length_in_bytes, value, first_byte_makes_sense] = Utf8View::decode_leading_byte(*m_ptr);
// If any of these tests fail, we will output a replacement character for this byte and treat it as a code point of size 1.
if (!first_byte_makes_sense)
return 1;
if (code_point_length_in_bytes > m_length)
return 1;
for (size_t offset = 1; offset < code_point_length_in_bytes; offset++) {
if (m_ptr[offset] >> 6 != 2)
return 1;
}
return code_point_length_in_bytes;
}
inline u32 Utf8CodePointIterator::operator*() const
{
VERIFY(m_length > 0);
// OPTIMIZATION: Fast path for ASCII characters.
if (*m_ptr <= 0x7F)
return *m_ptr;
auto [code_point_length_in_bytes, code_point_value_so_far, first_byte_makes_sense] = Utf8View::decode_leading_byte(*m_ptr);
if (!first_byte_makes_sense) {
// The first byte of the code point doesn't make sense: output a replacement character
dbgln_if(UTF8_DEBUG, "First byte doesn't make sense: {:#02x}.", m_ptr[0]);
return 0xFFFD;
}
if (code_point_length_in_bytes > m_length) {
// There is not enough data left for the full code point: output a replacement character
dbgln_if(UTF8_DEBUG, "Not enough bytes (need {}, have {}), first byte is: {:#02x}.", code_point_length_in_bytes, m_length, m_ptr[0]);
return 0xFFFD;
}
for (size_t offset = 1; offset < code_point_length_in_bytes; offset++) {
if (m_ptr[offset] >> 6 != 2) {
// One of the extension bytes of the code point doesn't make sense: output a replacement character
dbgln_if(UTF8_DEBUG, "Extension byte {:#02x} in {} position after first byte {:#02x} doesn't make sense.", m_ptr[offset], offset, m_ptr[0]);
return 0xFFFD;
}
code_point_value_so_far <<= 6;
code_point_value_so_far |= m_ptr[offset] & 63;
}
if (code_point_value_so_far > 0x10FFFF) {
dbgln_if(UTF8_DEBUG, "Multi-byte sequence is otherwise valid, but code point {:#x} is not permissible.", code_point_value_so_far);
return 0xFFFD;
}
return code_point_value_so_far;
}
}
#if USING_AK_GLOBALLY
using AK::Utf8CodePointIterator;
using AK::Utf8View;
#endif