diff --git a/libraries/libstratosphere/include/stratosphere/fs/fs_memory_management.hpp b/libraries/libstratosphere/include/stratosphere/fs/fs_memory_management.hpp index cf8f62304..b0a357d0d 100644 --- a/libraries/libstratosphere/include/stratosphere/fs/fs_memory_management.hpp +++ b/libraries/libstratosphere/include/stratosphere/fs/fs_memory_management.hpp @@ -18,6 +18,9 @@ namespace ams::fs { + /* Controls whether MakeUniqueBuffer uses a custom buffer wrapper which wraps the size inline. */ + #define AMS_FS_IMPL_MAKE_UNIQUE_BUFFER_WITH_INLINE_SIZE + /* ACCURATE_TO_VERSION: Unknown */ using AllocateFunction = void *(*)(size_t); using DeallocateFunction = void (*)(void *, size_t); @@ -26,6 +29,8 @@ namespace ams::fs { namespace impl { + class Newable; + void *Allocate(size_t size); void Deallocate(void *ptr, size_t size); @@ -129,21 +134,175 @@ namespace ams::fs { } }; + #if defined(AMS_FS_IMPL_MAKE_UNIQUE_BUFFER_WITH_INLINE_SIZE) + template BufferEntryType> + class BufferWithInlineSize final { + static_assert(sizeof(BufferEntryType) == 1); + NON_COPYABLE(BufferWithInlineSize); + NON_MOVEABLE(BufferWithInlineSize); + private: + template static constexpr inline T EncodedSizeMask = util::IsLittleEndian() ? (static_cast(3u) << (BITSIZEOF(T) - 2)) : static_cast(3u); + + template static constexpr inline T EncodedSize1 = util::IsLittleEndian() ? (static_cast(0u) << (BITSIZEOF(T) - 2)) : static_cast(0u); + template static constexpr inline T EncodedSize2 = util::IsLittleEndian() ? (static_cast(1u) << (BITSIZEOF(T) - 2)) : static_cast(1u); + template static constexpr inline T EncodedSize4 = util::IsLittleEndian() ? (static_cast(2u) << (BITSIZEOF(T) - 2)) : static_cast(2u); + template static constexpr inline T EncodedSize8 = util::IsLittleEndian() ? (static_cast(3u) << (BITSIZEOF(T) - 2)) : static_cast(3u); + + static constexpr inline u64 TestSize1Mask = ~((static_cast(1u) << (BITSIZEOF(u8) - 2)) - static_cast(1)); + static constexpr inline u64 TestSize2Mask = ~((static_cast(1u) << (BITSIZEOF(u16) - 2)) - static_cast(1)); + static constexpr inline u64 TestSize4Mask = ~((static_cast(1u) << (BITSIZEOF(u32) - 2)) - static_cast(1)); + static constexpr inline u64 TestSize8Mask = ~((static_cast(1u) << (BITSIZEOF(u64) - 2)) - static_cast(1)); + + template + static constexpr ALWAYS_INLINE SizeType EncodeSize(SizeType type, size_t size) noexcept { + if constexpr (util::IsLittleEndian()) { + return type | static_cast(size); + } else { + return type | (static_cast(size) << 2); + } + } + + template + static constexpr ALWAYS_INLINE size_t DecodeSize(const SizeType encoded) noexcept { + if constexpr (util::IsLittleEndian()) { + /* Small optimization: 1-byte size has size type field == 0 and no shifting, can return the value directly. */ + if constexpr (sizeof(SizeType) == 1) { + static_assert(EncodedSize1 == 0); + return encoded; + } else { + /* On little endian, we want to mask out the high bits storing the size field. */ + constexpr SizeType DecodedSizeMask = static_cast(~EncodedSizeMask); + + return encoded & DecodedSizeMask; + } + } else { + /* On big endian, we want to shift out the low bits storing the size type field. */ + return encoded >> 2; + } + } + + template + static ALWAYS_INLINE void DeleteBufferImpl(BufferEntryType *buffer) noexcept { + /* Get pointer to start of allocation. */ + SizeType *alloc = reinterpret_cast(buffer) - 1; + + /* Decode the size of the allocation. */ + const size_t alloc_size = sizeof(SizeType) + DecodeSize(*alloc); + + /* Delete the buffer. */ + return ::ams::fs::impl::Deallocate(alloc, alloc_size); + } + + template + static std::unique_ptr MakeBuffer(size_t size) noexcept { + /* Allocate a buffer. */ + SizeType *alloc = static_cast(::ams::fs::impl::Allocate(sizeof(SizeType) + size)); + if (AMS_UNLIKELY(alloc == nullptr)) { + return nullptr; + } + + /* Write the encoded size. */ + if constexpr (util::IsLittleEndian()) { + *alloc = EncodedSizeType | static_cast(size); + } else { + *alloc = EncodedSizeType | (static_cast(size) << 2); + } + + /* Return our buffer. */ + return std::unique_ptr(reinterpret_cast(alloc + 1)); + } + + static void DeleteBuffer(BufferEntryType *buffer) noexcept { + /* Convert to u8 pointer */ + const u8 *buffer_u8 = reinterpret_cast(buffer); + + /* Determine the storage size for the size. */ + const auto size_type = buffer_u8[-1] & EncodedSizeMask; + if (size_type == EncodedSize1) { + return DeleteBufferImpl(buffer); + } else if (size_type == EncodedSize2) { + return DeleteBufferImpl(buffer); + } else if (size_type == EncodedSize4) { + return DeleteBufferImpl(buffer); + } else /* if (size_type == EncodedSize8) */ { + return DeleteBufferImpl(buffer); + } + } + private: + BufferEntryType m_buffer[1]; + private: + ALWAYS_INLINE BufferWithInlineSize() noexcept { /* ... */ } + public: + ALWAYS_INLINE ~BufferWithInlineSize() noexcept { /* ... */ } + public: + ALWAYS_INLINE operator BufferEntryType *() noexcept { + return m_buffer; + } + + ALWAYS_INLINE operator const BufferEntryType *() const noexcept { + return m_buffer; + } + public: + static ALWAYS_INLINE std::unique_ptr Make(size_t size) noexcept { + /* Create based on overhead size. */ + if (!(size & TestSize1Mask)) { + return MakeBuffer>(size); + } else if (!(size & TestSize2Mask)) { + return MakeBuffer>(size); + } else if (!(size & TestSize4Mask)) { + return MakeBuffer>(size); + } else /* if (!(size & TestSize8Mask)) */ { + /* Check pre-condition. */ + AMS_ASSERT(!(size & TestSize8Mask)); + return MakeBuffer>(size); + } + } + public: + static ALWAYS_INLINE void *operator new(size_t) noexcept { AMS_ABORT(AMS_CURRENT_FUNCTION_NAME); } + + static ALWAYS_INLINE void *operator new(size_t size, Newable *placement) noexcept { AMS_ABORT(AMS_CURRENT_FUNCTION_NAME); } + + static ALWAYS_INLINE void operator delete(void *ptr, size_t) noexcept { + /* Delete the buffer. */ + DeleteBuffer(reinterpret_cast(ptr)->m_buffer); + } + + static void *operator new[](size_t size) noexcept = delete; + static void operator delete[](void *ptr, size_t size) noexcept = delete; + }; + #endif + template - std::unique_ptr MakeUnique() { - static_assert(util::is_pod::value); + auto MakeUnique() { + /* Check that we're not using MakeUnique unnecessarily. */ + static_assert(!std::derived_from); + return std::unique_ptr(static_cast(::ams::fs::impl::Allocate(sizeof(T))), Deleter(sizeof(T))); } template - std::unique_ptr MakeUnique(size_t size) { + auto MakeUnique(size_t size) { using T = typename std::remove_extent::type; static_assert(util::is_pod::value); static_assert(std::is_array::value); + /* Check that we're not using MakeUnique unnecessarily. */ + static_assert(!std::derived_from); + + using ReturnType = std::unique_ptr; + const size_t alloc_size = sizeof(T) * size; - return std::unique_ptr(static_cast(::ams::fs::impl::Allocate(alloc_size)), Deleter(alloc_size)); + return ReturnType(static_cast(::ams::fs::impl::Allocate(alloc_size)), Deleter(alloc_size)); + } + + template + auto MakeUniqueBuffer(size_t size) { + #if defined(AMS_FS_IMPL_MAKE_UNIQUE_BUFFER_WITH_INLINE_SIZE) + return BufferWithInlineSize::Make(size); + #else + return ::ams::fs::impl::MakeUnique(size); + #endif } } diff --git a/libraries/libstratosphere/include/stratosphere/fs/fs_path.hpp b/libraries/libstratosphere/include/stratosphere/fs/fs_path.hpp index c51ff32e7..a29fdff1d 100644 --- a/libraries/libstratosphere/include/stratosphere/fs/fs_path.hpp +++ b/libraries/libstratosphere/include/stratosphere/fs/fs_path.hpp @@ -33,20 +33,30 @@ namespace ams::fs { private: friend class DirectoryPathParser; private: - using WriteBuffer = std::unique_ptr; + using WriteBuffer = decltype(::ams::fs::impl::MakeUniqueBuffer(0)); + + template T> + static ALWAYS_INLINE char *GetBuffer(T &write_buffer) { + if constexpr (std::same_as(0))>) { + return write_buffer.get(); + } else { + return static_cast(*write_buffer); + } + } + + static ALWAYS_INLINE WriteBuffer MakeWriteBuffer(size_t size) { + return ::ams::fs::impl::MakeUniqueBuffer(size); + } private: const char *m_str; util::TypedStorage m_write_buffer; - size_t m_write_buffer_length; - bool m_is_normalized; + size_t m_write_buffer_length_and_is_normalized; public: - Path() : m_str(EmptyPath), m_write_buffer_length(0), m_is_normalized(false) { + Path() : m_str(EmptyPath), m_write_buffer_length_and_is_normalized(0) { util::ConstructAt(m_write_buffer, nullptr); } - constexpr Path(const char *s, util::ConstantInitializeTag) : m_str(s), m_write_buffer(), m_write_buffer_length(0), m_is_normalized(true) { - /* ... */ - } + constexpr Path(const char *s, util::ConstantInitializeTag) : m_str(s), m_write_buffer(), m_write_buffer_length_and_is_normalized(1) { } constexpr ~Path() { if (!std::is_constant_evaluated()) { @@ -59,8 +69,8 @@ namespace ams::fs { AMS_ASSERT(util::GetReference(m_write_buffer) != nullptr); /* Reset. */ - m_str = EmptyPath; - m_write_buffer_length = 0; + m_str = EmptyPath; + this->SetWriteBufferLength(0); /* Return our write buffer. */ return std::move(util::GetReference(m_write_buffer)); @@ -68,7 +78,7 @@ namespace ams::fs { constexpr Result SetShallowBuffer(const char *buffer) { /* Check pre-conditions. */ - AMS_ASSERT(m_write_buffer_length == 0); + AMS_ASSERT(this->GetWriteBufferLength() == 0); /* Check the buffer is valid. */ R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); @@ -77,14 +87,14 @@ namespace ams::fs { this->SetReadOnlyBuffer(buffer); /* Note that we're normalized. */ - m_is_normalized = true; + this->SetNormalized(); R_SUCCEED(); } const char *GetString() const { /* Check pre-conditions. */ - AMS_ASSERT(m_is_normalized); + AMS_ASSERT(this->IsNormalized()); return m_str; } @@ -103,18 +113,19 @@ namespace ams::fs { Result Initialize(const Path &rhs) { /* Check the other path is normalized. */ - R_UNLESS(rhs.m_is_normalized, fs::ResultNotNormalized()); + const bool normalized = rhs.IsNormalized(); + R_UNLESS(normalized, fs::ResultNotNormalized()); /* Allocate buffer for our path. */ const auto len = rhs.GetLength(); R_TRY(this->Preallocate(len + 1)); /* Copy the path. */ - const size_t copied = util::Strlcpy(util::GetReference(m_write_buffer).get(), rhs.GetString(), len + 1); + const size_t copied = util::Strlcpy(GetBuffer(util::GetReference(m_write_buffer)), rhs.GetString(), len + 1); R_UNLESS(copied == len, fs::ResultUnexpectedInPathA()); /* Set normalized. */ - m_is_normalized = rhs.m_is_normalized; + this->SetNormalized(); R_SUCCEED(); } @@ -126,7 +137,7 @@ namespace ams::fs { R_TRY(this->InitializeImpl(path, len)); /* Set not normalized. */ - m_is_normalized = false; + this->SetNotNormalized(); R_SUCCEED(); } @@ -154,7 +165,7 @@ namespace ams::fs { R_TRY(this->Preallocate(len + 1)); /* Format our path into our new buffer. */ - const auto real_len = util::VSNPrintf(util::GetReference(m_write_buffer).get(), m_write_buffer_length, fmt, vl); + const auto real_len = util::VSNPrintf(GetBuffer(util::GetReference(m_write_buffer)), this->GetWriteBufferLength(), fmt, vl); AMS_ASSERT(real_len == len); AMS_UNUSED(real_len); @@ -162,7 +173,7 @@ namespace ams::fs { va_end(vl); /* Set not normalized. */ - m_is_normalized = false; + this->SetNotNormalized(); R_SUCCEED(); } @@ -175,12 +186,12 @@ namespace ams::fs { R_TRY(this->InitializeImpl(path, std::strlen(path))); /* Replace slashes as desired. */ - if (m_write_buffer_length > 1) { - fs::Replace(this->GetWriteBuffer(), m_write_buffer_length - 1, '\\', '/'); + if (const auto write_buffer_length = this->GetWriteBufferLength(); write_buffer_length > 1) { + fs::Replace(this->GetWriteBuffer(), write_buffer_length - 1, '\\', '/'); } /* Set not normalized. */ - m_is_normalized = false; + this->SetNotNormalized(); R_SUCCEED(); } @@ -193,7 +204,7 @@ namespace ams::fs { R_TRY(this->InitializeImpl(path, std::strlen(path))); /* Replace slashes as desired. */ - if (m_write_buffer_length > 1) { + if (this->GetWriteBufferLength() > 1) { if (auto *p = this->GetWriteBuffer(); p[0] == '/' && p[1] == '/') { p[0] = '\\'; p[1] = '\\'; @@ -201,7 +212,7 @@ namespace ams::fs { } /* Set not normalized. */ - m_is_normalized = false; + this->SetNotNormalized(); R_SUCCEED(); } @@ -214,7 +225,7 @@ namespace ams::fs { R_TRY(this->InitializeImpl(path, std::strlen(path))); /* Set not normalized. */ - m_is_normalized = false; + this->SetNotNormalized(); /* Replace unc as desired. */ if (m_str[0]) { @@ -251,7 +262,7 @@ namespace ams::fs { R_TRY(this->InitializeImpl(path, size)); /* Set not normalized. */ - m_is_normalized = false; + this->SetNotNormalized(); /* Perform normalization. */ fs::PathFlags path_flags; @@ -263,16 +274,17 @@ namespace ams::fs { /* NOTE: In this case, Nintendo checks is normalized, then sets is normalized, then returns success. */ /* This seems like a bug. */ size_t dummy; - R_TRY(PathFormatter::IsNormalized(std::addressof(m_is_normalized), std::addressof(dummy), m_str)); + bool normalized; + R_TRY(PathFormatter::IsNormalized(std::addressof(normalized), std::addressof(dummy), m_str)); - m_is_normalized = true; + this->SetNormalized(); R_SUCCEED(); } /* Normalize. */ R_TRY(this->Normalize(path_flags)); - m_is_normalized = true; + this->SetNormalized(); R_SUCCEED(); } @@ -288,7 +300,7 @@ namespace ams::fs { this->ClearBuffer(); /* Set normalized. */ - m_is_normalized = true; + this->SetNormalized(); R_SUCCEED(); } @@ -335,7 +347,7 @@ namespace ams::fs { /* Get our write buffer. */ auto *dst = this->GetWriteBuffer(); if (old_write_buffer != nullptr && cur_len > 0) { - util::Strlcpy(dst, old_write_buffer.get(), cur_len + 1); + util::Strlcpy(dst, static_cast(*old_write_buffer), cur_len + 1); } /* Add separator. */ @@ -379,7 +391,7 @@ namespace ams::fs { if (util::GetReference(m_write_buffer) == nullptr) { if (const auto len = std::strlen(m_str); len > 0) { R_TRY(this->Preallocate(len)); - util::Strlcpy(util::GetReference(m_write_buffer).get(), m_str, len + 1); + util::Strlcpy(GetBuffer(util::GetReference(m_write_buffer)), m_str, len + 1); } } @@ -414,7 +426,7 @@ namespace ams::fs { Result Normalize(const PathFlags &flags) { /* If we're already normalized, nothing to do. */ - R_SUCCEED_IF(m_is_normalized); + R_SUCCEED_IF(this->IsNormalized()); /* Check if we're normalized. */ bool normalized; @@ -424,7 +436,7 @@ namespace ams::fs { /* If we're not normalized, normalize. */ if (!normalized) { /* Determine necessary buffer length. */ - auto len = m_write_buffer_length; + auto len = this->GetWriteBufferLength(); if (flags.IsRelativePathAllowed() && fs::IsPathRelative(m_str)) { len += 2; } @@ -434,24 +446,24 @@ namespace ams::fs { /* Allocate a new buffer. */ const size_t size = util::AlignUp(len, WriteBufferAlignmentLength); - auto buf = fs::impl::MakeUnique(size); + auto buf = MakeWriteBuffer(size); R_UNLESS(buf != nullptr, fs::ResultAllocationMemoryFailedMakeUnique()); /* Normalize into it. */ - R_TRY(PathFormatter::Normalize(buf.get(), size, util::GetReference(m_write_buffer).get(), m_write_buffer_length, flags)); + R_TRY(PathFormatter::Normalize(*buf, size, GetBuffer(util::GetReference(m_write_buffer)), this->GetWriteBufferLength(), flags)); /* Set the normalized buffer as our buffer. */ this->SetModifiableBuffer(std::move(buf), size); } /* Set normalized. */ - m_is_normalized = true; + this->SetNormalized(); R_SUCCEED(); } private: void ClearBuffer() { util::GetReference(m_write_buffer).reset(); - m_write_buffer_length = 0; + this->SetWriteBufferLength(0); m_str = EmptyPath; } @@ -463,24 +475,24 @@ namespace ams::fs { /* Set write buffer. */ util::GetReference(m_write_buffer) = std::move(buffer); - m_write_buffer_length = size; - m_str = util::GetReference(m_write_buffer).get(); + this->SetWriteBufferLength(size); + m_str = GetBuffer(util::GetReference(m_write_buffer)); } constexpr void SetReadOnlyBuffer(const char *buffer) { m_str = buffer; if (!std::is_constant_evaluated()) { util::GetReference(m_write_buffer) = nullptr; - m_write_buffer_length = 0; + this->SetWriteBufferLength(0); } } Result Preallocate(size_t length) { /* Allocate additional space, if needed. */ - if (length > m_write_buffer_length) { + if (length > this->GetWriteBufferLength()) { /* Allocate buffer. */ const size_t size = util::AlignUp(length, WriteBufferAlignmentLength); - auto buf = fs::impl::MakeUnique(size); + auto buf = MakeWriteBuffer(size); R_UNLESS(buf != nullptr, fs::ResultAllocationMemoryFailedMakeUnique()); /* Set write buffer. */ @@ -508,11 +520,27 @@ namespace ams::fs { char *GetWriteBuffer() { AMS_ASSERT(util::GetReference(m_write_buffer) != nullptr); - return util::GetReference(m_write_buffer).get(); + return GetBuffer(util::GetReference(m_write_buffer)); } - size_t GetWriteBufferLength() const { - return m_write_buffer_length; + constexpr ALWAYS_INLINE size_t GetWriteBufferLength() const { + return m_write_buffer_length_and_is_normalized >> 1; + } + + constexpr ALWAYS_INLINE void SetWriteBufferLength(size_t size) { + m_write_buffer_length_and_is_normalized = (m_write_buffer_length_and_is_normalized & 1) | (size << 1); + } + + constexpr ALWAYS_INLINE bool IsNormalized() const { + return static_cast(m_write_buffer_length_and_is_normalized & 1); + } + + constexpr ALWAYS_INLINE void SetNormalized() { + m_write_buffer_length_and_is_normalized |= static_cast(1); + } + + constexpr ALWAYS_INLINE void SetNotNormalized() { + m_write_buffer_length_and_is_normalized &= ~static_cast(1); } public: ALWAYS_INLINE bool operator==(const fs::Path &rhs) const { return std::strcmp(this->GetString(), rhs.GetString()) == 0; }