diff --git a/Kernel/API/Syscall.h b/Kernel/API/Syscall.h index 24d544d8fa0..114a159c2b9 100644 --- a/Kernel/API/Syscall.h +++ b/Kernel/API/Syscall.h @@ -49,7 +49,6 @@ enum class NeedsBigProcessLock { S(accept4, NeedsBigProcessLock::No) \ S(adjtime, NeedsBigProcessLock::No) \ S(alarm, NeedsBigProcessLock::No) \ - S(allocate_tls, NeedsBigProcessLock::No) \ S(archctl, NeedsBigProcessLock::No) \ S(anon_create, NeedsBigProcessLock::No) \ S(annotate_mapping, NeedsBigProcessLock::No) \ @@ -375,6 +374,7 @@ struct SC_create_thread_params { void* stack_location; // nullptr means any, o.w. process virtual address void* (*entry)(void*); void* entry_argument; + void* tls_pointer; }; struct SC_realpath_params { diff --git a/Kernel/API/archctl_numbers.h b/Kernel/API/archctl_numbers.h index d9338348729..cb341b45de6 100644 --- a/Kernel/API/archctl_numbers.h +++ b/Kernel/API/archctl_numbers.h @@ -5,3 +5,5 @@ */ #pragma once + +#define ARCHCTL_X86_64_SET_FS_BASE_FOR_CURRENT_THREAD 1 diff --git a/Kernel/Arch/Processor.h b/Kernel/Arch/Processor.h index d96b2ce4a3a..0b6e4959b80 100644 --- a/Kernel/Arch/Processor.h +++ b/Kernel/Arch/Processor.h @@ -165,8 +165,6 @@ public: static void deferred_call_queue(Function callback); - static void set_thread_specific_data(VirtualAddress thread_specific_data); - [[noreturn]] void initialize_context_switching(Thread& initial_thread); NEVER_INLINE void switch_context(Thread*& from_thread, Thread*& to_thread); [[noreturn]] static void assume_context(Thread& thread, InterruptsState new_interrupts_state); diff --git a/Kernel/Arch/ProcessorFunctions.include b/Kernel/Arch/ProcessorFunctions.include index 0e1dfd310d0..c0eb89237ee 100644 --- a/Kernel/Arch/ProcessorFunctions.include +++ b/Kernel/Arch/ProcessorFunctions.include @@ -26,7 +26,6 @@ template bool ProcessorBase::are_interrupts_enabled(); template void ProcessorBase::wait_for_interrupt() const; template Processor& ProcessorBase::by_id(u32 id); template StringView ProcessorBase::platform_string(); -template void ProcessorBase::set_thread_specific_data(VirtualAddress thread_specific_data); template void ProcessorBase::initialize_context_switching(Thread& initial_thread); template void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread); template void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state); diff --git a/Kernel/Arch/aarch64/Interrupts.cpp b/Kernel/Arch/aarch64/Interrupts.cpp index 5826d718a9b..a7d054d2c0f 100644 --- a/Kernel/Arch/aarch64/Interrupts.cpp +++ b/Kernel/Arch/aarch64/Interrupts.cpp @@ -43,6 +43,7 @@ void dump_registers(RegisterState const& regs) dbgln("Saved Program Status: (NZCV({:#b}) DAIF({:#b}) M({:#b})) / {:#x}", ((regs.spsr_el1 >> 28) & 0b1111), ((regs.spsr_el1 >> 6) & 0b1111), regs.spsr_el1 & 0b1111, regs.spsr_el1); dbgln("Exception Link Register: {:#x}", regs.elr_el1); dbgln("Stack Pointer (EL0): {:#x}", regs.sp_el0); + dbgln("Software Thread ID Register (EL0): {:#x}", regs.tpidr_el0); dbgln(" x0={:p} x1={:p} x2={:p} x3={:p} x4={:p}", regs.x[0], regs.x[1], regs.x[2], regs.x[3], regs.x[4]); dbgln(" x5={:p} x6={:p} x7={:p} x8={:p} x9={:p}", regs.x[5], regs.x[6], regs.x[7], regs.x[8], regs.x[9]); diff --git a/Kernel/Arch/aarch64/Processor.cpp b/Kernel/Arch/aarch64/Processor.cpp index 7287e2cdf0f..0388fb011b1 100644 --- a/Kernel/Arch/aarch64/Processor.cpp +++ b/Kernel/Arch/aarch64/Processor.cpp @@ -337,6 +337,7 @@ FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) } eretframe.elr_el1 = thread_regs.elr_el1; eretframe.sp_el0 = thread_regs.sp_el0; + eretframe.tpidr_el0 = thread_regs.tpidr_el0; eretframe.spsr_el1 = thread_regs.spsr_el1; // Push a TrapFrame onto the stack @@ -472,8 +473,6 @@ extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) to_thread->set_cpu(Processor::current().id()); - Processor::set_thread_specific_data(to_thread->thread_specific_data()); - auto in_critical = to_thread->saved_critical(); VERIFY(in_critical > 0); Processor::restore_critical(in_critical); @@ -487,12 +486,6 @@ StringView ProcessorBase::platform_string() return "aarch64"sv; } -template -void ProcessorBase::set_thread_specific_data(VirtualAddress thread_specific_data) -{ - Aarch64::Asm::set_tpidr_el0(thread_specific_data.get()); -} - template void ProcessorBase::wait_for_interrupt() const { diff --git a/Kernel/Arch/aarch64/RegisterState.h b/Kernel/Arch/aarch64/RegisterState.h index 4f0caf90b8b..b794ef8b74b 100644 --- a/Kernel/Arch/aarch64/RegisterState.h +++ b/Kernel/Arch/aarch64/RegisterState.h @@ -17,11 +17,12 @@ VALIDATE_IS_AARCH64() namespace Kernel { -struct RegisterState { - u64 x[31]; // Saved general purpose registers - u64 spsr_el1; // Save Processor Status Register, EL1 - u64 elr_el1; // Exception Link Register, EL1 - u64 sp_el0; // EL0 stack pointer +struct alignas(16) RegisterState { + u64 x[31]; // Saved general purpose registers + u64 spsr_el1; // Save Processor Status Register, EL1 + u64 elr_el1; // Exception Link Register, EL1 + u64 sp_el0; // EL0 stack pointer + u64 tpidr_el0; // EL0 Software Thread ID Register FlatPtr userspace_sp() const { return sp_el0; } void set_userspace_sp(FlatPtr value) @@ -51,7 +52,7 @@ struct RegisterState { } }; -#define REGISTER_STATE_SIZE (34 * 8) +#define REGISTER_STATE_SIZE (36 * 8) static_assert(AssertSize()); inline void copy_kernel_registers_into_ptrace_registers(PtraceRegisters& ptrace_regs, RegisterState const& kernel_regs) diff --git a/Kernel/Arch/aarch64/ThreadRegisters.h b/Kernel/Arch/aarch64/ThreadRegisters.h index 30843c5324b..f9f217a929b 100644 --- a/Kernel/Arch/aarch64/ThreadRegisters.h +++ b/Kernel/Arch/aarch64/ThreadRegisters.h @@ -17,6 +17,7 @@ struct ThreadRegisters { u64 spsr_el1; u64 elr_el1; u64 sp_el0; + u64 tpidr_el0; u64 ttbr0_el1; FlatPtr ip() const { return elr_el1; } diff --git a/Kernel/Arch/aarch64/vector_table.S b/Kernel/Arch/aarch64/vector_table.S index ceaa0213aa1..6f3e2be6736 100644 --- a/Kernel/Arch/aarch64/vector_table.S +++ b/Kernel/Arch/aarch64/vector_table.S @@ -8,7 +8,7 @@ // NOTE: This size must be a multiple of 16 bytes, to ensure that the stack pointer // stays 16 byte aligned. -#define REGISTER_STATE_SIZE 272 +#define REGISTER_STATE_SIZE (36 * 8) #if REGISTER_STATE_SIZE % 16 != 0 # error "REGISTER_STATE_SIZE is not a multiple of 16 bytes!" #endif @@ -16,6 +16,7 @@ #define SPSR_EL1_SLOT (31 * 8) #define ELR_EL1_SLOT (32 * 8) #define SP_EL0_SLOT (33 * 8) +#define TPIDR_EL0_SLOT (34 * 8) // Vector Table Entry macro. Each entry is aligned at 128 bytes, meaning we have // at most that many instructions. @@ -65,6 +66,8 @@ str x0, [sp, #ELR_EL1_SLOT] mrs x0, sp_el0 str x0, [sp, #SP_EL0_SLOT] + mrs x0, tpidr_el0 + str x0, [sp, #TPIDR_EL0_SLOT] // Set up TrapFrame struct on the stack mov x0, sp @@ -88,6 +91,8 @@ msr elr_el1, x0 ldr x0, [sp, #SP_EL0_SLOT] msr sp_el0, x0 + ldr x0, [sp, #TPIDR_EL0_SLOT] + msr tpidr_el0, x0 ldp x0, x1, [sp, #(0 * 0)] ldp x2, x3, [sp, #(2 * 8)] diff --git a/Kernel/Arch/riscv64/Processor.cpp b/Kernel/Arch/riscv64/Processor.cpp index eaf5ec570a9..17f69d13a38 100644 --- a/Kernel/Arch/riscv64/Processor.cpp +++ b/Kernel/Arch/riscv64/Processor.cpp @@ -499,8 +499,6 @@ extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) to_thread->set_cpu(Processor::current().id()); - Processor::set_thread_specific_data(to_thread->thread_specific_data()); - auto in_critical = to_thread->saved_critical(); VERIFY(in_critical > 0); Processor::restore_critical(in_critical); @@ -546,12 +544,6 @@ StringView ProcessorBase::platform_string() return "riscv64"sv; } -template -void ProcessorBase::set_thread_specific_data(VirtualAddress) -{ - // FIXME: Add support for thread-local storage on RISC-V -} - template void ProcessorBase::wait_for_interrupt() const { diff --git a/Kernel/Arch/x86_64/ArchSpecificThreadData.h b/Kernel/Arch/x86_64/ArchSpecificThreadData.h index 5dbe07b8d27..ffee9d5c8fd 100644 --- a/Kernel/Arch/x86_64/ArchSpecificThreadData.h +++ b/Kernel/Arch/x86_64/ArchSpecificThreadData.h @@ -11,6 +11,7 @@ namespace Kernel { struct ArchSpecificThreadData { + FlatPtr fs_base { 0 }; }; } diff --git a/Kernel/Arch/x86_64/Processor.cpp b/Kernel/Arch/x86_64/Processor.cpp index 15f3e840997..c4883860644 100644 --- a/Kernel/Arch/x86_64/Processor.cpp +++ b/Kernel/Arch/x86_64/Processor.cpp @@ -1390,7 +1390,7 @@ extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) } auto& processor = Processor::current(); - Processor::set_thread_specific_data(to_thread->thread_specific_data()); + Processor::set_fs_base(to_thread->arch_specific_data().fs_base); if (from_regs.cr3 != to_regs.cr3) write_cr3(to_regs.cr3); @@ -1717,11 +1717,10 @@ UNMAP_AFTER_INIT void ProcessorBase::initialize_context_switching(Thread& ini VERIFY_NOT_REACHED(); } -template -void ProcessorBase::set_thread_specific_data(VirtualAddress thread_specific_data) +void Processor::set_fs_base(FlatPtr fs_base) { MSR fs_base_msr(MSR_FS_BASE); - fs_base_msr.set(thread_specific_data.get()); + fs_base_msr.set(fs_base); } template diff --git a/Kernel/Arch/x86_64/Processor.h b/Kernel/Arch/x86_64/Processor.h index a2a1cf7f74d..3781265b15d 100644 --- a/Kernel/Arch/x86_64/Processor.h +++ b/Kernel/Arch/x86_64/Processor.h @@ -153,6 +153,8 @@ public: static void smp_unicast(u32 cpu, Function, bool async); static void smp_broadcast_flush_tlb(Memory::PageDirectory const*, VirtualAddress, size_t); + + static void set_fs_base(FlatPtr); }; template diff --git a/Kernel/Arch/x86_64/archctl.cpp b/Kernel/Arch/x86_64/archctl.cpp index 98b0eab4d19..7e471746142 100644 --- a/Kernel/Arch/x86_64/archctl.cpp +++ b/Kernel/Arch/x86_64/archctl.cpp @@ -4,17 +4,24 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include namespace Kernel { ErrorOr Process::sys$archctl(int option, FlatPtr arg1) { - (void)option; - (void)arg1; - VERIFY_NO_PROCESS_BIG_LOCK(this); - return ENOSYS; + switch (option) { + case ARCHCTL_X86_64_SET_FS_BASE_FOR_CURRENT_THREAD: { + Thread::current()->arch_specific_data().fs_base = arg1; + Processor::set_fs_base(arg1); + return 0; + } + + default: + return EINVAL; + } } } diff --git a/Kernel/Syscalls/execve.cpp b/Kernel/Syscalls/execve.cpp index e8452ac7251..0e9cdccfb02 100644 --- a/Kernel/Syscalls/execve.cpp +++ b/Kernel/Syscalls/execve.cpp @@ -33,9 +33,6 @@ struct LoadResult { FlatPtr load_base { 0 }; FlatPtr entry_eip { 0 }; size_t size { 0 }; - LockWeakPtr tls_region; - size_t tls_size { 0 }; - size_t tls_alignment { 0 }; LockWeakPtr stack_region; }; @@ -258,18 +255,13 @@ static ErrorOr get_load_offset(Elf_Ehdr const& main_program_header, Ope return random_load_offset_in_range(selected_range.start, selected_range.end - selected_range.start); } -enum class ShouldAllocateTls { - No, - Yes, -}; - enum class ShouldAllowSyscalls { No, Yes, }; static ErrorOr load_elf_object(Memory::AddressSpace& new_space, OpenFileDescription& object_description, - FlatPtr load_offset, ShouldAllocateTls should_allocate_tls, ShouldAllowSyscalls should_allow_syscalls, Optional minimum_stack_size = {}) + FlatPtr load_offset, ShouldAllowSyscalls should_allow_syscalls, Optional minimum_stack_size = {}) { auto& inode = *(object_description.inode()); auto vmobject = TRY(Memory::SharedInodeVMObject::try_create_with_inode(inode)); @@ -288,9 +280,6 @@ static ErrorOr load_elf_object(Memory::AddressSpace& new_space, Open if (!elf_image.is_valid()) return ENOEXEC; - Memory::Region* master_tls_region { nullptr }; - size_t master_tls_size = 0; - size_t master_tls_alignment = 0; FlatPtr load_base_address = 0; size_t stack_size = Thread::default_userspace_stack_size; @@ -302,24 +291,6 @@ static ErrorOr load_elf_object(Memory::AddressSpace& new_space, Open Memory::MemoryManager::enter_address_space(new_space); - auto load_tls_section = [&](auto& program_header) -> ErrorOr { - VERIFY(should_allocate_tls == ShouldAllocateTls::Yes); - VERIFY(program_header.size_in_memory()); - - if (!elf_image.is_within_image(program_header.raw_data(), program_header.size_in_image())) { - dbgln("Shenanigans! ELF PT_TLS header sneaks outside of executable."); - return ENOEXEC; - } - - auto region_name = TRY(KString::formatted("{} (master-tls)", elf_name)); - master_tls_region = TRY(new_space.allocate_region(Memory::RandomizeVirtualAddress::Yes, {}, program_header.size_in_memory(), PAGE_SIZE, region_name->view(), PROT_READ | PROT_WRITE, AllocationStrategy::Reserve)); - master_tls_size = program_header.size_in_memory(); - master_tls_alignment = program_header.alignment(); - - TRY(copy_to_user(master_tls_region->vaddr().as_ptr(), program_header.raw_data(), program_header.size_in_image())); - return {}; - }; - auto load_writable_section = [&](auto& program_header) -> ErrorOr { // Writable section: create a copy in memory. VERIFY(program_header.alignment() % PAGE_SIZE == 0); @@ -385,9 +356,6 @@ static ErrorOr load_elf_object(Memory::AddressSpace& new_space, Open }; auto load_elf_program_header = [&](auto& program_header) -> ErrorOr { - if (program_header.type() == PT_TLS) - return load_tls_section(program_header); - if (program_header.type() == PT_LOAD) return load_section(program_header); @@ -416,9 +384,6 @@ static ErrorOr load_elf_object(Memory::AddressSpace& new_space, Open load_base_address, elf_image.entry().offset(load_offset).get(), executable_size, - TRY(AK::try_make_weak_ptr_if_nonnull(master_tls_region)), - master_tls_size, - master_tls_alignment, TRY(stack_region->try_make_weak_ptr()) }; } @@ -429,24 +394,10 @@ Process::load(Memory::AddressSpace& new_space, NonnullRefPtr Process::do_exec(NonnullRefPtr main_program_d auto allocated_space = TRY(Memory::AddressSpace::try_create(*this, nullptr)); OwnPtr old_space; - auto old_master_tls = m_master_tls.with([](auto& master_tls) { - auto old = master_tls; - master_tls.region = nullptr; - master_tls.size = 0; - master_tls.alignment = 0; - return old; - }); auto& new_space = m_space.with([&](auto& space) -> Memory::AddressSpace& { old_space = move(space); space = move(allocated_space); @@ -509,9 +453,6 @@ ErrorOr Process::do_exec(NonnullRefPtr main_program_d m_space.with([&](auto& space) { space = old_space.release_nonnull(); }); - m_master_tls.with([&](auto& master_tls) { - master_tls = old_master_tls; - }); Memory::MemoryManager::enter_process_address_space(*this); }); @@ -703,11 +644,6 @@ ErrorOr Process::do_exec(NonnullRefPtr main_program_d protected_data.pid = new_main_thread->tid().value(); }); - auto tsr_result = new_main_thread->make_thread_specific_region({}); - if (tsr_result.is_error()) { - // FIXME: We cannot fail this late. Refactor this so the allocation happens before we commit to the new executable. - VERIFY_NOT_REACHED(); - } new_main_thread->reset_fpu_state(); auto& regs = new_main_thread->m_regs; diff --git a/Kernel/Syscalls/fork.cpp b/Kernel/Syscalls/fork.cpp index 9470442cc74..85a1fad4b2d 100644 --- a/Kernel/Syscalls/fork.cpp +++ b/Kernel/Syscalls/fork.cpp @@ -149,6 +149,7 @@ ErrorOr Process::sys$fork(RegisterState& regs) child_regs.spsr_el1 = regs.spsr_el1; child_regs.elr_el1 = regs.elr_el1; child_regs.sp_el0 = regs.sp_el0; + child_regs.tpidr_el0 = regs.tpidr_el0; #elif ARCH(RISCV64) for (size_t i = 0; i < array_size(child_regs.x); ++i) child_regs.x[i] = regs.x[i]; @@ -162,27 +163,15 @@ ErrorOr Process::sys$fork(RegisterState& regs) #endif TRY(address_space().with([&](auto& parent_space) { - return m_master_tls.with([&](auto& parent_master_tls) -> ErrorOr { - return child->address_space().with([&](auto& child_space) -> ErrorOr { - child_space->set_enforces_syscall_regions(parent_space->enforces_syscall_regions()); - for (auto& region : parent_space->region_tree().regions()) { - dbgln_if(FORK_DEBUG, "fork: cloning Region '{}' @ {}", region.name(), region.vaddr()); - auto region_clone = TRY(region.try_clone()); - TRY(region_clone->map(child_space->page_directory(), Memory::ShouldFlushTLB::No)); - TRY(child_space->region_tree().place_specifically(*region_clone, region.range())); - auto* child_region = region_clone.leak_ptr(); - - if (®ion == parent_master_tls.region.unsafe_ptr()) { - TRY(child->m_master_tls.with([&](auto& child_master_tls) -> ErrorOr { - child_master_tls.region = TRY(child_region->try_make_weak_ptr()); - child_master_tls.size = parent_master_tls.size; - child_master_tls.alignment = parent_master_tls.alignment; - return {}; - })); - } - } - return {}; - }); + return child->address_space().with([&](auto& child_space) -> ErrorOr { + child_space->set_enforces_syscall_regions(parent_space->enforces_syscall_regions()); + for (auto& region : parent_space->region_tree().regions()) { + dbgln_if(FORK_DEBUG, "fork: cloning Region '{}' @ {}", region.name(), region.vaddr()); + auto region_clone = TRY(region.try_clone()); + TRY(region_clone->map(child_space->page_directory(), Memory::ShouldFlushTLB::No)); + TRY(child_space->region_tree().place_specifically(*region_clone, region.range())); + (void)region_clone.leak_ptr(); + } return {}; }); })); diff --git a/Kernel/Syscalls/mmap.cpp b/Kernel/Syscalls/mmap.cpp index 12baf992ec3..72a849f165e 100644 --- a/Kernel/Syscalls/mmap.cpp +++ b/Kernel/Syscalls/mmap.cpp @@ -522,57 +522,6 @@ ErrorOr Process::sys$mremap(Userspace }); } -ErrorOr Process::sys$allocate_tls(Userspace initial_data, size_t size) -{ - VERIFY_NO_PROCESS_BIG_LOCK(this); - TRY(require_promise(Pledge::stdio)); - - if (!size || size % PAGE_SIZE != 0) - return EINVAL; - - return m_master_tls.with([&](auto& master_tls) -> ErrorOr { - if (!master_tls.region.is_null()) - return EEXIST; - - if (thread_count() != 1) - return EFAULT; - - Thread* main_thread = nullptr; - bool multiple_threads = false; - for_each_thread([&main_thread, &multiple_threads](auto& thread) { - if (main_thread) - multiple_threads = true; - main_thread = &thread; - return IterationDecision::Break; - }); - VERIFY(main_thread); - - if (multiple_threads) - return EINVAL; - - return address_space().with([&](auto& space) -> ErrorOr { - auto* region = TRY(space->allocate_region(Memory::RandomizeVirtualAddress::Yes, {}, size, PAGE_SIZE, "Master TLS"sv, PROT_READ | PROT_WRITE)); - - master_tls.region = TRY(region->try_make_weak_ptr()); - master_tls.size = size; - master_tls.alignment = PAGE_SIZE; - - { - Kernel::SmapDisabler disabler; - void* fault_at; - if (!Kernel::safe_memcpy((char*)master_tls.region.unsafe_ptr()->vaddr().as_ptr(), (char*)initial_data.ptr(), size, fault_at)) - return EFAULT; - } - - TRY(main_thread->make_thread_specific_region({})); - - Processor::set_thread_specific_data(main_thread->thread_specific_data()); - - return master_tls.region.unsafe_ptr()->vaddr().get(); - }); - }); -} - ErrorOr Process::sys$annotate_mapping(Userspace address, int flags) { VERIFY_NO_PROCESS_BIG_LOCK(this); diff --git a/Kernel/Syscalls/thread.cpp b/Kernel/Syscalls/thread.cpp index 31dd0560ad5..779c3782fd8 100644 --- a/Kernel/Syscalls/thread.cpp +++ b/Kernel/Syscalls/thread.cpp @@ -68,6 +68,8 @@ ErrorOr Process::sys$create_thread(void* (*entry)(void*), Userspacearch_specific_data().fs_base = bit_cast(params.tls_pointer); #elif ARCH(AARCH64) regs.ttbr0_el1 = address_space().with([](auto& space) { return space->page_directory().ttbr0(); }); @@ -76,6 +78,8 @@ ErrorOr Process::sys$create_thread(void* (*entry)(void*), Userspace(params.tls_pointer); #elif ARCH(RISCV64) regs.satp = address_space().with([](auto& space) { return space->page_directory().satp(); }); @@ -84,12 +88,12 @@ ErrorOr Process::sys$create_thread(void* (*entry)(void*), Userspace(params.tls_pointer); #else # error Unknown architecture #endif - TRY(thread->make_thread_specific_region({})); - PerformanceManager::add_thread_created_event(*thread); SpinlockLocker lock(g_scheduler_lock); diff --git a/Kernel/Tasks/Process.h b/Kernel/Tasks/Process.h index f0611439d6d..7edbc79f13d 100644 --- a/Kernel/Tasks/Process.h +++ b/Kernel/Tasks/Process.h @@ -471,7 +471,6 @@ public: ErrorOr sys$recvfd(int sockfd, int options); ErrorOr sys$sysconf(int name); ErrorOr sys$disown(ProcessID); - ErrorOr sys$allocate_tls(Userspace initial_data, size_t); ErrorOr sys$prctl(int option, FlatPtr arg1, FlatPtr arg2, FlatPtr arg3); ErrorOr sys$anon_create(size_t, int options); ErrorOr sys$statvfs(Userspace user_params); @@ -955,13 +954,6 @@ private: SpinlockProtected, LockRank::None> m_jail_process_list; SpinlockProtected, LockRank::Process> m_attached_jail {}; - struct MasterThreadLocalStorage { - LockWeakPtr region; - size_t size { 0 }; - size_t alignment { 0 }; - }; - SpinlockProtected m_master_tls; - Mutex m_big_lock { "Process"sv, Mutex::MutexBehavior::BigLock }; Mutex m_ptrace_lock { "ptrace"sv }; diff --git a/Kernel/Tasks/Thread.cpp b/Kernel/Tasks/Thread.cpp index 67aeec0f055..133b0933639 100644 --- a/Kernel/Tasks/Thread.cpp +++ b/Kernel/Tasks/Thread.cpp @@ -434,12 +434,6 @@ void Thread::exit(void* exit_value) set_should_die(); u32 unlock_count; [[maybe_unused]] auto rc = unlock_process_if_locked(unlock_count); - if (m_thread_specific_range.has_value()) { - process().address_space().with([&](auto& space) { - auto* region = space->find_region_from_range(m_thread_specific_range.value()); - space->deallocate_region(*region); - }); - } die_if_needed(); } @@ -1209,7 +1203,6 @@ ErrorOr> Thread::clone(NonnullRefPtr process) m_signal_action_masks.span().copy_to(clone->m_signal_action_masks); clone->m_signal_mask = m_signal_mask; clone->m_fpu_state = m_fpu_state; - clone->m_thread_specific_data = m_thread_specific_data; clone->m_arch_specific_data = m_arch_specific_data; return clone; } @@ -1356,34 +1349,6 @@ void Thread::print_backtrace() } } -ErrorOr Thread::make_thread_specific_region(Badge) -{ - return process().m_master_tls.with([&](auto& master_tls) -> ErrorOr { - // The process may not require a TLS region, or allocate TLS later with sys$allocate_tls (which is what dynamically loaded programs do) - if (!master_tls.region) - return {}; - - return process().address_space().with([&](auto& space) -> ErrorOr { - auto region_alignment = max(master_tls.alignment, alignof(ThreadSpecificData)); - auto region_size = align_up_to(master_tls.size, region_alignment) + sizeof(ThreadSpecificData); - auto* region = TRY(space->allocate_region(Memory::RandomizeVirtualAddress::Yes, {}, region_size, PAGE_SIZE, "Thread-specific"sv, PROT_READ | PROT_WRITE)); - - m_thread_specific_range = region->range(); - - SmapDisabler disabler; - auto* thread_specific_data = (ThreadSpecificData*)region->vaddr().offset(align_up_to(master_tls.size, region_alignment)).as_ptr(); - auto* thread_local_storage = (u8*)((u8*)thread_specific_data) - align_up_to(master_tls.size, master_tls.size); - m_thread_specific_data = VirtualAddress(thread_specific_data); - thread_specific_data->self = thread_specific_data; - - if (master_tls.size != 0) - memcpy(thread_local_storage, master_tls.region.unsafe_ptr()->vaddr().as_ptr(), master_tls.size); - - return {}; - }); - }); -} - RefPtr Thread::from_tid_in_same_jail(ThreadID tid) { return Thread::all_instances().with([&](auto& list) -> RefPtr { diff --git a/Kernel/Tasks/Thread.h b/Kernel/Tasks/Thread.h index 85129a9eddb..addf4bf36dc 100644 --- a/Kernel/Tasks/Thread.h +++ b/Kernel/Tasks/Thread.h @@ -790,7 +790,6 @@ public: State state() const { return m_state; } StringView state_string() const; - VirtualAddress thread_specific_data() const { return m_thread_specific_data; } ArchSpecificThreadData& arch_specific_data() { return m_arch_specific_data; } ArchSpecificThreadData const& arch_specific_data() const { return m_arch_specific_data; } @@ -892,8 +891,6 @@ public: FPUState& fpu_state() { return m_fpu_state; } - ErrorOr make_thread_specific_region(Badge); - unsigned syscall_count() const { return m_syscall_count; } void did_syscall() { ++m_syscall_count; } unsigned inode_faults() const { return m_inode_faults; } @@ -1186,8 +1183,6 @@ private: FlatPtr m_kernel_stack_base { 0 }; FlatPtr m_kernel_stack_top { 0 }; NonnullOwnPtr m_kernel_stack_region; - VirtualAddress m_thread_specific_data; - Optional m_thread_specific_range; Array, NSIG> m_signal_action_masks; Array m_signal_senders; Blocker* m_blocker { nullptr }; diff --git a/Userland/Libraries/LibC/pthread.cpp b/Userland/Libraries/LibC/pthread.cpp index cd2760fa71c..e84aac481c5 100644 --- a/Userland/Libraries/LibC/pthread.cpp +++ b/Userland/Libraries/LibC/pthread.cpp @@ -55,9 +55,13 @@ static __thread bool pending_cancellation = false; extern "C" { +[[gnu::weak]] ErrorOr (*__create_new_tls_region)(); +[[gnu::weak]] ErrorOr (*__free_tls_region)(FlatPtr thread_pointer); + [[noreturn]] static void exit_thread(void* code, void* stack_location, size_t stack_size) { __pthread_key_destroy_for_current_thread(); + MUST(__free_tls_region(bit_cast(__builtin_thread_pointer()))); syscall(SC_exit_thread, code, stack_location, stack_size); VERIFY_NOT_REACHED(); } @@ -94,6 +98,12 @@ static int create_thread(pthread_t* thread, void* (*entry)(void*), void* argumen thread_params->entry = entry; thread_params->entry_argument = argument; + auto maybe_thread_pointer = __create_new_tls_region(); + if (maybe_thread_pointer.is_error()) + return maybe_thread_pointer.error().code(); + + thread_params->tls_pointer = bit_cast(maybe_thread_pointer.release_value()); + VERIFY((uintptr_t)stack % 16 == 0); // Push a fake return address @@ -102,6 +112,9 @@ static int create_thread(pthread_t* thread, void* (*entry)(void*), void* argumen int rc = syscall(SC_create_thread, pthread_create_helper, thread_params); if (rc >= 0) *thread = rc; + else + MUST(__free_tls_region(bit_cast(thread_params->tls_pointer))); + __RETURN_PTHREAD_ERROR(rc); } diff --git a/Userland/Libraries/LibC/sys/mman.cpp b/Userland/Libraries/LibC/sys/mman.cpp index 20c0db4c45e..66fe64bb424 100644 --- a/Userland/Libraries/LibC/sys/mman.cpp +++ b/Userland/Libraries/LibC/sys/mman.cpp @@ -84,16 +84,6 @@ int posix_madvise(void* address, size_t len, int advice) return madvise(address, len, advice); } -void* allocate_tls(char const* initial_data, size_t size) -{ - ptrdiff_t rc = syscall(SC_allocate_tls, initial_data, size); - if (rc < 0 && rc > -EMAXERRNO) { - errno = -rc; - return MAP_FAILED; - } - return (void*)rc; -} - // https://pubs.opengroup.org/onlinepubs/9699919799/functions/mlock.html int mlock(void const*, size_t) { diff --git a/Userland/Libraries/LibC/sys/mman.h b/Userland/Libraries/LibC/sys/mman.h index c3aaf1302d6..76d460b0a51 100644 --- a/Userland/Libraries/LibC/sys/mman.h +++ b/Userland/Libraries/LibC/sys/mman.h @@ -20,7 +20,6 @@ int mprotect(void*, size_t, int prot); int set_mmap_name(void*, size_t, char const*); int madvise(void*, size_t, int advice); int posix_madvise(void*, size_t, int advice); -void* allocate_tls(char const* initial_data, size_t); int mlock(void const*, size_t); int munlock(void const*, size_t); int msync(void*, size_t, int flags); diff --git a/Userland/Libraries/LibELF/Arch/x86_64/tls.cpp b/Userland/Libraries/LibELF/Arch/x86_64/tls.cpp index 5f5f7eb9e94..ae31698d0ed 100644 --- a/Userland/Libraries/LibELF/Arch/x86_64/tls.cpp +++ b/Userland/Libraries/LibELF/Arch/x86_64/tls.cpp @@ -10,9 +10,10 @@ namespace ELF { -void set_thread_pointer_register(FlatPtr) +void set_thread_pointer_register(FlatPtr value) { - TODO(); + // TODO: Consider if we want to support the FSGSBASE extension: https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/best-practices/guidance-enabling-fsgsbase.html + VERIFY(archctl(ARCHCTL_X86_64_SET_FS_BASE_FOR_CURRENT_THREAD, value) == 0); } } diff --git a/Userland/Libraries/LibELF/DynamicLinker.cpp b/Userland/Libraries/LibELF/DynamicLinker.cpp index 305dd29a650..2a5d5817559 100644 --- a/Userland/Libraries/LibELF/DynamicLinker.cpp +++ b/Userland/Libraries/LibELF/DynamicLinker.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,9 @@ extern "C" [[noreturn]] void _invoke_entry(int argc, char** argv, char** envp, E struct TLSData { size_t total_tls_size { 0 }; + void* tls_template { nullptr }; size_t tls_template_size { 0 }; + size_t alignment { 0 }; }; static TLSData s_tls_data; @@ -234,6 +237,40 @@ static Result map_dependencies(ByteString const& path) return {}; } +struct ThreadSpecificData { + ThreadSpecificData* self; +}; + +static ErrorOr __create_new_tls_region() +{ + auto static_tls_region_alignment = max(s_tls_data.alignment, alignof(ThreadSpecificData)); + auto static_tls_region_size = align_up_to(s_tls_data.tls_template_size, static_tls_region_alignment) + sizeof(ThreadSpecificData); + void* thread_specific_ptr = serenity_mmap(nullptr, static_tls_region_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0, static_tls_region_alignment, "Static TLS Data"); + if (thread_specific_ptr == MAP_FAILED) + return Error::from_syscall("mmap"sv, -errno); + + auto* thread_specific_data = bit_cast(bit_cast(thread_specific_ptr) + (align_up_to(s_tls_data.tls_template_size, static_tls_region_alignment))); + thread_specific_data->self = thread_specific_data; + + auto* thread_local_storage = bit_cast(bit_cast(thread_specific_data) - align_up_to(s_tls_data.tls_template_size, s_tls_data.alignment)); + + if (s_tls_data.tls_template_size != 0) + memcpy(thread_local_storage, s_tls_data.tls_template, s_tls_data.tls_template_size); + + return bit_cast(thread_specific_data); +} + +static ErrorOr __free_tls_region(FlatPtr thread_pointer) +{ + auto static_tls_region_alignment = max(s_tls_data.alignment, alignof(ThreadSpecificData)); + auto static_tls_region_size = align_up_to(s_tls_data.tls_template_size, static_tls_region_alignment) + sizeof(ThreadSpecificData); + + if (munmap(bit_cast(bit_cast(thread_pointer) - align_up_to(s_tls_data.tls_template_size, s_tls_data.alignment)), static_tls_region_size) != 0) + return Error::from_syscall("mmap"sv, -errno); + + return {}; +} + static void allocate_tls() { for (auto const& data : s_loaders) { @@ -244,25 +281,23 @@ static void allocate_tls() if (s_tls_data.total_tls_size == 0) return; - auto page_aligned_size = align_up_to(s_tls_data.total_tls_size, PAGE_SIZE); - auto initial_tls_data_result = ByteBuffer::create_zeroed(page_aligned_size); - if (initial_tls_data_result.is_error()) { - dbgln("Failed to allocate initial TLS data"); + s_tls_data.tls_template_size = align_up_to(s_tls_data.total_tls_size, PAGE_SIZE); + s_tls_data.alignment = PAGE_SIZE; + s_tls_data.tls_template = mmap_with_name(nullptr, s_tls_data.tls_template_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0, "TLS Template"); + + if (s_tls_data.tls_template == MAP_FAILED) { + dbgln("Failed to allocate memory for the TLS template"); VERIFY_NOT_REACHED(); } - auto& initial_tls_data = initial_tls_data_result.value(); + auto tls_template = Bytes(s_tls_data.tls_template, s_tls_data.tls_template_size); // Initialize TLS data for (auto const& entry : s_loaders) { - entry.value->copy_initial_tls_data_into(initial_tls_data); + entry.value->copy_initial_tls_data_into(tls_template); } - void* master_tls = ::allocate_tls((char*)initial_tls_data.data(), initial_tls_data.size()); - VERIFY(master_tls != (void*)-1); - dbgln_if(DYNAMIC_LOAD_DEBUG, "from userspace, master_tls: {:p}", master_tls); - - s_tls_data.tls_template_size = initial_tls_data.size(); + set_thread_pointer_register(MUST(__create_new_tls_region())); } static int __dl_iterate_phdr(DlIteratePhdrCallbackFunction callback, void* data) @@ -648,6 +683,8 @@ void ELF::DynamicLinker::linker_main(ByteString&& main_program_path, int main_pr s_magic_weak_symbols.set("environ"sv, make_ref_counted(STT_OBJECT, s_envp)); s_magic_weak_symbols.set("__stack_chk_guard"sv, make_ref_counted(STT_OBJECT, stack_guard)); s_magic_weak_symbols.set("__call_fini_functions"sv, make_ref_counted(STT_FUNC, __call_fini_functions)); + s_magic_weak_symbols.set("__create_new_tls_region"sv, make_ref_counted(STT_FUNC, __create_new_tls_region)); + s_magic_weak_symbols.set("__free_tls_region"sv, make_ref_counted(STT_FUNC, __free_tls_region)); s_magic_weak_symbols.set("__dl_iterate_phdr"sv, make_ref_counted(STT_FUNC, __dl_iterate_phdr)); s_magic_weak_symbols.set("__dlclose"sv, make_ref_counted(STT_FUNC, __dlclose)); s_magic_weak_symbols.set("__dlopen"sv, make_ref_counted(STT_FUNC, __dlopen));