From 02de35efb0f946e2e370ac808b872a8716a14997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABlle=20van=20Essen?= Date: Thu, 14 Aug 2025 08:45:48 +0200 Subject: [PATCH 1/4] comgr: refactor --- comgr/src/lib.rs | 62 ++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/comgr/src/lib.rs b/comgr/src/lib.rs index 9c5671b..40b2447 100644 --- a/comgr/src/lib.rs +++ b/comgr/src/lib.rs @@ -138,11 +138,23 @@ impl<'a> DataSet<'a> { Ok(()) } + pub fn from_data(comgr: &'a Comgr, data: impl Iterator) -> Result { + let dataset = Self::new(comgr)?; + for data in data { + dataset.add(&data)?; + } + Ok(dataset) + } + fn get_data(&self, kind: DataKind, index: usize) -> Result { let mut handle = 0u64; call_dispatch!(self.comgr => amd_comgr_action_data_get_data(self, kind, { index }, { std::ptr::from_mut(&mut handle).cast() })); Ok(Data(handle)) } + + fn get_content(&self, comgr: &Comgr, kind: DataKind, index: usize) -> Result, Error> { + self.get_data(kind, index).map(|data| data.copy_content(comgr))? + } } struct Data(u64); @@ -196,28 +208,24 @@ pub fn compile_bitcode( attributes_buffer: &[u8], compiler_hook: Option<&dyn Fn(&Vec, String)>, ) -> Result, Error> { - let bitcode_data_set = DataSet::new(comgr)?; - let main_bitcode_data = Data::new(comgr, DataKind::Bc, c"zluda.bc", main_buffer)?; - bitcode_data_set.add(&main_bitcode_data)?; - let stdlib_bitcode_data = Data::new(comgr, DataKind::Bc, c"ptx_impl.bc", ptx_impl)?; - bitcode_data_set.add(&stdlib_bitcode_data)?; - let attributes_bitcode_data = - Data::new(comgr, DataKind::Bc, c"attributes.bc", attributes_buffer)?; - bitcode_data_set.add(&attributes_bitcode_data)?; + let bitcode_data_set = DataSet::from_data( + comgr, + [ + Data::new(comgr, DataKind::Bc, c"zluda.bc", main_buffer)?, + Data::new(comgr, DataKind::Bc, c"ptx_impl.bc", ptx_impl)?, + Data::new(comgr, DataKind::Bc, c"attributes.bc", attributes_buffer)?, + ].into_iter(), + )?; let linking_info = ActionInfo::new(comgr)?; let linked_data_set = comgr.do_action(ActionKind::LinkBcToBc, &linking_info, &bitcode_data_set)?; if let Some(hook) = compiler_hook { // Run compiler hook on human-readable LLVM IR - let data = linked_data_set.get_data(DataKind::Bc, 0)?; - let data = data.copy_content(comgr)?; + let data = linked_data_set.get_content(comgr, DataKind::Bc, 0)?; let data = ptx::bitcode_to_ir(data); hook(&data, String::from("linked.ll")); } - let compile_to_exec = ActionInfo::new(comgr)?; - compile_to_exec.set_isa_name(gcn_arch)?; - compile_to_exec.set_language(Language::LlvmIr)?; let common_options = [ // This makes no sense, but it makes ockl linking work c"-Xclang", @@ -249,14 +257,16 @@ pub fn compile_bitcode( c"-inlinehint-threshold=3250", ] }; + let compile_to_exec = ActionInfo::new(comgr)?; + compile_to_exec.set_isa_name(gcn_arch)?; + compile_to_exec.set_language(Language::LlvmIr)?; compile_to_exec.set_options(common_options.chain(opt_options))?; let exec_data_set = comgr.do_action( ActionKind::CompileSourceToExecutable, &compile_to_exec, &linked_data_set, )?; - let executable = exec_data_set.get_data(DataKind::Executable, 0)?; - let executable = executable.copy_content(comgr); + let executable = exec_data_set.get_content(comgr, DataKind::Executable, 0); if let Some(hook) = compiler_hook { // Run compiler hook for executable hook( @@ -272,9 +282,8 @@ pub fn compile_bitcode( &action_info, &exec_data_set, )?; - let disassembly = disassembly.get_data(DataKind::Source, 0)?; - let disassembly = disassembly.copy_content(comgr); - hook(&disassembly.unwrap_or(Vec::new()), String::from("asm")) + let disassembly = disassembly.get_content(comgr, DataKind::Source, 0)?; + hook(&disassembly, String::from("asm")) } executable } @@ -296,14 +305,15 @@ pub fn get_symbols(comgr: &Comgr, elf: &[u8]) -> Result, Erro } pub fn get_clang_version(comgr: &Comgr) -> Result { - let version_string_set = DataSet::new(comgr)?; - let version_string = Data::new( + let version_string_set = DataSet::from_data( comgr, - DataKind::Source, - c"version.cpp", - b"__clang_version__", + iter::once(Data::new( + comgr, + DataKind::Source, + c"version.cpp", + b"__clang_version__", + )?), )?; - version_string_set.add(&version_string)?; let preprocessor_info = ActionInfo::new(comgr)?; preprocessor_info.set_language(Language::Hip)?; preprocessor_info.set_options(iter::once(c"-P"))?; @@ -312,8 +322,8 @@ pub fn get_clang_version(comgr: &Comgr) -> Result { &preprocessor_info, &version_string_set, )?; - let data = preprocessed.get_data(DataKind::Source, 0)?; - String::from_utf8(trim_whitespace_and_quotes(data.copy_content(comgr)?)?) + let data = preprocessed.get_content(comgr, DataKind::Source, 0)?; + String::from_utf8(trim_whitespace_and_quotes(data)?) .map_err(|_| Error::UNKNOWN) } From 2cd7f3e359d5e68c4df9f3b5a95298cf6e7f8619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABlle=20van=20Essen?= Date: Wed, 27 Aug 2025 13:50:17 +0200 Subject: [PATCH 2/4] comgr: Prepare for splitting source-to-executable in substeps --- comgr/src/lib.rs | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/comgr/src/lib.rs b/comgr/src/lib.rs index 40b2447..c662c93 100644 --- a/comgr/src/lib.rs +++ b/comgr/src/lib.rs @@ -153,7 +153,8 @@ impl<'a> DataSet<'a> { } fn get_content(&self, comgr: &Comgr, kind: DataKind, index: usize) -> Result, Error> { - self.get_data(kind, index).map(|data| data.copy_content(comgr))? + self.get_data(kind, index) + .map(|data| data.copy_content(comgr))? } } @@ -214,7 +215,8 @@ pub fn compile_bitcode( Data::new(comgr, DataKind::Bc, c"zluda.bc", main_buffer)?, Data::new(comgr, DataKind::Bc, c"ptx_impl.bc", ptx_impl)?, Data::new(comgr, DataKind::Bc, c"attributes.bc", attributes_buffer)?, - ].into_iter(), + ] + .into_iter(), )?; let linking_info = ActionInfo::new(comgr)?; let linked_data_set = @@ -257,16 +259,8 @@ pub fn compile_bitcode( c"-inlinehint-threshold=3250", ] }; - let compile_to_exec = ActionInfo::new(comgr)?; - compile_to_exec.set_isa_name(gcn_arch)?; - compile_to_exec.set_language(Language::LlvmIr)?; - compile_to_exec.set_options(common_options.chain(opt_options))?; - let exec_data_set = comgr.do_action( - ActionKind::CompileSourceToExecutable, - &compile_to_exec, - &linked_data_set, - )?; - let executable = exec_data_set.get_content(comgr, DataKind::Executable, 0); + let exec_data_set: DataSet; + let executable: Result, Error>; if let Some(hook) = compiler_hook { // Run compiler hook for executable hook( @@ -284,6 +278,17 @@ pub fn compile_bitcode( )?; let disassembly = disassembly.get_content(comgr, DataKind::Source, 0)?; hook(&disassembly, String::from("asm")) + } else { + let compile_to_exec = ActionInfo::new(comgr)?; + compile_to_exec.set_isa_name(gcn_arch)?; + compile_to_exec.set_language(Language::LlvmIr)?; + compile_to_exec.set_options(common_options.chain(opt_options))?; + let exec_data_set = comgr.do_action( + ActionKind::CompileSourceToExecutable, + &compile_to_exec, + &linked_data_set, + )?; + executable = exec_data_set.get_content(comgr, DataKind::Executable, 0); } executable } @@ -323,8 +328,7 @@ pub fn get_clang_version(comgr: &Comgr) -> Result { &version_string_set, )?; let data = preprocessed.get_content(comgr, DataKind::Source, 0)?; - String::from_utf8(trim_whitespace_and_quotes(data)?) - .map_err(|_| Error::UNKNOWN) + String::from_utf8(trim_whitespace_and_quotes(data)?).map_err(|_| Error::UNKNOWN) } // When running the preprocessor to expand the macro the output is surrounded by @@ -550,6 +554,9 @@ impl_into!( [ LinkBcToBc => AMD_COMGR_ACTION_LINK_BC_TO_BC, CompileSourceToExecutable => AMD_COMGR_ACTION_COMPILE_SOURCE_TO_EXECUTABLE, + CompileSourceWithDeviceLibsToBc => AMD_COMGR_ACTION_COMPILE_SOURCE_WITH_DEVICE_LIBS_TO_BC, + CodegenBcToRelocatable => AMD_COMGR_ACTION_CODEGEN_BC_TO_RELOCATABLE, + LinkRelocatableToExecutable => AMD_COMGR_ACTION_LINK_RELOCATABLE_TO_EXECUTABLE, DisassembleExecutableToSource => AMD_COMGR_ACTION_DISASSEMBLE_EXECUTABLE_TO_SOURCE, SourceToPreprocessor => AMD_COMGR_ACTION_SOURCE_TO_PREPROCESSOR ] From 96f66d3e2749d8f21495b61fc4d6c7d22d6306cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABlle=20van=20Essen?= Date: Wed, 27 Aug 2025 14:53:43 +0200 Subject: [PATCH 3/4] comgr: Stepwise compilation for zoc --- comgr/src/lib.rs | 61 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/comgr/src/lib.rs b/comgr/src/lib.rs index c662c93..cb95d94 100644 --- a/comgr/src/lib.rs +++ b/comgr/src/lib.rs @@ -259,30 +259,74 @@ pub fn compile_bitcode( c"-inlinehint-threshold=3250", ] }; - let exec_data_set: DataSet; + let options = common_options.chain(opt_options); let executable: Result, Error>; if let Some(hook) = compiler_hook { - // Run compiler hook for executable + // CompileSourceWithDeviceLibsToBc + let action_info = ActionInfo::new(comgr)?; + action_info.set_isa_name(gcn_arch)?; + action_info.set_language(Language::LlvmIr)?; + action_info.set_options(options.clone())?; + let dataset = comgr.do_action( + ActionKind::CompileSourceWithDeviceLibsToBc, + &action_info, + &linked_data_set, + )?; + let data = dataset.get_content(comgr, DataKind::Bc, 0)?; + let data = ptx::bitcode_to_ir(data); + hook(&data, String::from("with-device-libs.ll")); + + // CodegenBcToRelocatable + let action_info = ActionInfo::new(comgr)?; + action_info.set_isa_name(gcn_arch)?; + action_info.set_options(options.clone())?; + let dataset = + comgr.do_action(ActionKind::CodegenBcToRelocatable, &action_info, &dataset)?; + let data = dataset.get_content(comgr, DataKind::Relocatable, 0)?; + hook(&data, String::from("relocatable.elf")); + + // Disassemble relocatable + let action_info = ActionInfo::new(comgr)?; + action_info.set_isa_name(gcn_arch)?; + let disassembled_dataset = comgr.do_action( + ActionKind::DisassembleRelocatableToSource, + &action_info, + &dataset, + )?; + let disassembly = disassembled_dataset.get_content(comgr, DataKind::Source, 0)?; + hook(&disassembly, String::from("relocatable.asm")); + + // LinkRelocatableToExecutable + let action_info = ActionInfo::new(comgr)?; + action_info.set_isa_name(gcn_arch)?; + action_info.set_language(Language::LlvmIr)?; + action_info.set_options(options.clone())?; + let dataset = comgr.do_action( + ActionKind::LinkRelocatableToExecutable, + &action_info, + &dataset, + )?; + executable = dataset.get_content(comgr, DataKind::Executable, 0); hook( executable.as_ref().unwrap_or(&Vec::new()), String::from("elf"), ); - // Disassemble executable and run compiler hook + // Disassemble executable let action_info = ActionInfo::new(comgr)?; action_info.set_isa_name(gcn_arch)?; - let disassembly = comgr.do_action( + let disassembled_dataset = comgr.do_action( ActionKind::DisassembleExecutableToSource, &action_info, - &exec_data_set, + &dataset, )?; - let disassembly = disassembly.get_content(comgr, DataKind::Source, 0)?; - hook(&disassembly, String::from("asm")) + let disassembly = disassembled_dataset.get_content(comgr, DataKind::Source, 0)?; + hook(&disassembly, String::from("asm")); } else { let compile_to_exec = ActionInfo::new(comgr)?; compile_to_exec.set_isa_name(gcn_arch)?; compile_to_exec.set_language(Language::LlvmIr)?; - compile_to_exec.set_options(common_options.chain(opt_options))?; + compile_to_exec.set_options(options.clone())?; let exec_data_set = comgr.do_action( ActionKind::CompileSourceToExecutable, &compile_to_exec, @@ -556,6 +600,7 @@ impl_into!( CompileSourceToExecutable => AMD_COMGR_ACTION_COMPILE_SOURCE_TO_EXECUTABLE, CompileSourceWithDeviceLibsToBc => AMD_COMGR_ACTION_COMPILE_SOURCE_WITH_DEVICE_LIBS_TO_BC, CodegenBcToRelocatable => AMD_COMGR_ACTION_CODEGEN_BC_TO_RELOCATABLE, + DisassembleRelocatableToSource => AMD_COMGR_ACTION_DISASSEMBLE_RELOCATABLE_TO_SOURCE, LinkRelocatableToExecutable => AMD_COMGR_ACTION_LINK_RELOCATABLE_TO_EXECUTABLE, DisassembleExecutableToSource => AMD_COMGR_ACTION_DISASSEMBLE_EXECUTABLE_TO_SOURCE, SourceToPreprocessor => AMD_COMGR_ACTION_SOURCE_TO_PREPROCESSOR From ee008ec9ea7d82ea6dd85d9018ead5abd5a9079c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABlle=20van=20Essen?= Date: Wed, 27 Aug 2025 15:16:57 +0200 Subject: [PATCH 4/4] comgr: Unnecessary (and incorrect) lines removed --- comgr/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/comgr/src/lib.rs b/comgr/src/lib.rs index cb95d94..d3a5f23 100644 --- a/comgr/src/lib.rs +++ b/comgr/src/lib.rs @@ -299,8 +299,6 @@ pub fn compile_bitcode( // LinkRelocatableToExecutable let action_info = ActionInfo::new(comgr)?; action_info.set_isa_name(gcn_arch)?; - action_info.set_language(Language::LlvmIr)?; - action_info.set_options(options.clone())?; let dataset = comgr.do_action( ActionKind::LinkRelocatableToExecutable, &action_info,