diff options
| author | Dave Airlie <airlied@redhat.com> | 2026-03-13 10:39:57 +1000 |
|---|---|---|
| committer | Dave Airlie <airlied@redhat.com> | 2026-03-13 10:40:17 +1000 |
| commit | b28913e897edfeedc4e33b03b28068b27d002e6c (patch) | |
| tree | 37f1b32aed1ddcc88cf6dab0f4e98bed628a3e82 | |
| parent | dd0365021be3e8693680bb1d2616edf7d5e17800 (diff) | |
| parent | 0073a17b466684413ac87cf8ff6c19560db44e7a (diff) | |
Merge tag 'drm-rust-fixes-2026-03-12' of https://gitlab.freedesktop.org/drm/rust/kernel into drm-fixes
Core Changes:
- Fix safety issue in dma_read! and dma_write!.
Driver Changes (Nova Core):
- Fix UB in DmaGspMem pointer accessors.
- Fix stack overflow in GSP memory allocation.
Signed-off-by: Dave Airlie <airlied@redhat.com>
From: Alice Ryhl <aliceryhl@google.com>
Link: https://patch.msgid.link/abNBSol3CLRCqlkZ@google.com
| -rw-r--r-- | drivers/gpu/nova-core/gsp.rs | 46 | ||||
| -rw-r--r-- | drivers/gpu/nova-core/gsp/boot.rs | 2 | ||||
| -rw-r--r-- | drivers/gpu/nova-core/gsp/cmdq.rs | 93 | ||||
| -rw-r--r-- | drivers/gpu/nova-core/gsp/fw.rs | 101 | ||||
| -rw-r--r-- | rust/kernel/dma.rs | 114 | ||||
| -rw-r--r-- | rust/kernel/lib.rs | 4 | ||||
| -rw-r--r-- | rust/kernel/ptr.rs | 30 | ||||
| -rw-r--r-- | rust/kernel/ptr/projection.rs | 305 | ||||
| -rw-r--r-- | samples/rust/rust_dma.rs | 30 | ||||
| -rw-r--r-- | scripts/Makefile.build | 4 |
10 files changed, 534 insertions, 195 deletions
diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 174feaca0a6b..c69adaa92bbe 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -47,16 +47,12 @@ struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]); unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {} impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> { - /// Creates a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`. - fn new(start: DmaAddress) -> Result<Self> { - let mut ptes = [0u64; NUM_PAGES]; - for (i, pte) in ptes.iter_mut().enumerate() { - *pte = start - .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT) - .ok_or(EOVERFLOW)?; - } - - Ok(Self(ptes)) + /// Returns the page table entry for `index`, for a mapping starting at `start`. + // TODO: Replace with `IoView` projection once available. + fn entry(start: DmaAddress, index: usize) -> Result<u64> { + start + .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT) + .ok_or(EOVERFLOW) } } @@ -86,16 +82,22 @@ impl LogBuffer { NUM_PAGES * GSP_PAGE_SIZE, GFP_KERNEL | __GFP_ZERO, )?); - let ptes = PteArray::<NUM_PAGES>::new(obj.0.dma_handle())?; + + let start_addr = obj.0.dma_handle(); // SAFETY: `obj` has just been created and we are its sole user. - unsafe { - // Copy the self-mapping PTE at the expected location. + let pte_region = unsafe { obj.0 - .as_slice_mut(size_of::<u64>(), size_of_val(&ptes))? - .copy_from_slice(ptes.as_bytes()) + .as_slice_mut(size_of::<u64>(), NUM_PAGES * size_of::<u64>())? }; + // Write values one by one to avoid an on-stack instance of `PteArray`. + for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() { + let pte_value = PteArray::<0>::entry(start_addr, i)?; + + chunk.copy_from_slice(&pte_value.to_ne_bytes()); + } + Ok(obj) } } @@ -143,14 +145,14 @@ impl Gsp { // _kgspInitLibosLoggingStructures (allocates memory for buffers) // kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array) dma_write!( - libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0) - )?; + libos, [0]?, LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0) + ); dma_write!( - libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0) - )?; - dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?; - dma_write!(rmargs[0].inner = fw::GspArgumentsCached::new(cmdq))?; - dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", rmargs))?; + libos, [1]?, LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0) + ); + dma_write!(libos, [2]?, LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0)); + dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(cmdq)); + dma_write!(libos, [3]?, LibosMemoryRegionInitArgument::new("RMARGS", rmargs)); }, })) }) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index be427fe26a58..94833f7996e8 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -157,7 +157,7 @@ impl super::Gsp { let wpr_meta = CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?; - dma_write!(wpr_meta[0] = GspFwWprMeta::new(&gsp_fw, &fb_layout))?; + dma_write!(wpr_meta, [0]?, GspFwWprMeta::new(&gsp_fw, &fb_layout)); self.cmdq .send_command(bar, commands::SetSystemInfo::new(pdev))?; diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 46819a82a51a..03a4f3599849 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,11 +2,7 @@ use core::{ cmp, - mem, - sync::atomic::{ - fence, - Ordering, // - }, // + mem, // }; use kernel::{ @@ -146,30 +142,36 @@ static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); #[repr(C)] // There is no struct defined for this in the open-gpu-kernel-source headers. // Instead it is defined by code in `GspMsgQueuesInit()`. -struct Msgq { +// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. +pub(super) struct Msgq { /// Header for sending messages, including the write pointer. - tx: MsgqTxHeader, + pub(super) tx: MsgqTxHeader, /// Header for receiving messages, including the read pointer. - rx: MsgqRxHeader, + pub(super) rx: MsgqRxHeader, /// The message queue proper. msgq: MsgqData, } /// Structure shared between the driver and the GSP and containing the command and message queues. #[repr(C)] -struct GspMem { +// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. +pub(super) struct GspMem { /// Self-mapping page table entries. - ptes: PteArray<{ GSP_PAGE_SIZE / size_of::<u64>() }>, + ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the /// write and read pointers that the CPU updates. /// /// This member is read-only for the GSP. - cpuq: Msgq, + pub(super) cpuq: Msgq, /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the /// write and read pointers that the GSP updates. /// /// This member is read-only for the driver. - gspq: Msgq, + pub(super) gspq: Msgq, +} + +impl GspMem { + const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); } // SAFETY: These structs don't meet the no-padding requirements of AsBytes but @@ -201,9 +203,19 @@ impl DmaGspMem { let gsp_mem = CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?; - dma_write!(gsp_mem[0].ptes = PteArray::new(gsp_mem.dma_handle())?)?; - dma_write!(gsp_mem[0].cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES))?; - dma_write!(gsp_mem[0].cpuq.rx = MsgqRxHeader::new())?; + + let start = gsp_mem.dma_handle(); + // Write values one by one to avoid an on-stack instance of `PteArray`. + for i in 0..GspMem::PTE_ARRAY_SIZE { + dma_write!(gsp_mem, [0]?.ptes.0[i], PteArray::<0>::entry(start, i)?); + } + + dma_write!( + gsp_mem, + [0]?.cpuq.tx, + MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) + ); + dma_write!(gsp_mem, [0]?.cpuq.rx, MsgqRxHeader::new()); Ok(Self(gsp_mem)) } @@ -317,12 +329,7 @@ impl DmaGspMem { // // - The returned value is between `0` and `MSGQ_NUM_PAGES`. fn gsp_write_ptr(&self) -> u32 { - let gsp_mem = self.0.start_ptr(); - - // SAFETY: - // - The 'CoherentAllocation' contains at least one object. - // - By the invariants of `CoherentAllocation` the pointer is valid. - (unsafe { (*gsp_mem).gspq.tx.write_ptr() } % MSGQ_NUM_PAGES) + super::fw::gsp_mem::gsp_write_ptr(&self.0) } // Returns the index of the memory page the GSP will read the next command from. @@ -331,12 +338,7 @@ impl DmaGspMem { // // - The returned value is between `0` and `MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - let gsp_mem = self.0.start_ptr(); - - // SAFETY: - // - The 'CoherentAllocation' contains at least one object. - // - By the invariants of `CoherentAllocation` the pointer is valid. - (unsafe { (*gsp_mem).gspq.rx.read_ptr() } % MSGQ_NUM_PAGES) + super::fw::gsp_mem::gsp_read_ptr(&self.0) } // Returns the index of the memory page the CPU can read the next message from. @@ -345,27 +347,12 @@ impl DmaGspMem { // // - The returned value is between `0` and `MSGQ_NUM_PAGES`. fn cpu_read_ptr(&self) -> u32 { - let gsp_mem = self.0.start_ptr(); - - // SAFETY: - // - The ['CoherentAllocation'] contains at least one object. - // - By the invariants of CoherentAllocation the pointer is valid. - (unsafe { (*gsp_mem).cpuq.rx.read_ptr() } % MSGQ_NUM_PAGES) + super::fw::gsp_mem::cpu_read_ptr(&self.0) } // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { - let rptr = self.cpu_read_ptr().wrapping_add(elem_count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - - let gsp_mem = self.0.start_ptr_mut(); - - // SAFETY: - // - The 'CoherentAllocation' contains at least one object. - // - By the invariants of `CoherentAllocation` the pointer is valid. - unsafe { (*gsp_mem).cpuq.rx.set_read_ptr(rptr) }; + super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) } // Returns the index of the memory page the CPU can write the next command to. @@ -374,26 +361,12 @@ impl DmaGspMem { // // - The returned value is between `0` and `MSGQ_NUM_PAGES`. fn cpu_write_ptr(&self) -> u32 { - let gsp_mem = self.0.start_ptr(); - - // SAFETY: - // - The 'CoherentAllocation' contains at least one object. - // - By the invariants of `CoherentAllocation` the pointer is valid. - (unsafe { (*gsp_mem).cpuq.tx.write_ptr() } % MSGQ_NUM_PAGES) + super::fw::gsp_mem::cpu_write_ptr(&self.0) } // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { - let wptr = self.cpu_write_ptr().wrapping_add(elem_count) & MSGQ_NUM_PAGES; - let gsp_mem = self.0.start_ptr_mut(); - - // SAFETY: - // - The 'CoherentAllocation' contains at least one object. - // - By the invariants of `CoherentAllocation` the pointer is valid. - unsafe { (*gsp_mem).cpuq.tx.set_write_ptr(wptr) }; - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); + super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) } } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 83ff91614e36..040b30ec3089 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -40,6 +40,75 @@ use crate::{ }, }; +// TODO: Replace with `IoView` projections once available; the `unwrap()` calls go away once we +// switch to the new `dma::Coherent` API. +pub(super) mod gsp_mem { + use core::sync::atomic::{ + fence, + Ordering, // + }; + + use kernel::{ + dma::CoherentAllocation, + dma_read, + dma_write, + prelude::*, // + }; + + use crate::gsp::cmdq::{ + GspMem, + MSGQ_NUM_PAGES, // + }; + + pub(in crate::gsp) fn gsp_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { + // PANIC: A `dma::CoherentAllocation` always contains at least one element. + || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap() + } + + pub(in crate::gsp) fn gsp_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { + // PANIC: A `dma::CoherentAllocation` always contains at least one element. + || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap() + } + + pub(in crate::gsp) fn cpu_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { + // PANIC: A `dma::CoherentAllocation` always contains at least one element. + || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap() + } + + pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &CoherentAllocation<GspMem>, count: u32) { + let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; + + // Ensure read pointer is properly ordered. + fence(Ordering::SeqCst); + + // PANIC: A `dma::CoherentAllocation` always contains at least one element. + || -> Result { + dma_write!(qs, [0]?.cpuq.rx.0.readPtr, rptr); + Ok(()) + }() + .unwrap() + } + + pub(in crate::gsp) fn cpu_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { + // PANIC: A `dma::CoherentAllocation` always contains at least one element. + || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap() + } + + pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &CoherentAllocation<GspMem>, count: u32) { + let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; + + // PANIC: A `dma::CoherentAllocation` always contains at least one element. + || -> Result { + dma_write!(qs, [0]?.cpuq.tx.0.writePtr, wptr); + Ok(()) + }() + .unwrap(); + + // Ensure all command data is visible before triggering the GSP read. + fence(Ordering::SeqCst); + } +} + /// Empty type to group methods related to heap parameters for running the GSP firmware. enum GspFwHeapParams {} @@ -708,22 +777,6 @@ impl MsgqTxHeader { entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), }) } - - /// Returns the value of the write pointer for this queue. - pub(crate) fn write_ptr(&self) -> u32 { - let ptr = core::ptr::from_ref(&self.0.writePtr); - - // SAFETY: `ptr` is a valid pointer to a `u32`. - unsafe { ptr.read_volatile() } - } - - /// Sets the value of the write pointer for this queue. - pub(crate) fn set_write_ptr(&mut self, val: u32) { - let ptr = core::ptr::from_mut(&mut self.0.writePtr); - - // SAFETY: `ptr` is a valid pointer to a `u32`. - unsafe { ptr.write_volatile(val) } - } } // SAFETY: Padding is explicit and does not contain uninitialized data. @@ -739,22 +792,6 @@ impl MsgqRxHeader { pub(crate) fn new() -> Self { Self(Default::default()) } - - /// Returns the value of the read pointer for this queue. - pub(crate) fn read_ptr(&self) -> u32 { - let ptr = core::ptr::from_ref(&self.0.readPtr); - - // SAFETY: `ptr` is a valid pointer to a `u32`. - unsafe { ptr.read_volatile() } - } - - /// Sets the value of the read pointer for this queue. - pub(crate) fn set_read_ptr(&mut self, val: u32) { - let ptr = core::ptr::from_mut(&mut self.0.readPtr); - - // SAFETY: `ptr` is a valid pointer to a `u32`. - unsafe { ptr.write_volatile(val) } - } } // SAFETY: Padding is explicit and does not contain uninitialized data. diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 909d56fd5118..a396f8435739 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -461,6 +461,19 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { self.count * core::mem::size_of::<T>() } + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. + #[inline] + pub fn as_ptr(&self) -> *const [T] { + core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count) + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as + /// a mutable pointer. + #[inline] + pub fn as_mut_ptr(&self) -> *mut [T] { + core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count) + } + /// Returns the base address to the allocated region in the CPU's virtual address space. pub fn start_ptr(&self) -> *const T { self.cpu_addr.as_ptr() @@ -581,23 +594,6 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { Ok(()) } - /// Returns a pointer to an element from the region with bounds checking. `offset` is in - /// units of `T`, not the number of bytes. - /// - /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros. - #[doc(hidden)] - pub fn item_from_index(&self, offset: usize) -> Result<*mut T> { - if offset >= self.count { - return Err(EINVAL); - } - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation` - // and we've just checked that the range and index is within bounds. - // - `offset` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - Ok(unsafe { self.cpu_addr.as_ptr().add(offset) }) - } - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. /// /// # Safety @@ -670,6 +666,9 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} /// Reads a field of an item from an allocated region of structs. /// +/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating +/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!). +/// /// # Examples /// /// ``` @@ -684,36 +683,29 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { -/// let whole = kernel::dma_read!(alloc[2]); -/// let field = kernel::dma_read!(alloc[1].field); +/// let whole = kernel::dma_read!(alloc, [2]?); +/// let field = kernel::dma_read!(alloc, [1]?.field); /// # Ok::<(), Error>(()) } /// ``` #[macro_export] macro_rules! dma_read { - ($dma:expr, $idx: expr, $($field:tt)*) => {{ - (|| -> ::core::result::Result<_, $crate::error::Error> { - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; - // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be - // dereferenced. The compiler also further validates the expression on whether `field` - // is a member of `item` when expanded by the macro. - unsafe { - let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); - ::core::result::Result::Ok( - $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) - ) - } - })() + ($dma:expr, $($proj:tt)*) => {{ + let dma = &$dma; + let ptr = $crate::ptr::project!( + $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)* + ); + // SAFETY: The pointer created by the projection is within the DMA region. + unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) } }}; - ($dma:ident [ $idx:expr ] $($field:tt)* ) => { - $crate::dma_read!($dma, $idx, $($field)*) - }; - ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { - $crate::dma_read!($($dma).*, $idx, $($field)*) - }; } /// Writes to a field of an item from an allocated region of structs. /// +/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression +/// evaluating to a [`CoherentAllocation`], `proj` is a +/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the +/// projected location. +/// /// # Examples /// /// ``` @@ -728,37 +720,31 @@ macro_rules! dma_read { /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { -/// kernel::dma_write!(alloc[2].member = 0xf); -/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf }); +/// kernel::dma_write!(alloc, [2]?.member, 0xf); +/// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf }); /// # Ok::<(), Error>(()) } /// ``` #[macro_export] macro_rules! dma_write { - ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ - $crate::dma_write!($dma, $idx, $($field)*) - }}; - ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ - $crate::dma_write!($($dma).*, $idx, $($field)*) + (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ + let dma = &$dma; + let ptr = $crate::ptr::project!( + mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)* + ); + let val = $val; + // SAFETY: The pointer created by the projection is within the DMA region. + unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) } }}; - ($dma:expr, $idx: expr, = $val:expr) => { - (|| -> ::core::result::Result<_, $crate::error::Error> { - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; - // SAFETY: `item_from_index` ensures that `item` is always a valid item. - unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } - ::core::result::Result::Ok(()) - })() + (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr]? $($rest:tt)*]) => { + $crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*]) + }; + (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr] $($rest:tt)*]) => { + $crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*]) }; - ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { - (|| -> ::core::result::Result<_, $crate::error::Error> { - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; - // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be - // dereferenced. The compiler also further validates the expression on whether `field` - // is a member of `item` when expanded by the macro. - unsafe { - let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); - $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) - } - ::core::result::Result::Ok(()) - })() + ($dma:expr, $($rest:tt)*) => { + $crate::dma_write!(@parse [$dma] [] [$($rest)*]) }; } diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 3da92f18f4ee..d93292d47420 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -20,6 +20,7 @@ #![feature(generic_nonzero)] #![feature(inline_const)] #![feature(pointer_is_aligned)] +#![feature(slice_ptr_len)] // // Stable since Rust 1.80.0. #![feature(slice_flatten)] @@ -37,6 +38,9 @@ #![feature(const_ptr_write)] #![feature(const_refs_to_cell)] // +// Stable since Rust 1.84.0. +#![feature(strict_provenance)] +// // Expected to become stable. #![feature(arbitrary_self_types)] // diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 5b6a382637fe..bdc2d79ff669 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -2,7 +2,13 @@ //! Types and functions to work with pointers and addresses. -use core::mem::align_of; +pub mod projection; +pub use crate::project_pointer as project; + +use core::mem::{ + align_of, + size_of, // +}; use core::num::NonZero; /// Type representing an alignment, which is always a power of two. @@ -225,3 +231,25 @@ macro_rules! impl_alignable_uint { } impl_alignable_uint!(u8, u16, u32, u64, usize); + +/// Trait to represent compile-time known size information. +/// +/// This is a generalization of [`size_of`] that works for dynamically sized types. +pub trait KnownSize { + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. + fn size(p: *const Self) -> usize; +} + +impl<T> KnownSize for T { + #[inline(always)] + fn size(_: *const Self) -> usize { + size_of::<T>() + } +} + +impl<T> KnownSize for [T] { + #[inline(always)] + fn size(p: *const Self) -> usize { + p.len() * size_of::<T>() + } +} diff --git a/rust/kernel/ptr/projection.rs b/rust/kernel/ptr/projection.rs new file mode 100644 index 000000000000..140ea8e21617 --- /dev/null +++ b/rust/kernel/ptr/projection.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for handling projections. + +use core::{ + mem::MaybeUninit, + ops::Deref, // +}; + +use crate::prelude::*; + +/// Error raised when a projection is attempted on an array or slice out of bounds. +pub struct OutOfBound; + +impl From<OutOfBound> for Error { + #[inline(always)] + fn from(_: OutOfBound) -> Self { + ERANGE + } +} + +/// A helper trait to perform index projection. +/// +/// This is similar to [`core::slice::SliceIndex`], but operates on raw pointers safely and +/// fallibly. +/// +/// # Safety +/// +/// The implementation of `index` and `get` (if [`Some`] is returned) must ensure that, if provided +/// input pointer `slice` and returned pointer `output`, then: +/// - `output` has the same provenance as `slice`; +/// - `output.byte_offset_from(slice)` is between 0 to +/// `KnownSize::size(slice) - KnownSize::size(output)`. +/// +/// This means that if the input pointer is valid, then pointer returned by `get` or `index` is +/// also valid. +#[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")] +#[doc(hidden)] +pub unsafe trait ProjectIndex<T: ?Sized>: Sized { + type Output: ?Sized; + + /// Returns an index-projected pointer, if in bounds. + fn get(self, slice: *mut T) -> Option<*mut Self::Output>; + + /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds. + #[inline(always)] + fn index(self, slice: *mut T) -> *mut Self::Output { + Self::get(self, slice).unwrap_or_else(|| build_error!()) + } +} + +// Forward array impl to slice impl. +// +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl<T, I, const N: usize> ProjectIndex<[T; N]> for I +where + I: ProjectIndex<[T]>, +{ + type Output = <I as ProjectIndex<[T]>>::Output; + + #[inline(always)] + fn get(self, slice: *mut [T; N]) -> Option<*mut Self::Output> { + <I as ProjectIndex<[T]>>::get(self, slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T; N]) -> *mut Self::Output { + <I as ProjectIndex<[T]>>::index(self, slice) + } +} + +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to +// not exceed the required bound. +unsafe impl<T> ProjectIndex<[T]> for usize { + type Output = T; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut T> { + if self >= slice.len() { + None + } else { + Some(slice.cast::<T>().wrapping_add(self)) + } + } +} + +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to +// not exceed the required bound. +unsafe impl<T> ProjectIndex<[T]> for core::ops::Range<usize> { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + let new_len = self.end.checked_sub(self.start)?; + if self.end > slice.len() { + return None; + } + Some(core::ptr::slice_from_raw_parts_mut( + slice.cast::<T>().wrapping_add(self.start), + new_len, + )) + } +} + +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeTo<usize> { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + (0..self.end).get(slice) + } +} + +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFrom<usize> { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + (self.start..slice.len()).get(slice) + } +} + +// SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0. +unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFull { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + Some(slice) + } +} + +/// A helper trait to perform field projection. +/// +/// This trait has a `DEREF` generic parameter so it can be implemented twice for types that +/// implement [`Deref`]. This will cause an ambiguity error and thus block [`Deref`] types being +/// used as base of projection, as they can inject unsoundness. Users therefore must not specify +/// `DEREF` and should always leave it to be inferred. +/// +/// # Safety +/// +/// `proj` may only invoke `f` with a valid allocation, as the documentation of [`Self::proj`] +/// describes. +#[doc(hidden)] +pub unsafe trait ProjectField<const DEREF: bool> { + /// Project a pointer to a type to a pointer of a field. + /// + /// `f` may only be invoked with a valid allocation so it can safely obtain raw pointers to + /// fields using `&raw mut`. + /// + /// This is needed because `base` might not point to a valid allocation, while `&raw mut` + /// requires pointers to be in bounds of a valid allocation. + /// + /// # Safety + /// + /// `f` must return a pointer in bounds of the provided pointer. + unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F; +} + +// NOTE: in theory, this API should work for `T: ?Sized` and `F: ?Sized`, too. However, we cannot +// currently support that as we need to obtain a valid allocation that `&raw const` can operate on. +// +// SAFETY: `proj` invokes `f` with valid allocation. +unsafe impl<T> ProjectField<false> for T { + #[inline(always)] + unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F { + // Create a valid allocation to start projection, as `base` is not necessarily so. The + // memory is never actually used so it will be optimized out, so it should work even for + // very large `T` (`memoffset` crate also relies on this). To be extra certain, we also + // annotate `f` closure with `#[inline(always)]` in the macro. + let mut place = MaybeUninit::uninit(); + let place_base = place.as_mut_ptr(); + let field = f(place_base); + // SAFETY: `field` is in bounds from `base` per safety requirement. + let offset = unsafe { field.byte_offset_from(place_base) }; + // Use `wrapping_byte_offset` as `base` does not need to be of valid allocation. + base.wrapping_byte_offset(offset).cast() + } +} + +// SAFETY: Vacuously satisfied. +unsafe impl<T: Deref> ProjectField<true> for T { + #[inline(always)] + unsafe fn proj<F>(_: *mut Self, _: impl FnOnce(*mut Self) -> *mut F) -> *mut F { + build_error!("this function is a guard against `Deref` impl and is never invoked"); + } +} + +/// Create a projection from a raw pointer. +/// +/// The projected pointer is within the memory region marked by the input pointer. There is no +/// requirement that the input raw pointer needs to be valid, so this macro may be used for +/// projecting pointers outside normal address space, e.g. I/O pointers. However, if the input +/// pointer is valid, the projected pointer is also valid. +/// +/// Supported projections include field projections and index projections. +/// It is not allowed to project into types that implement custom [`Deref`] or +/// [`Index`](core::ops::Index). +/// +/// The macro has basic syntax of `kernel::ptr::project!(ptr, projection)`, where `ptr` is an +/// expression that evaluates to a raw pointer which serves as the base of projection. `projection` +/// can be a projection expression of form `.field` (normally identifier, or numeral in case of +/// tuple structs) or of form `[index]`. +/// +/// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e. +/// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created. +/// +/// `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. +/// `[index]` form performs build-time bounds checking; if compiler fails to prove `[index]` is in +/// bounds, compilation will fail. `[index]?` can be used to perform runtime bounds checking; +/// `OutOfBound` error is raised via `?` if the index is out of bounds. +/// +/// # Examples +/// +/// Field projections are performed with `.field_name`: +/// +/// ``` +/// struct MyStruct { field: u32, } +/// let ptr: *const MyStruct = core::ptr::dangling(); +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .field); +/// +/// struct MyTupleStruct(u32, u32); +/// +/// fn proj(ptr: *const MyTupleStruct) { +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .1); +/// } +/// ``` +/// +/// Index projections are performed with `[index]`: +/// +/// ``` +/// fn proj(ptr: *const [u8; 32]) -> Result { +/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [1]); +/// // The following invocation, if uncommented, would fail the build. +/// // +/// // kernel::ptr::project!(ptr, [128]); +/// +/// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`). +/// kernel::ptr::project!(ptr, [128]?); +/// Ok(()) +/// } +/// ``` +/// +/// If you need to match on the error instead of propagate, put the invocation inside a closure: +/// +/// ``` +/// let ptr: *const [u8; 32] = core::ptr::dangling(); +/// let field_ptr: Result<*const u8> = (|| -> Result<_> { +/// Ok(kernel::ptr::project!(ptr, [128]?)) +/// })(); +/// assert!(field_ptr.is_err()); +/// ``` +/// +/// For mutable pointers, put `mut` as the first token in macro invocation. +/// +/// ``` +/// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut(); +/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [1].1); +/// ``` +#[macro_export] +macro_rules! project_pointer { + (@gen $ptr:ident, ) => {}; + // Field projection. `$field` needs to be `tt` to support tuple index like `.0`. + (@gen $ptr:ident, .$field:tt $($rest:tt)*) => { + // SAFETY: The provided closure always returns an in-bounds pointer. + let $ptr = unsafe { + $crate::ptr::projection::ProjectField::proj($ptr, #[inline(always)] |ptr| { + // Check unaligned field. Not all users (e.g. DMA) can handle unaligned + // projections. + if false { + let _ = &(*ptr).$field; + } + // SAFETY: `$field` is in bounds, and no implicit `Deref` is possible (if the + // type implements `Deref`, Rust cannot infer the generic parameter `DEREF`). + &raw mut (*ptr).$field + }) + }; + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Fallible index projection. + (@gen $ptr:ident, [$index:expr]? $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr) + .ok_or($crate::ptr::projection::OutOfBound)?; + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Build-time checked index projection. + (@gen $ptr:ident, [$index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr); + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + (mut $ptr:expr, $($proj:tt)*) => {{ + let ptr: *mut _ = $ptr; + $crate::ptr::project!(@gen ptr, $($proj)*); + ptr + }}; + ($ptr:expr, $($proj:tt)*) => {{ + let ptr = <*const _>::cast_mut($ptr); + // We currently always project using mutable pointer, as it is not decided whether `&raw + // const` allows the resulting pointer to be mutated (see documentation of `addr_of!`). + $crate::ptr::project!(@gen ptr, $($proj)*); + ptr.cast_const() + }}; +} diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 9c45851c876e..ce39b5545097 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -68,7 +68,7 @@ impl pci::Driver for DmaSampleDriver { CoherentAllocation::alloc_coherent(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; for (i, value) in TEST_VALUES.into_iter().enumerate() { - kernel::dma_write!(ca[i] = MyStruct::new(value.0, value.1))?; + kernel::dma_write!(ca, [i]?, MyStruct::new(value.0, value.1)); } let size = 4 * page::PAGE_SIZE; @@ -85,24 +85,26 @@ impl pci::Driver for DmaSampleDriver { } } +impl DmaSampleDriver { + fn check_dma(&self) -> Result { + for (i, value) in TEST_VALUES.into_iter().enumerate() { + let val0 = kernel::dma_read!(self.ca, [i]?.h); + let val1 = kernel::dma_read!(self.ca, [i]?.b); + + assert_eq!(val0, value.0); + assert_eq!(val1, value.1); + } + + Ok(()) + } +} + #[pinned_drop] impl PinnedDrop for DmaSampleDriver { fn drop(self: Pin<&mut Self>) { dev_info!(self.pdev, "Unload DMA test driver.\n"); - for (i, value) in TEST_VALUES.into_iter().enumerate() { - let val0 = kernel::dma_read!(self.ca[i].h); - let val1 = kernel::dma_read!(self.ca[i].b); - assert!(val0.is_ok()); - assert!(val1.is_ok()); - - if let Ok(val0) = val0 { - assert_eq!(val0, value.0); - } - if let Ok(val1) = val1 { - assert_eq!(val1, value.1); - } - } + assert!(self.check_dma().is_ok()); for (i, entry) in self.sgt.iter().enumerate() { dev_info!( diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 32e209bc7985..3652b85be545 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -310,16 +310,18 @@ $(obj)/%.lst: $(obj)/%.c FORCE # The features in this list are the ones allowed for non-`rust/` code. # +# - Stable since Rust 1.79.0: `feature(slice_ptr_len)`. # - Stable since Rust 1.81.0: `feature(lint_reasons)`. # - Stable since Rust 1.82.0: `feature(asm_const)`, # `feature(offset_of_nested)`, `feature(raw_ref_op)`. +# - Stable since Rust 1.84.0: `feature(strict_provenance)`. # - Stable since Rust 1.87.0: `feature(asm_goto)`. # - Expected to become stable: `feature(arbitrary_self_types)`. # - To be determined: `feature(used_with_arg)`. # # Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on # the unstable features in use. -rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,used_with_arg +rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,slice_ptr_len,strict_provenance,used_with_arg # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree |
