summaryrefslogtreecommitdiff
path: root/rust/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'rust/kernel')
-rw-r--r--rust/kernel/alloc.rs54
-rw-r--r--rust/kernel/alloc/allocator.rs105
-rw-r--r--rust/kernel/alloc/kbox.rs4
-rw-r--r--rust/kernel/alloc/kvec.rs11
-rw-r--r--rust/kernel/cpufreq.rs7
-rw-r--r--rust/kernel/cred.rs6
-rw-r--r--rust/kernel/dma.rs3
-rw-r--r--rust/kernel/fs.rs3
-rw-r--r--rust/kernel/fs/kiocb.rs68
-rw-r--r--rust/kernel/iov.rs314
-rw-r--r--rust/kernel/lib.rs2
-rw-r--r--rust/kernel/maple_tree.rs647
-rw-r--r--rust/kernel/miscdevice.rs63
-rw-r--r--rust/kernel/mm.rs3
-rw-r--r--rust/kernel/mm/mmput_async.rs2
-rw-r--r--rust/kernel/page.rs6
-rw-r--r--rust/kernel/security.rs37
-rw-r--r--rust/kernel/usb.rs456
18 files changed, 1750 insertions, 41 deletions
diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs
index 9c154209423c..e38720349dcf 100644
--- a/rust/kernel/alloc.rs
+++ b/rust/kernel/alloc.rs
@@ -21,6 +21,8 @@ pub use self::kvec::Vec;
/// Indicates an allocation error.
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct AllocError;
+
+use crate::error::{code::EINVAL, Result};
use core::{alloc::Layout, ptr::NonNull};
/// Flags to be used when allocating memory.
@@ -108,6 +110,31 @@ pub mod flags {
pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN);
}
+/// Non Uniform Memory Access (NUMA) node identifier.
+#[derive(Clone, Copy, PartialEq)]
+pub struct NumaNode(i32);
+
+impl NumaNode {
+ /// Create a new NUMA node identifier (non-negative integer).
+ ///
+ /// Returns [`EINVAL`] if a negative id or an id exceeding [`bindings::MAX_NUMNODES`] is
+ /// specified.
+ pub fn new(node: i32) -> Result<Self> {
+ // MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10.
+ if node < 0 || node >= bindings::MAX_NUMNODES as i32 {
+ return Err(EINVAL);
+ }
+ Ok(Self(node))
+ }
+}
+
+/// Specify necessary constant to pass the information to Allocator that the caller doesn't care
+/// about the NUMA node to allocate memory from.
+impl NumaNode {
+ /// No node preference.
+ pub const NO_NODE: NumaNode = NumaNode(bindings::NUMA_NO_NODE);
+}
+
/// The kernel's [`Allocator`] trait.
///
/// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described
@@ -138,7 +165,7 @@ pub unsafe trait Allocator {
/// the requested layout has a smaller alignment.
const MIN_ALIGN: usize;
- /// Allocate memory based on `layout` and `flags`.
+ /// Allocate memory based on `layout`, `flags` and `nid`.
///
/// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout
/// constraints (i.e. minimum size and alignment as specified by `layout`).
@@ -154,13 +181,21 @@ pub unsafe trait Allocator {
///
/// Additionally, `Flags` are honored as documented in
/// <https://docs.kernel.org/core-api/mm-api.html#mm-api-gfp-flags>.
- fn alloc(layout: Layout, flags: Flags) -> Result<NonNull<[u8]>, AllocError> {
+ fn alloc(layout: Layout, flags: Flags, nid: NumaNode) -> Result<NonNull<[u8]>, AllocError> {
// SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a
// new memory allocation.
- unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) }
+ unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, nid) }
}
- /// Re-allocate an existing memory allocation to satisfy the requested `layout`.
+ /// Re-allocate an existing memory allocation to satisfy the requested `layout` and
+ /// a specific NUMA node request to allocate the memory for.
+ ///
+ /// Systems employing a Non Uniform Memory Access (NUMA) architecture contain collections of
+ /// hardware resources including processors, memory, and I/O buses, that comprise what is
+ /// commonly known as a NUMA node.
+ ///
+ /// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a non-negative integer
+ /// if a node needs to be specified, or [`NumaNode::NO_NODE`] if the caller doesn't care.
///
/// If the requested size is zero, `realloc` behaves equivalent to `free`.
///
@@ -197,6 +232,7 @@ pub unsafe trait Allocator {
layout: Layout,
old_layout: Layout,
flags: Flags,
+ nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError>;
/// Free an existing memory allocation.
@@ -212,7 +248,15 @@ pub unsafe trait Allocator {
// SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this
// allocator. We are passing a `Layout` with the smallest possible alignment, so it is
// smaller than or equal to the alignment previously used with this allocation.
- let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) };
+ let _ = unsafe {
+ Self::realloc(
+ Some(ptr),
+ Layout::new::<()>(),
+ layout,
+ Flags(0),
+ NumaNode::NO_NODE,
+ )
+ };
}
}
diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs
index 08fd31bf72d2..63bfb91b3671 100644
--- a/rust/kernel/alloc/allocator.rs
+++ b/rust/kernel/alloc/allocator.rs
@@ -13,10 +13,9 @@ use core::alloc::Layout;
use core::ptr;
use core::ptr::NonNull;
-use crate::alloc::{AllocError, Allocator};
+use crate::alloc::{AllocError, Allocator, NumaNode};
use crate::bindings;
use crate::page;
-use crate::pr_warn;
const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN;
@@ -51,20 +50,26 @@ pub struct KVmalloc;
/// # Invariants
///
-/// One of the following: `krealloc`, `vrealloc`, `kvrealloc`.
+/// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `kvrealloc_node_align`.
struct ReallocFunc(
- unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void,
+ unsafe extern "C" fn(
+ *const crate::ffi::c_void,
+ usize,
+ crate::ffi::c_ulong,
+ u32,
+ crate::ffi::c_int,
+ ) -> *mut crate::ffi::c_void,
);
impl ReallocFunc {
- // INVARIANT: `krealloc` satisfies the type invariants.
- const KREALLOC: Self = Self(bindings::krealloc);
+ // INVARIANT: `krealloc_node_align` satisfies the type invariants.
+ const KREALLOC: Self = Self(bindings::krealloc_node_align);
- // INVARIANT: `vrealloc` satisfies the type invariants.
- const VREALLOC: Self = Self(bindings::vrealloc);
+ // INVARIANT: `vrealloc_node_align` satisfies the type invariants.
+ const VREALLOC: Self = Self(bindings::vrealloc_node_align);
- // INVARIANT: `kvrealloc` satisfies the type invariants.
- const KVREALLOC: Self = Self(bindings::kvrealloc);
+ // INVARIANT: `kvrealloc_node_align` satisfies the type invariants.
+ const KVREALLOC: Self = Self(bindings::kvrealloc_node_align);
/// # Safety
///
@@ -82,6 +87,7 @@ impl ReallocFunc {
layout: Layout,
old_layout: Layout,
flags: Flags,
+ nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
let size = layout.size();
let ptr = match ptr {
@@ -105,7 +111,7 @@ impl ReallocFunc {
// - Those functions provide the guarantees of this function.
let raw_ptr = unsafe {
// If `size == 0` and `ptr != NULL` the memory behind the pointer is freed.
- self.0(ptr.cast(), size, flags.0).cast()
+ self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast()
};
let ptr = if size == 0 {
@@ -142,11 +148,12 @@ unsafe impl Allocator for Kmalloc {
layout: Layout,
old_layout: Layout,
flags: Flags,
+ nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
let layout = Kmalloc::aligned_layout(layout);
// SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`.
- unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) }
+ unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) }
}
}
@@ -211,16 +218,11 @@ unsafe impl Allocator for Vmalloc {
layout: Layout,
old_layout: Layout,
flags: Flags,
+ nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
- // TODO: Support alignments larger than PAGE_SIZE.
- if layout.align() > bindings::PAGE_SIZE {
- pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n");
- return Err(AllocError);
- }
-
// SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
// allocated with this `Allocator`.
- unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) }
+ unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) }
}
}
@@ -237,19 +239,70 @@ unsafe impl Allocator for KVmalloc {
layout: Layout,
old_layout: Layout,
flags: Flags,
+ nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
// `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc`
// compatible layout.
let layout = Kmalloc::aligned_layout(layout);
- // TODO: Support alignments larger than PAGE_SIZE.
- if layout.align() > bindings::PAGE_SIZE {
- pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n");
- return Err(AllocError);
- }
-
// SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
// allocated with this `Allocator`.
- unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) }
+ unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) }
+ }
+}
+
+#[macros::kunit_tests(rust_allocator)]
+mod tests {
+ use super::*;
+ use core::mem::MaybeUninit;
+ use kernel::prelude::*;
+
+ #[test]
+ fn test_alignment() -> Result {
+ const TEST_SIZE: usize = 1024;
+ const TEST_LARGE_ALIGN_SIZE: usize = kernel::page::PAGE_SIZE * 4;
+
+ // These two structs are used to test allocating aligned memory.
+ // they don't need to be accessed, so they're marked as dead_code.
+ #[expect(dead_code)]
+ #[repr(align(128))]
+ struct Blob([u8; TEST_SIZE]);
+ #[expect(dead_code)]
+ #[repr(align(8192))]
+ struct LargeAlignBlob([u8; TEST_LARGE_ALIGN_SIZE]);
+
+ struct TestAlign<T, A: Allocator>(Box<MaybeUninit<T>, A>);
+ impl<T, A: Allocator> TestAlign<T, A> {
+ fn new() -> Result<Self> {
+ Ok(Self(Box::<_, A>::new_uninit(GFP_KERNEL)?))
+ }
+
+ fn is_aligned_to(&self, align: usize) -> bool {
+ assert!(align.is_power_of_two());
+
+ let addr = self.0.as_ptr() as usize;
+ addr & (align - 1) == 0
+ }
+ }
+
+ let ta = TestAlign::<Blob, Kmalloc>::new()?;
+ assert!(ta.is_aligned_to(128));
+
+ let ta = TestAlign::<LargeAlignBlob, Kmalloc>::new()?;
+ assert!(ta.is_aligned_to(8192));
+
+ let ta = TestAlign::<Blob, Vmalloc>::new()?;
+ assert!(ta.is_aligned_to(128));
+
+ let ta = TestAlign::<LargeAlignBlob, Vmalloc>::new()?;
+ assert!(ta.is_aligned_to(8192));
+
+ let ta = TestAlign::<Blob, KVmalloc>::new()?;
+ assert!(ta.is_aligned_to(128));
+
+ let ta = TestAlign::<LargeAlignBlob, KVmalloc>::new()?;
+ assert!(ta.is_aligned_to(8192));
+
+ Ok(())
}
}
diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs
index 2137c3700004..622b3529edfc 100644
--- a/rust/kernel/alloc/kbox.rs
+++ b/rust/kernel/alloc/kbox.rs
@@ -4,7 +4,7 @@
#[allow(unused_imports)] // Used in doc comments.
use super::allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter};
-use super::{AllocError, Allocator, Flags};
+use super::{AllocError, Allocator, Flags, NumaNode};
use core::alloc::Layout;
use core::borrow::{Borrow, BorrowMut};
use core::marker::PhantomData;
@@ -274,7 +274,7 @@ where
/// ```
pub fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>, A>, AllocError> {
let layout = Layout::new::<MaybeUninit<T>>();
- let ptr = A::alloc(layout, flags)?;
+ let ptr = A::alloc(layout, flags, NumaNode::NO_NODE)?;
// INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`,
// which is sufficient in size and alignment for storing a `T`.
diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs
index 5c3496b31e8b..e94aebd084c8 100644
--- a/rust/kernel/alloc/kvec.rs
+++ b/rust/kernel/alloc/kvec.rs
@@ -5,7 +5,7 @@
use super::{
allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter},
layout::ArrayLayout,
- AllocError, Allocator, Box, Flags,
+ AllocError, Allocator, Box, Flags, NumaNode,
};
use crate::{
fmt,
@@ -647,6 +647,7 @@ where
layout.into(),
self.layout.into(),
flags,
+ NumaNode::NO_NODE,
)?
};
@@ -1161,7 +1162,13 @@ where
// the type invariant to be smaller than `cap`. Depending on `realloc` this operation
// may shrink the buffer or leave it as it is.
ptr = match unsafe {
- A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags)
+ A::realloc(
+ Some(buf.cast()),
+ layout.into(),
+ old_layout.into(),
+ flags,
+ NumaNode::NO_NODE,
+ )
} {
// If we fail to shrink, which likely can't even happen, continue with the existing
// buffer.
diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs
index 29bdf7fe80a1..21b5b9b8acc1 100644
--- a/rust/kernel/cpufreq.rs
+++ b/rust/kernel/cpufreq.rs
@@ -38,7 +38,8 @@ use macros::vtable;
const CPUFREQ_NAME_LEN: usize = bindings::CPUFREQ_NAME_LEN as usize;
/// Default transition latency value in nanoseconds.
-pub const ETERNAL_LATENCY_NS: u32 = bindings::CPUFREQ_ETERNAL as u32;
+pub const DEFAULT_TRANSITION_LATENCY_NS: u32 =
+ bindings::CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS;
/// CPU frequency driver flags.
pub mod flags {
@@ -399,13 +400,13 @@ impl TableBuilder {
/// The following example demonstrates how to create a CPU frequency table.
///
/// ```
-/// use kernel::cpufreq::{ETERNAL_LATENCY_NS, Policy};
+/// use kernel::cpufreq::{DEFAULT_TRANSITION_LATENCY_NS, Policy};
///
/// fn update_policy(policy: &mut Policy) {
/// policy
/// .set_dvfs_possible_from_any_cpu(true)
/// .set_fast_switch_possible(true)
-/// .set_transition_latency_ns(ETERNAL_LATENCY_NS);
+/// .set_transition_latency_ns(DEFAULT_TRANSITION_LATENCY_NS);
///
/// pr_info!("The policy details are: {:?}\n", (policy.cpu(), policy.cur()));
/// }
diff --git a/rust/kernel/cred.rs b/rust/kernel/cred.rs
index 4a2229542fb7..ffa156b9df37 100644
--- a/rust/kernel/cred.rs
+++ b/rust/kernel/cred.rs
@@ -50,6 +50,12 @@ impl Credential {
unsafe { &*ptr.cast() }
}
+ /// Returns a raw pointer to the inner credential.
+ #[inline]
+ pub fn as_ptr(&self) -> *const bindings::cred {
+ self.0.get()
+ }
+
/// Get the id for this security context.
#[inline]
pub fn get_secid(&self) -> u32 {
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 2569c21208e3..4e0af3e1a3b9 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -252,6 +252,9 @@ pub mod attrs {
/// Indicates that the buffer is fully accessible at an elevated privilege level (and
/// ideally inaccessible or at least read-only at lesser-privileged levels).
pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);
+
+ /// Indicates that the buffer is MMIO memory.
+ pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO);
}
/// DMA data direction.
diff --git a/rust/kernel/fs.rs b/rust/kernel/fs.rs
index 0121b38c59e6..6ba6bdf143cb 100644
--- a/rust/kernel/fs.rs
+++ b/rust/kernel/fs.rs
@@ -6,3 +6,6 @@
pub mod file;
pub use self::file::{File, LocalFile};
+
+mod kiocb;
+pub use self::kiocb::Kiocb;
diff --git a/rust/kernel/fs/kiocb.rs b/rust/kernel/fs/kiocb.rs
new file mode 100644
index 000000000000..84c936cd69b0
--- /dev/null
+++ b/rust/kernel/fs/kiocb.rs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Kernel IO callbacks.
+//!
+//! C headers: [`include/linux/fs.h`](srctree/include/linux/fs.h)
+
+use core::marker::PhantomData;
+use core::ptr::NonNull;
+use kernel::types::ForeignOwnable;
+
+/// Wrapper for the kernel's `struct kiocb`.
+///
+/// Currently this abstractions is incomplete and is essentially just a tuple containing a
+/// reference to a file and a file position.
+///
+/// The type `T` represents the filesystem or driver specific data associated with the file.
+///
+/// # Invariants
+///
+/// `inner` points at a valid `struct kiocb` whose file has the type `T` as its private data.
+pub struct Kiocb<'a, T> {
+ inner: NonNull<bindings::kiocb>,
+ _phantom: PhantomData<&'a T>,
+}
+
+impl<'a, T: ForeignOwnable> Kiocb<'a, T> {
+ /// Create a `Kiocb` from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// The pointer must reference a valid `struct kiocb` for the duration of `'a`. The private
+ /// data of the file must be `T`.
+ pub unsafe fn from_raw(kiocb: *mut bindings::kiocb) -> Self {
+ Self {
+ // SAFETY: If a pointer is valid it is not null.
+ inner: unsafe { NonNull::new_unchecked(kiocb) },
+ _phantom: PhantomData,
+ }
+ }
+
+ /// Access the underlying `struct kiocb` directly.
+ pub fn as_raw(&self) -> *mut bindings::kiocb {
+ self.inner.as_ptr()
+ }
+
+ /// Get the filesystem or driver specific data associated with the file.
+ pub fn file(&self) -> <T as ForeignOwnable>::Borrowed<'a> {
+ // SAFETY: We have shared access to this kiocb and hence the underlying file, so we can
+ // read the file's private data.
+ let private = unsafe { (*(*self.as_raw()).ki_filp).private_data };
+ // SAFETY: The kiocb has shared access to the private data.
+ unsafe { <T as ForeignOwnable>::borrow(private) }
+ }
+
+ /// Gets the current value of `ki_pos`.
+ pub fn ki_pos(&self) -> i64 {
+ // SAFETY: We have shared access to the kiocb, so we can read its `ki_pos` field.
+ unsafe { (*self.as_raw()).ki_pos }
+ }
+
+ /// Gets a mutable reference to the `ki_pos` field.
+ pub fn ki_pos_mut(&mut self) -> &mut i64 {
+ // SAFETY: We have exclusive access to the kiocb, so we can write to `ki_pos`.
+ unsafe { &mut (*self.as_raw()).ki_pos }
+ }
+}
diff --git a/rust/kernel/iov.rs b/rust/kernel/iov.rs
new file mode 100644
index 000000000000..43bae8923c46
--- /dev/null
+++ b/rust/kernel/iov.rs
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2025 Google LLC.
+
+//! IO vectors.
+//!
+//! C headers: [`include/linux/iov_iter.h`](srctree/include/linux/iov_iter.h),
+//! [`include/linux/uio.h`](srctree/include/linux/uio.h)
+
+use crate::{
+ alloc::{Allocator, Flags},
+ bindings,
+ prelude::*,
+ types::Opaque,
+};
+use core::{marker::PhantomData, mem::MaybeUninit, ptr, slice};
+
+const ITER_SOURCE: bool = bindings::ITER_SOURCE != 0;
+const ITER_DEST: bool = bindings::ITER_DEST != 0;
+
+// Compile-time assertion for the above constants.
+const _: () = {
+ build_assert!(
+ ITER_SOURCE != ITER_DEST,
+ "ITER_DEST and ITER_SOURCE should be different."
+ );
+};
+
+/// An IO vector that acts as a source of data.
+///
+/// The data may come from many different sources. This includes both things in kernel-space and
+/// reading from userspace. It's not necessarily the case that the data source is immutable, so
+/// rewinding the IO vector to read the same data twice is not guaranteed to result in the same
+/// bytes. It's also possible that the data source is mapped in a thread-local manner using e.g.
+/// `kmap_local_page()`, so this type is not `Send` to ensure that the mapping is read from the
+/// right context in that scenario.
+///
+/// # Invariants
+///
+/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_SOURCE`. For the duration
+/// of `'data`, it must be safe to read from this IO vector using the standard C methods for this
+/// purpose.
+#[repr(transparent)]
+pub struct IovIterSource<'data> {
+ iov: Opaque<bindings::iov_iter>,
+ /// Represent to the type system that this value contains a pointer to readable data it does
+ /// not own.
+ _source: PhantomData<&'data [u8]>,
+}
+
+impl<'data> IovIterSource<'data> {
+ /// Obtain an `IovIterSource` from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// * The referenced `struct iov_iter` must be valid and must only be accessed through the
+ /// returned reference for the duration of `'iov`.
+ /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_SOURCE`.
+ /// * For the duration of `'data`, it must be safe to read from this IO vector using the
+ /// standard C methods for this purpose.
+ #[track_caller]
+ #[inline]
+ pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterSource<'data> {
+ // SAFETY: The caller ensures that `ptr` is valid.
+ let data_source = unsafe { (*ptr).data_source };
+ assert_eq!(data_source, ITER_SOURCE);
+
+ // SAFETY: The caller ensures the type invariants for the right durations, and
+ // `IovIterSource` is layout compatible with `struct iov_iter`.
+ unsafe { &mut *ptr.cast::<IovIterSource<'data>>() }
+ }
+
+ /// Access this as a raw `struct iov_iter`.
+ #[inline]
+ pub fn as_raw(&mut self) -> *mut bindings::iov_iter {
+ self.iov.get()
+ }
+
+ /// Returns the number of bytes available in this IO vector.
+ ///
+ /// Note that this may overestimate the number of bytes. For example, reading from userspace
+ /// memory could fail with `EFAULT`, which will be treated as the end of the IO vector.
+ #[inline]
+ pub fn len(&self) -> usize {
+ // SAFETY: We have shared access to this IO vector, so we can read its `count` field.
+ unsafe {
+ (*self.iov.get())
+ .__bindgen_anon_1
+ .__bindgen_anon_1
+ .as_ref()
+ .count
+ }
+ }
+
+ /// Returns whether there are any bytes left in this IO vector.
+ ///
+ /// This may return `true` even if there are no more bytes available. For example, reading from
+ /// userspace memory could fail with `EFAULT`, which will be treated as the end of the IO vector.
+ #[inline]
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ /// Advance this IO vector by `bytes` bytes.
+ ///
+ /// If `bytes` is larger than the size of this IO vector, it is advanced to the end.
+ #[inline]
+ pub fn advance(&mut self, bytes: usize) {
+ // SAFETY: By the type invariants, `self.iov` is a valid IO vector.
+ unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) };
+ }
+
+ /// Advance this IO vector backwards by `bytes` bytes.
+ ///
+ /// # Safety
+ ///
+ /// The IO vector must not be reverted to before its beginning.
+ #[inline]
+ pub unsafe fn revert(&mut self, bytes: usize) {
+ // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller
+ // ensures that `bytes` is in bounds.
+ unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) };
+ }
+
+ /// Read data from this IO vector.
+ ///
+ /// Returns the number of bytes that have been copied.
+ #[inline]
+ pub fn copy_from_iter(&mut self, out: &mut [u8]) -> usize {
+ // SAFETY: `Self::copy_from_iter_raw` guarantees that it will not write any uninitialized
+ // bytes in the provided buffer, so `out` is still a valid `u8` slice after this call.
+ let out = unsafe { &mut *(ptr::from_mut(out) as *mut [MaybeUninit<u8>]) };
+
+ self.copy_from_iter_raw(out).len()
+ }
+
+ /// Read data from this IO vector and append it to a vector.
+ ///
+ /// Returns the number of bytes that have been copied.
+ #[inline]
+ pub fn copy_from_iter_vec<A: Allocator>(
+ &mut self,
+ out: &mut Vec<u8, A>,
+ flags: Flags,
+ ) -> Result<usize> {
+ out.reserve(self.len(), flags)?;
+ let len = self.copy_from_iter_raw(out.spare_capacity_mut()).len();
+ // SAFETY:
+ // - `len` is the length of a subslice of the spare capacity, so `len` is at most the
+ // length of the spare capacity.
+ // - `Self::copy_from_iter_raw` guarantees that the first `len` bytes of the spare capacity
+ // have been initialized.
+ unsafe { out.inc_len(len) };
+ Ok(len)
+ }
+
+ /// Read data from this IO vector into potentially uninitialized memory.
+ ///
+ /// Returns the sub-slice of the output that has been initialized. If the returned slice is
+ /// shorter than the input buffer, then the entire IO vector has been read.
+ ///
+ /// This will never write uninitialized bytes to the provided buffer.
+ #[inline]
+ pub fn copy_from_iter_raw(&mut self, out: &mut [MaybeUninit<u8>]) -> &mut [u8] {
+ let capacity = out.len();
+ let out = out.as_mut_ptr().cast::<u8>();
+
+ // GUARANTEES: The C API guarantees that it does not write uninitialized bytes to the
+ // provided buffer.
+ // SAFETY:
+ // * By the type invariants, it is still valid to read from this IO vector.
+ // * `out` is valid for writing for `capacity` bytes because it comes from a slice of
+ // that length.
+ let len = unsafe { bindings::_copy_from_iter(out.cast(), capacity, self.as_raw()) };
+
+ // SAFETY: The underlying C api guarantees that initialized bytes have been written to the
+ // first `len` bytes of the spare capacity.
+ unsafe { slice::from_raw_parts_mut(out, len) }
+ }
+}
+
+/// An IO vector that acts as a destination for data.
+///
+/// IO vectors support many different types of destinations. This includes both buffers in
+/// kernel-space and writing to userspace. It's possible that the destination buffer is mapped in a
+/// thread-local manner using e.g. `kmap_local_page()`, so this type is not `Send` to ensure that
+/// the mapping is written to the right context in that scenario.
+///
+/// # Invariants
+///
+/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_DEST`. For the duration of
+/// `'data`, it must be safe to write to this IO vector using the standard C methods for this
+/// purpose.
+#[repr(transparent)]
+pub struct IovIterDest<'data> {
+ iov: Opaque<bindings::iov_iter>,
+ /// Represent to the type system that this value contains a pointer to writable data it does
+ /// not own.
+ _source: PhantomData<&'data mut [u8]>,
+}
+
+impl<'data> IovIterDest<'data> {
+ /// Obtain an `IovIterDest` from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// * The referenced `struct iov_iter` must be valid and must only be accessed through the
+ /// returned reference for the duration of `'iov`.
+ /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_DEST`.
+ /// * For the duration of `'data`, it must be safe to write to this IO vector using the
+ /// standard C methods for this purpose.
+ #[track_caller]
+ #[inline]
+ pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterDest<'data> {
+ // SAFETY: The caller ensures that `ptr` is valid.
+ let data_source = unsafe { (*ptr).data_source };
+ assert_eq!(data_source, ITER_DEST);
+
+ // SAFETY: The caller ensures the type invariants for the right durations, and
+ // `IovIterSource` is layout compatible with `struct iov_iter`.
+ unsafe { &mut *ptr.cast::<IovIterDest<'data>>() }
+ }
+
+ /// Access this as a raw `struct iov_iter`.
+ #[inline]
+ pub fn as_raw(&mut self) -> *mut bindings::iov_iter {
+ self.iov.get()
+ }
+
+ /// Returns the number of bytes available in this IO vector.
+ ///
+ /// Note that this may overestimate the number of bytes. For example, reading from userspace
+ /// memory could fail with EFAULT, which will be treated as the end of the IO vector.
+ #[inline]
+ pub fn len(&self) -> usize {
+ // SAFETY: We have shared access to this IO vector, so we can read its `count` field.
+ unsafe {
+ (*self.iov.get())
+ .__bindgen_anon_1
+ .__bindgen_anon_1
+ .as_ref()
+ .count
+ }
+ }
+
+ /// Returns whether there are any bytes left in this IO vector.
+ ///
+ /// This may return `true` even if there are no more bytes available. For example, reading from
+ /// userspace memory could fail with EFAULT, which will be treated as the end of the IO vector.
+ #[inline]
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ /// Advance this IO vector by `bytes` bytes.
+ ///
+ /// If `bytes` is larger than the size of this IO vector, it is advanced to the end.
+ #[inline]
+ pub fn advance(&mut self, bytes: usize) {
+ // SAFETY: By the type invariants, `self.iov` is a valid IO vector.
+ unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) };
+ }
+
+ /// Advance this IO vector backwards by `bytes` bytes.
+ ///
+ /// # Safety
+ ///
+ /// The IO vector must not be reverted to before its beginning.
+ #[inline]
+ pub unsafe fn revert(&mut self, bytes: usize) {
+ // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller
+ // ensures that `bytes` is in bounds.
+ unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) };
+ }
+
+ /// Write data to this IO vector.
+ ///
+ /// Returns the number of bytes that were written. If this is shorter than the provided slice,
+ /// then no more bytes can be written.
+ #[inline]
+ pub fn copy_to_iter(&mut self, input: &[u8]) -> usize {
+ // SAFETY:
+ // * By the type invariants, it is still valid to write to this IO vector.
+ // * `input` is valid for `input.len()` bytes.
+ unsafe { bindings::_copy_to_iter(input.as_ptr().cast(), input.len(), self.as_raw()) }
+ }
+
+ /// Utility for implementing `read_iter` given the full contents of the file.
+ ///
+ /// The full contents of the file being read from is represented by `contents`. This call will
+ /// write the appropriate sub-slice of `contents` and update the file position in `ppos` so
+ /// that the file will appear to contain `contents` even if takes multiple reads to read the
+ /// entire file.
+ #[inline]
+ pub fn simple_read_from_buffer(&mut self, ppos: &mut i64, contents: &[u8]) -> Result<usize> {
+ if *ppos < 0 {
+ return Err(EINVAL);
+ }
+ let Ok(pos) = usize::try_from(*ppos) else {
+ return Ok(0);
+ };
+ if pos >= contents.len() {
+ return Ok(0);
+ }
+
+ // BOUNDS: We just checked that `pos < contents.len()` above.
+ let num_written = self.copy_to_iter(&contents[pos..]);
+
+ // OVERFLOW: `pos+num_written <= contents.len() <= isize::MAX <= i64::MAX`.
+ *ppos = (pos + num_written) as i64;
+
+ Ok(num_written)
+ }
+}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 2b0cb6df8e0f..3dd7bebe7888 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -98,11 +98,13 @@ pub mod id_pool;
pub mod init;
pub mod io;
pub mod ioctl;
+pub mod iov;
pub mod irq;
pub mod jump_label;
#[cfg(CONFIG_KUNIT)]
pub mod kunit;
pub mod list;
+pub mod maple_tree;
pub mod miscdevice;
pub mod mm;
#[cfg(CONFIG_NET)]
diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs
new file mode 100644
index 000000000000..e72eec56bf57
--- /dev/null
+++ b/rust/kernel/maple_tree.rs
@@ -0,0 +1,647 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Maple trees.
+//!
+//! C header: [`include/linux/maple_tree.h`](srctree/include/linux/maple_tree.h)
+//!
+//! Reference: <https://docs.kernel.org/core-api/maple_tree.html>
+
+use core::{
+ marker::PhantomData,
+ ops::{Bound, RangeBounds},
+ ptr,
+};
+
+use kernel::{
+ alloc::Flags,
+ error::to_result,
+ prelude::*,
+ types::{ForeignOwnable, Opaque},
+};
+
+/// A maple tree optimized for storing non-overlapping ranges.
+///
+/// # Invariants
+///
+/// Each range in the maple tree owns an instance of `T`.
+#[pin_data(PinnedDrop)]
+#[repr(transparent)]
+pub struct MapleTree<T: ForeignOwnable> {
+ #[pin]
+ tree: Opaque<bindings::maple_tree>,
+ _p: PhantomData<T>,
+}
+
+/// A maple tree with `MT_FLAGS_ALLOC_RANGE` set.
+///
+/// All methods on [`MapleTree`] are also accessible on this type.
+#[pin_data]
+#[repr(transparent)]
+pub struct MapleTreeAlloc<T: ForeignOwnable> {
+ #[pin]
+ tree: MapleTree<T>,
+}
+
+// Make MapleTree methods usable on MapleTreeAlloc.
+impl<T: ForeignOwnable> core::ops::Deref for MapleTreeAlloc<T> {
+ type Target = MapleTree<T>;
+
+ #[inline]
+ fn deref(&self) -> &MapleTree<T> {
+ &self.tree
+ }
+}
+
+#[inline]
+fn to_maple_range(range: impl RangeBounds<usize>) -> Option<(usize, usize)> {
+ let first = match range.start_bound() {
+ Bound::Included(start) => *start,
+ Bound::Excluded(start) => start.checked_add(1)?,
+ Bound::Unbounded => 0,
+ };
+
+ let last = match range.end_bound() {
+ Bound::Included(end) => *end,
+ Bound::Excluded(end) => end.checked_sub(1)?,
+ Bound::Unbounded => usize::MAX,
+ };
+
+ if last < first {
+ return None;
+ }
+
+ Some((first, last))
+}
+
+impl<T: ForeignOwnable> MapleTree<T> {
+ /// Create a new maple tree.
+ ///
+ /// The tree will use the regular implementation with a higher branching factor, rather than
+ /// the allocation tree.
+ #[inline]
+ pub fn new() -> impl PinInit<Self> {
+ pin_init!(MapleTree {
+ // SAFETY: This initializes a maple tree into a pinned slot. The maple tree will be
+ // destroyed in Drop before the memory location becomes invalid.
+ tree <- Opaque::ffi_init(|slot| unsafe { bindings::mt_init_flags(slot, 0) }),
+ _p: PhantomData,
+ })
+ }
+
+ /// Insert the value at the given index.
+ ///
+ /// # Errors
+ ///
+ /// If the maple tree already contains a range using the given index, then this call will
+ /// return an [`InsertErrorKind::Occupied`]. It may also fail if memory allocation fails.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::maple_tree::{InsertErrorKind, MapleTree};
+ ///
+ /// let tree = KBox::pin_init(MapleTree::<KBox<i32>>::new(), GFP_KERNEL)?;
+ ///
+ /// let ten = KBox::new(10, GFP_KERNEL)?;
+ /// let twenty = KBox::new(20, GFP_KERNEL)?;
+ /// let the_answer = KBox::new(42, GFP_KERNEL)?;
+ ///
+ /// // These calls will succeed.
+ /// tree.insert(100, ten, GFP_KERNEL)?;
+ /// tree.insert(101, twenty, GFP_KERNEL)?;
+ ///
+ /// // This will fail because the index is already in use.
+ /// assert_eq!(
+ /// tree.insert(100, the_answer, GFP_KERNEL).unwrap_err().cause,
+ /// InsertErrorKind::Occupied,
+ /// );
+ /// # Ok::<_, Error>(())
+ /// ```
+ #[inline]
+ pub fn insert(&self, index: usize, value: T, gfp: Flags) -> Result<(), InsertError<T>> {
+ self.insert_range(index..=index, value, gfp)
+ }
+
+ /// Insert a value to the specified range, failing on overlap.
+ ///
+ /// This accepts the usual types of Rust ranges using the `..` and `..=` syntax for exclusive
+ /// and inclusive ranges respectively. The range must not be empty, and must not overlap with
+ /// any existing range.
+ ///
+ /// # Errors
+ ///
+ /// If the maple tree already contains an overlapping range, then this call will return an
+ /// [`InsertErrorKind::Occupied`]. It may also fail if memory allocation fails or if the
+ /// requested range is invalid (e.g. empty).
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::maple_tree::{InsertErrorKind, MapleTree};
+ ///
+ /// let tree = KBox::pin_init(MapleTree::<KBox<i32>>::new(), GFP_KERNEL)?;
+ ///
+ /// let ten = KBox::new(10, GFP_KERNEL)?;
+ /// let twenty = KBox::new(20, GFP_KERNEL)?;
+ /// let the_answer = KBox::new(42, GFP_KERNEL)?;
+ /// let hundred = KBox::new(100, GFP_KERNEL)?;
+ ///
+ /// // Insert the value 10 at the indices 100 to 499.
+ /// tree.insert_range(100..500, ten, GFP_KERNEL)?;
+ ///
+ /// // Insert the value 20 at the indices 500 to 1000.
+ /// tree.insert_range(500..=1000, twenty, GFP_KERNEL)?;
+ ///
+ /// // This will fail due to overlap with the previous range on index 1000.
+ /// assert_eq!(
+ /// tree.insert_range(1000..1200, the_answer, GFP_KERNEL).unwrap_err().cause,
+ /// InsertErrorKind::Occupied,
+ /// );
+ ///
+ /// // When using .. to specify the range, you must be careful to ensure that the range is
+ /// // non-empty.
+ /// assert_eq!(
+ /// tree.insert_range(72..72, hundred, GFP_KERNEL).unwrap_err().cause,
+ /// InsertErrorKind::InvalidRequest,
+ /// );
+ /// # Ok::<_, Error>(())
+ /// ```
+ pub fn insert_range<R>(&self, range: R, value: T, gfp: Flags) -> Result<(), InsertError<T>>
+ where
+ R: RangeBounds<usize>,
+ {
+ let Some((first, last)) = to_maple_range(range) else {
+ return Err(InsertError {
+ value,
+ cause: InsertErrorKind::InvalidRequest,
+ });
+ };
+
+ let ptr = T::into_foreign(value);
+
+ // SAFETY: The tree is valid, and we are passing a pointer to an owned instance of `T`.
+ let res = to_result(unsafe {
+ bindings::mtree_insert_range(self.tree.get(), first, last, ptr, gfp.as_raw())
+ });
+
+ if let Err(err) = res {
+ // SAFETY: As `mtree_insert_range` failed, it is safe to take back ownership.
+ let value = unsafe { T::from_foreign(ptr) };
+
+ let cause = if err == ENOMEM {
+ InsertErrorKind::AllocError(kernel::alloc::AllocError)
+ } else if err == EEXIST {
+ InsertErrorKind::Occupied
+ } else {
+ InsertErrorKind::InvalidRequest
+ };
+ Err(InsertError { value, cause })
+ } else {
+ Ok(())
+ }
+ }
+
+ /// Erase the range containing the given index.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::maple_tree::MapleTree;
+ ///
+ /// let tree = KBox::pin_init(MapleTree::<KBox<i32>>::new(), GFP_KERNEL)?;
+ ///
+ /// let ten = KBox::new(10, GFP_KERNEL)?;
+ /// let twenty = KBox::new(20, GFP_KERNEL)?;
+ ///
+ /// tree.insert_range(100..500, ten, GFP_KERNEL)?;
+ /// tree.insert(67, twenty, GFP_KERNEL)?;
+ ///
+ /// assert_eq!(tree.erase(67).map(|v| *v), Some(20));
+ /// assert_eq!(tree.erase(275).map(|v| *v), Some(10));
+ ///
+ /// // The previous call erased the entire range, not just index 275.
+ /// assert!(tree.erase(127).is_none());
+ /// # Ok::<_, Error>(())
+ /// ```
+ #[inline]
+ pub fn erase(&self, index: usize) -> Option<T> {
+ // SAFETY: `self.tree` contains a valid maple tree.
+ let ret = unsafe { bindings::mtree_erase(self.tree.get(), index) };
+
+ // SAFETY: If the pointer is not null, then we took ownership of a valid instance of `T`
+ // from the tree.
+ unsafe { T::try_from_foreign(ret) }
+ }
+
+ /// Lock the internal spinlock.
+ #[inline]
+ pub fn lock(&self) -> MapleGuard<'_, T> {
+ // SAFETY: It's safe to lock the spinlock in a maple tree.
+ unsafe { bindings::spin_lock(self.ma_lock()) };
+
+ // INVARIANT: We just took the spinlock.
+ MapleGuard(self)
+ }
+
+ #[inline]
+ fn ma_lock(&self) -> *mut bindings::spinlock_t {
+ // SAFETY: This pointer offset operation stays in-bounds.
+ let lock_ptr = unsafe { &raw mut (*self.tree.get()).__bindgen_anon_1.ma_lock };
+ lock_ptr.cast()
+ }
+
+ /// Free all `T` instances in this tree.
+ ///
+ /// # Safety
+ ///
+ /// This frees Rust data referenced by the maple tree without removing it from the maple tree,
+ /// leaving it in an invalid state. The caller must ensure that this invalid state cannot be
+ /// observed by the end-user.
+ unsafe fn free_all_entries(self: Pin<&mut Self>) {
+ // SAFETY: The caller provides exclusive access to the entire maple tree, so we have
+ // exclusive access to the entire maple tree despite not holding the lock.
+ let mut ma_state = unsafe { MaState::new_raw(self.into_ref().get_ref(), 0, usize::MAX) };
+
+ loop {
+ // This uses the raw accessor because we're destroying pointers without removing them
+ // from the maple tree, which is only valid because this is the destructor.
+ let ptr = ma_state.mas_find_raw(usize::MAX);
+ if ptr.is_null() {
+ break;
+ }
+ // SAFETY: By the type invariants, this pointer references a valid value of type `T`.
+ // By the safety requirements, it is okay to free it without removing it from the maple
+ // tree.
+ drop(unsafe { T::from_foreign(ptr) });
+ }
+ }
+}
+
+#[pinned_drop]
+impl<T: ForeignOwnable> PinnedDrop for MapleTree<T> {
+ #[inline]
+ fn drop(mut self: Pin<&mut Self>) {
+ // We only iterate the tree if the Rust value has a destructor.
+ if core::mem::needs_drop::<T>() {
+ // SAFETY: Other than the below `mtree_destroy` call, the tree will not be accessed
+ // after this call.
+ unsafe { self.as_mut().free_all_entries() };
+ }
+
+ // SAFETY: The tree is valid, and will not be accessed after this call.
+ unsafe { bindings::mtree_destroy(self.tree.get()) };
+ }
+}
+
+/// A reference to a [`MapleTree`] that owns the inner lock.
+///
+/// # Invariants
+///
+/// This guard owns the inner spinlock.
+#[must_use = "if unused, the lock will be immediately unlocked"]
+pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree<T>);
+
+impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: By the type invariants, we hold this spinlock.
+ unsafe { bindings::spin_unlock(self.0.ma_lock()) };
+ }
+}
+
+impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> {
+ /// Create a [`MaState`] protected by this lock guard.
+ pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> {
+ // SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s
+ // read/write permissions to the maple tree.
+ unsafe { MaState::new_raw(self.0, first, end) }
+ }
+
+ /// Load the value at the given index.
+ ///
+ /// # Examples
+ ///
+ /// Read the value while holding the spinlock.
+ ///
+ /// ```
+ /// use kernel::maple_tree::MapleTree;
+ ///
+ /// let tree = KBox::pin_init(MapleTree::<KBox<i32>>::new(), GFP_KERNEL)?;
+ ///
+ /// let ten = KBox::new(10, GFP_KERNEL)?;
+ /// let twenty = KBox::new(20, GFP_KERNEL)?;
+ /// tree.insert(100, ten, GFP_KERNEL)?;
+ /// tree.insert(200, twenty, GFP_KERNEL)?;
+ ///
+ /// let mut lock = tree.lock();
+ /// assert_eq!(lock.load(100).map(|v| *v), Some(10));
+ /// assert_eq!(lock.load(200).map(|v| *v), Some(20));
+ /// assert_eq!(lock.load(300).map(|v| *v), None);
+ /// # Ok::<_, Error>(())
+ /// ```
+ ///
+ /// Increment refcount under the lock, to keep value alive afterwards.
+ ///
+ /// ```
+ /// use kernel::maple_tree::MapleTree;
+ /// use kernel::sync::Arc;
+ ///
+ /// let tree = KBox::pin_init(MapleTree::<Arc<i32>>::new(), GFP_KERNEL)?;
+ ///
+ /// let ten = Arc::new(10, GFP_KERNEL)?;
+ /// let twenty = Arc::new(20, GFP_KERNEL)?;
+ /// tree.insert(100, ten, GFP_KERNEL)?;
+ /// tree.insert(200, twenty, GFP_KERNEL)?;
+ ///
+ /// // Briefly take the lock to increment the refcount.
+ /// let value = tree.lock().load(100).map(Arc::from);
+ ///
+ /// // At this point, another thread might remove the value.
+ /// tree.erase(100);
+ ///
+ /// // But we can still access it because we took a refcount.
+ /// assert_eq!(value.map(|v| *v), Some(10));
+ /// # Ok::<_, Error>(())
+ /// ```
+ #[inline]
+ pub fn load(&mut self, index: usize) -> Option<T::BorrowedMut<'_>> {
+ // SAFETY: `self.tree` contains a valid maple tree.
+ let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) };
+ if ret.is_null() {
+ return None;
+ }
+
+ // SAFETY: If the pointer is not null, then it references a valid instance of `T`. It is
+ // safe to borrow the instance mutably because the signature of this function enforces that
+ // the mutable borrow is not used after the spinlock is dropped.
+ Some(unsafe { T::borrow_mut(ret) })
+ }
+}
+
+impl<T: ForeignOwnable> MapleTreeAlloc<T> {
+ /// Create a new allocation tree.
+ pub fn new() -> impl PinInit<Self> {
+ let tree = pin_init!(MapleTree {
+ // SAFETY: This initializes a maple tree into a pinned slot. The maple tree will be
+ // destroyed in Drop before the memory location becomes invalid.
+ tree <- Opaque::ffi_init(|slot| unsafe {
+ bindings::mt_init_flags(slot, bindings::MT_FLAGS_ALLOC_RANGE)
+ }),
+ _p: PhantomData,
+ });
+
+ pin_init!(MapleTreeAlloc { tree <- tree })
+ }
+
+ /// Insert an entry with the given size somewhere in the given range.
+ ///
+ /// The maple tree will search for a location in the given range where there is space to insert
+ /// the new range. If there is not enough available space, then an error will be returned.
+ ///
+ /// The index of the new range is returned.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::maple_tree::{MapleTreeAlloc, AllocErrorKind};
+ ///
+ /// let tree = KBox::pin_init(MapleTreeAlloc::<KBox<i32>>::new(), GFP_KERNEL)?;
+ ///
+ /// let ten = KBox::new(10, GFP_KERNEL)?;
+ /// let twenty = KBox::new(20, GFP_KERNEL)?;
+ /// let thirty = KBox::new(30, GFP_KERNEL)?;
+ /// let hundred = KBox::new(100, GFP_KERNEL)?;
+ ///
+ /// // Allocate three ranges.
+ /// let idx1 = tree.alloc_range(100, ten, ..1000, GFP_KERNEL)?;
+ /// let idx2 = tree.alloc_range(100, twenty, ..1000, GFP_KERNEL)?;
+ /// let idx3 = tree.alloc_range(100, thirty, ..1000, GFP_KERNEL)?;
+ ///
+ /// assert_eq!(idx1, 0);
+ /// assert_eq!(idx2, 100);
+ /// assert_eq!(idx3, 200);
+ ///
+ /// // This will fail because the remaining space is too small.
+ /// assert_eq!(
+ /// tree.alloc_range(800, hundred, ..1000, GFP_KERNEL).unwrap_err().cause,
+ /// AllocErrorKind::Busy,
+ /// );
+ /// # Ok::<_, Error>(())
+ /// ```
+ pub fn alloc_range<R>(
+ &self,
+ size: usize,
+ value: T,
+ range: R,
+ gfp: Flags,
+ ) -> Result<usize, AllocError<T>>
+ where
+ R: RangeBounds<usize>,
+ {
+ let Some((min, max)) = to_maple_range(range) else {
+ return Err(AllocError {
+ value,
+ cause: AllocErrorKind::InvalidRequest,
+ });
+ };
+
+ let ptr = T::into_foreign(value);
+ let mut index = 0;
+
+ // SAFETY: The tree is valid, and we are passing a pointer to an owned instance of `T`.
+ let res = to_result(unsafe {
+ bindings::mtree_alloc_range(
+ self.tree.tree.get(),
+ &mut index,
+ ptr,
+ size,
+ min,
+ max,
+ gfp.as_raw(),
+ )
+ });
+
+ if let Err(err) = res {
+ // SAFETY: As `mtree_alloc_range` failed, it is safe to take back ownership.
+ let value = unsafe { T::from_foreign(ptr) };
+
+ let cause = if err == ENOMEM {
+ AllocErrorKind::AllocError(kernel::alloc::AllocError)
+ } else if err == EBUSY {
+ AllocErrorKind::Busy
+ } else {
+ AllocErrorKind::InvalidRequest
+ };
+ Err(AllocError { value, cause })
+ } else {
+ Ok(index)
+ }
+ }
+}
+
+/// A helper type used for navigating a [`MapleTree`].
+///
+/// # Invariants
+///
+/// For the duration of `'tree`:
+///
+/// * The `ma_state` references a valid `MapleTree<T>`.
+/// * The `ma_state` has read/write access to the tree.
+pub struct MaState<'tree, T: ForeignOwnable> {
+ state: bindings::ma_state,
+ _phantom: PhantomData<&'tree mut MapleTree<T>>,
+}
+
+impl<'tree, T: ForeignOwnable> MaState<'tree, T> {
+ /// Initialize a new `MaState` with the given tree.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that this `MaState` has read/write access to the maple tree.
+ #[inline]
+ unsafe fn new_raw(mt: &'tree MapleTree<T>, first: usize, end: usize) -> Self {
+ // INVARIANT:
+ // * Having a reference ensures that the `MapleTree<T>` is valid for `'tree`.
+ // * The caller ensures that we have read/write access.
+ Self {
+ state: bindings::ma_state {
+ tree: mt.tree.get(),
+ index: first,
+ last: end,
+ node: ptr::null_mut(),
+ status: bindings::maple_status_ma_start,
+ min: 0,
+ max: usize::MAX,
+ alloc: ptr::null_mut(),
+ mas_flags: 0,
+ store_type: bindings::store_type_wr_invalid,
+ ..Default::default()
+ },
+ _phantom: PhantomData,
+ }
+ }
+
+ #[inline]
+ fn as_raw(&mut self) -> *mut bindings::ma_state {
+ &raw mut self.state
+ }
+
+ #[inline]
+ fn mas_find_raw(&mut self, max: usize) -> *mut c_void {
+ // SAFETY: By the type invariants, the `ma_state` is active and we have read/write access
+ // to the tree.
+ unsafe { bindings::mas_find(self.as_raw(), max) }
+ }
+
+ /// Find the next entry in the maple tree.
+ ///
+ /// # Examples
+ ///
+ /// Iterate the maple tree.
+ ///
+ /// ```
+ /// use kernel::maple_tree::MapleTree;
+ /// use kernel::sync::Arc;
+ ///
+ /// let tree = KBox::pin_init(MapleTree::<Arc<i32>>::new(), GFP_KERNEL)?;
+ ///
+ /// let ten = Arc::new(10, GFP_KERNEL)?;
+ /// let twenty = Arc::new(20, GFP_KERNEL)?;
+ /// tree.insert(100, ten, GFP_KERNEL)?;
+ /// tree.insert(200, twenty, GFP_KERNEL)?;
+ ///
+ /// let mut ma_lock = tree.lock();
+ /// let mut iter = ma_lock.ma_state(0, usize::MAX);
+ ///
+ /// assert_eq!(iter.find(usize::MAX).map(|v| *v), Some(10));
+ /// assert_eq!(iter.find(usize::MAX).map(|v| *v), Some(20));
+ /// assert!(iter.find(usize::MAX).is_none());
+ /// # Ok::<_, Error>(())
+ /// ```
+ #[inline]
+ pub fn find(&mut self, max: usize) -> Option<T::BorrowedMut<'_>> {
+ let ret = self.mas_find_raw(max);
+ if ret.is_null() {
+ return None;
+ }
+
+ // SAFETY: If the pointer is not null, then it references a valid instance of `T`. It's
+ // safe to access it mutably as the returned reference borrows this `MaState`, and the
+ // `MaState` has read/write access to the maple tree.
+ Some(unsafe { T::borrow_mut(ret) })
+ }
+}
+
+/// Error type for failure to insert a new value.
+pub struct InsertError<T> {
+ /// The value that could not be inserted.
+ pub value: T,
+ /// The reason for the failure to insert.
+ pub cause: InsertErrorKind,
+}
+
+/// The reason for the failure to insert.
+#[derive(PartialEq, Eq, Copy, Clone, Debug)]
+pub enum InsertErrorKind {
+ /// There is already a value in the requested range.
+ Occupied,
+ /// Failure to allocate memory.
+ AllocError(kernel::alloc::AllocError),
+ /// The insertion request was invalid.
+ InvalidRequest,
+}
+
+impl From<InsertErrorKind> for Error {
+ #[inline]
+ fn from(kind: InsertErrorKind) -> Error {
+ match kind {
+ InsertErrorKind::Occupied => EEXIST,
+ InsertErrorKind::AllocError(kernel::alloc::AllocError) => ENOMEM,
+ InsertErrorKind::InvalidRequest => EINVAL,
+ }
+ }
+}
+
+impl<T> From<InsertError<T>> for Error {
+ #[inline]
+ fn from(insert_err: InsertError<T>) -> Error {
+ Error::from(insert_err.cause)
+ }
+}
+
+/// Error type for failure to insert a new value.
+pub struct AllocError<T> {
+ /// The value that could not be inserted.
+ pub value: T,
+ /// The reason for the failure to insert.
+ pub cause: AllocErrorKind,
+}
+
+/// The reason for the failure to insert.
+#[derive(PartialEq, Eq, Copy, Clone)]
+pub enum AllocErrorKind {
+ /// There is not enough space for the requested allocation.
+ Busy,
+ /// Failure to allocate memory.
+ AllocError(kernel::alloc::AllocError),
+ /// The insertion request was invalid.
+ InvalidRequest,
+}
+
+impl From<AllocErrorKind> for Error {
+ #[inline]
+ fn from(kind: AllocErrorKind) -> Error {
+ match kind {
+ AllocErrorKind::Busy => EBUSY,
+ AllocErrorKind::AllocError(kernel::alloc::AllocError) => ENOMEM,
+ AllocErrorKind::InvalidRequest => EINVAL,
+ }
+ }
+}
+
+impl<T> From<AllocError<T>> for Error {
+ #[inline]
+ fn from(insert_err: AllocError<T>) -> Error {
+ Error::from(insert_err.cause)
+ }
+}
diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs
index d3aa7d25afad..d698cddcb4a5 100644
--- a/rust/kernel/miscdevice.rs
+++ b/rust/kernel/miscdevice.rs
@@ -13,7 +13,8 @@ use crate::{
device::Device,
error::{to_result, Error, Result, VTABLE_DEFAULT_ERROR},
ffi::{c_int, c_long, c_uint, c_ulong},
- fs::File,
+ fs::{File, Kiocb},
+ iov::{IovIterDest, IovIterSource},
mm::virt::VmaNew,
prelude::*,
seq_file::SeqFile,
@@ -141,6 +142,16 @@ pub trait MiscDevice: Sized {
build_error!(VTABLE_DEFAULT_ERROR)
}
+ /// Read from this miscdevice.
+ fn read_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterDest<'_>) -> Result<usize> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Write to this miscdevice.
+ fn write_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterSource<'_>) -> Result<usize> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
/// Handler for ioctls.
///
/// The `cmd` argument is usually manipulated using the utilities in [`kernel::ioctl`].
@@ -247,6 +258,46 @@ impl<T: MiscDevice> MiscdeviceVTable<T> {
/// # Safety
///
+ /// `kiocb` must be correspond to a valid file that is associated with a
+ /// `MiscDeviceRegistration<T>`. `iter` must be a valid `struct iov_iter` for writing.
+ unsafe extern "C" fn read_iter(
+ kiocb: *mut bindings::kiocb,
+ iter: *mut bindings::iov_iter,
+ ) -> isize {
+ // SAFETY: The caller provides a valid `struct kiocb` associated with a
+ // `MiscDeviceRegistration<T>` file.
+ let kiocb = unsafe { Kiocb::from_raw(kiocb) };
+ // SAFETY: This is a valid `struct iov_iter` for writing.
+ let iov = unsafe { IovIterDest::from_raw(iter) };
+
+ match T::read_iter(kiocb, iov) {
+ Ok(res) => res as isize,
+ Err(err) => err.to_errno() as isize,
+ }
+ }
+
+ /// # Safety
+ ///
+ /// `kiocb` must be correspond to a valid file that is associated with a
+ /// `MiscDeviceRegistration<T>`. `iter` must be a valid `struct iov_iter` for writing.
+ unsafe extern "C" fn write_iter(
+ kiocb: *mut bindings::kiocb,
+ iter: *mut bindings::iov_iter,
+ ) -> isize {
+ // SAFETY: The caller provides a valid `struct kiocb` associated with a
+ // `MiscDeviceRegistration<T>` file.
+ let kiocb = unsafe { Kiocb::from_raw(kiocb) };
+ // SAFETY: This is a valid `struct iov_iter` for reading.
+ let iov = unsafe { IovIterSource::from_raw(iter) };
+
+ match T::write_iter(kiocb, iov) {
+ Ok(res) => res as isize,
+ Err(err) => err.to_errno() as isize,
+ }
+ }
+
+ /// # Safety
+ ///
/// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
/// `vma` must be a vma that is currently being mmap'ed with this file.
unsafe extern "C" fn mmap(
@@ -341,6 +392,16 @@ impl<T: MiscDevice> MiscdeviceVTable<T> {
open: Some(Self::open),
release: Some(Self::release),
mmap: if T::HAS_MMAP { Some(Self::mmap) } else { None },
+ read_iter: if T::HAS_READ_ITER {
+ Some(Self::read_iter)
+ } else {
+ None
+ },
+ write_iter: if T::HAS_WRITE_ITER {
+ Some(Self::write_iter)
+ } else {
+ None
+ },
unlocked_ioctl: if T::HAS_IOCTL {
Some(Self::ioctl)
} else {
diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs
index 43f525c0d16c..4764d7b68f2a 100644
--- a/rust/kernel/mm.rs
+++ b/rust/kernel/mm.rs
@@ -13,7 +13,8 @@
use crate::{
bindings,
- types::{ARef, AlwaysRefCounted, NotThreadSafe, Opaque},
+ sync::aref::{ARef, AlwaysRefCounted},
+ types::{NotThreadSafe, Opaque},
};
use core::{ops::Deref, ptr::NonNull};
diff --git a/rust/kernel/mm/mmput_async.rs b/rust/kernel/mm/mmput_async.rs
index 9289e05f7a67..b8d2f051225c 100644
--- a/rust/kernel/mm/mmput_async.rs
+++ b/rust/kernel/mm/mmput_async.rs
@@ -10,7 +10,7 @@
use crate::{
bindings,
mm::MmWithUser,
- types::{ARef, AlwaysRefCounted},
+ sync::aref::{ARef, AlwaysRefCounted},
};
use core::{ops::Deref, ptr::NonNull};
diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs
index 75ef096075cb..432fc0297d4a 100644
--- a/rust/kernel/page.rs
+++ b/rust/kernel/page.rs
@@ -170,6 +170,12 @@ impl Page {
self.page.as_ptr()
}
+ /// Get the node id containing this page.
+ pub fn nid(&self) -> i32 {
+ // SAFETY: Always safe to call with a valid page.
+ unsafe { bindings::page_to_nid(self.as_ptr()) }
+ }
+
/// Runs a piece of code with this page mapped to an address.
///
/// The page is unmapped when this call returns.
diff --git a/rust/kernel/security.rs b/rust/kernel/security.rs
index 0c63e9e7e564..9d271695265f 100644
--- a/rust/kernel/security.rs
+++ b/rust/kernel/security.rs
@@ -8,9 +8,46 @@
use crate::{
bindings,
+ cred::Credential,
error::{to_result, Result},
+ fs::File,
};
+/// Calls the security modules to determine if the given task can become the manager of a binder
+/// context.
+#[inline]
+pub fn binder_set_context_mgr(mgr: &Credential) -> Result {
+ // SAFETY: `mrg.0` is valid because the shared reference guarantees a nonzero refcount.
+ to_result(unsafe { bindings::security_binder_set_context_mgr(mgr.as_ptr()) })
+}
+
+/// Calls the security modules to determine if binder transactions are allowed from task `from` to
+/// task `to`.
+#[inline]
+pub fn binder_transaction(from: &Credential, to: &Credential) -> Result {
+ // SAFETY: `from` and `to` are valid because the shared references guarantee nonzero refcounts.
+ to_result(unsafe { bindings::security_binder_transaction(from.as_ptr(), to.as_ptr()) })
+}
+
+/// Calls the security modules to determine if task `from` is allowed to send binder objects
+/// (owned by itself or other processes) to task `to` through a binder transaction.
+#[inline]
+pub fn binder_transfer_binder(from: &Credential, to: &Credential) -> Result {
+ // SAFETY: `from` and `to` are valid because the shared references guarantee nonzero refcounts.
+ to_result(unsafe { bindings::security_binder_transfer_binder(from.as_ptr(), to.as_ptr()) })
+}
+
+/// Calls the security modules to determine if task `from` is allowed to send the given file to
+/// task `to` (which would get its own file descriptor) through a binder transaction.
+#[inline]
+pub fn binder_transfer_file(from: &Credential, to: &Credential, file: &File) -> Result {
+ // SAFETY: `from`, `to` and `file` are valid because the shared references guarantee nonzero
+ // refcounts.
+ to_result(unsafe {
+ bindings::security_binder_transfer_file(from.as_ptr(), to.as_ptr(), file.as_ptr())
+ })
+}
+
/// A security context string.
///
/// # Invariants
diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs
new file mode 100644
index 000000000000..14ddb711bab3
--- /dev/null
+++ b/rust/kernel/usb.rs
@@ -0,0 +1,456 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (C) 2025 Collabora Ltd.
+
+//! Abstractions for the USB bus.
+//!
+//! C header: [`include/linux/usb.h`](srctree/include/linux/usb.h)
+
+use crate::{
+ bindings, device,
+ device_id::{RawDeviceId, RawDeviceIdIndex},
+ driver,
+ error::{from_result, to_result, Result},
+ prelude::*,
+ str::CStr,
+ types::{AlwaysRefCounted, Opaque},
+ ThisModule,
+};
+use core::{marker::PhantomData, mem::MaybeUninit, ptr::NonNull};
+
+/// An adapter for the registration of USB drivers.
+pub struct Adapter<T: Driver>(T);
+
+// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if
+// a preceding call to `register` has been successful.
+unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
+ type RegType = bindings::usb_driver;
+
+ unsafe fn register(
+ udrv: &Opaque<Self::RegType>,
+ name: &'static CStr,
+ module: &'static ThisModule,
+ ) -> Result {
+ // SAFETY: It's safe to set the fields of `struct usb_driver` on initialization.
+ unsafe {
+ (*udrv.get()).name = name.as_char_ptr();
+ (*udrv.get()).probe = Some(Self::probe_callback);
+ (*udrv.get()).disconnect = Some(Self::disconnect_callback);
+ (*udrv.get()).id_table = T::ID_TABLE.as_ptr();
+ }
+
+ // SAFETY: `udrv` is guaranteed to be a valid `RegType`.
+ to_result(unsafe {
+ bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr())
+ })
+ }
+
+ unsafe fn unregister(udrv: &Opaque<Self::RegType>) {
+ // SAFETY: `udrv` is guaranteed to be a valid `RegType`.
+ unsafe { bindings::usb_deregister(udrv.get()) };
+ }
+}
+
+impl<T: Driver + 'static> Adapter<T> {
+ extern "C" fn probe_callback(
+ intf: *mut bindings::usb_interface,
+ id: *const bindings::usb_device_id,
+ ) -> kernel::ffi::c_int {
+ // SAFETY: The USB core only ever calls the probe callback with a valid pointer to a
+ // `struct usb_interface` and `struct usb_device_id`.
+ //
+ // INVARIANT: `intf` is valid for the duration of `probe_callback()`.
+ let intf = unsafe { &*intf.cast::<Interface<device::CoreInternal>>() };
+
+ from_result(|| {
+ // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct usb_device_id` and
+ // does not add additional invariants, so it's safe to transmute.
+ let id = unsafe { &*id.cast::<DeviceId>() };
+
+ let info = T::ID_TABLE.info(id.index());
+ let data = T::probe(intf, id, info)?;
+
+ let dev: &device::Device<device::CoreInternal> = intf.as_ref();
+ dev.set_drvdata(data);
+ Ok(0)
+ })
+ }
+
+ extern "C" fn disconnect_callback(intf: *mut bindings::usb_interface) {
+ // SAFETY: The USB core only ever calls the disconnect callback with a valid pointer to a
+ // `struct usb_interface`.
+ //
+ // INVARIANT: `intf` is valid for the duration of `disconnect_callback()`.
+ let intf = unsafe { &*intf.cast::<Interface<device::CoreInternal>>() };
+
+ let dev: &device::Device<device::CoreInternal> = intf.as_ref();
+
+ // SAFETY: `disconnect_callback` is only ever called after a successful call to
+ // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
+ // and stored a `Pin<KBox<T>>`.
+ let data = unsafe { dev.drvdata_obtain::<Pin<KBox<T>>>() };
+
+ T::disconnect(intf, data.as_ref());
+ }
+}
+
+/// Abstraction for the USB device ID structure, i.e. [`struct usb_device_id`].
+///
+/// [`struct usb_device_id`]: https://docs.kernel.org/driver-api/basics.html#c.usb_device_id
+#[repr(transparent)]
+#[derive(Clone, Copy)]
+pub struct DeviceId(bindings::usb_device_id);
+
+impl DeviceId {
+ /// Equivalent to C's `USB_DEVICE` macro.
+ pub const fn from_id(vendor: u16, product: u16) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE as u16,
+ idVendor: vendor,
+ idProduct: product,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+
+ /// Equivalent to C's `USB_DEVICE_VER` macro.
+ pub const fn from_device_ver(vendor: u16, product: u16, bcd_lo: u16, bcd_hi: u16) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE_AND_VERSION as u16,
+ idVendor: vendor,
+ idProduct: product,
+ bcdDevice_lo: bcd_lo,
+ bcdDevice_hi: bcd_hi,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+
+ /// Equivalent to C's `USB_DEVICE_INFO` macro.
+ pub const fn from_device_info(class: u8, subclass: u8, protocol: u8) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: bindings::USB_DEVICE_ID_MATCH_DEV_INFO as u16,
+ bDeviceClass: class,
+ bDeviceSubClass: subclass,
+ bDeviceProtocol: protocol,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+
+ /// Equivalent to C's `USB_INTERFACE_INFO` macro.
+ pub const fn from_interface_info(class: u8, subclass: u8, protocol: u8) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: bindings::USB_DEVICE_ID_MATCH_INT_INFO as u16,
+ bInterfaceClass: class,
+ bInterfaceSubClass: subclass,
+ bInterfaceProtocol: protocol,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+
+ /// Equivalent to C's `USB_DEVICE_INTERFACE_CLASS` macro.
+ pub const fn from_device_interface_class(vendor: u16, product: u16, class: u8) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE
+ | bindings::USB_DEVICE_ID_MATCH_INT_CLASS) as u16,
+ idVendor: vendor,
+ idProduct: product,
+ bInterfaceClass: class,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+
+ /// Equivalent to C's `USB_DEVICE_INTERFACE_PROTOCOL` macro.
+ pub const fn from_device_interface_protocol(vendor: u16, product: u16, protocol: u8) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE
+ | bindings::USB_DEVICE_ID_MATCH_INT_PROTOCOL) as u16,
+ idVendor: vendor,
+ idProduct: product,
+ bInterfaceProtocol: protocol,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+
+ /// Equivalent to C's `USB_DEVICE_INTERFACE_NUMBER` macro.
+ pub const fn from_device_interface_number(vendor: u16, product: u16, number: u8) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE
+ | bindings::USB_DEVICE_ID_MATCH_INT_NUMBER) as u16,
+ idVendor: vendor,
+ idProduct: product,
+ bInterfaceNumber: number,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+
+ /// Equivalent to C's `USB_DEVICE_AND_INTERFACE_INFO` macro.
+ pub const fn from_device_and_interface_info(
+ vendor: u16,
+ product: u16,
+ class: u8,
+ subclass: u8,
+ protocol: u8,
+ ) -> Self {
+ Self(bindings::usb_device_id {
+ match_flags: (bindings::USB_DEVICE_ID_MATCH_INT_INFO
+ | bindings::USB_DEVICE_ID_MATCH_DEVICE) as u16,
+ idVendor: vendor,
+ idProduct: product,
+ bInterfaceClass: class,
+ bInterfaceSubClass: subclass,
+ bInterfaceProtocol: protocol,
+ // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ })
+ }
+}
+
+// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `usb_device_id` and does not add
+// additional invariants, so it's safe to transmute to `RawType`.
+unsafe impl RawDeviceId for DeviceId {
+ type RawType = bindings::usb_device_id;
+}
+
+// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_info` field.
+unsafe impl RawDeviceIdIndex for DeviceId {
+ const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::usb_device_id, driver_info);
+
+ fn index(&self) -> usize {
+ self.0.driver_info
+ }
+}
+
+/// [`IdTable`](kernel::device_id::IdTable) type for USB.
+pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>;
+
+/// Create a USB `IdTable` with its alias for modpost.
+#[macro_export]
+macro_rules! usb_device_table {
+ ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => {
+ const $table_name: $crate::device_id::IdArray<
+ $crate::usb::DeviceId,
+ $id_info_type,
+ { $table_data.len() },
+ > = $crate::device_id::IdArray::new($table_data);
+
+ $crate::module_device_table!("usb", $module_table_name, $table_name);
+ };
+}
+
+/// The USB driver trait.
+///
+/// # Examples
+///
+///```
+/// # use kernel::{bindings, device::Core, usb};
+/// use kernel::prelude::*;
+///
+/// struct MyDriver;
+///
+/// kernel::usb_device_table!(
+/// USB_TABLE,
+/// MODULE_USB_TABLE,
+/// <MyDriver as usb::Driver>::IdInfo,
+/// [
+/// (usb::DeviceId::from_id(0x1234, 0x5678), ()),
+/// (usb::DeviceId::from_id(0xabcd, 0xef01), ()),
+/// ]
+/// );
+///
+/// impl usb::Driver for MyDriver {
+/// type IdInfo = ();
+/// const ID_TABLE: usb::IdTable<Self::IdInfo> = &USB_TABLE;
+///
+/// fn probe(
+/// _interface: &usb::Interface<Core>,
+/// _id: &usb::DeviceId,
+/// _info: &Self::IdInfo,
+/// ) -> Result<Pin<KBox<Self>>> {
+/// Err(ENODEV)
+/// }
+///
+/// fn disconnect(_interface: &usb::Interface<Core>, _data: Pin<&Self>) {}
+/// }
+///```
+pub trait Driver {
+ /// The type holding information about each one of the device ids supported by the driver.
+ type IdInfo: 'static;
+
+ /// The table of device ids supported by the driver.
+ const ID_TABLE: IdTable<Self::IdInfo>;
+
+ /// USB driver probe.
+ ///
+ /// Called when a new USB interface is bound to this driver.
+ /// Implementers should attempt to initialize the interface here.
+ fn probe(
+ interface: &Interface<device::Core>,
+ id: &DeviceId,
+ id_info: &Self::IdInfo,
+ ) -> Result<Pin<KBox<Self>>>;
+
+ /// USB driver disconnect.
+ ///
+ /// Called when the USB interface is about to be unbound from this driver.
+ fn disconnect(interface: &Interface<device::Core>, data: Pin<&Self>);
+}
+
+/// A USB interface.
+///
+/// This structure represents the Rust abstraction for a C [`struct usb_interface`].
+/// The implementation abstracts the usage of a C [`struct usb_interface`] passed
+/// in from the C side.
+///
+/// # Invariants
+///
+/// An [`Interface`] instance represents a valid [`struct usb_interface`] created
+/// by the C portion of the kernel.
+///
+/// [`struct usb_interface`]: https://www.kernel.org/doc/html/latest/driver-api/usb/usb.html#c.usb_interface
+#[repr(transparent)]
+pub struct Interface<Ctx: device::DeviceContext = device::Normal>(
+ Opaque<bindings::usb_interface>,
+ PhantomData<Ctx>,
+);
+
+impl<Ctx: device::DeviceContext> Interface<Ctx> {
+ fn as_raw(&self) -> *mut bindings::usb_interface {
+ self.0.get()
+ }
+}
+
+// SAFETY: `Interface` is a transparent wrapper of a type that doesn't depend on
+// `Interface`'s generic argument.
+kernel::impl_device_context_deref!(unsafe { Interface });
+kernel::impl_device_context_into_aref!(Interface);
+
+impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Interface<Ctx> {
+ fn as_ref(&self) -> &device::Device<Ctx> {
+ // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
+ // `struct usb_interface`.
+ let dev = unsafe { &raw mut ((*self.as_raw()).dev) };
+
+ // SAFETY: `dev` points to a valid `struct device`.
+ unsafe { device::Device::from_raw(dev) }
+ }
+}
+
+impl<Ctx: device::DeviceContext> AsRef<Device> for Interface<Ctx> {
+ fn as_ref(&self) -> &Device {
+ // SAFETY: `self.as_raw()` is valid by the type invariants.
+ let usb_dev = unsafe { bindings::interface_to_usbdev(self.as_raw()) };
+
+ // SAFETY: For a valid `struct usb_interface` pointer, the above call to
+ // `interface_to_usbdev()` guarantees to return a valid pointer to a `struct usb_device`.
+ unsafe { &*(usb_dev.cast()) }
+ }
+}
+
+// SAFETY: Instances of `Interface` are always reference-counted.
+unsafe impl AlwaysRefCounted for Interface {
+ fn inc_ref(&self) {
+ // SAFETY: The invariants of `Interface` guarantee that `self.as_raw()`
+ // returns a valid `struct usb_interface` pointer, for which we will
+ // acquire a new refcount.
+ unsafe { bindings::usb_get_intf(self.as_raw()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The safety requirements guarantee that the refcount is non-zero.
+ unsafe { bindings::usb_put_intf(obj.cast().as_ptr()) }
+ }
+}
+
+// SAFETY: A `Interface` is always reference-counted and can be released from any thread.
+unsafe impl Send for Interface {}
+
+// SAFETY: It is safe to send a &Interface to another thread because we do not
+// allow any mutation through a shared reference.
+unsafe impl Sync for Interface {}
+
+/// A USB device.
+///
+/// This structure represents the Rust abstraction for a C [`struct usb_device`].
+/// The implementation abstracts the usage of a C [`struct usb_device`] passed in
+/// from the C side.
+///
+/// # Invariants
+///
+/// A [`Device`] instance represents a valid [`struct usb_device`] created by the C portion of the
+/// kernel.
+///
+/// [`struct usb_device`]: https://www.kernel.org/doc/html/latest/driver-api/usb/usb.html#c.usb_device
+#[repr(transparent)]
+struct Device<Ctx: device::DeviceContext = device::Normal>(
+ Opaque<bindings::usb_device>,
+ PhantomData<Ctx>,
+);
+
+impl<Ctx: device::DeviceContext> Device<Ctx> {
+ fn as_raw(&self) -> *mut bindings::usb_device {
+ self.0.get()
+ }
+}
+
+// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
+// argument.
+kernel::impl_device_context_deref!(unsafe { Device });
+kernel::impl_device_context_into_aref!(Device);
+
+// SAFETY: Instances of `Device` are always reference-counted.
+unsafe impl AlwaysRefCounted for Device {
+ fn inc_ref(&self) {
+ // SAFETY: The invariants of `Device` guarantee that `self.as_raw()`
+ // returns a valid `struct usb_device` pointer, for which we will
+ // acquire a new refcount.
+ unsafe { bindings::usb_get_dev(self.as_raw()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The safety requirements guarantee that the refcount is non-zero.
+ unsafe { bindings::usb_put_dev(obj.cast().as_ptr()) }
+ }
+}
+
+impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
+ fn as_ref(&self) -> &device::Device<Ctx> {
+ // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
+ // `struct usb_device`.
+ let dev = unsafe { &raw mut ((*self.as_raw()).dev) };
+
+ // SAFETY: `dev` points to a valid `struct device`.
+ unsafe { device::Device::from_raw(dev) }
+ }
+}
+
+// SAFETY: A `Device` is always reference-counted and can be released from any thread.
+unsafe impl Send for Device {}
+
+// SAFETY: It is safe to send a &Device to another thread because we do not
+// allow any mutation through a shared reference.
+unsafe impl Sync for Device {}
+
+/// Declares a kernel module that exposes a single USB driver.
+///
+/// # Examples
+///
+/// ```ignore
+/// module_usb_driver! {
+/// type: MyDriver,
+/// name: "Module name",
+/// author: ["Author name"],
+/// description: "Description",
+/// license: "GPL v2",
+/// }
+/// ```
+#[macro_export]
+macro_rules! module_usb_driver {
+ ($($f:tt)*) => {
+ $crate::module_driver!(<T>, $crate::usb::Adapter<T>, { $($f)* });
+ }
+}
generated by cgit v1.2.3 (git 2.46.0) at 2025-11-09 18:52:14 +0000