mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-18 00:24:58 +08:00
52a7f2deb4
This makes it possible to use `T` as a `impl Init<T, E>` for every error type `E` instead of just `Infallible`. Signed-off-by: Benno Lossin <benno.lossin@proton.me> Reviewed-by: Gary Guo <gary@garyguo.net> Reviewed-by: Martin Rodriguez Reboredo <yakoyoku@gmail.com> Link: https://lore.kernel.org/r/20230413100157.740697-1-benno.lossin@proton.me Signed-off-by: Miguel Ojeda <ojeda@kernel.org>
1428 lines
48 KiB
Rust
1428 lines
48 KiB
Rust
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
|
|
//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
|
|
//!
|
|
//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
|
|
//! overflow.
|
|
//!
|
|
//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
|
|
//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
|
|
//!
|
|
//! # Overview
|
|
//!
|
|
//! To initialize a `struct` with an in-place constructor you will need two things:
|
|
//! - an in-place constructor,
|
|
//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
|
|
//! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]).
|
|
//!
|
|
//! To get an in-place constructor there are generally three options:
|
|
//! - directly creating an in-place constructor using the [`pin_init!`] macro,
|
|
//! - a custom function/macro returning an in-place constructor provided by someone else,
|
|
//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
|
|
//!
|
|
//! Aside from pinned initialization, this API also supports in-place construction without pinning,
|
|
//! the macros/types/functions are generally named like the pinned variants without the `pin`
|
|
//! prefix.
|
|
//!
|
|
//! # Examples
|
|
//!
|
|
//! ## Using the [`pin_init!`] macro
|
|
//!
|
|
//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
|
|
//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
|
|
//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
|
|
//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
|
|
//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
|
|
//!
|
|
//! ```rust
|
|
//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
//! use kernel::{prelude::*, sync::Mutex, new_mutex};
|
|
//! # use core::pin::Pin;
|
|
//! #[pin_data]
|
|
//! struct Foo {
|
|
//! #[pin]
|
|
//! a: Mutex<usize>,
|
|
//! b: u32,
|
|
//! }
|
|
//!
|
|
//! let foo = pin_init!(Foo {
|
|
//! a <- new_mutex!(42, "Foo::a"),
|
|
//! b: 24,
|
|
//! });
|
|
//! ```
|
|
//!
|
|
//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
|
|
//! (or just the stack) to actually initialize a `Foo`:
|
|
//!
|
|
//! ```rust
|
|
//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
//! # use kernel::{prelude::*, sync::Mutex, new_mutex};
|
|
//! # use core::pin::Pin;
|
|
//! # #[pin_data]
|
|
//! # struct Foo {
|
|
//! # #[pin]
|
|
//! # a: Mutex<usize>,
|
|
//! # b: u32,
|
|
//! # }
|
|
//! # let foo = pin_init!(Foo {
|
|
//! # a <- new_mutex!(42, "Foo::a"),
|
|
//! # b: 24,
|
|
//! # });
|
|
//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo);
|
|
//! ```
|
|
//!
|
|
//! For more information see the [`pin_init!`] macro.
|
|
//!
|
|
//! ## Using a custom function/macro that returns an initializer
|
|
//!
|
|
//! Many types from the kernel supply a function/macro that returns an initializer, because the
|
|
//! above method only works for types where you can access the fields.
|
|
//!
|
|
//! ```rust
|
|
//! # use kernel::{new_mutex, sync::{Arc, Mutex}};
|
|
//! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx"));
|
|
//! ```
|
|
//!
|
|
//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
|
|
//!
|
|
//! ```rust
|
|
//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
//! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init};
|
|
//! #[pin_data]
|
|
//! struct DriverData {
|
|
//! #[pin]
|
|
//! status: Mutex<i32>,
|
|
//! buffer: Box<[u8; 1_000_000]>,
|
|
//! }
|
|
//!
|
|
//! impl DriverData {
|
|
//! fn new() -> impl PinInit<Self, Error> {
|
|
//! try_pin_init!(Self {
|
|
//! status <- new_mutex!(0, "DriverData::status"),
|
|
//! buffer: Box::init(kernel::init::zeroed())?,
|
|
//! })
|
|
//! }
|
|
//! }
|
|
//! ```
|
|
//!
|
|
//! ## Manual creation of an initializer
|
|
//!
|
|
//! Often when working with primitives the previous approaches are not sufficient. That is where
|
|
//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
|
|
//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
|
|
//! actually does the initialization in the correct way. Here are the things to look out for
|
|
//! (we are calling the parameter to the closure `slot`):
|
|
//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
|
|
//! `slot` now contains a valid bit pattern for the type `T`,
|
|
//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
|
|
//! you need to take care to clean up anything if your initialization fails mid-way,
|
|
//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
|
|
//! `slot` gets called.
|
|
//!
|
|
//! ```rust
|
|
//! use kernel::{prelude::*, init};
|
|
//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
|
|
//! # mod bindings {
|
|
//! # pub struct foo;
|
|
//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
|
|
//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
|
|
//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
|
|
//! # }
|
|
//! /// # Invariants
|
|
//! ///
|
|
//! /// `foo` is always initialized
|
|
//! #[pin_data(PinnedDrop)]
|
|
//! pub struct RawFoo {
|
|
//! #[pin]
|
|
//! foo: Opaque<bindings::foo>,
|
|
//! #[pin]
|
|
//! _p: PhantomPinned,
|
|
//! }
|
|
//!
|
|
//! impl RawFoo {
|
|
//! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
|
|
//! // SAFETY:
|
|
//! // - when the closure returns `Ok(())`, then it has successfully initialized and
|
|
//! // enabled `foo`,
|
|
//! // - when it returns `Err(e)`, then it has cleaned up before
|
|
//! unsafe {
|
|
//! init::pin_init_from_closure(move |slot: *mut Self| {
|
|
//! // `slot` contains uninit memory, avoid creating a reference.
|
|
//! let foo = addr_of_mut!((*slot).foo);
|
|
//!
|
|
//! // Initialize the `foo`
|
|
//! bindings::init_foo(Opaque::raw_get(foo));
|
|
//!
|
|
//! // Try to enable it.
|
|
//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
|
|
//! if err != 0 {
|
|
//! // Enabling has failed, first clean up the foo and then return the error.
|
|
//! bindings::destroy_foo(Opaque::raw_get(foo));
|
|
//! return Err(Error::from_kernel_errno(err));
|
|
//! }
|
|
//!
|
|
//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
|
|
//! Ok(())
|
|
//! })
|
|
//! }
|
|
//! }
|
|
//! }
|
|
//!
|
|
//! #[pinned_drop]
|
|
//! impl PinnedDrop for RawFoo {
|
|
//! fn drop(self: Pin<&mut Self>) {
|
|
//! // SAFETY: Since `foo` is initialized, destroying is safe.
|
|
//! unsafe { bindings::destroy_foo(self.foo.get()) };
|
|
//! }
|
|
//! }
|
|
//! ```
|
|
//!
|
|
//! For the special case where initializing a field is a single FFI-function call that cannot fail,
|
|
//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
|
|
//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
|
|
//! with [`pin_init!`].
|
|
//!
|
|
//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
|
|
//! the `kernel` crate. The [`sync`] module is a good starting point.
|
|
//!
|
|
//! [`sync`]: kernel::sync
|
|
//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
|
|
//! [structurally pinned fields]:
|
|
//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
|
|
//! [stack]: crate::stack_pin_init
|
|
//! [`Arc<T>`]: crate::sync::Arc
|
|
//! [`impl PinInit<Foo>`]: PinInit
|
|
//! [`impl PinInit<T, E>`]: PinInit
|
|
//! [`impl Init<T, E>`]: Init
|
|
//! [`Opaque`]: kernel::types::Opaque
|
|
//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
|
|
//! [`pin_data`]: ::macros::pin_data
|
|
|
|
use crate::{
|
|
error::{self, Error},
|
|
sync::UniqueArc,
|
|
};
|
|
use alloc::boxed::Box;
|
|
use core::{
|
|
alloc::AllocError,
|
|
cell::Cell,
|
|
convert::Infallible,
|
|
marker::PhantomData,
|
|
mem::MaybeUninit,
|
|
num::*,
|
|
pin::Pin,
|
|
ptr::{self, NonNull},
|
|
};
|
|
|
|
#[doc(hidden)]
|
|
pub mod __internal;
|
|
#[doc(hidden)]
|
|
pub mod macros;
|
|
|
|
/// Initialize and pin a type directly on the stack.
|
|
///
|
|
/// # Examples
|
|
///
|
|
/// ```rust
|
|
/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
/// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
|
|
/// # use macros::pin_data;
|
|
/// # use core::pin::Pin;
|
|
/// #[pin_data]
|
|
/// struct Foo {
|
|
/// #[pin]
|
|
/// a: Mutex<usize>,
|
|
/// b: Bar,
|
|
/// }
|
|
///
|
|
/// #[pin_data]
|
|
/// struct Bar {
|
|
/// x: u32,
|
|
/// }
|
|
///
|
|
/// stack_pin_init!(let foo = pin_init!(Foo {
|
|
/// a <- new_mutex!(42),
|
|
/// b: Bar {
|
|
/// x: 64,
|
|
/// },
|
|
/// }));
|
|
/// let foo: Pin<&mut Foo> = foo;
|
|
/// pr_info!("a: {}", &*foo.a.lock());
|
|
/// ```
|
|
///
|
|
/// # Syntax
|
|
///
|
|
/// A normal `let` binding with optional type annotation. The expression is expected to implement
|
|
/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
|
|
/// type, then use [`stack_try_pin_init!`].
|
|
#[macro_export]
|
|
macro_rules! stack_pin_init {
|
|
(let $var:ident $(: $t:ty)? = $val:expr) => {
|
|
let val = $val;
|
|
let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
|
|
let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
|
|
Ok(res) => res,
|
|
Err(x) => {
|
|
let x: ::core::convert::Infallible = x;
|
|
match x {}
|
|
}
|
|
};
|
|
};
|
|
}
|
|
|
|
/// Initialize and pin a type directly on the stack.
|
|
///
|
|
/// # Examples
|
|
///
|
|
/// ```rust
|
|
/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
|
|
/// # use macros::pin_data;
|
|
/// # use core::{alloc::AllocError, pin::Pin};
|
|
/// #[pin_data]
|
|
/// struct Foo {
|
|
/// #[pin]
|
|
/// a: Mutex<usize>,
|
|
/// b: Box<Bar>,
|
|
/// }
|
|
///
|
|
/// struct Bar {
|
|
/// x: u32,
|
|
/// }
|
|
///
|
|
/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
|
|
/// a <- new_mutex!(42),
|
|
/// b: Box::try_new(Bar {
|
|
/// x: 64,
|
|
/// })?,
|
|
/// }));
|
|
/// let foo = foo.unwrap();
|
|
/// pr_info!("a: {}", &*foo.a.lock());
|
|
/// ```
|
|
///
|
|
/// ```rust
|
|
/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
|
|
/// # use macros::pin_data;
|
|
/// # use core::{alloc::AllocError, pin::Pin};
|
|
/// #[pin_data]
|
|
/// struct Foo {
|
|
/// #[pin]
|
|
/// a: Mutex<usize>,
|
|
/// b: Box<Bar>,
|
|
/// }
|
|
///
|
|
/// struct Bar {
|
|
/// x: u32,
|
|
/// }
|
|
///
|
|
/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
|
|
/// a <- new_mutex!(42),
|
|
/// b: Box::try_new(Bar {
|
|
/// x: 64,
|
|
/// })?,
|
|
/// }));
|
|
/// pr_info!("a: {}", &*foo.a.lock());
|
|
/// # Ok::<_, AllocError>(())
|
|
/// ```
|
|
///
|
|
/// # Syntax
|
|
///
|
|
/// A normal `let` binding with optional type annotation. The expression is expected to implement
|
|
/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
|
|
/// `=` will propagate this error.
|
|
#[macro_export]
|
|
macro_rules! stack_try_pin_init {
|
|
(let $var:ident $(: $t:ty)? = $val:expr) => {
|
|
let val = $val;
|
|
let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
|
|
let mut $var = $crate::init::__internal::StackInit::init($var, val);
|
|
};
|
|
(let $var:ident $(: $t:ty)? =? $val:expr) => {
|
|
let val = $val;
|
|
let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
|
|
let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
|
|
};
|
|
}
|
|
|
|
/// Construct an in-place, pinned initializer for `struct`s.
|
|
///
|
|
/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
|
|
/// [`try_pin_init!`].
|
|
///
|
|
/// The syntax is almost identical to that of a normal `struct` initializer:
|
|
///
|
|
/// ```rust
|
|
/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
/// # use kernel::{init, pin_init, macros::pin_data, init::*};
|
|
/// # use core::pin::Pin;
|
|
/// #[pin_data]
|
|
/// struct Foo {
|
|
/// a: usize,
|
|
/// b: Bar,
|
|
/// }
|
|
///
|
|
/// #[pin_data]
|
|
/// struct Bar {
|
|
/// x: u32,
|
|
/// }
|
|
///
|
|
/// # fn demo() -> impl PinInit<Foo> {
|
|
/// let a = 42;
|
|
///
|
|
/// let initializer = pin_init!(Foo {
|
|
/// a,
|
|
/// b: Bar {
|
|
/// x: 64,
|
|
/// },
|
|
/// });
|
|
/// # initializer }
|
|
/// # Box::pin_init(demo()).unwrap();
|
|
/// ```
|
|
///
|
|
/// Arbitrary Rust expressions can be used to set the value of a variable.
|
|
///
|
|
/// The fields are initialized in the order that they appear in the initializer. So it is possible
|
|
/// to read already initialized fields using raw pointers.
|
|
///
|
|
/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
|
|
/// initializer.
|
|
///
|
|
/// # Init-functions
|
|
///
|
|
/// When working with this API it is often desired to let others construct your types without
|
|
/// giving access to all fields. This is where you would normally write a plain function `new`
|
|
/// that would return a new instance of your type. With this API that is also possible.
|
|
/// However, there are a few extra things to keep in mind.
|
|
///
|
|
/// To create an initializer function, simply declare it like this:
|
|
///
|
|
/// ```rust
|
|
/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
/// # use kernel::{init, pin_init, prelude::*, init::*};
|
|
/// # use core::pin::Pin;
|
|
/// # #[pin_data]
|
|
/// # struct Foo {
|
|
/// # a: usize,
|
|
/// # b: Bar,
|
|
/// # }
|
|
/// # #[pin_data]
|
|
/// # struct Bar {
|
|
/// # x: u32,
|
|
/// # }
|
|
/// impl Foo {
|
|
/// fn new() -> impl PinInit<Self> {
|
|
/// pin_init!(Self {
|
|
/// a: 42,
|
|
/// b: Bar {
|
|
/// x: 64,
|
|
/// },
|
|
/// })
|
|
/// }
|
|
/// }
|
|
/// ```
|
|
///
|
|
/// Users of `Foo` can now create it like this:
|
|
///
|
|
/// ```rust
|
|
/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
/// # use kernel::{init, pin_init, macros::pin_data, init::*};
|
|
/// # use core::pin::Pin;
|
|
/// # #[pin_data]
|
|
/// # struct Foo {
|
|
/// # a: usize,
|
|
/// # b: Bar,
|
|
/// # }
|
|
/// # #[pin_data]
|
|
/// # struct Bar {
|
|
/// # x: u32,
|
|
/// # }
|
|
/// # impl Foo {
|
|
/// # fn new() -> impl PinInit<Self> {
|
|
/// # pin_init!(Self {
|
|
/// # a: 42,
|
|
/// # b: Bar {
|
|
/// # x: 64,
|
|
/// # },
|
|
/// # })
|
|
/// # }
|
|
/// # }
|
|
/// let foo = Box::pin_init(Foo::new());
|
|
/// ```
|
|
///
|
|
/// They can also easily embed it into their own `struct`s:
|
|
///
|
|
/// ```rust
|
|
/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
|
|
/// # use kernel::{init, pin_init, macros::pin_data, init::*};
|
|
/// # use core::pin::Pin;
|
|
/// # #[pin_data]
|
|
/// # struct Foo {
|
|
/// # a: usize,
|
|
/// # b: Bar,
|
|
/// # }
|
|
/// # #[pin_data]
|
|
/// # struct Bar {
|
|
/// # x: u32,
|
|
/// # }
|
|
/// # impl Foo {
|
|
/// # fn new() -> impl PinInit<Self> {
|
|
/// # pin_init!(Self {
|
|
/// # a: 42,
|
|
/// # b: Bar {
|
|
/// # x: 64,
|
|
/// # },
|
|
/// # })
|
|
/// # }
|
|
/// # }
|
|
/// #[pin_data]
|
|
/// struct FooContainer {
|
|
/// #[pin]
|
|
/// foo1: Foo,
|
|
/// #[pin]
|
|
/// foo2: Foo,
|
|
/// other: u32,
|
|
/// }
|
|
///
|
|
/// impl FooContainer {
|
|
/// fn new(other: u32) -> impl PinInit<Self> {
|
|
/// pin_init!(Self {
|
|
/// foo1 <- Foo::new(),
|
|
/// foo2 <- Foo::new(),
|
|
/// other,
|
|
/// })
|
|
/// }
|
|
/// }
|
|
/// ```
|
|
///
|
|
/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
|
|
/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
|
|
/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
|
|
///
|
|
/// # Syntax
|
|
///
|
|
/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
|
|
/// the following modifications is expected:
|
|
/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
|
|
/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
|
|
/// pointer named `this` inside of the initializer.
|
|
///
|
|
/// For instance:
|
|
///
|
|
/// ```rust
|
|
/// # use kernel::pin_init;
|
|
/// # use macros::pin_data;
|
|
/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
|
|
/// #[pin_data]
|
|
/// struct Buf {
|
|
/// // `ptr` points into `buf`.
|
|
/// ptr: *mut u8,
|
|
/// buf: [u8; 64],
|
|
/// #[pin]
|
|
/// pin: PhantomPinned,
|
|
/// }
|
|
/// pin_init!(&this in Buf {
|
|
/// buf: [0; 64],
|
|
/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
|
|
/// pin: PhantomPinned,
|
|
/// });
|
|
/// ```
|
|
///
|
|
/// [`try_pin_init!`]: kernel::try_pin_init
|
|
/// [`NonNull<Self>`]: core::ptr::NonNull
|
|
// For a detailed example of how this macro works, see the module documentation of the hidden
|
|
// module `__internal` inside of `init/__internal.rs`.
|
|
#[macro_export]
|
|
macro_rules! pin_init {
|
|
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
|
|
$($fields:tt)*
|
|
}) => {
|
|
$crate::try_pin_init!(
|
|
@this($($this)?),
|
|
@typ($t $(::<$($generics),*>)?),
|
|
@fields($($fields)*),
|
|
@error(::core::convert::Infallible),
|
|
)
|
|
};
|
|
}
|
|
|
|
/// Construct an in-place, fallible pinned initializer for `struct`s.
|
|
///
|
|
/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
|
|
///
|
|
/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
|
|
/// initialization and return the error.
|
|
///
|
|
/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
|
|
/// initialization fails, the memory can be safely deallocated without any further modifications.
|
|
///
|
|
/// This macro defaults the error to [`Error`].
|
|
///
|
|
/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
|
|
/// after the `struct` initializer to specify the error type you want to use.
|
|
///
|
|
/// # Examples
|
|
///
|
|
/// ```rust
|
|
/// # #![feature(new_uninit)]
|
|
/// use kernel::{init::{self, PinInit}, error::Error};
|
|
/// #[pin_data]
|
|
/// struct BigBuf {
|
|
/// big: Box<[u8; 1024 * 1024 * 1024]>,
|
|
/// small: [u8; 1024 * 1024],
|
|
/// ptr: *mut u8,
|
|
/// }
|
|
///
|
|
/// impl BigBuf {
|
|
/// fn new() -> impl PinInit<Self, Error> {
|
|
/// try_pin_init!(Self {
|
|
/// big: Box::init(init::zeroed())?,
|
|
/// small: [0; 1024 * 1024],
|
|
/// ptr: core::ptr::null_mut(),
|
|
/// }? Error)
|
|
/// }
|
|
/// }
|
|
/// ```
|
|
// For a detailed example of how this macro works, see the module documentation of the hidden
|
|
// module `__internal` inside of `init/__internal.rs`.
|
|
#[macro_export]
|
|
macro_rules! try_pin_init {
|
|
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
|
|
$($fields:tt)*
|
|
}) => {
|
|
$crate::try_pin_init!(
|
|
@this($($this)?),
|
|
@typ($t $(::<$($generics),*>)? ),
|
|
@fields($($fields)*),
|
|
@error($crate::error::Error),
|
|
)
|
|
};
|
|
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
|
|
$($fields:tt)*
|
|
}? $err:ty) => {
|
|
$crate::try_pin_init!(
|
|
@this($($this)?),
|
|
@typ($t $(::<$($generics),*>)? ),
|
|
@fields($($fields)*),
|
|
@error($err),
|
|
)
|
|
};
|
|
(
|
|
@this($($this:ident)?),
|
|
@typ($t:ident $(::<$($generics:ty),*>)?),
|
|
@fields($($fields:tt)*),
|
|
@error($err:ty),
|
|
) => {{
|
|
// We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
|
|
// type and shadow it later when we insert the arbitrary user code. That way there will be
|
|
// no possibility of returning without `unsafe`.
|
|
struct __InitOk;
|
|
// Get the pin data from the supplied type.
|
|
let data = unsafe {
|
|
use $crate::init::__internal::HasPinData;
|
|
$t$(::<$($generics),*>)?::__pin_data()
|
|
};
|
|
// Ensure that `data` really is of type `PinData` and help with type inference:
|
|
let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>(
|
|
data,
|
|
move |slot| {
|
|
{
|
|
// Shadow the structure so it cannot be used to return early.
|
|
struct __InitOk;
|
|
// Create the `this` so it can be referenced by the user inside of the
|
|
// expressions creating the individual fields.
|
|
$(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
|
|
// Initialize every field.
|
|
$crate::try_pin_init!(init_slot:
|
|
@data(data),
|
|
@slot(slot),
|
|
@munch_fields($($fields)*,),
|
|
);
|
|
// We use unreachable code to ensure that all fields have been mentioned exactly
|
|
// once, this struct initializer will still be type-checked and complain with a
|
|
// very natural error message if a field is forgotten/mentioned more than once.
|
|
#[allow(unreachable_code, clippy::diverging_sub_expression)]
|
|
if false {
|
|
$crate::try_pin_init!(make_initializer:
|
|
@slot(slot),
|
|
@type_name($t),
|
|
@munch_fields($($fields)*,),
|
|
@acc(),
|
|
);
|
|
}
|
|
// Forget all guards, since initialization was a success.
|
|
$crate::try_pin_init!(forget_guards:
|
|
@munch_fields($($fields)*,),
|
|
);
|
|
}
|
|
Ok(__InitOk)
|
|
}
|
|
);
|
|
let init = move |slot| -> ::core::result::Result<(), $err> {
|
|
init(slot).map(|__InitOk| ())
|
|
};
|
|
let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) };
|
|
init
|
|
}};
|
|
(init_slot:
|
|
@data($data:ident),
|
|
@slot($slot:ident),
|
|
@munch_fields($(,)?),
|
|
) => {
|
|
// Endpoint of munching, no fields are left.
|
|
};
|
|
(init_slot:
|
|
@data($data:ident),
|
|
@slot($slot:ident),
|
|
// In-place initialization syntax.
|
|
@munch_fields($field:ident <- $val:expr, $($rest:tt)*),
|
|
) => {
|
|
let $field = $val;
|
|
// Call the initializer.
|
|
//
|
|
// SAFETY: `slot` is valid, because we are inside of an initializer closure, we
|
|
// return when an error/panic occurs.
|
|
// We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
|
|
unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? };
|
|
// Create the drop guard.
|
|
//
|
|
// We only give access to `&DropGuard`, so it cannot be forgotten via safe code.
|
|
//
|
|
// SAFETY: We forget the guard later when initialization has succeeded.
|
|
let $field = &unsafe {
|
|
$crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
|
|
};
|
|
|
|
$crate::try_pin_init!(init_slot:
|
|
@data($data),
|
|
@slot($slot),
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
(init_slot:
|
|
@data($data:ident),
|
|
@slot($slot:ident),
|
|
// Direct value init, this is safe for every field.
|
|
@munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
|
|
) => {
|
|
$(let $field = $val;)?
|
|
// Initialize the field.
|
|
//
|
|
// SAFETY: The memory at `slot` is uninitialized.
|
|
unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
|
|
// Create the drop guard:
|
|
//
|
|
// We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
|
|
//
|
|
// SAFETY: We forget the guard later when initialization has succeeded.
|
|
let $field = &unsafe {
|
|
$crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
|
|
};
|
|
|
|
$crate::try_pin_init!(init_slot:
|
|
@data($data),
|
|
@slot($slot),
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
(make_initializer:
|
|
@slot($slot:ident),
|
|
@type_name($t:ident),
|
|
@munch_fields($(,)?),
|
|
@acc($($acc:tt)*),
|
|
) => {
|
|
// Endpoint, nothing more to munch, create the initializer.
|
|
// Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
|
|
// get the correct type inference here:
|
|
unsafe {
|
|
::core::ptr::write($slot, $t {
|
|
$($acc)*
|
|
});
|
|
}
|
|
};
|
|
(make_initializer:
|
|
@slot($slot:ident),
|
|
@type_name($t:ident),
|
|
@munch_fields($field:ident <- $val:expr, $($rest:tt)*),
|
|
@acc($($acc:tt)*),
|
|
) => {
|
|
$crate::try_pin_init!(make_initializer:
|
|
@slot($slot),
|
|
@type_name($t),
|
|
@munch_fields($($rest)*),
|
|
@acc($($acc)* $field: ::core::panic!(),),
|
|
);
|
|
};
|
|
(make_initializer:
|
|
@slot($slot:ident),
|
|
@type_name($t:ident),
|
|
@munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
|
|
@acc($($acc:tt)*),
|
|
) => {
|
|
$crate::try_pin_init!(make_initializer:
|
|
@slot($slot),
|
|
@type_name($t),
|
|
@munch_fields($($rest)*),
|
|
@acc($($acc)* $field: ::core::panic!(),),
|
|
);
|
|
};
|
|
(forget_guards:
|
|
@munch_fields($(,)?),
|
|
) => {
|
|
// Munching finished.
|
|
};
|
|
(forget_guards:
|
|
@munch_fields($field:ident <- $val:expr, $($rest:tt)*),
|
|
) => {
|
|
unsafe { $crate::init::__internal::DropGuard::forget($field) };
|
|
|
|
$crate::try_pin_init!(forget_guards:
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
(forget_guards:
|
|
@munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
|
|
) => {
|
|
unsafe { $crate::init::__internal::DropGuard::forget($field) };
|
|
|
|
$crate::try_pin_init!(forget_guards:
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
}
|
|
|
|
/// Construct an in-place initializer for `struct`s.
|
|
///
|
|
/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
|
|
/// [`try_init!`].
|
|
///
|
|
/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
|
|
/// - `unsafe` code must guarantee either full initialization or return an error and allow
|
|
/// deallocation of the memory.
|
|
/// - the fields are initialized in the order given in the initializer.
|
|
/// - no references to fields are allowed to be created inside of the initializer.
|
|
///
|
|
/// This initializer is for initializing data in-place that might later be moved. If you want to
|
|
/// pin-initialize, use [`pin_init!`].
|
|
// For a detailed example of how this macro works, see the module documentation of the hidden
|
|
// module `__internal` inside of `init/__internal.rs`.
|
|
#[macro_export]
|
|
macro_rules! init {
|
|
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
|
|
$($fields:tt)*
|
|
}) => {
|
|
$crate::try_init!(
|
|
@this($($this)?),
|
|
@typ($t $(::<$($generics),*>)?),
|
|
@fields($($fields)*),
|
|
@error(::core::convert::Infallible),
|
|
)
|
|
}
|
|
}
|
|
|
|
/// Construct an in-place fallible initializer for `struct`s.
|
|
///
|
|
/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
|
|
/// [`init!`].
|
|
///
|
|
/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
|
|
/// append `? $type` after the `struct` initializer.
|
|
/// The safety caveats from [`try_pin_init!`] also apply:
|
|
/// - `unsafe` code must guarantee either full initialization or return an error and allow
|
|
/// deallocation of the memory.
|
|
/// - the fields are initialized in the order given in the initializer.
|
|
/// - no references to fields are allowed to be created inside of the initializer.
|
|
///
|
|
/// # Examples
|
|
///
|
|
/// ```rust
|
|
/// use kernel::{init::PinInit, error::Error, InPlaceInit};
|
|
/// struct BigBuf {
|
|
/// big: Box<[u8; 1024 * 1024 * 1024]>,
|
|
/// small: [u8; 1024 * 1024],
|
|
/// }
|
|
///
|
|
/// impl BigBuf {
|
|
/// fn new() -> impl Init<Self, Error> {
|
|
/// try_init!(Self {
|
|
/// big: Box::init(zeroed())?,
|
|
/// small: [0; 1024 * 1024],
|
|
/// }? Error)
|
|
/// }
|
|
/// }
|
|
/// ```
|
|
// For a detailed example of how this macro works, see the module documentation of the hidden
|
|
// module `__internal` inside of `init/__internal.rs`.
|
|
#[macro_export]
|
|
macro_rules! try_init {
|
|
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
|
|
$($fields:tt)*
|
|
}) => {
|
|
$crate::try_init!(
|
|
@this($($this)?),
|
|
@typ($t $(::<$($generics),*>)?),
|
|
@fields($($fields)*),
|
|
@error($crate::error::Error),
|
|
)
|
|
};
|
|
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
|
|
$($fields:tt)*
|
|
}? $err:ty) => {
|
|
$crate::try_init!(
|
|
@this($($this)?),
|
|
@typ($t $(::<$($generics),*>)?),
|
|
@fields($($fields)*),
|
|
@error($err),
|
|
)
|
|
};
|
|
(
|
|
@this($($this:ident)?),
|
|
@typ($t:ident $(::<$($generics:ty),*>)?),
|
|
@fields($($fields:tt)*),
|
|
@error($err:ty),
|
|
) => {{
|
|
// We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
|
|
// type and shadow it later when we insert the arbitrary user code. That way there will be
|
|
// no possibility of returning without `unsafe`.
|
|
struct __InitOk;
|
|
// Get the init data from the supplied type.
|
|
let data = unsafe {
|
|
use $crate::init::__internal::HasInitData;
|
|
$t$(::<$($generics),*>)?::__init_data()
|
|
};
|
|
// Ensure that `data` really is of type `InitData` and help with type inference:
|
|
let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>(
|
|
data,
|
|
move |slot| {
|
|
{
|
|
// Shadow the structure so it cannot be used to return early.
|
|
struct __InitOk;
|
|
// Create the `this` so it can be referenced by the user inside of the
|
|
// expressions creating the individual fields.
|
|
$(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
|
|
// Initialize every field.
|
|
$crate::try_init!(init_slot:
|
|
@slot(slot),
|
|
@munch_fields($($fields)*,),
|
|
);
|
|
// We use unreachable code to ensure that all fields have been mentioned exactly
|
|
// once, this struct initializer will still be type-checked and complain with a
|
|
// very natural error message if a field is forgotten/mentioned more than once.
|
|
#[allow(unreachable_code, clippy::diverging_sub_expression)]
|
|
if false {
|
|
$crate::try_init!(make_initializer:
|
|
@slot(slot),
|
|
@type_name($t),
|
|
@munch_fields($($fields)*,),
|
|
@acc(),
|
|
);
|
|
}
|
|
// Forget all guards, since initialization was a success.
|
|
$crate::try_init!(forget_guards:
|
|
@munch_fields($($fields)*,),
|
|
);
|
|
}
|
|
Ok(__InitOk)
|
|
}
|
|
);
|
|
let init = move |slot| -> ::core::result::Result<(), $err> {
|
|
init(slot).map(|__InitOk| ())
|
|
};
|
|
let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) };
|
|
init
|
|
}};
|
|
(init_slot:
|
|
@slot($slot:ident),
|
|
@munch_fields( $(,)?),
|
|
) => {
|
|
// Endpoint of munching, no fields are left.
|
|
};
|
|
(init_slot:
|
|
@slot($slot:ident),
|
|
@munch_fields($field:ident <- $val:expr, $($rest:tt)*),
|
|
) => {
|
|
let $field = $val;
|
|
// Call the initializer.
|
|
//
|
|
// SAFETY: `slot` is valid, because we are inside of an initializer closure, we
|
|
// return when an error/panic occurs.
|
|
unsafe {
|
|
$crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?;
|
|
}
|
|
// Create the drop guard.
|
|
//
|
|
// We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
|
|
//
|
|
// SAFETY: We forget the guard later when initialization has succeeded.
|
|
let $field = &unsafe {
|
|
$crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
|
|
};
|
|
|
|
$crate::try_init!(init_slot:
|
|
@slot($slot),
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
(init_slot:
|
|
@slot($slot:ident),
|
|
// Direct value init.
|
|
@munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
|
|
) => {
|
|
$(let $field = $val;)?
|
|
// Call the initializer.
|
|
//
|
|
// SAFETY: The memory at `slot` is uninitialized.
|
|
unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
|
|
// Create the drop guard.
|
|
//
|
|
// We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
|
|
//
|
|
// SAFETY: We forget the guard later when initialization has succeeded.
|
|
let $field = &unsafe {
|
|
$crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
|
|
};
|
|
|
|
$crate::try_init!(init_slot:
|
|
@slot($slot),
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
(make_initializer:
|
|
@slot($slot:ident),
|
|
@type_name($t:ident),
|
|
@munch_fields( $(,)?),
|
|
@acc($($acc:tt)*),
|
|
) => {
|
|
// Endpoint, nothing more to munch, create the initializer.
|
|
// Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
|
|
// get the correct type inference here:
|
|
unsafe {
|
|
::core::ptr::write($slot, $t {
|
|
$($acc)*
|
|
});
|
|
}
|
|
};
|
|
(make_initializer:
|
|
@slot($slot:ident),
|
|
@type_name($t:ident),
|
|
@munch_fields($field:ident <- $val:expr, $($rest:tt)*),
|
|
@acc($($acc:tt)*),
|
|
) => {
|
|
$crate::try_init!(make_initializer:
|
|
@slot($slot),
|
|
@type_name($t),
|
|
@munch_fields($($rest)*),
|
|
@acc($($acc)*$field: ::core::panic!(),),
|
|
);
|
|
};
|
|
(make_initializer:
|
|
@slot($slot:ident),
|
|
@type_name($t:ident),
|
|
@munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
|
|
@acc($($acc:tt)*),
|
|
) => {
|
|
$crate::try_init!(make_initializer:
|
|
@slot($slot),
|
|
@type_name($t),
|
|
@munch_fields($($rest)*),
|
|
@acc($($acc)*$field: ::core::panic!(),),
|
|
);
|
|
};
|
|
(forget_guards:
|
|
@munch_fields($(,)?),
|
|
) => {
|
|
// Munching finished.
|
|
};
|
|
(forget_guards:
|
|
@munch_fields($field:ident <- $val:expr, $($rest:tt)*),
|
|
) => {
|
|
unsafe { $crate::init::__internal::DropGuard::forget($field) };
|
|
|
|
$crate::try_init!(forget_guards:
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
(forget_guards:
|
|
@munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
|
|
) => {
|
|
unsafe { $crate::init::__internal::DropGuard::forget($field) };
|
|
|
|
$crate::try_init!(forget_guards:
|
|
@munch_fields($($rest)*),
|
|
);
|
|
};
|
|
}
|
|
|
|
/// A pin-initializer for the type `T`.
|
|
///
|
|
/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
|
|
/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
|
|
/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
|
|
///
|
|
/// Also see the [module description](self).
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// When implementing this type you will need to take great care. Also there are probably very few
|
|
/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
|
|
///
|
|
/// The [`PinInit::__pinned_init`] function
|
|
/// - returns `Ok(())` if it initialized every field of `slot`,
|
|
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
|
|
/// - `slot` can be deallocated without UB occurring,
|
|
/// - `slot` does not need to be dropped,
|
|
/// - `slot` is not partially initialized.
|
|
/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
|
|
///
|
|
/// [`Arc<T>`]: crate::sync::Arc
|
|
/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
|
|
#[must_use = "An initializer must be used in order to create its value."]
|
|
pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
|
|
/// Initializes `slot`.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// - `slot` is a valid pointer to uninitialized memory.
|
|
/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
|
|
/// deallocate.
|
|
/// - `slot` will not move until it is dropped, i.e. it will be pinned.
|
|
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
|
|
}
|
|
|
|
/// An initializer for `T`.
|
|
///
|
|
/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
|
|
/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
|
|
/// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
|
|
/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
|
|
///
|
|
/// Also see the [module description](self).
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// When implementing this type you will need to take great care. Also there are probably very few
|
|
/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
|
|
///
|
|
/// The [`Init::__init`] function
|
|
/// - returns `Ok(())` if it initialized every field of `slot`,
|
|
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
|
|
/// - `slot` can be deallocated without UB occurring,
|
|
/// - `slot` does not need to be dropped,
|
|
/// - `slot` is not partially initialized.
|
|
/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
|
|
///
|
|
/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
|
|
/// code as `__init`.
|
|
///
|
|
/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
|
|
/// move the pointee after initialization.
|
|
///
|
|
/// [`Arc<T>`]: crate::sync::Arc
|
|
#[must_use = "An initializer must be used in order to create its value."]
|
|
pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized {
|
|
/// Initializes `slot`.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// - `slot` is a valid pointer to uninitialized memory.
|
|
/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
|
|
/// deallocate.
|
|
unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
|
|
}
|
|
|
|
// SAFETY: Every in-place initializer can also be used as a pin-initializer.
|
|
unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I
|
|
where
|
|
I: Init<T, E>,
|
|
{
|
|
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
|
|
// SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not
|
|
// require `slot` to not move after init.
|
|
unsafe { self.__init(slot) }
|
|
}
|
|
}
|
|
|
|
/// Creates a new [`PinInit<T, E>`] from the given closure.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// The closure:
|
|
/// - returns `Ok(())` if it initialized every field of `slot`,
|
|
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
|
|
/// - `slot` can be deallocated without UB occurring,
|
|
/// - `slot` does not need to be dropped,
|
|
/// - `slot` is not partially initialized.
|
|
/// - may assume that the `slot` does not move if `T: !Unpin`,
|
|
/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
|
|
#[inline]
|
|
pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
|
|
f: impl FnOnce(*mut T) -> Result<(), E>,
|
|
) -> impl PinInit<T, E> {
|
|
__internal::InitClosure(f, PhantomData)
|
|
}
|
|
|
|
/// Creates a new [`Init<T, E>`] from the given closure.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// The closure:
|
|
/// - returns `Ok(())` if it initialized every field of `slot`,
|
|
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
|
|
/// - `slot` can be deallocated without UB occurring,
|
|
/// - `slot` does not need to be dropped,
|
|
/// - `slot` is not partially initialized.
|
|
/// - the `slot` may move after initialization.
|
|
/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
|
|
#[inline]
|
|
pub const unsafe fn init_from_closure<T: ?Sized, E>(
|
|
f: impl FnOnce(*mut T) -> Result<(), E>,
|
|
) -> impl Init<T, E> {
|
|
__internal::InitClosure(f, PhantomData)
|
|
}
|
|
|
|
/// An initializer that leaves the memory uninitialized.
|
|
///
|
|
/// The initializer is a no-op. The `slot` memory is not changed.
|
|
#[inline]
|
|
pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
|
|
// SAFETY: The memory is allowed to be uninitialized.
|
|
unsafe { init_from_closure(|_| Ok(())) }
|
|
}
|
|
|
|
// SAFETY: Every type can be initialized by-value.
|
|
unsafe impl<T, E> Init<T, E> for T {
|
|
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
|
|
unsafe { slot.write(self) };
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Smart pointer that can initialize memory in-place.
|
|
pub trait InPlaceInit<T>: Sized {
|
|
/// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
|
|
/// type.
|
|
///
|
|
/// If `T: !Unpin` it will not be able to move afterwards.
|
|
fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
|
|
where
|
|
E: From<AllocError>;
|
|
|
|
/// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
|
|
/// type.
|
|
///
|
|
/// If `T: !Unpin` it will not be able to move afterwards.
|
|
fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>>
|
|
where
|
|
Error: From<E>,
|
|
{
|
|
// SAFETY: We delegate to `init` and only change the error type.
|
|
let init = unsafe {
|
|
pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
|
|
};
|
|
Self::try_pin_init(init)
|
|
}
|
|
|
|
/// Use the given initializer to in-place initialize a `T`.
|
|
fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
|
|
where
|
|
E: From<AllocError>;
|
|
|
|
/// Use the given initializer to in-place initialize a `T`.
|
|
fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
|
|
where
|
|
Error: From<E>,
|
|
{
|
|
// SAFETY: We delegate to `init` and only change the error type.
|
|
let init = unsafe {
|
|
init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
|
|
};
|
|
Self::try_init(init)
|
|
}
|
|
}
|
|
|
|
impl<T> InPlaceInit<T> for Box<T> {
|
|
#[inline]
|
|
fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
|
|
where
|
|
E: From<AllocError>,
|
|
{
|
|
let mut this = Box::try_new_uninit()?;
|
|
let slot = this.as_mut_ptr();
|
|
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
|
|
// slot is valid and will not be moved, because we pin it later.
|
|
unsafe { init.__pinned_init(slot)? };
|
|
// SAFETY: All fields have been initialized.
|
|
Ok(unsafe { this.assume_init() }.into())
|
|
}
|
|
|
|
#[inline]
|
|
fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
|
|
where
|
|
E: From<AllocError>,
|
|
{
|
|
let mut this = Box::try_new_uninit()?;
|
|
let slot = this.as_mut_ptr();
|
|
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
|
|
// slot is valid.
|
|
unsafe { init.__init(slot)? };
|
|
// SAFETY: All fields have been initialized.
|
|
Ok(unsafe { this.assume_init() })
|
|
}
|
|
}
|
|
|
|
impl<T> InPlaceInit<T> for UniqueArc<T> {
|
|
#[inline]
|
|
fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
|
|
where
|
|
E: From<AllocError>,
|
|
{
|
|
let mut this = UniqueArc::try_new_uninit()?;
|
|
let slot = this.as_mut_ptr();
|
|
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
|
|
// slot is valid and will not be moved, because we pin it later.
|
|
unsafe { init.__pinned_init(slot)? };
|
|
// SAFETY: All fields have been initialized.
|
|
Ok(unsafe { this.assume_init() }.into())
|
|
}
|
|
|
|
#[inline]
|
|
fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
|
|
where
|
|
E: From<AllocError>,
|
|
{
|
|
let mut this = UniqueArc::try_new_uninit()?;
|
|
let slot = this.as_mut_ptr();
|
|
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
|
|
// slot is valid.
|
|
unsafe { init.__init(slot)? };
|
|
// SAFETY: All fields have been initialized.
|
|
Ok(unsafe { this.assume_init() })
|
|
}
|
|
}
|
|
|
|
/// Trait facilitating pinned destruction.
|
|
///
|
|
/// Use [`pinned_drop`] to implement this trait safely:
|
|
///
|
|
/// ```rust
|
|
/// # use kernel::sync::Mutex;
|
|
/// use kernel::macros::pinned_drop;
|
|
/// use core::pin::Pin;
|
|
/// #[pin_data(PinnedDrop)]
|
|
/// struct Foo {
|
|
/// #[pin]
|
|
/// mtx: Mutex<usize>,
|
|
/// }
|
|
///
|
|
/// #[pinned_drop]
|
|
/// impl PinnedDrop for Foo {
|
|
/// fn drop(self: Pin<&mut Self>) {
|
|
/// pr_info!("Foo is being dropped!");
|
|
/// }
|
|
/// }
|
|
/// ```
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
|
|
///
|
|
/// [`pinned_drop`]: kernel::macros::pinned_drop
|
|
pub unsafe trait PinnedDrop: __internal::HasPinData {
|
|
/// Executes the pinned destructor of this type.
|
|
///
|
|
/// While this function is marked safe, it is actually unsafe to call it manually. For this
|
|
/// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
|
|
/// and thus prevents this function from being called where it should not.
|
|
///
|
|
/// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
|
|
/// automatically.
|
|
fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
|
|
}
|
|
|
|
/// Marker trait for types that can be initialized by writing just zeroes.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
|
|
/// this is not UB:
|
|
///
|
|
/// ```rust,ignore
|
|
/// let val: Self = unsafe { core::mem::zeroed() };
|
|
/// ```
|
|
pub unsafe trait Zeroable {}
|
|
|
|
/// Create a new zeroed T.
|
|
///
|
|
/// The returned initializer will write `0x00` to every byte of the given `slot`.
|
|
#[inline]
|
|
pub fn zeroed<T: Zeroable>() -> impl Init<T> {
|
|
// SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
|
|
// and because we write all zeroes, the memory is initialized.
|
|
unsafe {
|
|
init_from_closure(|slot: *mut T| {
|
|
slot.write_bytes(0, 1);
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
macro_rules! impl_zeroable {
|
|
($($({$($generics:tt)*})? $t:ty, )*) => {
|
|
$(unsafe impl$($($generics)*)? Zeroable for $t {})*
|
|
};
|
|
}
|
|
|
|
impl_zeroable! {
|
|
// SAFETY: All primitives that are allowed to be zero.
|
|
bool,
|
|
char,
|
|
u8, u16, u32, u64, u128, usize,
|
|
i8, i16, i32, i64, i128, isize,
|
|
f32, f64,
|
|
|
|
// SAFETY: These are ZSTs, there is nothing to zero.
|
|
{<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (),
|
|
|
|
// SAFETY: Type is allowed to take any value, including all zeros.
|
|
{<T>} MaybeUninit<T>,
|
|
|
|
// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
|
|
Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
|
|
Option<NonZeroU128>, Option<NonZeroUsize>,
|
|
Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
|
|
Option<NonZeroI128>, Option<NonZeroIsize>,
|
|
|
|
// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
|
|
//
|
|
// In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
|
|
{<T: ?Sized>} Option<NonNull<T>>,
|
|
{<T: ?Sized>} Option<Box<T>>,
|
|
|
|
// SAFETY: `null` pointer is valid.
|
|
//
|
|
// We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
|
|
// null.
|
|
//
|
|
// When `Pointee` gets stabilized, we could use
|
|
// `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
|
|
{<T>} *mut T, {<T>} *const T,
|
|
|
|
// SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
|
|
// zero.
|
|
{<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
|
|
|
|
// SAFETY: `T` is `Zeroable`.
|
|
{<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
|
|
}
|
|
|
|
macro_rules! impl_tuple_zeroable {
|
|
($(,)?) => {};
|
|
($first:ident, $($t:ident),* $(,)?) => {
|
|
// SAFETY: All elements are zeroable and padding can be zero.
|
|
unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
|
|
impl_tuple_zeroable!($($t),* ,);
|
|
}
|
|
}
|
|
|
|
impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
|