diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index 8fbd4b612bde0..7e09a88156a09 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -22,11 +22,16 @@ unsafe extern "Rust" { #[rustc_deallocator] #[rustc_nounwind] #[rustc_std_internal_symbol] - fn __rust_dealloc(ptr: *mut u8, size: usize, align: Alignment); + fn __rust_dealloc(ptr: NonNull, size: usize, align: Alignment); #[rustc_reallocator] #[rustc_nounwind] #[rustc_std_internal_symbol] - fn __rust_realloc(ptr: *mut u8, old_size: usize, align: Alignment, new_size: usize) -> *mut u8; + fn __rust_realloc( + ptr: NonNull, + old_size: usize, + align: Alignment, + new_size: usize, + ) -> *mut u8; #[rustc_allocator_zeroed] #[rustc_nounwind] #[rustc_std_internal_symbol] @@ -112,6 +117,13 @@ pub unsafe fn alloc(layout: Layout) -> *mut u8 { #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { + unsafe { dealloc_nonnull(NonNull::new_unchecked(ptr), layout) } +} + +/// Same as [`dealloc`] but when you already have a non-null pointer +#[inline] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +unsafe fn dealloc_nonnull(ptr: NonNull, layout: Layout) { unsafe { __rust_dealloc(ptr, layout.size(), layout.alignment()) } } @@ -132,6 +144,13 @@ pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + unsafe { realloc_nonnull(NonNull::new_unchecked(ptr), layout, new_size) } +} + +/// Same as [`realloc`] but when you already have a non-null pointer +#[inline] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +unsafe fn realloc_nonnull(ptr: NonNull, layout: Layout, new_size: usize) -> *mut u8 { unsafe { __rust_realloc(ptr, layout.size(), layout.alignment(), new_size) } } @@ -206,7 +225,7 @@ impl Global { // allocation than requested. // * Other conditions must be upheld by the caller, as per `Allocator::deallocate()`'s // safety documentation. - unsafe { dealloc(ptr.as_ptr(), layout) } + unsafe { dealloc_nonnull(ptr, layout) } } } @@ -236,7 +255,7 @@ impl Global { // `realloc` probably checks for `new_size >= old_layout.size()` or something similar. hint::assert_unchecked(new_size >= old_layout.size()); - let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); + let raw_ptr = realloc_nonnull(ptr, old_layout, new_size); let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; if zeroed { raw_ptr.add(old_size).write_bytes(0, new_size - old_size); @@ -285,7 +304,7 @@ impl Global { // `realloc` probably checks for `new_size <= old_layout.size()` or something similar. hint::assert_unchecked(new_size <= old_layout.size()); - let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); + let raw_ptr = realloc_nonnull(ptr, old_layout, new_size); let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; Ok(NonNull::slice_from_raw_parts(ptr, new_size)) }, diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-abort.mir index ca548be1362d0..1190f7812c11a 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-abort.mir @@ -12,8 +12,8 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { scope 17 (inlined Layout::size) { } scope 18 (inlined std::ptr::Unique::<[u8; 1024]>::cast::) { + let mut _3: std::ptr::NonNull; scope 19 (inlined NonNull::<[u8; 1024]>::cast::) { - let mut _3: *mut u8; scope 20 (inlined NonNull::<[u8; 1024]>::as_ptr) { } } @@ -27,12 +27,10 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -68,12 +66,14 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { } bb0: { + StorageLive(_3); _2 = copy (((*_1).0: std::ptr::Unique<[u8; 1024]>).0: std::ptr::NonNull<[u8; 1024]>); - _3 = copy _2 as *mut u8 (Transmute); + _3 = copy _2 as std::ptr::NonNull (Transmute); _4 = alloc::alloc::__rust_dealloc(move _3, const 1024_usize, const std::ptr::Alignment {{ _inner_repr_trick: std::ptr::alignment::AlignmentEnum::_Align1Shl0 }}) -> [return: bb1, unwind unreachable]; } bb1: { + StorageDead(_3); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-unwind.mir index ca548be1362d0..1190f7812c11a 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.PreCodegen.after.panic-unwind.mir @@ -12,8 +12,8 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { scope 17 (inlined Layout::size) { } scope 18 (inlined std::ptr::Unique::<[u8; 1024]>::cast::) { + let mut _3: std::ptr::NonNull; scope 19 (inlined NonNull::<[u8; 1024]>::cast::) { - let mut _3: *mut u8; scope 20 (inlined NonNull::<[u8; 1024]>::as_ptr) { } } @@ -27,12 +27,10 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -68,12 +66,14 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { } bb0: { + StorageLive(_3); _2 = copy (((*_1).0: std::ptr::Unique<[u8; 1024]>).0: std::ptr::NonNull<[u8; 1024]>); - _3 = copy _2 as *mut u8 (Transmute); + _3 = copy _2 as std::ptr::NonNull (Transmute); _4 = alloc::alloc::__rust_dealloc(move _3, const 1024_usize, const std::ptr::Alignment {{ _inner_repr_trick: std::ptr::alignment::AlignmentEnum::_Align1Shl0 }}) -> [return: bb1, unwind unreachable]; } bb1: { + StorageDead(_3); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-abort.mir index a85adbd90b45e..b5c25035bdc40 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-abort.mir @@ -12,8 +12,8 @@ fn drop_generic(_1: *mut Box) -> () { scope 17 (inlined Layout::size) { } scope 18 (inlined std::ptr::Unique::::cast::) { + let mut _4: std::ptr::NonNull; scope 19 (inlined NonNull::::cast::) { - let mut _4: *mut u8; scope 20 (inlined NonNull::::as_ptr) { } } @@ -27,12 +27,10 @@ fn drop_generic(_1: *mut Box) -> () { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -69,13 +67,14 @@ fn drop_generic(_1: *mut Box) -> () { } bb0: { + StorageLive(_4); _2 = copy (((*_1).0: std::ptr::Unique).0: std::ptr::NonNull); _3 = const ::ALIGN as std::ptr::Alignment (Transmute); switchInt(const ::SIZE) -> [0: bb3, otherwise: bb1]; } bb1: { - _4 = copy _2 as *mut u8 (Transmute); + _4 = copy _2 as std::ptr::NonNull (Transmute); switchInt(const ::SIZE) -> [0: bb3, otherwise: bb2]; } @@ -84,6 +83,7 @@ fn drop_generic(_1: *mut Box) -> () { } bb3: { + StorageDead(_4); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-unwind.mir index a85adbd90b45e..b5c25035bdc40 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.PreCodegen.after.panic-unwind.mir @@ -12,8 +12,8 @@ fn drop_generic(_1: *mut Box) -> () { scope 17 (inlined Layout::size) { } scope 18 (inlined std::ptr::Unique::::cast::) { + let mut _4: std::ptr::NonNull; scope 19 (inlined NonNull::::cast::) { - let mut _4: *mut u8; scope 20 (inlined NonNull::::as_ptr) { } } @@ -27,12 +27,10 @@ fn drop_generic(_1: *mut Box) -> () { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -69,13 +67,14 @@ fn drop_generic(_1: *mut Box) -> () { } bb0: { + StorageLive(_4); _2 = copy (((*_1).0: std::ptr::Unique).0: std::ptr::NonNull); _3 = const ::ALIGN as std::ptr::Alignment (Transmute); switchInt(const ::SIZE) -> [0: bb3, otherwise: bb1]; } bb1: { - _4 = copy _2 as *mut u8 (Transmute); + _4 = copy _2 as std::ptr::NonNull (Transmute); switchInt(const ::SIZE) -> [0: bb3, otherwise: bb2]; } @@ -84,6 +83,7 @@ fn drop_generic(_1: *mut Box) -> () { } bb3: { + StorageDead(_4); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir index 9a734a90d0d1f..3bff170fff5de 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir @@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { let _2: std::ptr::NonNull<[T]>; let mut _3: *mut [T]; let mut _4: *const [T]; - let _10: (); + let _9: (); scope 3 { scope 4 { scope 17 (inlined Layout::size) { @@ -28,15 +28,12 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { scope 23 (inlined ::deallocate) { scope 24 (inlined std::alloc::Global::deallocate_impl) { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { - let mut _9: *mut u8; scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -87,7 +84,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { bb1: { _6 = const ::ALIGN as std::ptr::Alignment (Transmute); StorageDead(_4); - switchInt(copy _5) -> [0: bb4, otherwise: bb2]; + switchInt(copy _5) -> [0: bb3, otherwise: bb2]; } bb2: { @@ -95,17 +92,10 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { _7 = copy _3 as *mut u8 (PtrToPtr); _8 = copy _7 as std::ptr::NonNull (Transmute); StorageDead(_7); - StorageLive(_9); - _9 = copy _8 as *mut u8 (Transmute); - _10 = alloc::alloc::__rust_dealloc(move _9, move _5, move _6) -> [return: bb3, unwind unreachable]; + _9 = alloc::alloc::__rust_dealloc(move _8, move _5, move _6) -> [return: bb3, unwind unreachable]; } bb3: { - StorageDead(_9); - goto -> bb4; - } - - bb4: { StorageDead(_2); StorageDead(_8); StorageDead(_3); diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir index 9a734a90d0d1f..3bff170fff5de 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir @@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { let _2: std::ptr::NonNull<[T]>; let mut _3: *mut [T]; let mut _4: *const [T]; - let _10: (); + let _9: (); scope 3 { scope 4 { scope 17 (inlined Layout::size) { @@ -28,15 +28,12 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { scope 23 (inlined ::deallocate) { scope 24 (inlined std::alloc::Global::deallocate_impl) { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { - let mut _9: *mut u8; scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -87,7 +84,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { bb1: { _6 = const ::ALIGN as std::ptr::Alignment (Transmute); StorageDead(_4); - switchInt(copy _5) -> [0: bb4, otherwise: bb2]; + switchInt(copy _5) -> [0: bb3, otherwise: bb2]; } bb2: { @@ -95,17 +92,10 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { _7 = copy _3 as *mut u8 (PtrToPtr); _8 = copy _7 as std::ptr::NonNull (Transmute); StorageDead(_7); - StorageLive(_9); - _9 = copy _8 as *mut u8 (Transmute); - _10 = alloc::alloc::__rust_dealloc(move _9, move _5, move _6) -> [return: bb3, unwind unreachable]; + _9 = alloc::alloc::__rust_dealloc(move _8, move _5, move _6) -> [return: bb3, unwind unreachable]; } bb3: { - StorageDead(_9); - goto -> bb4; - } - - bb4: { StorageDead(_2); StorageDead(_8); StorageDead(_3); diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir index 9a734a90d0d1f..3bff170fff5de 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir @@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { let _2: std::ptr::NonNull<[T]>; let mut _3: *mut [T]; let mut _4: *const [T]; - let _10: (); + let _9: (); scope 3 { scope 4 { scope 17 (inlined Layout::size) { @@ -28,15 +28,12 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { scope 23 (inlined ::deallocate) { scope 24 (inlined std::alloc::Global::deallocate_impl) { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { - let mut _9: *mut u8; scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -87,7 +84,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { bb1: { _6 = const ::ALIGN as std::ptr::Alignment (Transmute); StorageDead(_4); - switchInt(copy _5) -> [0: bb4, otherwise: bb2]; + switchInt(copy _5) -> [0: bb3, otherwise: bb2]; } bb2: { @@ -95,17 +92,10 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { _7 = copy _3 as *mut u8 (PtrToPtr); _8 = copy _7 as std::ptr::NonNull (Transmute); StorageDead(_7); - StorageLive(_9); - _9 = copy _8 as *mut u8 (Transmute); - _10 = alloc::alloc::__rust_dealloc(move _9, move _5, move _6) -> [return: bb3, unwind unreachable]; + _9 = alloc::alloc::__rust_dealloc(move _8, move _5, move _6) -> [return: bb3, unwind unreachable]; } bb3: { - StorageDead(_9); - goto -> bb4; - } - - bb4: { StorageDead(_2); StorageDead(_8); StorageDead(_3); diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir index 9a734a90d0d1f..3bff170fff5de 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir @@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { let _2: std::ptr::NonNull<[T]>; let mut _3: *mut [T]; let mut _4: *const [T]; - let _10: (); + let _9: (); scope 3 { scope 4 { scope 17 (inlined Layout::size) { @@ -28,15 +28,12 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { scope 23 (inlined ::deallocate) { scope 24 (inlined std::alloc::Global::deallocate_impl) { scope 25 (inlined std::alloc::Global::deallocate_impl_runtime) { - let mut _9: *mut u8; scope 26 (inlined Layout::size) { } - scope 27 (inlined NonNull::::as_ptr) { - } - scope 28 (inlined std::alloc::dealloc) { - scope 29 (inlined Layout::size) { + scope 27 (inlined alloc::alloc::dealloc_nonnull) { + scope 28 (inlined Layout::size) { } - scope 30 (inlined Layout::alignment) { + scope 29 (inlined Layout::alignment) { } } } @@ -87,7 +84,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { bb1: { _6 = const ::ALIGN as std::ptr::Alignment (Transmute); StorageDead(_4); - switchInt(copy _5) -> [0: bb4, otherwise: bb2]; + switchInt(copy _5) -> [0: bb3, otherwise: bb2]; } bb2: { @@ -95,17 +92,10 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { _7 = copy _3 as *mut u8 (PtrToPtr); _8 = copy _7 as std::ptr::NonNull (Transmute); StorageDead(_7); - StorageLive(_9); - _9 = copy _8 as *mut u8 (Transmute); - _10 = alloc::alloc::__rust_dealloc(move _9, move _5, move _6) -> [return: bb3, unwind unreachable]; + _9 = alloc::alloc::__rust_dealloc(move _8, move _5, move _6) -> [return: bb3, unwind unreachable]; } bb3: { - StorageDead(_9); - goto -> bb4; - } - - bb4: { StorageDead(_2); StorageDead(_8); StorageDead(_3);