Path: blob/main/crates/bevy_ecs/src/storage/thin_array_ptr.rs
6604 views
use crate::query::DebugCheckedUnwrap;1use alloc::{2alloc::{alloc, handle_alloc_error, realloc},3boxed::Box,4};5use core::{6alloc::Layout,7mem::{needs_drop, size_of},8num::NonZeroUsize,9ptr::{self, NonNull},10};1112/// Similar to [`Vec<T>`], but with the capacity and length cut out for performance reasons.13///14/// This type can be treated as a `ManuallyDrop<Box<[T]>>` without a built in length. To avoid15/// memory leaks, [`drop`](Self::drop) must be called when no longer in use.16///17/// [`Vec<T>`]: alloc::vec::Vec18pub struct ThinArrayPtr<T> {19data: NonNull<T>,20#[cfg(debug_assertions)]21capacity: usize,22}2324impl<T> ThinArrayPtr<T> {25fn empty() -> Self {26#[cfg(debug_assertions)]27{28Self {29data: NonNull::dangling(),30capacity: 0,31}32}33#[cfg(not(debug_assertions))]34{35Self {36data: NonNull::dangling(),37}38}39}4041#[inline(always)]42fn set_capacity(&mut self, _capacity: usize) {43#[cfg(debug_assertions)]44{45self.capacity = _capacity;46}47}4849/// Create a new [`ThinArrayPtr`] with a given capacity. If the `capacity` is 0, this will no allocate any memory.50#[inline]51pub fn with_capacity(capacity: usize) -> Self {52let mut arr = Self::empty();53if capacity > 0 {54// SAFETY:55// - The `current_capacity` is 0 because it was just created56unsafe { arr.alloc(NonZeroUsize::new_unchecked(capacity)) };57}58arr59}6061/// Allocate memory for the array, this should only be used if not previous allocation has been made (capacity = 0)62/// The caller should update their saved `capacity` value to reflect the fact that it was changed63///64/// # Panics65/// - Panics if the new capacity overflows `usize`66pub fn alloc(&mut self, capacity: NonZeroUsize) {67self.set_capacity(capacity.get());68if size_of::<T>() != 0 {69let new_layout = Layout::array::<T>(capacity.get())70.expect("layout should be valid (arithmetic overflow)");71// SAFETY:72// - layout has non-zero size, `capacity` > 0, `size` > 0 (`size_of::<T>() != 0`)73self.data = NonNull::new(unsafe { alloc(new_layout) })74.unwrap_or_else(|| handle_alloc_error(new_layout))75.cast();76}77}7879/// Reallocate memory for the array, this should only be used if a previous allocation for this array has been made (capacity > 0).80///81/// # Panics82/// - Panics if the new capacity overflows `usize`83///84/// # Safety85/// - The current capacity is indeed greater than 086/// - The caller should update their saved `capacity` value to reflect the fact that it was changed87pub unsafe fn realloc(&mut self, current_capacity: NonZeroUsize, new_capacity: NonZeroUsize) {88#[cfg(debug_assertions)]89assert_eq!(self.capacity, current_capacity.get());90self.set_capacity(new_capacity.get());91if size_of::<T>() != 0 {92let new_layout =93Layout::array::<T>(new_capacity.get()).expect("overflow while allocating memory");94// SAFETY:95// - ptr was be allocated via this allocator96// - the layout of the array is the same as `Layout::array::<T>(current_capacity)`97// - the size of `T` is non 0, and `new_capacity` > 098// - "new_size, when rounded up to the nearest multiple of layout.align(), must not overflow (i.e., the rounded value must be less than usize::MAX)",99// since the item size is always a multiple of its align, the rounding cannot happen100// here and the overflow is handled in `Layout::array`101self.data = NonNull::new(unsafe {102realloc(103self.data.cast().as_ptr(),104// We can use `unwrap_unchecked` because this is the Layout of the current allocation, it must be valid105Layout::array::<T>(current_capacity.get()).debug_checked_unwrap(),106new_layout.size(),107)108})109.unwrap_or_else(|| handle_alloc_error(new_layout))110.cast();111}112}113114/// Initializes the value at `index` to `value`. This function does not do any bounds checking.115///116/// # Safety117/// `index` must be in bounds i.e. within the `capacity`.118/// if `index` = `len` the caller should update their saved `len` value to reflect the fact that it was changed119#[inline]120pub unsafe fn initialize_unchecked(&mut self, index: usize, value: T) {121// SAFETY: `index` is in bounds122let ptr = unsafe { self.get_unchecked_raw(index) };123// SAFETY: `index` is in bounds, therefore the pointer to that location in the array is valid, and aligned.124unsafe { ptr::write(ptr, value) };125}126127/// Get a raw pointer to the element at `index`. This method doesn't do any bounds checking.128///129/// # Safety130/// - `index` must be safe to access.131#[inline]132pub unsafe fn get_unchecked_raw(&mut self, index: usize) -> *mut T {133// SAFETY:134// - `self.data` and the resulting pointer are in the same allocated object135// - the memory address of the last element doesn't overflow `isize`, so if `index` is in bounds, it won't overflow either136unsafe { self.data.as_ptr().add(index) }137}138139/// Get a reference to the element at `index`. This method doesn't do any bounds checking.140///141/// # Safety142/// - `index` must be safe to read.143#[inline]144pub unsafe fn get_unchecked(&self, index: usize) -> &'_ T {145// SAFETY:146// - `self.data` and the resulting pointer are in the same allocated object147// - the memory address of the last element doesn't overflow `isize`, so if `index` is in bounds, it won't overflow either148let ptr = unsafe { self.data.as_ptr().add(index) };149// SAFETY:150// - The pointer is properly aligned151// - It is dereferenceable (all in the same allocation)152// - `index` < `len` and the element is safe to write to, so its valid153// - We have a reference to self, so no other mutable accesses to the element can occur154unsafe {155ptr.as_ref()156// SAFETY: We can use `unwarp_unchecked` because the pointer isn't null)157.debug_checked_unwrap()158}159}160161/// Get a mutable reference to the element at `index`. This method doesn't do any bounds checking.162///163/// # Safety164/// - `index` must be safe to write to.165#[inline]166pub unsafe fn get_unchecked_mut(&mut self, index: usize) -> &'_ mut T {167// SAFETY:168// - `self.data` and the resulting pointer are in the same allocated object169// - the memory address of the last element doesn't overflow `isize`, so if `index` is in bounds, it won't overflow either170let ptr = unsafe { self.data.as_ptr().add(index) };171// SAFETY:172// - The pointer is properly aligned173// - It is dereferenceable (all in the same allocation)174// - `index` < `len` and the element is safe to write to, so its valid175// - We have a mutable reference to `self`176unsafe {177ptr.as_mut()178// SAFETY: We can use `unwarp_unchecked` because the pointer isn't null)179.unwrap_unchecked()180}181}182183/// Perform a [`swap-remove`](https://doc.rust-lang.org/std/vec/struct.Vec.html#method.swap_remove) and return the removed value.184///185/// # Safety186/// - `index_to_keep` must be safe to access (within the bounds of the length of the array).187/// - `index_to_remove` must be safe to access (within the bounds of the length of the array).188/// - `index_to_remove` != `index_to_keep`189/// - The caller should address the inconsistent state of the array that has occurred after the swap, either:190/// 1) initialize a different value in `index_to_keep`191/// 2) update the saved length of the array if `index_to_keep` was the last element.192#[inline]193pub unsafe fn swap_remove_unchecked_nonoverlapping(194&mut self,195index_to_remove: usize,196index_to_keep: usize,197) -> T {198#[cfg(debug_assertions)]199{200debug_assert!(self.capacity > index_to_keep);201debug_assert!(self.capacity > index_to_remove);202debug_assert_ne!(index_to_keep, index_to_remove);203}204let base_ptr = self.data.as_ptr();205let value = ptr::read(base_ptr.add(index_to_remove));206ptr::copy_nonoverlapping(207base_ptr.add(index_to_keep),208base_ptr.add(index_to_remove),2091,210);211value212}213214/// Perform a [`swap-remove`](https://doc.rust-lang.org/std/vec/struct.Vec.html#method.swap_remove) and return the removed value.215///216/// # Safety217/// - `index_to_keep` must be safe to access (within the bounds of the length of the array).218/// - `index_to_remove` must be safe to access (within the bounds of the length of the array).219/// - `index_to_remove` != `index_to_keep`220/// - The caller should address the inconsistent state of the array that has occurred after the swap, either:221/// 1) initialize a different value in `index_to_keep`222/// 2) update the saved length of the array if `index_to_keep` was the last element.223#[inline]224pub unsafe fn swap_remove_unchecked(225&mut self,226index_to_remove: usize,227index_to_keep: usize,228) -> T {229if index_to_remove != index_to_keep {230return self.swap_remove_unchecked_nonoverlapping(index_to_remove, index_to_keep);231}232ptr::read(self.data.as_ptr().add(index_to_remove))233}234235/// Perform a [`swap-remove`](https://doc.rust-lang.org/std/vec/struct.Vec.html#method.swap_remove) and drop the removed value.236///237/// # Safety238/// - `index_to_keep` must be safe to access (within the bounds of the length of the array).239/// - `index_to_remove` must be safe to access (within the bounds of the length of the array).240/// - `index_to_remove` != `index_to_keep`241/// - The caller should address the inconsistent state of the array that has occurred after the swap, either:242/// 1) initialize a different value in `index_to_keep`243/// 2) update the saved length of the array if `index_to_keep` was the last element.244#[inline]245pub unsafe fn swap_remove_and_drop_unchecked(246&mut self,247index_to_remove: usize,248index_to_keep: usize,249) {250let val = &mut self.swap_remove_unchecked(index_to_remove, index_to_keep);251ptr::drop_in_place(ptr::from_mut(val));252}253254/// Get a raw pointer to the last element of the array, return `None` if the length is 0255///256/// # Safety257/// - ensure that `current_len` is indeed the len of the array258#[inline]259unsafe fn last_element(&mut self, current_len: usize) -> Option<*mut T> {260(current_len != 0).then_some(self.data.as_ptr().add(current_len - 1))261}262263/// Clears the array, removing (and dropping) Note that this method has no effect on the allocated capacity of the vector.264///265/// # Safety266/// - `current_len` is indeed the length of the array267/// - The caller should update their saved length value268pub unsafe fn clear_elements(&mut self, mut current_len: usize) {269if needs_drop::<T>() {270while let Some(to_drop) = self.last_element(current_len) {271ptr::drop_in_place(to_drop);272current_len -= 1;273}274}275}276277/// Drop the entire array and all its elements.278///279/// # Safety280/// - `current_len` is indeed the length of the array281/// - `current_capacity` is indeed the capacity of the array282/// - The caller must not use this `ThinArrayPtr` in any way after calling this function283pub unsafe fn drop(&mut self, current_capacity: usize, current_len: usize) {284#[cfg(debug_assertions)]285assert_eq!(self.capacity, current_capacity);286if current_capacity != 0 {287self.clear_elements(current_len);288let layout = Layout::array::<T>(current_capacity).expect("layout should be valid");289alloc::alloc::dealloc(self.data.as_ptr().cast(), layout);290}291self.set_capacity(0);292}293294/// Get the [`ThinArrayPtr`] as a slice with a given length.295///296/// # Safety297/// - `slice_len` must match the actual length of the array298#[inline]299pub unsafe fn as_slice(&self, slice_len: usize) -> &[T] {300// SAFETY:301// - the data is valid - allocated with the same allocator302// - non-null and well-aligned303// - we have a shared reference to self - the data will not be mutated during 'a304unsafe { core::slice::from_raw_parts(self.data.as_ptr(), slice_len) }305}306}307308impl<T> From<Box<[T]>> for ThinArrayPtr<T> {309fn from(value: Box<[T]>) -> Self {310let _len = value.len();311let slice_ptr = Box::<[T]>::into_raw(value);312// SAFETY: We just got the pointer from a reference313let first_element_ptr = unsafe { (*slice_ptr).as_mut_ptr() };314Self {315// SAFETY: The pointer can't be null, it came from a reference316data: unsafe { NonNull::new_unchecked(first_element_ptr) },317#[cfg(debug_assertions)]318capacity: _len,319}320}321}322323324