Path: blob/main/crates/polars-arrow/src/io/avro/read/nested.rs
7884 views
use polars_error::{PolarsResult, polars_err};12use crate::array::*;3use crate::bitmap::*;4use crate::datatypes::*;5use crate::offset::{Offset, Offsets};67/// Auxiliary struct8#[derive(Debug)]9pub struct DynMutableListArray<O: Offset> {10dtype: ArrowDataType,11offsets: Offsets<O>,12values: Box<dyn MutableArray>,13validity: Option<MutableBitmap>,14}1516impl<O: Offset> DynMutableListArray<O> {17pub fn new_from(values: Box<dyn MutableArray>, dtype: ArrowDataType, capacity: usize) -> Self {18assert_eq!(values.len(), 0);19ListArray::<O>::get_child_field(&dtype);20Self {21dtype,22offsets: Offsets::<O>::with_capacity(capacity),23values,24validity: None,25}26}2728/// The values29pub fn mut_values(&mut self) -> &mut dyn MutableArray {30self.values.as_mut()31}3233#[inline]34pub fn try_push_valid(&mut self) -> PolarsResult<()> {35let total_length = self.values.len();36let offset = self.offsets.last().to_usize();37let length = total_length38.checked_sub(offset)39.ok_or_else(|| polars_err!(ComputeError: "overflow"))?;4041self.offsets.try_push(length)?;42if let Some(validity) = &mut self.validity {43validity.push(true)44}45Ok(())46}4748#[inline]49fn push_null(&mut self) {50self.offsets.extend_constant(1);51match &mut self.validity {52Some(validity) => validity.push(false),53None => self.init_validity(),54}55}5657fn init_validity(&mut self) {58let len = self.offsets.len_proxy();5960let mut validity = MutableBitmap::new();61validity.extend_constant(len, true);62validity.set(len - 1, false);63self.validity = Some(validity)64}65}6667impl<O: Offset> MutableArray for DynMutableListArray<O> {68fn len(&self) -> usize {69self.offsets.len_proxy()70}7172fn validity(&self) -> Option<&MutableBitmap> {73self.validity.as_ref()74}7576fn as_box(&mut self) -> Box<dyn Array> {77ListArray::new(78self.dtype.clone(),79std::mem::take(&mut self.offsets).into(),80self.values.as_box(),81std::mem::take(&mut self.validity).map(|x| x.into()),82)83.boxed()84}8586fn as_arc(&mut self) -> std::sync::Arc<dyn Array> {87ListArray::new(88self.dtype.clone(),89std::mem::take(&mut self.offsets).into(),90self.values.as_box(),91std::mem::take(&mut self.validity).map(|x| x.into()),92)93.arced()94}9596fn dtype(&self) -> &ArrowDataType {97&self.dtype98}99100fn as_any(&self) -> &dyn std::any::Any {101self102}103104fn as_mut_any(&mut self) -> &mut dyn std::any::Any {105self106}107108#[inline]109fn push_null(&mut self) {110self.push_null()111}112113fn reserve(&mut self, _: usize) {114todo!();115}116117fn shrink_to_fit(&mut self) {118todo!();119}120}121122#[derive(Debug)]123pub struct FixedItemsUtf8Dictionary {124dtype: ArrowDataType,125keys: MutablePrimitiveArray<i32>,126values: Utf8Array<i32>,127}128129impl FixedItemsUtf8Dictionary {130pub fn with_capacity(values: Utf8Array<i32>, capacity: usize) -> Self {131Self {132dtype: ArrowDataType::Dictionary(133IntegerType::Int32,134Box::new(values.dtype().clone()),135false,136),137keys: MutablePrimitiveArray::<i32>::with_capacity(capacity),138values,139}140}141142pub fn push_valid(&mut self, key: i32) {143self.keys.push(Some(key))144}145146/// pushes a null value147pub fn push_null(&mut self) {148self.keys.push(None)149}150}151152impl MutableArray for FixedItemsUtf8Dictionary {153fn len(&self) -> usize {154self.keys.len()155}156157fn validity(&self) -> Option<&MutableBitmap> {158self.keys.validity()159}160161fn as_box(&mut self) -> Box<dyn Array> {162Box::new(163DictionaryArray::try_new(164self.dtype.clone(),165std::mem::take(&mut self.keys).into(),166Box::new(self.values.clone()),167)168.unwrap(),169)170}171172fn as_arc(&mut self) -> std::sync::Arc<dyn Array> {173std::sync::Arc::new(174DictionaryArray::try_new(175self.dtype.clone(),176std::mem::take(&mut self.keys).into(),177Box::new(self.values.clone()),178)179.unwrap(),180)181}182183fn dtype(&self) -> &ArrowDataType {184&self.dtype185}186187fn as_any(&self) -> &dyn std::any::Any {188self189}190191fn as_mut_any(&mut self) -> &mut dyn std::any::Any {192self193}194195#[inline]196fn push_null(&mut self) {197self.push_null()198}199200fn reserve(&mut self, _: usize) {201todo!();202}203204fn shrink_to_fit(&mut self) {205todo!();206}207}208209/// Auxiliary struct210#[derive(Debug)]211pub struct DynMutableStructArray {212dtype: ArrowDataType,213length: usize,214values: Vec<Box<dyn MutableArray>>,215validity: Option<MutableBitmap>,216}217218impl DynMutableStructArray {219pub fn new(values: Vec<Box<dyn MutableArray>>, dtype: ArrowDataType) -> Self {220Self {221dtype,222length: 0,223values,224validity: None,225}226}227228/// The values229pub fn mut_values(&mut self, field: usize) -> &mut dyn MutableArray {230self.values[field].as_mut()231}232233#[inline]234pub fn try_push_valid(&mut self) -> PolarsResult<()> {235if let Some(validity) = &mut self.validity {236validity.push(true)237}238self.length += 1;239Ok(())240}241242#[inline]243fn push_null(&mut self) {244self.values.iter_mut().for_each(|x| x.push_null());245self.length += 1;246match &mut self.validity {247Some(validity) => validity.push(false),248None => self.init_validity(),249}250}251252fn init_validity(&mut self) {253let len = self.len();254255let mut validity = MutableBitmap::new();256validity.extend_constant(len, true);257validity.set(len - 1, false);258self.validity = Some(validity)259}260}261262impl MutableArray for DynMutableStructArray {263fn len(&self) -> usize {264self.length265}266267fn validity(&self) -> Option<&MutableBitmap> {268self.validity.as_ref()269}270271fn as_box(&mut self) -> Box<dyn Array> {272let values = self.values.iter_mut().map(|x| x.as_box()).collect();273274Box::new(StructArray::new(275self.dtype.clone(),276self.length,277values,278std::mem::take(&mut self.validity).map(|x| x.into()),279))280}281282fn as_arc(&mut self) -> std::sync::Arc<dyn Array> {283let values = self.values.iter_mut().map(|x| x.as_box()).collect();284285std::sync::Arc::new(StructArray::new(286self.dtype.clone(),287self.length,288values,289std::mem::take(&mut self.validity).map(|x| x.into()),290))291}292293fn dtype(&self) -> &ArrowDataType {294&self.dtype295}296297fn as_any(&self) -> &dyn std::any::Any {298self299}300301fn as_mut_any(&mut self) -> &mut dyn std::any::Any {302self303}304305#[inline]306fn push_null(&mut self) {307self.push_null()308}309310fn reserve(&mut self, _: usize) {311todo!();312}313314fn shrink_to_fit(&mut self) {315todo!();316}317}318319320