Path: blob/main/crates/polars-python/src/series/import.rs
7889 views
use arrow::array::{Array, PrimitiveArray};1use arrow::ffi;2use arrow::ffi::{ArrowArray, ArrowArrayStream, ArrowArrayStreamReader, ArrowSchema};3use polars::prelude::*;4use polars_ffi::version_0::SeriesExport;5use pyo3::exceptions::{PyTypeError, PyValueError};6use pyo3::prelude::*;7use pyo3::pybacked::PyBackedBytes;8use pyo3::types::{PyCapsule, PyTuple, PyType};910use super::PySeries;11use crate::error::PyPolarsErr;1213/// Validate PyCapsule has provided name14fn validate_pycapsule_name(capsule: &Bound<PyCapsule>, expected_name: &str) -> PyResult<()> {15let capsule_name = capsule.name()?;16if let Some(capsule_name) = capsule_name {17let capsule_name = capsule_name.to_str()?;18if capsule_name != expected_name {19return Err(PyValueError::new_err(format!(20"Expected name '{expected_name}' in PyCapsule, instead got '{capsule_name}'"21)));22}23} else {24return Err(PyValueError::new_err(25"Expected schema PyCapsule to have name set.",26));27}2829Ok(())30}3132/// Import `__arrow_c_array__` across Python boundary33pub(crate) fn call_arrow_c_array<'py>(34ob: &Bound<'py, PyAny>,35) -> PyResult<(Bound<'py, PyCapsule>, Bound<'py, PyCapsule>)> {36if !ob.hasattr("__arrow_c_array__")? {37return Err(PyValueError::new_err(38"Expected an object with dunder __arrow_c_array__",39));40}4142let tuple = ob.getattr("__arrow_c_array__")?.call0()?;43if !tuple.is_instance_of::<PyTuple>() {44return Err(PyTypeError::new_err(45"Expected __arrow_c_array__ to return a tuple.",46));47}4849let schema_capsule = tuple.get_item(0)?.downcast_into()?;50let array_capsule = tuple.get_item(1)?.downcast_into()?;51Ok((schema_capsule, array_capsule))52}5354pub(crate) fn import_array_pycapsules(55schema_capsule: &Bound<PyCapsule>,56array_capsule: &Bound<PyCapsule>,57) -> PyResult<(arrow::datatypes::Field, Box<dyn Array>)> {58let field = import_schema_pycapsule(schema_capsule)?;5960validate_pycapsule_name(array_capsule, "arrow_array")?;6162// # Safety63// array_capsule holds a valid C ArrowArray pointer, as defined by the Arrow PyCapsule64// Interface65unsafe {66let array_ptr = std::ptr::replace(array_capsule.pointer() as _, ArrowArray::empty());67let array = ffi::import_array_from_c(array_ptr, field.dtype().clone()).unwrap();6869Ok((field, array))70}71}7273pub(crate) fn import_schema_pycapsule(74schema_capsule: &Bound<PyCapsule>,75) -> PyResult<arrow::datatypes::Field> {76validate_pycapsule_name(schema_capsule, "arrow_schema")?;7778// # Safety79// schema_capsule holds a valid C ArrowSchema pointer, as defined by the Arrow PyCapsule80// Interface81unsafe {82let schema_ptr = schema_capsule.reference::<ArrowSchema>();83let field = ffi::import_field_from_c(schema_ptr).unwrap();8485Ok(field)86}87}8889/// Import `__arrow_c_stream__` across Python boundary.90fn call_arrow_c_stream<'py>(ob: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyCapsule>> {91if !ob.hasattr("__arrow_c_stream__")? {92return Err(PyValueError::new_err(93"Expected an object with dunder __arrow_c_stream__",94));95}9697let capsule = ob.getattr("__arrow_c_stream__")?.call0()?.downcast_into()?;98Ok(capsule)99}100101pub(crate) fn import_stream_pycapsule(capsule: &Bound<PyCapsule>) -> PyResult<PySeries> {102validate_pycapsule_name(capsule, "arrow_array_stream")?;103104// # Safety105// capsule holds a valid C ArrowArrayStream pointer, as defined by the Arrow PyCapsule106// Interface107let mut stream = unsafe {108// Takes ownership of the pointed to ArrowArrayStream109// This acts to move the data out of the capsule pointer, setting the release callback to NULL110let stream_ptr = Box::new(std::ptr::replace(111capsule.pointer() as _,112ArrowArrayStream::empty(),113));114ArrowArrayStreamReader::try_new(stream_ptr)115.map_err(|err| PyValueError::new_err(err.to_string()))?116};117118let mut produced_arrays: Vec<Box<dyn Array>> = vec![];119while let Some(array) = unsafe { stream.next() } {120produced_arrays.push(array.unwrap());121}122123// Series::try_from fails for an empty vec of chunks124let s = if produced_arrays.is_empty() {125let polars_dt = DataType::from_arrow_field(stream.field());126Series::new_empty(stream.field().name.clone(), &polars_dt)127} else {128Series::try_from((stream.field(), produced_arrays)).unwrap()129};130Ok(PySeries::new(s))131}132#[pymethods]133impl PySeries {134#[classmethod]135pub fn from_arrow_c_array(_cls: &Bound<PyType>, ob: &Bound<'_, PyAny>) -> PyResult<Self> {136let (schema_capsule, array_capsule) = call_arrow_c_array(ob)?;137let (field, array) = import_array_pycapsules(&schema_capsule, &array_capsule)?;138let s = Series::try_from((&field, array)).unwrap();139Ok(PySeries::new(s))140}141142#[classmethod]143pub fn from_arrow_c_stream(_cls: &Bound<PyType>, ob: &Bound<'_, PyAny>) -> PyResult<Self> {144let capsule = call_arrow_c_stream(ob)?;145import_stream_pycapsule(&capsule)146}147148#[classmethod]149/// Import a series via polars-ffi150/// Takes ownership of the [`SeriesExport`] at [`location`]151/// # Safety152/// [`location`] should be the address of an allocated and initialized [`SeriesExport`]153pub unsafe fn _import(_cls: &Bound<PyType>, location: usize) -> PyResult<Self> {154let location = location as *mut SeriesExport;155156// # Safety157// `location` should be valid for reading158let series = unsafe {159let export = location.read();160polars_ffi::version_0::import_series(export).map_err(PyPolarsErr::from)?161};162Ok(PySeries::from(series))163}164165#[staticmethod]166pub fn _import_decimal_from_iceberg_binary_repr(167bytes_list: &Bound<PyAny>, // list[bytes | None]168precision: usize,169scale: usize,170) -> PyResult<Self> {171// From iceberg spec:172// * Decimal(P, S): Stores unscaled value as two’s-complement173// big-endian binary, using the minimum number of bytes for the174// value.175let max_abs_decimal_value = 10_i128.pow(u32::try_from(precision).unwrap()) - 1;176177let out: Vec<i128> = bytes_list178.try_iter()?179.map(|bytes| {180let be_bytes: Option<PyBackedBytes> = bytes?.extract()?;181182let mut le_bytes: [u8; 16] = [0; _];183184if let Some(be_bytes) = be_bytes.as_deref() {185if be_bytes.len() > le_bytes.len() {186return Err(PyValueError::new_err(format!(187"iceberg binary data for decimal exceeded 16 bytes: {}",188be_bytes.len()189)));190}191192for (i, byte) in be_bytes.iter().rev().enumerate() {193le_bytes[i] = *byte;194}195}196197let value = i128::from_le_bytes(le_bytes);198199if value.abs() > max_abs_decimal_value {200return Err(PyValueError::new_err(format!(201"iceberg decoded value for decimal exceeded precision: \202value: {value}, precision: {precision}",203)));204}205206Ok(value)207})208.collect::<PyResult<_>>()?;209210Ok(PySeries::from(unsafe {211Series::from_chunks_and_dtype_unchecked(212PlSmallStr::EMPTY,213vec![PrimitiveArray::<i128>::from_vec(out).boxed()],214&DataType::Decimal(precision, scale),215)216}))217}218}219220221