Path: blob/main/crates/polars-python/src/functions/io.rs
7889 views
use std::io::BufReader;12#[cfg(any(feature = "ipc", feature = "parquet"))]3use polars::prelude::ArrowSchema;4use polars::prelude::CloudScheme;5use polars_io::cloud::CloudOptions;6use pyo3::prelude::*;7use pyo3::types::PyDict;89use crate::conversion::Wrap;10use crate::error::PyPolarsErr;11use crate::file::{EitherRustPythonFile, get_either_file};1213#[cfg(feature = "ipc")]14#[pyfunction]15pub fn read_ipc_schema(py: Python<'_>, py_f: Py<PyAny>) -> PyResult<Bound<'_, PyDict>> {16use arrow::io::ipc::read::read_file_metadata;17let metadata = match get_either_file(py_f, false)? {18EitherRustPythonFile::Rust(r) => {19read_file_metadata(&mut BufReader::new(r)).map_err(PyPolarsErr::from)?20},21EitherRustPythonFile::Py(mut r) => read_file_metadata(&mut r).map_err(PyPolarsErr::from)?,22};2324let dict = PyDict::new(py);25fields_to_pydict(&metadata.schema, &dict)?;26Ok(dict)27}2829#[cfg(feature = "parquet")]30#[pyfunction]31pub fn read_parquet_metadata(32py: Python,33py_f: Py<PyAny>,34storage_options: Option<Vec<(String, String)>>,35credential_provider: Option<Py<PyAny>>,36retries: usize,37) -> PyResult<Bound<PyDict>> {38use std::io::Cursor;3940use polars_error::feature_gated;41use polars_io::pl_async::get_runtime;42use polars_parquet::read::read_metadata;43use polars_parquet::read::schema::read_custom_key_value_metadata;44use polars_utils::plpath::PlPath;4546use crate::file::{PythonScanSourceInput, get_python_scan_source_input};4748let metadata = match get_python_scan_source_input(py_f, false)? {49PythonScanSourceInput::Buffer(buf) => {50read_metadata(&mut Cursor::new(buf)).map_err(PyPolarsErr::from)?51},52PythonScanSourceInput::Path(p) => {53let cloud_options = parse_cloud_options(54CloudScheme::from_uri(p.to_str()),55storage_options,56credential_provider,57retries,58)?;59match p {60PlPath::Local(local) => {61let file = polars_utils::open_file(&local).map_err(PyPolarsErr::from)?;62read_metadata(&mut BufReader::new(file)).map_err(PyPolarsErr::from)?63},64PlPath::Cloud(_) => {65use polars::prelude::ParquetObjectStore;66use polars_error::PolarsResult;6768feature_gated!("cloud", {69py.detach(|| {70get_runtime().block_on(async {71let mut reader = ParquetObjectStore::from_uri(72p.as_ref(),73cloud_options.as_ref(),74None,75)76.await?;77let result = reader.get_metadata().await?;78PolarsResult::Ok((**result).clone())79})80})81})82.map_err(PyPolarsErr::from)?83},84}85},86PythonScanSourceInput::File(f) => {87read_metadata(&mut BufReader::new(f)).map_err(PyPolarsErr::from)?88},89};9091let key_value_metadata = read_custom_key_value_metadata(metadata.key_value_metadata());92let dict = PyDict::new(py);93for (key, value) in key_value_metadata.into_iter() {94dict.set_item(key.as_str(), value.as_str())?;95}96Ok(dict)97}9899#[cfg(any(feature = "ipc", feature = "parquet"))]100fn fields_to_pydict(schema: &ArrowSchema, dict: &Bound<'_, PyDict>) -> PyResult<()> {101for field in schema.iter_values() {102let dt = Wrap(polars::prelude::DataType::from_arrow_field(field));103dict.set_item(field.name.as_str(), &dt)?;104}105Ok(())106}107108#[cfg(feature = "clipboard")]109#[pyfunction]110pub fn read_clipboard_string() -> PyResult<String> {111use arboard;112let mut clipboard =113arboard::Clipboard::new().map_err(|e| PyPolarsErr::Other(format!("{e}")))?;114let result = clipboard115.get_text()116.map_err(|e| PyPolarsErr::Other(format!("{e}")))?;117Ok(result)118}119120#[cfg(feature = "clipboard")]121#[pyfunction]122pub fn write_clipboard_string(s: &str) -> PyResult<()> {123use arboard;124let mut clipboard =125arboard::Clipboard::new().map_err(|e| PyPolarsErr::Other(format!("{e}")))?;126clipboard127.set_text(s)128.map_err(|e| PyPolarsErr::Other(format!("{e}")))?;129Ok(())130}131132pub fn parse_cloud_options(133cloud_scheme: Option<CloudScheme>,134storage_options: Option<Vec<(String, String)>>,135credential_provider: Option<Py<PyAny>>,136retries: usize,137) -> PyResult<Option<CloudOptions>> {138let result: Option<CloudOptions> = {139#[cfg(feature = "cloud")]140{141use polars_io::cloud::credential_provider::PlCredentialProvider;142143use crate::prelude::parse_cloud_options;144145let cloud_options =146parse_cloud_options(cloud_scheme, storage_options.unwrap_or_default())?;147148Some(149cloud_options150.with_max_retries(retries)151.with_credential_provider(152credential_provider.map(PlCredentialProvider::from_python_builder),153),154)155}156157#[cfg(not(feature = "cloud"))]158{159None160}161};162Ok(result)163}164165166