Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
pola-rs
GitHub Repository: pola-rs/polars
Path: blob/main/crates/polars-parquet/src/parquet/encoding/hybrid_rle/proptest.rs
7887 views
1
use proptest::prelude::*;
2
3
#[derive(Debug, Clone)]
4
enum Chunk {
5
Rle(u32, usize),
6
Bitpacked(Vec<u32>),
7
}
8
9
proptest::prop_compose! {
10
fn hybrid_rle_chunks
11
(max_idx: u32, size: usize)
12
(idxs in proptest::collection::vec(0..=max_idx, size),
13
mut chunk_offsets in proptest::collection::vec((0..=size, any::<bool>()), 2..=size.max(2)),
14
)
15
-> Vec<Chunk> {
16
if size == 0 {
17
return Vec::new();
18
}
19
20
chunk_offsets.sort_unstable();
21
chunk_offsets.first_mut().unwrap().0 = 0;
22
chunk_offsets.last_mut().unwrap().0 = idxs.len();
23
chunk_offsets.dedup_by_key(|(idx, _)| *idx);
24
25
chunk_offsets
26
.windows(2)
27
.map(|values| {
28
let (start, is_bitpacked) = values[0];
29
let (end, _) = values[1];
30
31
if is_bitpacked {
32
Chunk::Bitpacked(idxs[start..end].to_vec())
33
} else {
34
Chunk::Rle(idxs[start], end - start)
35
}
36
})
37
.collect::<Vec<Chunk>>()
38
}
39
}
40
41
proptest::prop_compose! {
42
pub fn hybrid_rle
43
(max_idx: u32, size: usize)
44
(chunks in hybrid_rle_chunks(max_idx, size))
45
-> Vec<u8> {
46
use super::encoder::Encoder;
47
let mut buffer = Vec::new();
48
let bit_width = 32 - max_idx.leading_zeros();
49
for chunk in chunks {
50
match chunk {
51
Chunk::Rle(value, size) => {
52
u32::run_length_encode(&mut buffer, size, value, bit_width).unwrap()
53
}
54
Chunk::Bitpacked(values) => {
55
u32::bitpacked_encode(&mut buffer, values.into_iter(), bit_width as usize).unwrap()
56
}
57
}
58
}
59
buffer
60
}
61
}
62
63