tessera/readers/
polars.rs1use polars::prelude::*;
8
9use super::ResolvedPartition;
10use crate::error::TesseraError;
11
12const EXPIRY_MARKERS: [&str; 5] = [
14 "403",
15 "expired",
16 "accessdenied",
17 "access denied",
18 "forbidden",
19];
20
21pub fn build_lazyframe(
32 parts: &[ResolvedPartition],
33 columns: Option<&[&str]>,
34) -> Result<LazyFrame, TesseraError> {
35 let multi = parts.len() > 1;
36 let objects: usize = parts.iter().map(|(_, urls)| urls.len()).sum();
37 let mut frames = Vec::with_capacity(objects);
38 for (partition, urls) in parts {
39 for url in urls {
40 let mut lazy = LazyFrame::scan_parquet(url.as_str().into(), ScanArgsParquet::default())
41 .map_err(|err| TesseraError::Network(err.to_string()))?;
42 if let Some(columns) = columns {
43 lazy = lazy.select(
44 columns
45 .iter()
46 .map(|column| col(*column))
47 .collect::<Vec<_>>(),
48 );
49 }
50 if multi {
51 lazy = lazy.with_columns(vec![
52 lit(partition.coin.as_str()).alias("coin"),
53 lit(partition.month.as_str()).alias("month"),
54 ]);
55 }
56 frames.push(lazy);
57 }
58 }
59 if let [single] = &frames[..] {
60 return Ok(single.clone());
61 }
62 concat(
63 frames,
64 UnionArgs {
65 rechunk: true,
66 parallel: true,
67 to_supertypes: true,
68 ..Default::default()
69 },
70 )
71 .map_err(|err| TesseraError::Network(err.to_string()))
72}
73
74pub fn collect(lazy: LazyFrame) -> Result<DataFrame, TesseraError> {
83 lazy.collect().map_err(|err| {
84 let message = err.to_string().to_lowercase();
85 if EXPIRY_MARKERS.iter().any(|marker| message.contains(marker)) {
86 TesseraError::PresignExpired
87 } else {
88 TesseraError::Network(err.to_string())
89 }
90 })
91}