Skip to main content

tessera/readers/
polars.rs

1//! Load Tessera partitions into Polars frames.
2//!
3//! Reads happen directly over the presigned HTTPS URL via range requests —
4//! only the Parquet footer and the row-groups/columns a query touches cross
5//! the wire.
6
7use polars::prelude::*;
8
9use super::ResolvedPartition;
10use crate::error::TesseraError;
11
12/// Substrings that signal a presigned URL was rejected (typically expired).
13const EXPIRY_MARKERS: [&str; 5] = [
14    "403",
15    "expired",
16    "accessdenied",
17    "access denied",
18    "forbidden",
19];
20
21/// Build a (possibly concatenated) `LazyFrame` over the resolved partitions.
22///
23/// A day-backed partition contributes one frame per day object; they are
24/// concatenated like any other partitions. For multi-partition reads, a `coin`
25/// and `month` column identifying the source partition are appended so rows
26/// stay attributable after concatenation.
27///
28/// # Errors
29///
30/// Returns [`TesseraError::Network`] when a Parquet path fails to register.
31pub fn build_lazyframe(
32    parts: &[ResolvedPartition],
33    columns: Option<&[&str]>,
34) -> Result<LazyFrame, TesseraError> {
35    let multi = parts.len() > 1;
36    let objects: usize = parts.iter().map(|(_, urls)| urls.len()).sum();
37    let mut frames = Vec::with_capacity(objects);
38    for (partition, urls) in parts {
39        for url in urls {
40            let mut lazy = LazyFrame::scan_parquet(url.as_str().into(), ScanArgsParquet::default())
41                .map_err(|err| TesseraError::Network(err.to_string()))?;
42            if let Some(columns) = columns {
43                lazy = lazy.select(
44                    columns
45                        .iter()
46                        .map(|column| col(*column))
47                        .collect::<Vec<_>>(),
48                );
49            }
50            if multi {
51                lazy = lazy.with_columns(vec![
52                    lit(partition.coin.as_str()).alias("coin"),
53                    lit(partition.month.as_str()).alias("month"),
54                ]);
55            }
56            frames.push(lazy);
57        }
58    }
59    if let [single] = &frames[..] {
60        return Ok(single.clone());
61    }
62    concat(
63        frames,
64        UnionArgs {
65            rechunk: true,
66            parallel: true,
67            to_supertypes: true,
68            ..Default::default()
69        },
70    )
71    .map_err(|err| TesseraError::Network(err.to_string()))
72}
73
74/// Collect a lazy frame, translating presign-expiry failures into a clear error.
75///
76/// Non-expiry Polars failures surface as [`TesseraError::Network`].
77///
78/// # Errors
79///
80/// Returns [`TesseraError::PresignExpired`] when the failure looks like a
81/// rejected presigned URL; other failures map to [`TesseraError::Network`].
82pub fn collect(lazy: LazyFrame) -> Result<DataFrame, TesseraError> {
83    lazy.collect().map_err(|err| {
84        let message = err.to_string().to_lowercase();
85        if EXPIRY_MARKERS.iter().any(|marker| message.contains(marker)) {
86            TesseraError::PresignExpired
87        } else {
88            TesseraError::Network(err.to_string())
89        }
90    })
91}