Skip to main content

tessera/readers/
duckdb.rs

1//! Load Tessera partitions into DuckDB relations.
2//!
3//! Uses DuckDB's `httpfs` to range-read the presigned Parquet directly; the
4//! returned connection carries a `tessera` view that can be filtered and
5//! aggregated in SQL with predicate pushdown.
6
7use duckdb::Connection;
8
9use super::ResolvedPartition;
10use crate::error::TesseraError;
11
12/// Quote a value as a SQL string literal.
13fn sql_str(value: &str) -> String {
14    format!("'{}'", value.replace('\'', "''"))
15}
16
17/// One Parquet path, or a DuckDB list literal of paths for a day-backed
18/// partition (the open month is one object per day).
19fn sql_url_list(urls: &[String]) -> String {
20    if let [single] = urls {
21        return sql_str(single);
22    }
23    let quoted: Vec<String> = urls.iter().map(|url| sql_str(url)).collect();
24    format!("[{}]", quoted.join(", "))
25}
26
27/// Build an in-memory DuckDB connection with a `tessera` view over the
28/// resolved partitions.
29///
30/// A day-backed partition (the open month) reads all of its day objects as one
31/// leaf. For multi-partition reads each leaf is unioned with a `coin` and
32/// `month` column identifying the source partition. Query it with
33/// `SELECT ... FROM tessera`.
34///
35/// # Errors
36///
37/// Returns [`TesseraError::Network`] when the connection or view creation fails.
38pub fn build_relation(
39    parts: &[ResolvedPartition],
40    columns: Option<&[&str]>,
41) -> Result<Connection, TesseraError> {
42    let connection =
43        Connection::open_in_memory().map_err(|err| TesseraError::Network(err.to_string()))?;
44
45    // Recent DuckDB autoloads httpfs for https paths; load explicitly but
46    // don't fail if the environment can't reach the extension repository.
47    let _ = connection.execute_batch("INSTALL httpfs; LOAD httpfs;");
48
49    let multi = parts.len() > 1;
50    let select_cols = columns.map_or_else(|| "*".to_string(), |columns| columns.join(", "));
51    let selects: Vec<String> = parts
52        .iter()
53        .map(|(partition, urls)| {
54            let projection = if multi {
55                format!(
56                    "{}, {} AS coin, {} AS month",
57                    select_cols,
58                    sql_str(&partition.coin),
59                    sql_str(&partition.month)
60                )
61            } else {
62                select_cols.clone()
63            };
64            format!(
65                "SELECT {projection} FROM read_parquet({})",
66                sql_url_list(urls)
67            )
68        })
69        .collect();
70    let query = selects.join("\nUNION ALL\n");
71    connection
72        .execute_batch(&format!("CREATE OR REPLACE TEMP VIEW tessera AS {query}"))
73        .map_err(|err| TesseraError::Network(err.to_string()))?;
74    Ok(connection)
75}