Skip to main content

datasus_arrow/
lib.rs

1//! Apache Arrow front end for [`datasus_core`]: reads a DBC/DBF table as a
2//! stream of record batches.
3//!
4//! Type mapping: integer → `Int32`, double → `Float64`, logical →
5//! `Boolean`, date → `Date32`, character → `Utf8` (decoded from the file
6//! encoding; never null, blank fields are empty strings).
7
8use std::sync::Arc;
9
10use arrow_array::builder::{
11    BooleanBuilder, Date32Builder, Float64Builder, Int32Builder, StringBuilder,
12};
13use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, RecordBatchReader};
14use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
15use datasus_core::{repair_names, ColType, Cursor, Encoding, Sink, Table};
16
17/// Default number of rows per record batch.
18pub const DEFAULT_BATCH_SIZE: usize = 65_536;
19
20/// Arrow schema of the selected columns.  Names are made unique and, with
21/// `clean_names`, lowercased (as datasusr does).
22pub fn schema(table: &Table, clean_names: bool) -> SchemaRef {
23    let names = repair_names(table.columns().map(|f| f.name.as_str()), clean_names);
24    let fields: Vec<Field> = table
25        .columns()
26        .zip(names)
27        .map(|(f, name)| {
28            let (dt, nullable) = match f.col_type {
29                Some(ColType::Integer) => (DataType::Int32, true),
30                Some(ColType::Double) => (DataType::Float64, true),
31                Some(ColType::Logical) => (DataType::Boolean, true),
32                Some(ColType::Date) => (DataType::Date32, true),
33                _ => (DataType::Utf8, false),
34            };
35            Field::new(name, dt, nullable)
36        })
37        .collect();
38    Arc::new(Schema::new(fields))
39}
40
41enum Builder {
42    Int(Int32Builder),
43    Double(Float64Builder),
44    Logical(BooleanBuilder),
45    Date(Date32Builder),
46    Text(StringBuilder),
47}
48
49struct Builders {
50    cols: Vec<Builder>,
51    encoding: Encoding,
52    scratch: String,
53}
54
55impl Sink for Builders {
56    #[inline]
57    fn int(&mut self, col: usize, _: usize, value: Option<i32>) {
58        if let Builder::Int(b) = &mut self.cols[col] {
59            b.append_option(value);
60        }
61    }
62    #[inline]
63    fn double(&mut self, col: usize, _: usize, value: Option<f64>) {
64        if let Builder::Double(b) = &mut self.cols[col] {
65            b.append_option(value);
66        }
67    }
68    #[inline]
69    fn logical(&mut self, col: usize, _: usize, value: Option<bool>) {
70        if let Builder::Logical(b) = &mut self.cols[col] {
71            b.append_option(value);
72        }
73    }
74    #[inline]
75    fn date(&mut self, col: usize, _: usize, value: Option<i32>) {
76        if let Builder::Date(b) = &mut self.cols[col] {
77            b.append_option(value);
78        }
79    }
80    #[inline]
81    fn text(&mut self, col: usize, _: usize, raw: &[u8]) {
82        if let Builder::Text(b) = &mut self.cols[col] {
83            self.scratch.clear();
84            self.encoding.decode_into(raw, &mut self.scratch);
85            b.append_value(&self.scratch);
86        }
87    }
88}
89
90/// Iterator of record batches over a [`Table`].
91pub struct BatchReader {
92    table: Table,
93    schema: SchemaRef,
94    cursor: Cursor,
95    batch_size: usize,
96    builders: Builders,
97}
98
99impl BatchReader {
100    pub fn new(table: Table, clean_names: bool, batch_size: usize) -> BatchReader {
101        let schema = schema(&table, clean_names);
102        let cols = table
103            .columns()
104            .map(|f| match f.col_type {
105                Some(ColType::Integer) => Builder::Int(Int32Builder::new()),
106                Some(ColType::Double) => Builder::Double(Float64Builder::new()),
107                Some(ColType::Logical) => Builder::Logical(BooleanBuilder::new()),
108                Some(ColType::Date) => Builder::Date(Date32Builder::new()),
109                _ => Builder::Text(StringBuilder::new()),
110            })
111            .collect();
112        let builders = Builders { cols, encoding: table.encoding(), scratch: String::new() };
113        BatchReader {
114            table,
115            schema,
116            cursor: Cursor::default(),
117            batch_size: batch_size.max(1),
118            builders,
119        }
120    }
121
122    /// The table being read (row count, warnings, unmatched `select` names).
123    pub fn table(&self) -> &Table {
124        &self.table
125    }
126}
127
128impl Iterator for BatchReader {
129    type Item = Result<RecordBatch, ArrowError>;
130
131    fn next(&mut self) -> Option<Self::Item> {
132        let n = self.table.read_rows(&mut self.cursor, self.batch_size, &mut self.builders);
133        if n == 0 {
134            return None;
135        }
136        let arrays: Vec<ArrayRef> = self
137            .builders
138            .cols
139            .iter_mut()
140            .map(|b| match b {
141                Builder::Int(b) => Arc::new(b.finish()) as ArrayRef,
142                Builder::Double(b) => Arc::new(b.finish()),
143                Builder::Logical(b) => Arc::new(b.finish()),
144                Builder::Date(b) => Arc::new(b.finish()),
145                Builder::Text(b) => Arc::new(b.finish()),
146            })
147            .collect();
148        // The row count is given explicitly so a selection that matched no
149        // column still yields batches with the right number of rows.
150        let options = RecordBatchOptions::new().with_row_count(Some(n));
151        Some(RecordBatch::try_new_with_options(self.schema.clone(), arrays, &options))
152    }
153}
154
155impl RecordBatchReader for BatchReader {
156    fn schema(&self) -> SchemaRef {
157        self.schema.clone()
158    }
159}