1use std::sync::Arc;
9
10use arrow_array::builder::{
11 BooleanBuilder, Date32Builder, Float64Builder, Int32Builder, StringBuilder,
12};
13use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, RecordBatchReader};
14use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
15use datasus_core::{repair_names, ColType, Cursor, Encoding, Sink, Table};
16
17pub const DEFAULT_BATCH_SIZE: usize = 65_536;
19
20pub fn schema(table: &Table, clean_names: bool) -> SchemaRef {
23 let names = repair_names(table.columns().map(|f| f.name.as_str()), clean_names);
24 let fields: Vec<Field> = table
25 .columns()
26 .zip(names)
27 .map(|(f, name)| {
28 let (dt, nullable) = match f.col_type {
29 Some(ColType::Integer) => (DataType::Int32, true),
30 Some(ColType::Double) => (DataType::Float64, true),
31 Some(ColType::Logical) => (DataType::Boolean, true),
32 Some(ColType::Date) => (DataType::Date32, true),
33 _ => (DataType::Utf8, false),
34 };
35 Field::new(name, dt, nullable)
36 })
37 .collect();
38 Arc::new(Schema::new(fields))
39}
40
41enum Builder {
42 Int(Int32Builder),
43 Double(Float64Builder),
44 Logical(BooleanBuilder),
45 Date(Date32Builder),
46 Text(StringBuilder),
47}
48
49struct Builders {
50 cols: Vec<Builder>,
51 encoding: Encoding,
52 scratch: String,
53}
54
55impl Sink for Builders {
56 #[inline]
57 fn int(&mut self, col: usize, _: usize, value: Option<i32>) {
58 if let Builder::Int(b) = &mut self.cols[col] {
59 b.append_option(value);
60 }
61 }
62 #[inline]
63 fn double(&mut self, col: usize, _: usize, value: Option<f64>) {
64 if let Builder::Double(b) = &mut self.cols[col] {
65 b.append_option(value);
66 }
67 }
68 #[inline]
69 fn logical(&mut self, col: usize, _: usize, value: Option<bool>) {
70 if let Builder::Logical(b) = &mut self.cols[col] {
71 b.append_option(value);
72 }
73 }
74 #[inline]
75 fn date(&mut self, col: usize, _: usize, value: Option<i32>) {
76 if let Builder::Date(b) = &mut self.cols[col] {
77 b.append_option(value);
78 }
79 }
80 #[inline]
81 fn text(&mut self, col: usize, _: usize, raw: &[u8]) {
82 if let Builder::Text(b) = &mut self.cols[col] {
83 self.scratch.clear();
84 self.encoding.decode_into(raw, &mut self.scratch);
85 b.append_value(&self.scratch);
86 }
87 }
88}
89
90pub struct BatchReader {
92 table: Table,
93 schema: SchemaRef,
94 cursor: Cursor,
95 batch_size: usize,
96 builders: Builders,
97}
98
99impl BatchReader {
100 pub fn new(table: Table, clean_names: bool, batch_size: usize) -> BatchReader {
101 let schema = schema(&table, clean_names);
102 let cols = table
103 .columns()
104 .map(|f| match f.col_type {
105 Some(ColType::Integer) => Builder::Int(Int32Builder::new()),
106 Some(ColType::Double) => Builder::Double(Float64Builder::new()),
107 Some(ColType::Logical) => Builder::Logical(BooleanBuilder::new()),
108 Some(ColType::Date) => Builder::Date(Date32Builder::new()),
109 _ => Builder::Text(StringBuilder::new()),
110 })
111 .collect();
112 let builders = Builders { cols, encoding: table.encoding(), scratch: String::new() };
113 BatchReader {
114 table,
115 schema,
116 cursor: Cursor::default(),
117 batch_size: batch_size.max(1),
118 builders,
119 }
120 }
121
122 pub fn table(&self) -> &Table {
124 &self.table
125 }
126}
127
128impl Iterator for BatchReader {
129 type Item = Result<RecordBatch, ArrowError>;
130
131 fn next(&mut self) -> Option<Self::Item> {
132 let n = self.table.read_rows(&mut self.cursor, self.batch_size, &mut self.builders);
133 if n == 0 {
134 return None;
135 }
136 let arrays: Vec<ArrayRef> = self
137 .builders
138 .cols
139 .iter_mut()
140 .map(|b| match b {
141 Builder::Int(b) => Arc::new(b.finish()) as ArrayRef,
142 Builder::Double(b) => Arc::new(b.finish()),
143 Builder::Logical(b) => Arc::new(b.finish()),
144 Builder::Date(b) => Arc::new(b.finish()),
145 Builder::Text(b) => Arc::new(b.finish()),
146 })
147 .collect();
148 let options = RecordBatchOptions::new().with_row_count(Some(n));
151 Some(RecordBatch::try_new_with_options(self.schema.clone(), arrays, &options))
152 }
153}
154
155impl RecordBatchReader for BatchReader {
156 fn schema(&self) -> SchemaRef {
157 self.schema.clone()
158 }
159}