1use crate::container::DbfData;
9use crate::Error;
10
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub enum ColType {
14 Character,
15 Integer,
16 Double,
17 Logical,
18 Date,
19}
20
21impl ColType {
22 pub fn from_name(x: &str) -> Option<ColType> {
24 match x {
25 "character" => Some(ColType::Character),
26 "integer" => Some(ColType::Integer),
27 "double" => Some(ColType::Double),
28 "logical" => Some(ColType::Logical),
29 "date" => Some(ColType::Date),
30 _ => None,
31 }
32 }
33
34 pub fn name(self) -> &'static str {
35 match self {
36 ColType::Character => "character",
37 ColType::Integer => "integer",
38 ColType::Double => "double",
39 ColType::Logical => "logical",
40 ColType::Date => "date",
41 }
42 }
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
47pub enum Encoding {
48 #[default]
50 Latin1,
51 Utf8,
52}
53
54impl Encoding {
55 pub fn from_name(x: &str) -> Option<Encoding> {
57 match x.to_ascii_lowercase().as_str() {
58 "latin1" | "latin-1" | "iso-8859-1" | "iso8859-1" => Some(Encoding::Latin1),
59 "utf-8" | "utf8" => Some(Encoding::Utf8),
60 _ => None,
61 }
62 }
63
64 pub fn decode_into(self, raw: &[u8], out: &mut String) {
67 match std::str::from_utf8(raw) {
68 Ok(s) if self == Encoding::Utf8 || raw.is_ascii() => out.push_str(s),
69 _ => match self {
70 Encoding::Latin1 => out.extend(raw.iter().map(|&b| b as char)),
71 Encoding::Utf8 => out.push_str(&String::from_utf8_lossy(raw)),
72 },
73 }
74 }
75
76 pub fn decode(self, raw: &[u8]) -> String {
77 let mut s = String::with_capacity(raw.len());
78 self.decode_into(raw, &mut s);
79 s
80 }
81}
82
83#[derive(Debug, Clone)]
85pub struct ReadOptions {
86 pub select: Option<Vec<String>>,
88 pub n_max: Option<u64>,
90 pub trim_ws: bool,
93 pub encoding: Encoding,
94 pub guess_types: bool,
96 pub col_types: Vec<(String, ColType)>,
98 pub parse_dates: bool,
100}
101
102impl Default for ReadOptions {
103 fn default() -> Self {
104 ReadOptions {
105 select: None,
106 n_max: None,
107 trim_ws: true,
108 encoding: Encoding::Latin1,
109 guess_types: true,
110 col_types: Vec::new(),
111 parse_dates: false,
112 }
113 }
114}
115
116#[derive(Debug, Clone)]
118pub struct Field {
119 pub name: String,
121 pub dbf_type: u8,
123 pub len: u8,
124 pub dec: u8,
125 pub col_type: Option<ColType>,
127 offset: usize,
128}
129
130pub trait Sink {
134 fn int(&mut self, col: usize, row: usize, value: Option<i32>);
135 fn double(&mut self, col: usize, row: usize, value: Option<f64>);
136 fn logical(&mut self, col: usize, row: usize, value: Option<bool>);
137 fn date(&mut self, col: usize, row: usize, value: Option<i32>);
139 fn text(&mut self, col: usize, row: usize, raw: &[u8]);
143}
144
145#[derive(Debug, Clone, Copy, Default)]
147pub struct Cursor {
148 raw: usize,
149 row: usize,
150}
151
152pub struct Table {
154 data: DbfData,
155 fields: Vec<Field>,
156 columns: Vec<usize>,
157 record_len: usize,
158 n_scan: usize,
159 n_rows: usize,
160 declared_records: u32,
161 trim_ws: bool,
162 encoding: Encoding,
163 warnings: Vec<String>,
164 unmatched_select: Vec<String>,
165}
166
167fn le_u16(p: &[u8]) -> u16 {
168 u16::from_le_bytes([p[0], p[1]])
169}
170
171fn le_u32(p: &[u8]) -> u32 {
172 u32::from_le_bytes([p[0], p[1], p[2], p[3]])
173}
174
175#[inline]
177fn is_space(b: u8) -> bool {
178 b == b' ' || (0x09..=0x0D).contains(&b)
179}
180
181#[inline]
182fn trim(src: &[u8]) -> &[u8] {
183 let mut s = 0;
184 let mut e = src.len();
185 while s < e && is_space(src[s]) {
186 s += 1;
187 }
188 while e > s && is_space(src[e - 1]) {
189 e -= 1;
190 }
191 &src[s..e]
192}
193
194#[inline]
197fn until_nul(src: &[u8]) -> &[u8] {
198 match src.iter().position(|&b| b == 0) {
199 Some(i) => &src[..i],
200 None => src,
201 }
202}
203
204fn parse_int(x: &[u8]) -> Option<i32> {
207 let (neg, digits) = match x.first() {
208 Some(b'-') => (true, &x[1..]),
209 Some(b'+') => (false, &x[1..]),
210 _ => (false, x),
211 };
212 if digits.is_empty() {
213 return None;
214 }
215 let mut val: i64 = 0;
216 for &b in digits {
217 if !b.is_ascii_digit() {
218 return None;
219 }
220 val = val * 10 + (b - b'0') as i64;
221 if val > i32::MAX as i64 + 1 {
222 return None;
223 }
224 }
225 let val = if neg { -val } else { val };
226 if val <= i32::MIN as i64 || val > i32::MAX as i64 {
227 return None;
228 }
229 Some(val as i32)
230}
231
232fn parse_double(x: &[u8]) -> Option<f64> {
235 std::str::from_utf8(x).ok()?.parse::<f64>().ok()
236}
237
238fn days_in_month(year: i32, month: i32) -> i32 {
239 const MDAYS: [i32; 12] = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
240 if month == 2 {
241 let leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0;
242 return if leap { 29 } else { 28 };
243 }
244 MDAYS[(month - 1) as usize]
245}
246
247fn parse_date(src: &[u8]) -> Option<i32> {
249 if src.len() < 8 || !src[..8].iter().all(u8::is_ascii_digit) {
250 return None;
251 }
252 let d = |i: usize| (src[i] - b'0') as i32;
253 let year = d(0) * 1000 + d(1) * 100 + d(2) * 10 + d(3);
254 let month = d(4) * 10 + d(5);
255 let day = d(6) * 10 + d(7);
256 if !(1..=12).contains(&month) || day < 1 || day > days_in_month(year, month) {
257 return None;
258 }
259 let y = year - (month <= 2) as i32;
261 let era = (if y >= 0 { y } else { y - 399 }) / 400;
262 let yoe = y - era * 400;
263 let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1;
264 let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
265 Some(era * 146097 + doe - 719468)
266}
267
268impl Table {
269 pub fn new(data: DbfData, opts: &ReadOptions) -> Result<Table, Error> {
272 let header = data.header();
273 let records = data.records();
274
275 if header.len() < 32 {
276 return Err(Error::Header("Invalid DBF header: too small.".into()));
277 }
278 let declared_records = le_u32(&header[4..]);
279 let header_len_decl = le_u16(&header[8..]) as usize;
280 let record_len = le_u16(&header[10..]) as usize;
281
282 if header_len_decl > header.len() || record_len == 0 {
283 return Err(Error::Header("Invalid DBF header.".into()));
284 }
285 if header_len_decl < 33 || (header_len_decl - 33) % 32 != 0 {
286 return Err(Error::Header("Invalid DBF header field descriptor section.".into()));
287 }
288 let n_fields = (header_len_decl - 33) / 32;
289 if n_fields == 0 {
290 return Err(Error::Header("No DBF fields found.".into()));
291 }
292
293 let mut sum_len = 1usize;
294 for i in 0..n_fields {
295 let flen = header[32 + i * 32 + 16] as usize;
296 if flen == 0 {
297 return Err(Error::Header(format!(
298 "Invalid DBF header: field {} has length 0.",
299 i + 1
300 )));
301 }
302 sum_len += flen;
303 }
304 if sum_len > record_len {
305 return Err(Error::Header(format!(
306 "Invalid DBF header: field lengths (1 + {} bytes) exceed the declared record length ({} bytes).",
307 sum_len - 1,
308 record_len
309 )));
310 }
311
312 let mut warnings = Vec::new();
313 let mut n_records = declared_records as usize;
314 let available = (records.len() / record_len).min(u32::MAX as usize);
315 if available < n_records {
316 warnings.push(format!(
317 "DBF declares {n_records} records but only {available} are present; reading {available}."
318 ));
319 n_records = available;
320 }
321
322 let n_max = opts.n_max.unwrap_or(u64::MAX);
324 let mut n_scan = 0usize;
325 let mut n_rows = 0usize;
326 for r in 0..n_records {
327 if n_rows as u64 >= n_max {
328 break;
329 }
330 n_scan = r + 1;
331 if records[r * record_len] != b'*' {
332 n_rows += 1;
333 }
334 }
335
336 let select: Vec<String> = opts
337 .select
338 .iter()
339 .flatten()
340 .map(|s| s.to_uppercase())
341 .collect();
342 let declared: Vec<(String, ColType)> = opts
343 .col_types
344 .iter()
345 .map(|(n, t)| (n.to_uppercase(), *t))
346 .collect();
347
348 let mut fields = Vec::with_capacity(n_fields);
349 let mut columns = Vec::new();
350 let mut upper_names = Vec::new();
351 let mut offset = 1usize;
352
353 for i in 0..n_fields {
354 let fd = &header[32 + i * 32..32 + i * 32 + 32];
355 let name = Encoding::Latin1.decode(until_nul(&fd[..11]));
356 let upper = name.to_ascii_uppercase();
357 let dbf_type = fd[11];
358 let len = fd[16];
359 let dec = fd[17];
360 let keep = select.is_empty() || select.contains(&upper);
361
362 let mut field = Field { name, dbf_type, len, dec, col_type: None, offset };
363 offset += len as usize;
364
365 if keep {
366 let target = match declared.iter().find(|(n, _)| *n == upper) {
367 Some((_, t)) => *t,
368 None => match dbf_type {
369 b'L' => ColType::Logical,
370 b'D' if opts.parse_dates => ColType::Date,
371 b'D' => ColType::Character,
372 b'N' | b'F' if opts.guess_types => {
373 infer_numeric(&field, records, n_scan, record_len)
374 }
375 b'N' | b'F' if dec > 0 => ColType::Double,
376 b'N' | b'F' => ColType::Integer,
377 _ => ColType::Character,
378 },
379 };
380 field.col_type = Some(target);
381 columns.push(i);
382 upper_names.push(upper);
383 }
384 fields.push(field);
385 }
386
387 let mut unmatched_select: Vec<String> = Vec::new();
388 for s in opts.select.iter().flatten() {
389 let up = s.to_uppercase();
390 if !upper_names.contains(&up) && !unmatched_select.contains(s) {
391 unmatched_select.push(s.clone());
392 }
393 }
394
395 Ok(Table {
396 data,
397 fields,
398 columns,
399 record_len,
400 n_scan,
401 n_rows,
402 declared_records,
403 trim_ws: opts.trim_ws,
404 encoding: opts.encoding,
405 warnings,
406 unmatched_select,
407 })
408 }
409
410 pub fn fields(&self) -> &[Field] {
412 &self.fields
413 }
414
415 pub fn columns(&self) -> impl ExactSizeIterator<Item = &Field> + '_ {
417 self.columns.iter().map(|&i| &self.fields[i])
418 }
419
420 pub fn n_cols(&self) -> usize {
421 self.columns.len()
422 }
423
424 pub fn n_rows(&self) -> usize {
427 self.n_rows
428 }
429
430 pub fn declared_records(&self) -> u32 {
432 self.declared_records
433 }
434
435 pub fn encoding(&self) -> Encoding {
436 self.encoding
437 }
438
439 pub fn warnings(&self) -> &[String] {
441 &self.warnings
442 }
443
444 pub fn unmatched_select(&self) -> &[String] {
446 &self.unmatched_select
447 }
448
449 pub fn data(&self) -> &DbfData {
450 &self.data
451 }
452
453 pub fn read_rows<S: Sink>(&self, cursor: &mut Cursor, max_rows: usize, sink: &mut S) -> usize {
456 let records = self.data.records();
457 let mut done = 0usize;
458
459 while cursor.raw < self.n_scan && cursor.row < self.n_rows && done < max_rows {
460 let start = cursor.raw * self.record_len;
461 let rec = &records[start..start + self.record_len];
462 cursor.raw += 1;
463
464 if rec[0] == b'*' {
465 continue;
466 }
467
468 for (c, &fi) in self.columns.iter().enumerate() {
469 let f = &self.fields[fi];
470 let raw = &rec[f.offset..f.offset + f.len as usize];
471
472 match f.col_type {
473 Some(ColType::Integer) => sink.int(c, done, parse_int(until_nul(trim(raw)))),
474 Some(ColType::Double) => {
475 sink.double(c, done, parse_double(until_nul(trim(raw))))
476 }
477 Some(ColType::Logical) => {
478 let v = match trim(raw).first() {
479 Some(b'Y' | b'y' | b'T' | b't') => Some(true),
480 Some(b'N' | b'n' | b'F' | b'f') => Some(false),
481 _ => None,
482 };
483 sink.logical(c, done, v);
484 }
485 Some(ColType::Date) => {
486 let v = if f.dbf_type == b'D' {
487 parse_date(trim(raw))
488 } else {
489 let t = until_nul(trim(raw));
490 if t.len() == 8 { parse_date(t) } else { None }
491 };
492 sink.date(c, done, v);
493 }
494 Some(ColType::Character) | None => {
495 let t = if self.trim_ws { trim(raw) } else { raw };
496 sink.text(c, done, until_nul(t));
497 }
498 }
499 }
500
501 done += 1;
502 cursor.row += 1;
503 }
504 done
505 }
506}
507
508fn infer_numeric(field: &Field, records: &[u8], n_scan: usize, record_len: usize) -> ColType {
511 if field.dec > 0 {
512 return ColType::Double;
513 }
514 for r in 0..n_scan {
515 let rec = &records[r * record_len..];
516 if rec[0] == b'*' {
517 continue;
518 }
519 let t = until_nul(trim(&rec[field.offset..field.offset + field.len as usize]));
520 if t.is_empty() {
521 continue;
522 }
523 if t.iter().any(|&b| b == b'.' || b == b'e' || b == b'E') || parse_int(t).is_none() {
524 return ColType::Double;
525 }
526 }
527 ColType::Integer
528}
529
530#[derive(Debug, Clone, PartialEq)]
532pub enum ColumnData {
533 Integer(Vec<Option<i32>>),
534 Double(Vec<Option<f64>>),
535 Logical(Vec<Option<bool>>),
536 Date(Vec<Option<i32>>),
538 Character(Vec<String>),
539}
540
541struct Collect {
542 cols: Vec<ColumnData>,
543 encoding: Encoding,
544}
545
546impl Sink for Collect {
547 fn int(&mut self, col: usize, _: usize, value: Option<i32>) {
548 if let ColumnData::Integer(v) = &mut self.cols[col] {
549 v.push(value);
550 }
551 }
552 fn double(&mut self, col: usize, _: usize, value: Option<f64>) {
553 if let ColumnData::Double(v) = &mut self.cols[col] {
554 v.push(value);
555 }
556 }
557 fn logical(&mut self, col: usize, _: usize, value: Option<bool>) {
558 if let ColumnData::Logical(v) = &mut self.cols[col] {
559 v.push(value);
560 }
561 }
562 fn date(&mut self, col: usize, _: usize, value: Option<i32>) {
563 if let ColumnData::Date(v) = &mut self.cols[col] {
564 v.push(value);
565 }
566 }
567 fn text(&mut self, col: usize, _: usize, raw: &[u8]) {
568 if let ColumnData::Character(v) = &mut self.cols[col] {
569 v.push(self.encoding.decode(raw));
570 }
571 }
572}
573
574impl Table {
575 pub fn collect(&self) -> Vec<ColumnData> {
577 let n = self.n_rows;
578 let cols = self
579 .columns()
580 .map(|f| match f.col_type {
581 Some(ColType::Integer) => ColumnData::Integer(Vec::with_capacity(n)),
582 Some(ColType::Double) => ColumnData::Double(Vec::with_capacity(n)),
583 Some(ColType::Logical) => ColumnData::Logical(Vec::with_capacity(n)),
584 Some(ColType::Date) => ColumnData::Date(Vec::with_capacity(n)),
585 _ => ColumnData::Character(Vec::with_capacity(n)),
586 })
587 .collect();
588 let mut sink = Collect { cols, encoding: self.encoding };
589 self.read_rows(&mut Cursor::default(), usize::MAX, &mut sink);
590 sink.cols
591 }
592}
593
594pub fn repair_names<'a>(names: impl Iterator<Item = &'a str>, clean: bool) -> Vec<String> {
598 let names: Vec<&str> = names.collect();
599 names
600 .iter()
601 .enumerate()
602 .map(|(i, n)| {
603 let dup = n.is_empty() || names.iter().filter(|m| m == &n).count() > 1;
604 let out = if dup { format!("{n}...{}", i + 1) } else { n.to_string() };
605 if clean { out.to_lowercase() } else { out }
606 })
607 .collect()
608}
609
610#[cfg(test)]
611mod tests {
612 use super::*;
613
614 #[test]
615 fn ints() {
616 assert_eq!(parse_int(b"12"), Some(12));
617 assert_eq!(parse_int(b"-7"), Some(-7));
618 assert_eq!(parse_int(b"+7"), Some(7));
619 assert_eq!(parse_int(b"2147483647"), Some(i32::MAX));
620 assert_eq!(parse_int(b"2147483648"), None);
621 assert_eq!(parse_int(b"-2147483647"), Some(-2147483647));
622 assert_eq!(parse_int(b"-2147483648"), None);
623 assert_eq!(parse_int(b"99999999999999999999999999"), None);
624 assert_eq!(parse_int(b""), None);
625 assert_eq!(parse_int(b"-"), None);
626 assert_eq!(parse_int(b"1 2"), None);
627 assert_eq!(parse_int(b"1.0"), None);
628 }
629
630 #[test]
631 fn dates() {
632 assert_eq!(parse_date(b"19700101"), Some(0));
633 assert_eq!(parse_date(b"20240115"), Some(19737));
634 assert_eq!(parse_date(b"20240229"), Some(19782));
635 assert_eq!(parse_date(b"20230229"), None);
636 assert_eq!(parse_date(b"19000229"), None);
637 assert_eq!(parse_date(b"2024130a"), None);
638 assert_eq!(parse_date(b"2024"), None);
639 }
640
641 #[test]
642 fn trimming() {
643 assert_eq!(trim(b" ab \t"), b"ab");
644 assert_eq!(trim(b" "), b"");
645 assert_eq!(until_nul(b"ab\0\0"), b"ab");
646 assert_eq!(until_nul(trim(b"\0\0\0")), b"");
647 }
648
649 #[test]
650 fn names() {
651 let r = repair_names(["A", "B", "A", ""].into_iter(), true);
652 assert_eq!(r, ["a...1", "b", "a...3", "...4"]);
653 }
654}