Skip to main content

datasus_core/
container.rs

1//! Container detection: turn the bytes of a `.dbc` (or `.dbf`) file into a
2//! DBF header plus an uncompressed record area.
3//!
4//! Mirrors `dbc_reader.c` from the datasusr R package, including the order
5//! in which the three layouts are tried.
6
7use std::ops::Range;
8
9use crate::blast::{blast, BlastError};
10use crate::Error;
11
12/// Bytes between the DBF header and the compressed payload in a DBC file.
13const CRC_OFFSET: usize = 4;
14
15/// Default bound on decompressed size used by [`decode`]: 4 GiB, twice the
16/// largest file dBase itself can address.
17pub const DEFAULT_MAX_DECOMPRESSED: u64 = 4 << 30;
18
19/// How the input was recognised.
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub enum Container {
22    /// Uncompressed DBF with all declared records present.
23    PlainDbf,
24    /// Uncompressed DBF with fewer bytes than the header declares.
25    TruncatedDbf,
26    /// DATASUS DBC: plain DBF header followed by a compressed record area.
27    Dbc,
28    /// A whole DBF file compressed as a single PKWare DCL stream.
29    WholeFileDbc,
30}
31
32enum Records {
33    InBuf(Range<usize>),
34    Owned(Vec<u8>),
35}
36
37/// An uncompressed DBF: header bytes and record area.
38pub struct DbfData {
39    buf: Vec<u8>,
40    header_len: usize,
41    records: Records,
42    container: Container,
43}
44
45impl DbfData {
46    pub fn header(&self) -> &[u8] {
47        &self.buf[..self.header_len]
48    }
49
50    pub fn records(&self) -> &[u8] {
51        match &self.records {
52            Records::InBuf(r) => &self.buf[r.clone()],
53            Records::Owned(v) => v,
54        }
55    }
56
57    pub fn container(&self) -> Container {
58        self.container
59    }
60}
61
62fn le_u16(p: &[u8]) -> u16 {
63    u16::from_le_bytes([p[0], p[1]])
64}
65
66fn le_u32(p: &[u8]) -> u32 {
67    u32::from_le_bytes([p[0], p[1], p[2], p[3]])
68}
69
70fn looks_like_dbf_header(buf: &[u8]) -> bool {
71    if buf.len() < 32 {
72        return false;
73    }
74    let version_ok = matches!(
75        buf[0],
76        0x02 | 0x03 | 0x04 | 0x05 | 0x30 | 0x31 | 0x32 | 0x43 | 0x63 | 0x7B | 0x83 | 0x8B | 0x8E
77            | 0xCB | 0xF5
78    );
79    if !version_ok {
80        return false;
81    }
82    let header_len = le_u16(&buf[8..]) as usize;
83    let record_len = le_u16(&buf[10..]) as usize;
84    if header_len < 33 || record_len < 1 || header_len >= buf.len() {
85        return false;
86    }
87    matches!(buf[header_len - 1], 0x0D | 0x00)
88}
89
90fn looks_like_complete_dbf(buf: &[u8]) -> bool {
91    if !looks_like_dbf_header(buf) {
92        return false;
93    }
94    let n_records = le_u32(&buf[4..]) as u64;
95    let header_len = le_u16(&buf[8..]) as u64;
96    let record_len = le_u16(&buf[10..]) as u64;
97    buf.len() as u64 >= header_len + n_records * record_len
98}
99
100/// Every record of a plain DBF starts with a deletion flag (`' '` live,
101/// `'*'` deleted), optionally followed by the 0x1A end-of-file marker.
102fn looks_like_plain_dbf_records(buf: &[u8], header_len: usize, record_len: usize) -> bool {
103    if header_len > buf.len() || record_len == 0 {
104        return false;
105    }
106    let mut off = header_len;
107    while off < buf.len() {
108        if off == buf.len() - 1 && buf[off] == 0x1A {
109            break;
110        }
111        if buf[off] != b' ' && buf[off] != b'*' {
112            return false;
113        }
114        off += record_len;
115    }
116    true
117}
118
119/// Capacity hint for the decompressed record area, bounded so a hostile
120/// header cannot trigger a huge allocation up front.
121fn capacity_hint(header: &[u8], compressed_len: usize) -> usize {
122    let declared = le_u32(&header[4..]) as u64 * le_u16(&header[10..]) as u64 + 1;
123    declared.min(compressed_len as u64 * 64).min(1 << 31) as usize
124}
125
126/// Decompress `payload` as the record area described by `header`.
127///
128/// Records past the count declared in the header are never read, so
129/// decompression stops once the declared area (plus the end-of-file marker)
130/// is complete; that also bounds what a crafted payload can expand to.
131fn blast_records(
132    header: &[u8],
133    payload: &[u8],
134    out: &mut Vec<u8>,
135    max_bytes: u64,
136) -> Result<(), BlastError> {
137    out.clear();
138    let declared = le_u32(&header[4..]) as u64 * le_u16(&header[10..]) as u64 + 1;
139    let limit = declared.min(max_bytes).min(usize::MAX as u64) as usize;
140    out.reserve(capacity_hint(header, payload.len()).min(limit));
141    match blast(payload, out, limit) {
142        Err(BlastError::OutputLimit) if declared <= max_bytes => Ok(()),
143        other => other,
144    }
145}
146
147/// Recognise the layout of `buf` and decompress the record area if needed.
148///
149/// Decompressed data is limited to [`DEFAULT_MAX_DECOMPRESSED`]; use
150/// [`decode_with_limit`] to change that.
151pub fn decode(buf: Vec<u8>) -> Result<DbfData, Error> {
152    decode_with_limit(buf, DEFAULT_MAX_DECOMPRESSED)
153}
154
155/// Same as [`decode`], failing with [`Error::TooLarge`] if the file would
156/// decompress to more than `max_bytes`.  Lower it when reading untrusted
157/// files: a PKWare DCL stream can expand to hundreds of times its size.
158pub fn decode_with_limit(buf: Vec<u8>, max_bytes: u64) -> Result<DbfData, Error> {
159    // A plain DBF must not only be long enough for the declared records:
160    // every record must also start with a deletion flag.  A small DBC whose
161    // compressed payload is larger than the uncompressed record area passes
162    // the size test alone.
163    if looks_like_complete_dbf(&buf) {
164        let header_len = le_u16(&buf[8..]) as usize;
165        let record_len = le_u16(&buf[10..]) as usize;
166        if looks_like_plain_dbf_records(&buf, header_len, record_len) {
167            let records = Records::InBuf(header_len..buf.len());
168            return Ok(DbfData { buf, header_len, records, container: Container::PlainDbf });
169        }
170    }
171
172    if looks_like_dbf_header(&buf) {
173        let header_len = le_u16(&buf[8..]) as usize;
174        let record_len = le_u16(&buf[10..]) as usize;
175
176        let mut out = Vec::new();
177        let mut result = Err(BlastError::OutOfInput);
178
179        if header_len + CRC_OFFSET < buf.len() {
180            result = blast_records(&buf, &buf[header_len + CRC_OFFSET..], &mut out, max_bytes);
181        }
182
183        // Some DBC files have no CRC bytes between header and payload.
184        if result.is_err() && result != Err(BlastError::OutputLimit) {
185            result = blast_records(&buf, &buf[header_len..], &mut out, max_bytes);
186        }
187
188        if let Err(e) = result {
189            if e == BlastError::OutputLimit {
190                return Err(Error::TooLarge { max_bytes });
191            }
192            let code = e.code();
193            // Not a DBC: a truncated plain DBF.  Read what is there.
194            if looks_like_plain_dbf_records(&buf, header_len, record_len) {
195                let records = Records::InBuf(header_len..buf.len());
196                return Ok(DbfData { buf, header_len, records, container: Container::TruncatedDbf });
197            }
198            return Err(Error::Blast { code, record_area: true });
199        }
200
201        return Ok(DbfData {
202            buf,
203            header_len,
204            records: Records::Owned(out),
205            container: Container::Dbc,
206        });
207    }
208
209    let mut out = Vec::new();
210    let limit = max_bytes.min(usize::MAX as u64) as usize;
211    match blast(&buf, &mut out, limit) {
212        Ok(()) => {}
213        Err(BlastError::OutputLimit) => return Err(Error::TooLarge { max_bytes }),
214        Err(e) => return Err(Error::Blast { code: e.code(), record_area: false }),
215    }
216    if !looks_like_complete_dbf(&out) {
217        return Err(Error::NotDbf);
218    }
219    let header_len = le_u16(&out[8..]) as usize;
220    let records = Records::InBuf(header_len..out.len());
221    Ok(DbfData { buf: out, header_len, records, container: Container::WholeFileDbc })
222}