From a3fc44fd3adc792de108b9b62618405033281b62 Mon Sep 17 00:00:00 2001 From: bchanot Date: Sun, 18 Oct 2026 15:44:52 +0200 Subject: [PATCH] feat(domain): R4.2a pdf::read scan, strings and stamp (port of scan.ts, JS regex semantics explicit) find_embedded_files, decode_pdf_string, has_archive_stamp on the frozen O7 oracle: 144 PDF inputs, 178 candidates, 19 string rows, all equal to the TypeScript. --- crates/core/src/pdf/read/scan.rs | 521 +++++++++++++++++++++++++++- crates/core/src/pdf/read/stamp.rs | 43 ++- crates/core/src/pdf/read/strings.rs | 122 ++++++- crates/core/tests/common/r4.rs | 13 +- crates/core/tests/pdf_scan.rs | 132 ++++++- 5 files changed, 819 insertions(+), 12 deletions(-) diff --git a/crates/core/src/pdf/read/scan.rs b/crates/core/src/pdf/read/scan.rs index ca5175ee..f10d3640 100644 --- a/crates/core/src/pdf/read/scan.rs +++ b/crates/core/src/pdf/read/scan.rs @@ -1,5 +1,39 @@ -//! R4.2a: `findEmbeddedFiles` of scan.ts (header, typed dictionaries, stream bounds, -//! Filespec names). Written by the R4.2a track; `index_of` is shared by the other files. +//! Every `/Type /EmbeddedFile` stream in clear: `findEmbeddedFiles` of +//! `packages/pdf/src/attachment/scan.ts` (typed dictionaries, `n 0 obj` headers, stream +//! bounds, Filespec names), byte for byte. +//! +//! Where the TypeScript ran a JavaScript regular expression on a Latin-1 window, a matcher +//! below walks the bytes with the same leftmost-match result: the start positions are tried +//! in order, and the backtracking of each pattern is resolved by hand where its classes +//! (JavaScript `\s`, digits, keywords) cannot overlap. The PDF helpers (`is_whitespace`, +//! `skip_whitespace`) keep their own whitespace set, NUL included and VT, NBSP excluded. + +use std::collections::BTreeMap; +use std::ops::Range; + +use super::strings::{decode_pdf_string, is_js_space, string_token_len}; +use super::{ + ATTACHMENT_NAME_PREFIX, EmbeddedFileCandidate, MAX_ATTACHMENT_BYTES, MAX_CANDIDATES, + MAX_DICT_BYTES, MAX_PDF_BYTES, StreamFilter, +}; + +/// Farthest `endstream` worth finding: a longer candidate is refused anyway. +pub const MEASURE_WINDOW: usize = MAX_ATTACHMENT_BYTES + 16; + +/// Window read before a dictionary for its `n 0 obj` header. +const HEADER_WINDOW: usize = 48; +/// The longest digit run `\d{1,10}` takes. +const MAX_DIGITS: usize = 10; + +const TYPE: &[u8] = b"/Type"; +const EMBEDDED_FILE: &[u8] = b"/EmbeddedFile"; +const FILESPEC: &[u8] = b"/Filespec"; +const STREAM: &[u8] = b"stream"; +const ENDSTREAM: &[u8] = b"endstream"; +const FLATE_DECODE: &[u8] = b"/FlateDecode"; + +// --------------------------------------------------------------------------- +// Byte helpers: every loop is bounded by `end` (at most MAX_PDF_BYTES) /// Naive forward search of `needle` in `bytes[from..end]`; the needle's first byte is the /// fast reject (scan.ts `indexOf`). `None` when absent or when the window is shorter than @@ -20,3 +54,486 @@ pub fn index_of(bytes: &[u8], needle: &[u8], from: usize, end: usize) -> Option< } None } + +/// PDF whitespace as the scanner reads it: NUL included, VT and NBSP excluded. +pub(super) fn is_whitespace(byte: u8) -> bool { + matches!(byte, 0x20 | 0x0A | 0x0D | 0x09 | 0x0C | 0x00) +} + +/// PDF delimiters and whitespace end a name token. +pub(super) fn is_delimiter(byte: u8) -> bool { + is_whitespace(byte) || b"()<>[]{}/%".contains(&byte) +} + +pub(super) fn skip_whitespace(bytes: &[u8], from: usize, end: usize) -> usize { + let mut i = from; + while i < end && bytes.get(i).is_some_and(|&b| is_whitespace(b)) { + i += 1; + } + i +} + +fn is_digit(byte: u8) -> bool { + byte.is_ascii_digit() +} + +/// `\w` without the `u` flag. +fn is_word(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} + +/// End of the run of `class` bytes starting at `from` (`from` itself when the run is empty). +fn run_end(text: &[u8], from: usize, class: fn(u8) -> bool) -> usize { + let mut i = from; + while text.get(i).is_some_and(|&b| class(b)) { + i += 1; + } + i +} + +/// Start of the run of `class` bytes ending at `to`. +fn run_start(text: &[u8], to: usize, class: fn(u8) -> bool) -> usize { + let mut i = to; + while i > 0 && text.get(i - 1).is_some_and(|&b| class(b)) { + i -= 1; + } + i +} + +/// `class+` forwards from `from`: the run end, `None` when the run is empty. +fn some_run_end(text: &[u8], from: usize, class: fn(u8) -> bool) -> Option { + let end = run_end(text, from, class); + (end > from).then_some(end) +} + +/// `class+` backwards from `to`: the run start, `None` when the run is empty. +fn some_run_start(text: &[u8], to: usize, class: fn(u8) -> bool) -> Option { + let start = run_start(text, to, class); + (start < to).then_some(start) +} + +/// `\d{1,10}` that no digit follows: the end of a whole run of one to ten digits (a longer +/// run fails, every backtracked length being followed by a digit). +fn short_number_end(text: &[u8], from: usize) -> Option { + let end = some_run_end(text, from, is_digit)?; + (end - from <= MAX_DIGITS).then_some(end) +} + +/// `Number()` of at most ten ASCII digits. +fn parse_digits(digits: &[u8]) -> u64 { + digits + .iter() + .fold(0, |n, &d| n * 10 + u64::from(d.wrapping_sub(b'0'))) +} + +/// Every start of `needle` in `text`, in order: the start positions a regular expression +/// beginning with that literal can match at. +fn occurrences<'a>(text: &'a [u8], needle: &'a [u8]) -> impl Iterator + 'a { + let mut from = 0; + std::iter::from_fn(move || { + let at = index_of(text, needle, from, text.len())?; + from = at + 1; + Some(at) + }) +} + +// --------------------------------------------------------------------------- +// Dictionaries: `/Type /Name` matches and their `<< … >>` bounds + +/// Every `/Type /` (space optional) whose name ends on a delimiter, at most +/// [`MAX_CANDIDATES`]; the positions are those of `/Type`. +fn find_typed_dicts(bytes: &[u8], name: &[u8], end: usize) -> Vec { + let mut hits = Vec::new(); + let mut from = 0; + while hits.len() < MAX_CANDIDATES { + let Some(at) = index_of(bytes, name, from, end) else { + break; + }; + from = at + name.len(); + if from < end && bytes.get(from).is_some_and(|&b| !is_delimiter(b)) { + continue; + } + if let Some(type_at) = type_key_before(bytes, at) { + hits.push(type_at); + } + } + hits +} + +/// The `/Type` key ending right before `at`, PDF whitespace between them skipped. +fn type_key_before(bytes: &[u8], at: usize) -> Option { + let before = run_start(bytes, at, is_whitespace); + let type_at = before.checked_sub(TYPE.len())?; + (bytes.get(type_at..before) == Some(TYPE)).then_some(type_at) +} + +/// Index of the `<<` opening the dictionary holding `pos` (nested dicts skipped). +fn dict_start(bytes: &[u8], pos: usize) -> Option { + let floor = pos.saturating_sub(MAX_DICT_BYTES); + let mut depth = 0usize; + let mut i = pos; + while i >= floor + 2 { + match bytes.get(i - 2..i)? { + [b'>', b'>'] => { + depth += 1; + i -= 2; + } + [b'<', b'<'] => { + if depth == 0 { + return Some(i - 2); + } + depth -= 1; + i -= 2; + } + _ => i -= 1, + } + } + None +} + +/// Index just past the `>>` closing the dictionary opened at `start`. +fn dict_end(bytes: &[u8], start: usize, end: usize) -> Option { + let limit = end.min(start + MAX_DICT_BYTES); + let mut depth = 0i64; + let mut i = start; + while i + 1 < limit { + match bytes.get(i..i + 2)? { + [b'<', b'<'] => { + depth += 1; + i += 2; + } + [b'>', b'>'] => { + depth -= 1; + if depth == 0 { + return Some(i + 2); + } + i += 2; + } + _ => i += 1, + } + } + None +} + +/// A dictionary's bounds and the object number of the `n 0 obj` header before it. +struct DictSpan { + start: usize, + end: usize, + object_number: Option, +} + +impl DictSpan { + fn text<'a>(&self, bytes: &'a [u8]) -> &'a [u8] { + bytes.get(self.start..self.end).unwrap_or_default() + } +} + +/// The dictionary around a `/Type` match and the header before it. +fn dict_around(bytes: &[u8], pos: usize, end: usize) -> Option { + let start = dict_start(bytes, pos)?; + let close = dict_end(bytes, start, end)?; + let window = bytes.get(start.saturating_sub(HEADER_WINDOW)..start)?; + Some(DictSpan { + start, + end: close, + object_number: object_number(window), + }) +} + +/// `/(\d{1,10})\s+\d{1,10}\s+obj\s*$/` on the window before a dictionary. The end anchor +/// fixes every part but the first: the leftmost match starts the object number up to ten +/// digits before its end, so a longer run keeps its last ten digits. +fn object_number(window: &[u8]) -> Option { + let keyword_end = run_start(window, window.len(), is_js_space); + let keyword = keyword_end.checked_sub(3)?; + if window.get(keyword..keyword_end) != Some(b"obj".as_slice()) { + return None; + } + let generation_end = some_run_start(window, keyword, is_js_space)?; + let generation = some_run_start(window, generation_end, is_digit)?; + if generation_end - generation > MAX_DIGITS { + return None; + } + let number_end = some_run_start(window, generation, is_js_space)?; + let run = some_run_start(window, number_end, is_digit)?; + let number = number_end - (number_end - run).min(MAX_DIGITS); + Some(parse_digits(window.get(number..number_end)?)) +} + +// --------------------------------------------------------------------------- +// Stream data bounds + +/// Data offset after `stream` + `\r\n` or `\n`. The EOL bytes are read from the whole file, +/// as `bytes[i]` did; an offset past `end` never yields a candidate. +fn stream_data_offset(bytes: &[u8], dict_close: usize, end: usize) -> Option { + let at = skip_whitespace(bytes, dict_close, end); + if index_of(bytes, STREAM, at, end.min(at + STREAM.len())) != Some(at) { + return None; + } + let mut i = at + STREAM.len(); + if bytes.get(i) == Some(&b'\r') { + i += 1; + } + (bytes.get(i) == Some(&b'\n')).then_some(i + 1) +} + +/// A direct `/Length N` (never an `N 0 R` reference), when it lands on `endstream`. +fn declared_length(bytes: &[u8], dict: &[u8], offset: usize, end: usize) -> Option { + let length = occurrences(dict, b"/Length").find_map(|at| direct_length_at(dict, at + 7))?; + let stop = offset.checked_add(usize::try_from(length).ok()?)?; + if stop > end { + return None; + } + let tail = skip_whitespace(bytes, stop, end); + let found = index_of(bytes, ENDSTREAM, tail, end.min(tail + ENDSTREAM.len())); + (found == Some(tail)).then_some(stop - offset) +} + +/// `\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)` right after a `/Length` key. +fn direct_length_at(dict: &[u8], from: usize) -> Option { + let digits = some_run_end(dict, from, is_js_space)?; + let after = short_number_end(dict, digits)?; + if reference_tail_at(dict, after) { + return None; + } + Some(parse_digits(dict.get(digits..after)?)) +} + +/// `\s+\d+\s+R` at `from` (every run whole: each class is followed by another). +fn reference_tail_at(text: &[u8], from: usize) -> bool { + let key = some_run_end(text, from, is_js_space) + .and_then(|digits| some_run_end(text, digits, is_digit)) + .and_then(|gap| some_run_end(text, gap, is_js_space)); + key.is_some_and(|key| text.get(key) == Some(&b'R')) +} + +/// Finds `endstream` once over the file instead of once per candidate (scan.ts +/// `endstreamFinder`): candidates come in offset order, so the last hit serves every offset +/// before it and a miss clears the bytes it covered; an offset out of order restarts. +struct EndstreamFinder<'a> { + bytes: &'a [u8], + end: usize, + last_hit: Option, + cleared_to: usize, + last_offset: usize, +} + +impl<'a> EndstreamFinder<'a> { + fn new(bytes: &'a [u8], end: usize) -> Self { + Self { + bytes, + end, + last_hit: None, + cleared_to: 0, + last_offset: 0, + } + } + + /// Index of the first `endstream` at or after `offset`, within [`MEASURE_WINDOW`]. + fn find(&mut self, offset: usize) -> Option { + if offset < self.last_offset { + self.last_hit = None; + self.cleared_to = 0; + } + self.last_offset = offset; + if let Some(hit) = self.last_hit.filter(|&hit| hit >= offset) { + return Some(hit); + } + let limit = self.end.min(offset.saturating_add(MEASURE_WINDOW)); + let at = index_of(self.bytes, ENDSTREAM, offset.max(self.cleared_to), limit); + match at { + Some(hit) => self.last_hit = Some(hit), + None => { + let covered = (limit + 1).saturating_sub(ENDSTREAM.len()); + self.cleared_to = self.cleared_to.max(covered); + } + } + at + } + + /// Fallback for an indirect or wrong `/Length`: up to `endstream`, one `\n` then one + /// `\r` before it dropped. + fn measured_length(&mut self, offset: usize) -> Option { + let mut stop = self.find(offset)?; + if stop > offset && self.bytes.get(stop - 1) == Some(&b'\n') { + stop -= 1; + } + if stop > offset && self.bytes.get(stop - 1) == Some(&b'\r') { + stop -= 1; + } + Some(stop - offset) + } +} + +/// `/\/Filter\s*(\[\s*)?\/FlateDecode\b/` anywhere in the dictionary. +fn filter_of(dict: &[u8]) -> StreamFilter { + if occurrences(dict, b"/Filter").any(|at| flate_at(dict, at + 7)) { + StreamFilter::FlateDecode + } else { + StreamFilter::None + } +} + +/// `\s*(\[\s*)?\/FlateDecode\b` at `from`; without the group a `[` cannot start the name. +fn flate_at(text: &[u8], from: usize) -> bool { + let mut name = run_end(text, from, is_js_space); + if text.get(name) == Some(&b'[') { + name = run_end(text, name + 1, is_js_space); + } + let rest = text.get(name..).unwrap_or_default(); + rest.starts_with(FLATE_DECODE) && !rest.get(FLATE_DECODE.len()).is_some_and(|&b| is_word(b)) +} + +/// An `EmbeddedFile` dictionary followed by `stream`, before its data is measured. +struct StreamHead { + dict: DictSpan, + offset: usize, +} + +/// Every `/Type /EmbeddedFile` dictionary followed by `stream`, sorted (stable) by data +/// offset: nested dictionaries list outermost first, and their descending offsets would +/// restart the `endstream` finder each time. +fn embedded_stream_heads(bytes: &[u8], end: usize) -> Vec { + let mut heads: Vec = find_typed_dicts(bytes, EMBEDDED_FILE, end) + .into_iter() + .filter_map(|pos| { + let dict = dict_around(bytes, pos, end)?; + let offset = stream_data_offset(bytes, dict.end, end)?; + Some(StreamHead { dict, offset }) + }) + .collect(); + heads.sort_by_key(|head| head.offset); + heads +} + +/// The head's data bounds, unnamed; the finder runs only when no direct length holds. +fn candidate_of( + bytes: &[u8], + head: &StreamHead, + end: usize, + finder: &mut EndstreamFinder<'_>, +) -> Option { + let dict = head.dict.text(bytes); + let length = declared_length(bytes, dict, head.offset, end) + .or_else(|| finder.measured_length(head.offset))?; + Some(EmbeddedFileCandidate { + offset: head.offset, + length, + filter: filter_of(dict), + name: None, + }) +} + +// --------------------------------------------------------------------------- +// Filespec names + +/// The first `/\/EF\s*<<[\s\S]*?>>/` match: its bounds and the bytes between `<<` and `>>`. +struct EfSpan { + start: usize, + inner: Range, + end: usize, +} + +fn ef_span(text: &[u8]) -> Option { + let (start, open) = occurrences(text, b"/EF").find_map(|at| { + let open = run_end(text, at + 3, is_js_space); + let opens = text.get(open..).is_some_and(|rest| rest.starts_with(b"<<")); + opens.then_some((at, open + 2)) + })?; + // Lazy: the first `>>`. A later `/EF` starts past this `<<`, so none can find a `>>` + // that this one missed. + let close = index_of(text, b">>", open, text.len())?; + Some(EfSpan { + start, + inner: open..close, + end: close + 2, + }) +} + +/// `/F` (else `/UF`) of a Filespec dictionary, the first `/EF` sub-dictionary set aside. +fn filespec_name(dict: &[u8]) -> Option { + let without_ef = match ef_span(dict) { + Some(ef) => [dict.get(..ef.start)?, dict.get(ef.end..)?].concat(), + None => dict.to_vec(), + }; + let token = keyed_token(&without_ef, b"/F").or_else(|| keyed_token(&without_ef, b"/UF"))?; + decode_pdf_string(token) +} + +/// `/\s*(STRING_TOKEN)`, leftmost: the token of the first key occurrence that has one. +fn keyed_token<'a>(text: &'a [u8], key: &[u8]) -> Option<&'a [u8]> { + occurrences(text, key).find_map(|at| { + let start = run_end(text, at + key.len(), is_js_space); + let length = string_token_len(text, start)?; + text.get(start..start + length) + }) +} + +/// Object number of the stream a Filespec's `/EF << /F n 0 R >>` points at. +fn filespec_stream_object(dict: &[u8]) -> Option { + let inner = dict.get(ef_span(dict)?.inner)?; + occurrences(inner, b"/F").find_map(|at| reference_at(inner, at + 2)) +} + +/// `\s+(\d{1,10})\s+\d{1,10}\s+R` at `from`: the referenced object number. +fn reference_at(text: &[u8], from: usize) -> Option { + let number = some_run_end(text, from, is_js_space)?; + let number_end = short_number_end(text, number)?; + let generation = some_run_end(text, number_end, is_js_space)?; + let generation_end = short_number_end(text, generation)?; + let key = some_run_end(text, generation_end, is_js_space)?; + if text.get(key) != Some(&b'R') { + return None; + } + Some(parse_digits(text.get(number..number_end)?)) +} + +/// Stream object number → file name, from every Filespec dictionary in clear; the first +/// Filespec naming an object wins. +fn filespec_names(bytes: &[u8], end: usize) -> BTreeMap { + let mut names = BTreeMap::new(); + for pos in find_typed_dicts(bytes, FILESPEC, end) { + let Some(dict) = dict_around(bytes, pos, end) else { + continue; + }; + let text = dict.text(bytes); + let target = filespec_stream_object(text); + let name = filespec_name(text); + if let (Some(target), Some(name)) = (target, name) { + names.entry(target).or_insert(name); + } + } + names +} + +// --------------------------------------------------------------------------- +// Public scan + +/// Every `/Type /EmbeddedFile` stream of the file, named when its Filespec is in clear +/// (export without object streams); `osteo-report*` names first, then data offset order. +/// At most [`MAX_CANDIDATES`], within [`MAX_PDF_BYTES`]. +pub fn find_embedded_files(bytes: &[u8]) -> Vec { + let end = bytes.len().min(MAX_PDF_BYTES); + let names = filespec_names(bytes, end); + let mut finder = EndstreamFinder::new(bytes, end); + let mut found: Vec = embedded_stream_heads(bytes, end) + .iter() + .filter_map(|head| { + let mut candidate = candidate_of(bytes, head, end, &mut finder)?; + candidate.name = head + .dict + .object_number + .and_then(|number| names.get(&number).cloned()); + Some(candidate) + }) + .collect(); + found.sort_by_key(|candidate| !is_attachment_name(candidate)); + found +} + +fn is_attachment_name(candidate: &EmbeddedFileCandidate) -> bool { + candidate + .name + .as_deref() + .is_some_and(|name| name.starts_with(ATTACHMENT_NAME_PREFIX)) +} diff --git a/crates/core/src/pdf/read/stamp.rs b/crates/core/src/pdf/read/stamp.rs index e32174b5..c0b7ad39 100644 --- a/crates/core/src/pdf/read/stamp.rs +++ b/crates/core/src/pdf/read/stamp.rs @@ -1 +1,42 @@ -//! R4.2a: `hasArchiveStamp` of scan.ts. Written by the R4.2a track. +//! The archive stamp: `hasArchiveStamp` and `keywordsAt` of +//! `packages/pdf/src/attachment/scan.ts`. Every archive carries `faunosteo-archive/` in +//! its `/Keywords`; a stamped file without a readable attachment was stripped by another tool. + +use super::scan::{index_of, is_delimiter, skip_whitespace}; +use super::strings::{decode_pdf_string, string_token_len}; +use super::{ARCHIVE_STAMP, MAX_CANDIDATES, MAX_DICT_BYTES, MAX_PDF_BYTES}; + +const KEYWORDS: &[u8] = b"/Keywords"; + +/// True when a visible `/Keywords` string carries the archive stamp. At most +/// [`MAX_CANDIDATES`] occurrences of the key are looked at, delimited or not. +pub fn has_archive_stamp(bytes: &[u8]) -> bool { + let end = bytes.len().min(MAX_PDF_BYTES); + let mut from = 0; + for _ in 0..MAX_CANDIDATES { + let Some(at) = index_of(bytes, KEYWORDS, from, end) else { + return false; + }; + from = at + KEYWORDS.len(); + // `bytes[from] ?? 0x20`: the file's own next byte, even past the size cap. + let next = bytes.get(from).copied().unwrap_or(b' '); + if is_delimiter(next) && keywords_at(bytes, at, end).is_some_and(stamped) { + return true; + } + } + false +} + +fn stamped(text: String) -> bool { + text.contains(ARCHIVE_STAMP) +} + +/// The string token right after the `/Keywords` key at `key_at`, as text; the token must +/// start the window and fit in its [`MAX_DICT_BYTES`]. +fn keywords_at(bytes: &[u8], key_at: usize, end: usize) -> Option { + let start = skip_whitespace(bytes, key_at + KEYWORDS.len(), end); + let limit = end.min(start + MAX_DICT_BYTES); + let window = bytes.get(start..limit)?; + let length = string_token_len(window, 0)?; + decode_pdf_string(window.get(..length)?) +} diff --git a/crates/core/src/pdf/read/strings.rs b/crates/core/src/pdf/read/strings.rs index f378fdc3..e3350768 100644 --- a/crates/core/src/pdf/read/strings.rs +++ b/crates/core/src/pdf/read/strings.rs @@ -1 +1,121 @@ -//! R4.2a: `decodePdfString` of scan.ts. Written by the R4.2a track. +//! PDF string tokens as text: `decodePdfString`, `decodeTextBytes` and the `STRING_TOKEN` +//! pattern of `packages/pdf/src/attachment/scan.ts`. +//! +//! The TypeScript matched `STRING_TOKEN` (`\((?:\\.|[^\\)])*\)|<[0-9a-fA-F\s]*>`) with a +//! JavaScript regular expression on a Latin-1 window, one character per byte. The matcher +//! below takes the bytes and gives the same answer: `\s` is JavaScript's set restricted to +//! Latin-1, and `.` refuses CR and LF only. + +/// JavaScript's `\s` on Latin-1 text: tab, LF, VT, FF, CR, space and NBSP, never NUL (unlike +/// the PDF whitespace of the scanner). +pub fn is_js_space(byte: u8) -> bool { + matches!(byte, 0x09..=0x0D | 0x20 | 0xA0) +} + +/// Length of the `STRING_TOKEN` starting at `at` in `text`, `None` when none starts there. +/// +/// In the literal branch the alternatives of `(?:\\.|[^\\)])*` never overlap (one takes a +/// backslash, the other refuses it) and no item starts on `)`, so backtracking cannot find +/// another closing parenthesis: a backslash before CR, LF or the end fails the token. +pub fn string_token_len(text: &[u8], at: usize) -> Option { + let end = match text.get(at)? { + b'(' => literal_end(text, at + 1)?, + b'<' => hex_end(text, at + 1)?, + _ => return None, + }; + Some(end - at) +} + +/// Index just past the `)` closing a literal whose body starts at `from`. +fn literal_end(text: &[u8], from: usize) -> Option { + let mut i = from; + loop { + match *text.get(i)? { + b')' => return Some(i + 1), + b'\\' => match text.get(i + 1) { + Some(&next) if next != b'\n' && next != b'\r' => i += 2, + _ => return None, + }, + _ => i += 1, + } + } +} + +/// Index just past the `>` closing a hex string whose body starts at `from`. +fn hex_end(text: &[u8], from: usize) -> Option { + let mut i = from; + while text + .get(i) + .is_some_and(|&b| b.is_ascii_hexdigit() || is_js_space(b)) + { + i += 1; + } + (text.get(i) == Some(&b'>')).then_some(i + 1) +} + +/// A `(literal)` or `` string token as text; `None` for anything else. `token` holds +/// the raw bytes (the TypeScript took them as a Latin-1 string). +pub fn decode_pdf_string(token: &[u8]) -> Option { + if let Some(hex) = enclosed(token, b'<', b'>') { + return Some(decode_text_bytes(&hex_bytes(hex))); + } + let literal = enclosed(token, b'(', b')')?; + Some(decode_text_bytes(&unescape_literal(literal))) +} + +/// The body of `token` when it starts with `open` and ends with `close`. +fn enclosed(token: &[u8], open: u8, close: u8) -> Option<&[u8]> { + match token { + [first, body @ .., last] if *first == open && *last == close => Some(body), + _ => None, + } +} + +/// Hex digits only, read in pairs; an odd trailing nibble is dropped. +fn hex_bytes(body: &[u8]) -> Vec { + let nibbles: Vec = body.iter().filter_map(|&b| nibble(b)).collect(); + let (pairs, _odd) = nibbles.as_chunks::<2>(); + pairs.iter().map(|&[high, low]| (high << 4) | low).collect() +} + +fn nibble(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +/// `body.replace(/\\([()\\])/g, '$1')`: only the three escapes the writer produces are +/// undone; any other backslash stays as is (`\n`, `\157` are kept literally). +fn unescape_literal(body: &[u8]) -> Vec { + let mut out = Vec::with_capacity(body.len()); + let mut i = 0; + while let Some(&byte) = body.get(i) { + match body.get(i + 1) { + Some(&next @ (b'(' | b')' | b'\\')) if byte == b'\\' => { + out.push(next); + i += 2; + } + _ => { + out.push(byte); + i += 1; + } + } + } + out +} + +/// UTF-16BE after a `FE FF` mark (how pdf-lib writes text strings; an odd trailing byte is +/// dropped, a lone surrogate becomes U+FFFD), else Latin-1 bytes. +pub fn decode_text_bytes(raw: &[u8]) -> String { + match raw { + [0xFE, 0xFF, rest @ ..] => { + let (pairs, _odd) = rest.as_chunks::<2>(); + let units: Vec = pairs.iter().map(|&pair| u16::from_be_bytes(pair)).collect(); + String::from_utf16_lossy(&units) + } + _ => raw.iter().map(|&b| char::from(b)).collect(), + } +} diff --git a/crates/core/tests/common/r4.rs b/crates/core/tests/common/r4.rs index 5b6d6469..dbdae274 100644 --- a/crates/core/tests/common/r4.rs +++ b/crates/core/tests/common/r4.rs @@ -39,11 +39,18 @@ pub fn sha256_hex(data: &[u8]) -> String { pub fn r4_input(dir: &str, file: &str) -> Vec { let base = repo_root().join("apps/app/tests/fixtures/pdf/r4").join(dir); let stored = base.join(file); - if stored.exists() { + if stored.exists() && !file.ends_with(".gen.json") { return std::fs::read(&stored).unwrap_or_else(|e| panic!("{stored:?}: {e}")); } - let slug = file.rsplit_once('.').map_or(file, |(stem, _)| stem); - let description = base.join(format!("{slug}.gen.json")); + // A generator row names its description (`.gen.json`); a stored name may also be + // asked for by its would-be byte name (`.pdf`). + let description = match file.strip_suffix(".gen.json") { + Some(_) => base.join(file), + None => base.join(format!( + "{}.gen.json", + file.rsplit_once('.').map_or(file, |(stem, _)| stem) + )), + }; let text = std::fs::read_to_string(&description) .unwrap_or_else(|e| panic!("{description:?}: {e} (no stored bytes for {file})")); let spec: Value = serde_json::from_str(&text).unwrap(); diff --git a/crates/core/tests/pdf_scan.rs b/crates/core/tests/pdf_scan.rs index 1f1317b7..71c788b3 100644 --- a/crates/core/tests/pdf_scan.rs +++ b/crates/core/tests/pdf_scan.rs @@ -1,10 +1,132 @@ -//! R4.2a placeholder: the O7 scan suite, written by its track. +//! O7 scan suite (R4.2a): `find_embedded_files`, `has_archive_stamp` and `decode_pdf_string` +//! against the outcomes the TypeScript recorded (`oracles/r4/read.json`), on every `pdf` +//! input of the oracle and every `decodePdfString` row. +#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] + mod common; +use common::r4::{latin1_bytes, r4_input, r4_oracle, sha256_hex}; +use faunosteo_core::pdf::read::scan::find_embedded_files; +use faunosteo_core::pdf::read::stamp::has_archive_stamp; +use faunosteo_core::pdf::read::strings::decode_pdf_string; +use faunosteo_core::pdf::read::{EmbeddedFileCandidate, StreamFilter}; +use serde_json::{Value, json}; + +/// The bytes of an oracle input, checked against the row's length and digest. A generator +/// row names its `.gen.json` description, which `r4_input` expands from any file name +/// of that slug (the description itself is stored under the row's name). +fn input_bytes(row: &Value) -> Vec { + let file = row["file"].as_str().unwrap(); + if row["stored"] == "generator" { + let slug = file.strip_suffix(".gen.json").unwrap(); + return r4_input("read", &format!("{slug}.pdf")); + } + let bytes = r4_input("read", file); + assert_eq!( + bytes.len() as u64, + row["length"].as_u64().unwrap(), + "{file}: length" + ); + assert_eq!( + sha256_hex(&bytes), + row["sha256"].as_str().unwrap(), + "{file}: digest" + ); + bytes +} + +fn candidate_json(candidate: &EmbeddedFileCandidate) -> Value { + let filter = match candidate.filter { + StreamFilter::FlateDecode => json!("FlateDecode"), + StreamFilter::None => Value::Null, + }; + json!({ + "name": candidate.name, + "offset": candidate.offset, + "length": candidate.length, + "filter": filter, + }) +} + +#[derive(Default)] +struct Tally { + inputs: usize, + candidates: usize, + stamps: usize, + failures: Vec, +} + +impl Tally { + fn check(&mut self, file: &str, field: &str, actual: Value, expected: &Value) { + if actual != *expected { + self.failures + .push(format!("{file}: {field}: got {actual}, oracle {expected}")); + } + } + + fn scan(&mut self, row: &Value) { + let file = row["file"].as_str().unwrap(); + let bytes = input_bytes(row); + let outcomes = &row["outcomes"]; + self.inputs += 1; + if let Some(expected) = outcomes.get("findEmbeddedFiles") { + let found: Vec = find_embedded_files(&bytes) + .iter() + .map(candidate_json) + .collect(); + self.candidates += found.len(); + self.check(file, "findEmbeddedFiles", Value::Array(found), expected); + } + if let Some(expected) = outcomes.get("hasArchiveStamp") { + self.stamps += 1; + self.check( + file, + "hasArchiveStamp", + json!(has_archive_stamp(&bytes)), + expected, + ); + } + } +} + #[test] -fn placeholder_reads_an_oracle() { - let oracle = common::r4::r4_oracle("read.json"); +fn every_pdf_input_scans_as_the_typescript_did() { + let oracle = r4_oracle("read.json"); assert_eq!(oracle["oracle"], "O7"); - let bytes = common::r4::r4_input("read", "scan.demo-export.pdf"); - assert_eq!(bytes.len(), 78463); + let mut tally = Tally::default(); + let rows = oracle["inputs"].as_array().unwrap(); + for row in rows.iter().filter(|row| row["kind"] == "pdf") { + tally.scan(row); + } + assert!(tally.failures.is_empty(), "{}", tally.failures.join("\n")); + assert_eq!(tally.inputs, 144, "pdf inputs"); + assert!(tally.candidates > 0 && tally.stamps == tally.inputs); + println!( + "O7 scan: {} pdf inputs of {}, {} candidates, {} stamp outcomes", + tally.inputs, + rows.len(), + tally.candidates, + tally.stamps + ); +} + +#[test] +fn decode_pdf_string_rows_match() { + let oracle = r4_oracle("read.json"); + let rows = oracle["decodePdfString"].as_array().unwrap(); + let failures: Vec = rows + .iter() + .filter_map(|row| { + let token = row["token"].as_str().unwrap(); + let actual = json!(decode_pdf_string(&latin1_bytes(token))); + (actual != row["result"]).then(|| { + format!( + "decodePdfString({token:?}): got {actual}, oracle {}", + row["result"] + ) + }) + }) + .collect(); + assert!(failures.is_empty(), "{}", failures.join("\n")); + assert_eq!(rows.len(), 19, "decodePdfString rows"); }