feat(domain): R4.2a pdf::read scan, strings and stamp (port of scan.ts, JS regex semantics explicit)

find_embedded_files, decode_pdf_string, has_archive_stamp on the frozen O7 oracle: 144 PDF
inputs, 178 candidates, 19 string rows, all equal to the TypeScript.
This commit is contained in:
bchanot committed 2026-10-18 15:44:52 +02:00
1 parent 09ce18b6dc
commit a3fc44fd3a
5 files changed
+819 -12

No files matched your search

+519 -2
View File
@@ -1,5 +1,39 @@
//! R4.2a: `findEmbeddedFiles` of scan.ts (header, typed dictionaries, stream bounds,
//! Filespec names). Written by the R4.2a track; `index_of` is shared by the other files.
//! Every `/Type /EmbeddedFile` stream in clear: `findEmbeddedFiles` of
//! `packages/pdf/src/attachment/scan.ts` (typed dictionaries, `n 0 obj` headers, stream
//! bounds, Filespec names), byte for byte.
//!
//! Where the TypeScript ran a JavaScript regular expression on a Latin-1 window, a matcher
//! below walks the bytes with the same leftmost-match result: the start positions are tried
//! in order, and the backtracking of each pattern is resolved by hand where its classes
//! (JavaScript `\s`, digits, keywords) cannot overlap. The PDF helpers (`is_whitespace`,
//! `skip_whitespace`) keep their own whitespace set, NUL included and VT, NBSP excluded.
use std::collections::BTreeMap;
use std::ops::Range;
use super::strings::{decode_pdf_string, is_js_space, string_token_len};
use super::{
ATTACHMENT_NAME_PREFIX, EmbeddedFileCandidate, MAX_ATTACHMENT_BYTES, MAX_CANDIDATES,
MAX_DICT_BYTES, MAX_PDF_BYTES, StreamFilter,
};
/// Farthest `endstream` worth finding: a longer candidate is refused anyway.
pub const MEASURE_WINDOW: usize = MAX_ATTACHMENT_BYTES + 16;
/// Window read before a dictionary for its `n 0 obj` header.
const HEADER_WINDOW: usize = 48;
/// The longest digit run `\d{1,10}` takes.
const MAX_DIGITS: usize = 10;
const TYPE: &[u8] = b"/Type";
const EMBEDDED_FILE: &[u8] = b"/EmbeddedFile";
const FILESPEC: &[u8] = b"/Filespec";
const STREAM: &[u8] = b"stream";
const ENDSTREAM: &[u8] = b"endstream";
const FLATE_DECODE: &[u8] = b"/FlateDecode";
// ---------------------------------------------------------------------------
// Byte helpers: every loop is bounded by `end` (at most MAX_PDF_BYTES)
/// Naive forward search of `needle` in `bytes[from..end]`; the needle's first byte is the
/// fast reject (scan.ts `indexOf`). `None` when absent or when the window is shorter than
@@ -20,3 +54,486 @@ pub fn index_of(bytes: &[u8], needle: &[u8], from: usize, end: usize) -> Option<
}
None
}
/// PDF whitespace as the scanner reads it: NUL included, VT and NBSP excluded.
pub(super) fn is_whitespace(byte: u8) -> bool {
matches!(byte, 0x20 | 0x0A | 0x0D | 0x09 | 0x0C | 0x00)
}
/// PDF delimiters and whitespace end a name token.
pub(super) fn is_delimiter(byte: u8) -> bool {
is_whitespace(byte) || b"()<>[]{}/%".contains(&byte)
}
pub(super) fn skip_whitespace(bytes: &[u8], from: usize, end: usize) -> usize {
let mut i = from;
while i < end && bytes.get(i).is_some_and(|&b| is_whitespace(b)) {
i += 1;
}
i
}
fn is_digit(byte: u8) -> bool {
byte.is_ascii_digit()
}
/// `\w` without the `u` flag.
fn is_word(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || byte == b'_'
}
/// End of the run of `class` bytes starting at `from` (`from` itself when the run is empty).
fn run_end(text: &[u8], from: usize, class: fn(u8) -> bool) -> usize {
let mut i = from;
while text.get(i).is_some_and(|&b| class(b)) {
i += 1;
}
i
}
/// Start of the run of `class` bytes ending at `to`.
fn run_start(text: &[u8], to: usize, class: fn(u8) -> bool) -> usize {
let mut i = to;
while i > 0 && text.get(i - 1).is_some_and(|&b| class(b)) {
i -= 1;
}
i
}
/// `class+` forwards from `from`: the run end, `None` when the run is empty.
fn some_run_end(text: &[u8], from: usize, class: fn(u8) -> bool) -> Option<usize> {
let end = run_end(text, from, class);
(end > from).then_some(end)
}
/// `class+` backwards from `to`: the run start, `None` when the run is empty.
fn some_run_start(text: &[u8], to: usize, class: fn(u8) -> bool) -> Option<usize> {
let start = run_start(text, to, class);
(start < to).then_some(start)
}
/// `\d{1,10}` that no digit follows: the end of a whole run of one to ten digits (a longer
/// run fails, every backtracked length being followed by a digit).
fn short_number_end(text: &[u8], from: usize) -> Option<usize> {
let end = some_run_end(text, from, is_digit)?;
(end - from <= MAX_DIGITS).then_some(end)
}
/// `Number()` of at most ten ASCII digits.
fn parse_digits(digits: &[u8]) -> u64 {
digits
.iter()
.fold(0, |n, &d| n * 10 + u64::from(d.wrapping_sub(b'0')))
}
/// Every start of `needle` in `text`, in order: the start positions a regular expression
/// beginning with that literal can match at.
fn occurrences<'a>(text: &'a [u8], needle: &'a [u8]) -> impl Iterator<Item = usize> + 'a {
let mut from = 0;
std::iter::from_fn(move || {
let at = index_of(text, needle, from, text.len())?;
from = at + 1;
Some(at)
})
}
// ---------------------------------------------------------------------------
// Dictionaries: `/Type /Name` matches and their `<< … >>` bounds
/// Every `/Type /<name>` (space optional) whose name ends on a delimiter, at most
/// [`MAX_CANDIDATES`]; the positions are those of `/Type`.
fn find_typed_dicts(bytes: &[u8], name: &[u8], end: usize) -> Vec<usize> {
let mut hits = Vec::new();
let mut from = 0;
while hits.len() < MAX_CANDIDATES {
let Some(at) = index_of(bytes, name, from, end) else {
break;
};
from = at + name.len();
if from < end && bytes.get(from).is_some_and(|&b| !is_delimiter(b)) {
continue;
}
if let Some(type_at) = type_key_before(bytes, at) {
hits.push(type_at);
}
}
hits
}
/// The `/Type` key ending right before `at`, PDF whitespace between them skipped.
fn type_key_before(bytes: &[u8], at: usize) -> Option<usize> {
let before = run_start(bytes, at, is_whitespace);
let type_at = before.checked_sub(TYPE.len())?;
(bytes.get(type_at..before) == Some(TYPE)).then_some(type_at)
}
/// Index of the `<<` opening the dictionary holding `pos` (nested dicts skipped).
fn dict_start(bytes: &[u8], pos: usize) -> Option<usize> {
let floor = pos.saturating_sub(MAX_DICT_BYTES);
let mut depth = 0usize;
let mut i = pos;
while i >= floor + 2 {
match bytes.get(i - 2..i)? {
[b'>', b'>'] => {
depth += 1;
i -= 2;
}
[b'<', b'<'] => {
if depth == 0 {
return Some(i - 2);
}
depth -= 1;
i -= 2;
}
_ => i -= 1,
}
}
None
}
/// Index just past the `>>` closing the dictionary opened at `start`.
fn dict_end(bytes: &[u8], start: usize, end: usize) -> Option<usize> {
let limit = end.min(start + MAX_DICT_BYTES);
let mut depth = 0i64;
let mut i = start;
while i + 1 < limit {
match bytes.get(i..i + 2)? {
[b'<', b'<'] => {
depth += 1;
i += 2;
}
[b'>', b'>'] => {
depth -= 1;
if depth == 0 {
return Some(i + 2);
}
i += 2;
}
_ => i += 1,
}
}
None
}
/// A dictionary's bounds and the object number of the `n 0 obj` header before it.
struct DictSpan {
start: usize,
end: usize,
object_number: Option<u64>,
}
impl DictSpan {
fn text<'a>(&self, bytes: &'a [u8]) -> &'a [u8] {
bytes.get(self.start..self.end).unwrap_or_default()
}
}
/// The dictionary around a `/Type` match and the header before it.
fn dict_around(bytes: &[u8], pos: usize, end: usize) -> Option<DictSpan> {
let start = dict_start(bytes, pos)?;
let close = dict_end(bytes, start, end)?;
let window = bytes.get(start.saturating_sub(HEADER_WINDOW)..start)?;
Some(DictSpan {
start,
end: close,
object_number: object_number(window),
})
}
/// `/(\d{1,10})\s+\d{1,10}\s+obj\s*$/` on the window before a dictionary. The end anchor
/// fixes every part but the first: the leftmost match starts the object number up to ten
/// digits before its end, so a longer run keeps its last ten digits.
fn object_number(window: &[u8]) -> Option<u64> {
let keyword_end = run_start(window, window.len(), is_js_space);
let keyword = keyword_end.checked_sub(3)?;
if window.get(keyword..keyword_end) != Some(b"obj".as_slice()) {
return None;
}
let generation_end = some_run_start(window, keyword, is_js_space)?;
let generation = some_run_start(window, generation_end, is_digit)?;
if generation_end - generation > MAX_DIGITS {
return None;
}
let number_end = some_run_start(window, generation, is_js_space)?;
let run = some_run_start(window, number_end, is_digit)?;
let number = number_end - (number_end - run).min(MAX_DIGITS);
Some(parse_digits(window.get(number..number_end)?))
}
// ---------------------------------------------------------------------------
// Stream data bounds
/// Data offset after `stream` + `\r\n` or `\n`. The EOL bytes are read from the whole file,
/// as `bytes[i]` did; an offset past `end` never yields a candidate.
fn stream_data_offset(bytes: &[u8], dict_close: usize, end: usize) -> Option<usize> {
let at = skip_whitespace(bytes, dict_close, end);
if index_of(bytes, STREAM, at, end.min(at + STREAM.len())) != Some(at) {
return None;
}
let mut i = at + STREAM.len();
if bytes.get(i) == Some(&b'\r') {
i += 1;
}
(bytes.get(i) == Some(&b'\n')).then_some(i + 1)
}
/// A direct `/Length N` (never an `N 0 R` reference), when it lands on `endstream`.
fn declared_length(bytes: &[u8], dict: &[u8], offset: usize, end: usize) -> Option<usize> {
let length = occurrences(dict, b"/Length").find_map(|at| direct_length_at(dict, at + 7))?;
let stop = offset.checked_add(usize::try_from(length).ok()?)?;
if stop > end {
return None;
}
let tail = skip_whitespace(bytes, stop, end);
let found = index_of(bytes, ENDSTREAM, tail, end.min(tail + ENDSTREAM.len()));
(found == Some(tail)).then_some(stop - offset)
}
/// `\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)` right after a `/Length` key.
fn direct_length_at(dict: &[u8], from: usize) -> Option<u64> {
let digits = some_run_end(dict, from, is_js_space)?;
let after = short_number_end(dict, digits)?;
if reference_tail_at(dict, after) {
return None;
}
Some(parse_digits(dict.get(digits..after)?))
}
/// `\s+\d+\s+R` at `from` (every run whole: each class is followed by another).
fn reference_tail_at(text: &[u8], from: usize) -> bool {
let key = some_run_end(text, from, is_js_space)
.and_then(|digits| some_run_end(text, digits, is_digit))
.and_then(|gap| some_run_end(text, gap, is_js_space));
key.is_some_and(|key| text.get(key) == Some(&b'R'))
}
/// Finds `endstream` once over the file instead of once per candidate (scan.ts
/// `endstreamFinder`): candidates come in offset order, so the last hit serves every offset
/// before it and a miss clears the bytes it covered; an offset out of order restarts.
struct EndstreamFinder<'a> {
bytes: &'a [u8],
end: usize,
last_hit: Option<usize>,
cleared_to: usize,
last_offset: usize,
}
impl<'a> EndstreamFinder<'a> {
fn new(bytes: &'a [u8], end: usize) -> Self {
Self {
bytes,
end,
last_hit: None,
cleared_to: 0,
last_offset: 0,
}
}
/// Index of the first `endstream` at or after `offset`, within [`MEASURE_WINDOW`].
fn find(&mut self, offset: usize) -> Option<usize> {
if offset < self.last_offset {
self.last_hit = None;
self.cleared_to = 0;
}
self.last_offset = offset;
if let Some(hit) = self.last_hit.filter(|&hit| hit >= offset) {
return Some(hit);
}
let limit = self.end.min(offset.saturating_add(MEASURE_WINDOW));
let at = index_of(self.bytes, ENDSTREAM, offset.max(self.cleared_to), limit);
match at {
Some(hit) => self.last_hit = Some(hit),
None => {
let covered = (limit + 1).saturating_sub(ENDSTREAM.len());
self.cleared_to = self.cleared_to.max(covered);
}
}
at
}
/// Fallback for an indirect or wrong `/Length`: up to `endstream`, one `\n` then one
/// `\r` before it dropped.
fn measured_length(&mut self, offset: usize) -> Option<usize> {
let mut stop = self.find(offset)?;
if stop > offset && self.bytes.get(stop - 1) == Some(&b'\n') {
stop -= 1;
}
if stop > offset && self.bytes.get(stop - 1) == Some(&b'\r') {
stop -= 1;
}
Some(stop - offset)
}
}
/// `/\/Filter\s*(\[\s*)?\/FlateDecode\b/` anywhere in the dictionary.
fn filter_of(dict: &[u8]) -> StreamFilter {
if occurrences(dict, b"/Filter").any(|at| flate_at(dict, at + 7)) {
StreamFilter::FlateDecode
} else {
StreamFilter::None
}
}
/// `\s*(\[\s*)?\/FlateDecode\b` at `from`; without the group a `[` cannot start the name.
fn flate_at(text: &[u8], from: usize) -> bool {
let mut name = run_end(text, from, is_js_space);
if text.get(name) == Some(&b'[') {
name = run_end(text, name + 1, is_js_space);
}
let rest = text.get(name..).unwrap_or_default();
rest.starts_with(FLATE_DECODE) && !rest.get(FLATE_DECODE.len()).is_some_and(|&b| is_word(b))
}
/// An `EmbeddedFile` dictionary followed by `stream`, before its data is measured.
struct StreamHead {
dict: DictSpan,
offset: usize,
}
/// Every `/Type /EmbeddedFile` dictionary followed by `stream`, sorted (stable) by data
/// offset: nested dictionaries list outermost first, and their descending offsets would
/// restart the `endstream` finder each time.
fn embedded_stream_heads(bytes: &[u8], end: usize) -> Vec<StreamHead> {
let mut heads: Vec<StreamHead> = find_typed_dicts(bytes, EMBEDDED_FILE, end)
.into_iter()
.filter_map(|pos| {
let dict = dict_around(bytes, pos, end)?;
let offset = stream_data_offset(bytes, dict.end, end)?;
Some(StreamHead { dict, offset })
})
.collect();
heads.sort_by_key(|head| head.offset);
heads
}
/// The head's data bounds, unnamed; the finder runs only when no direct length holds.
fn candidate_of(
bytes: &[u8],
head: &StreamHead,
end: usize,
finder: &mut EndstreamFinder<'_>,
) -> Option<EmbeddedFileCandidate> {
let dict = head.dict.text(bytes);
let length = declared_length(bytes, dict, head.offset, end)
.or_else(|| finder.measured_length(head.offset))?;
Some(EmbeddedFileCandidate {
offset: head.offset,
length,
filter: filter_of(dict),
name: None,
})
}
// ---------------------------------------------------------------------------
// Filespec names
/// The first `/\/EF\s*<<[\s\S]*?>>/` match: its bounds and the bytes between `<<` and `>>`.
struct EfSpan {
start: usize,
inner: Range<usize>,
end: usize,
}
fn ef_span(text: &[u8]) -> Option<EfSpan> {
let (start, open) = occurrences(text, b"/EF").find_map(|at| {
let open = run_end(text, at + 3, is_js_space);
let opens = text.get(open..).is_some_and(|rest| rest.starts_with(b"<<"));
opens.then_some((at, open + 2))
})?;
// Lazy: the first `>>`. A later `/EF` starts past this `<<`, so none can find a `>>`
// that this one missed.
let close = index_of(text, b">>", open, text.len())?;
Some(EfSpan {
start,
inner: open..close,
end: close + 2,
})
}
/// `/F` (else `/UF`) of a Filespec dictionary, the first `/EF` sub-dictionary set aside.
fn filespec_name(dict: &[u8]) -> Option<String> {
let without_ef = match ef_span(dict) {
Some(ef) => [dict.get(..ef.start)?, dict.get(ef.end..)?].concat(),
None => dict.to_vec(),
};
let token = keyed_token(&without_ef, b"/F").or_else(|| keyed_token(&without_ef, b"/UF"))?;
decode_pdf_string(token)
}
/// `/<key>\s*(STRING_TOKEN)`, leftmost: the token of the first key occurrence that has one.
fn keyed_token<'a>(text: &'a [u8], key: &[u8]) -> Option<&'a [u8]> {
occurrences(text, key).find_map(|at| {
let start = run_end(text, at + key.len(), is_js_space);
let length = string_token_len(text, start)?;
text.get(start..start + length)
})
}
/// Object number of the stream a Filespec's `/EF << /F n 0 R >>` points at.
fn filespec_stream_object(dict: &[u8]) -> Option<u64> {
let inner = dict.get(ef_span(dict)?.inner)?;
occurrences(inner, b"/F").find_map(|at| reference_at(inner, at + 2))
}
/// `\s+(\d{1,10})\s+\d{1,10}\s+R` at `from`: the referenced object number.
fn reference_at(text: &[u8], from: usize) -> Option<u64> {
let number = some_run_end(text, from, is_js_space)?;
let number_end = short_number_end(text, number)?;
let generation = some_run_end(text, number_end, is_js_space)?;
let generation_end = short_number_end(text, generation)?;
let key = some_run_end(text, generation_end, is_js_space)?;
if text.get(key) != Some(&b'R') {
return None;
}
Some(parse_digits(text.get(number..number_end)?))
}
/// Stream object number → file name, from every Filespec dictionary in clear; the first
/// Filespec naming an object wins.
fn filespec_names(bytes: &[u8], end: usize) -> BTreeMap<u64, String> {
let mut names = BTreeMap::new();
for pos in find_typed_dicts(bytes, FILESPEC, end) {
let Some(dict) = dict_around(bytes, pos, end) else {
continue;
};
let text = dict.text(bytes);
let target = filespec_stream_object(text);
let name = filespec_name(text);
if let (Some(target), Some(name)) = (target, name) {
names.entry(target).or_insert(name);
}
}
names
}
// ---------------------------------------------------------------------------
// Public scan
/// Every `/Type /EmbeddedFile` stream of the file, named when its Filespec is in clear
/// (export without object streams); `osteo-report*` names first, then data offset order.
/// At most [`MAX_CANDIDATES`], within [`MAX_PDF_BYTES`].
pub fn find_embedded_files(bytes: &[u8]) -> Vec<EmbeddedFileCandidate> {
let end = bytes.len().min(MAX_PDF_BYTES);
let names = filespec_names(bytes, end);
let mut finder = EndstreamFinder::new(bytes, end);
let mut found: Vec<EmbeddedFileCandidate> = embedded_stream_heads(bytes, end)
.iter()
.filter_map(|head| {
let mut candidate = candidate_of(bytes, head, end, &mut finder)?;
candidate.name = head
.dict
.object_number
.and_then(|number| names.get(&number).cloned());
Some(candidate)
})
.collect();
found.sort_by_key(|candidate| !is_attachment_name(candidate));
found
}
fn is_attachment_name(candidate: &EmbeddedFileCandidate) -> bool {
candidate
.name
.as_deref()
.is_some_and(|name| name.starts_with(ATTACHMENT_NAME_PREFIX))
}
+42 -1
View File
@@ -1 +1,42 @@
//! R4.2a: `hasArchiveStamp` of scan.ts. Written by the R4.2a track.
//! The archive stamp: `hasArchiveStamp` and `keywordsAt` of
//! `packages/pdf/src/attachment/scan.ts`. Every archive carries `faunosteo-archive/<n>` in
//! its `/Keywords`; a stamped file without a readable attachment was stripped by another tool.
use super::scan::{index_of, is_delimiter, skip_whitespace};
use super::strings::{decode_pdf_string, string_token_len};
use super::{ARCHIVE_STAMP, MAX_CANDIDATES, MAX_DICT_BYTES, MAX_PDF_BYTES};
const KEYWORDS: &[u8] = b"/Keywords";
/// True when a visible `/Keywords` string carries the archive stamp. At most
/// [`MAX_CANDIDATES`] occurrences of the key are looked at, delimited or not.
pub fn has_archive_stamp(bytes: &[u8]) -> bool {
let end = bytes.len().min(MAX_PDF_BYTES);
let mut from = 0;
for _ in 0..MAX_CANDIDATES {
let Some(at) = index_of(bytes, KEYWORDS, from, end) else {
return false;
};
from = at + KEYWORDS.len();
// `bytes[from] ?? 0x20`: the file's own next byte, even past the size cap.
let next = bytes.get(from).copied().unwrap_or(b' ');
if is_delimiter(next) && keywords_at(bytes, at, end).is_some_and(stamped) {
return true;
}
}
false
}
fn stamped(text: String) -> bool {
text.contains(ARCHIVE_STAMP)
}
/// The string token right after the `/Keywords` key at `key_at`, as text; the token must
/// start the window and fit in its [`MAX_DICT_BYTES`].
fn keywords_at(bytes: &[u8], key_at: usize, end: usize) -> Option<String> {
let start = skip_whitespace(bytes, key_at + KEYWORDS.len(), end);
let limit = end.min(start + MAX_DICT_BYTES);
let window = bytes.get(start..limit)?;
let length = string_token_len(window, 0)?;
decode_pdf_string(window.get(..length)?)
}
+121 -1
View File
@@ -1 +1,121 @@
//! R4.2a: `decodePdfString` of scan.ts. Written by the R4.2a track.
//! PDF string tokens as text: `decodePdfString`, `decodeTextBytes` and the `STRING_TOKEN`
//! pattern of `packages/pdf/src/attachment/scan.ts`.
//!
//! The TypeScript matched `STRING_TOKEN` (`\((?:\\.|[^\\)])*\)|<[0-9a-fA-F\s]*>`) with a
//! JavaScript regular expression on a Latin-1 window, one character per byte. The matcher
//! below takes the bytes and gives the same answer: `\s` is JavaScript's set restricted to
//! Latin-1, and `.` refuses CR and LF only.
/// JavaScript's `\s` on Latin-1 text: tab, LF, VT, FF, CR, space and NBSP, never NUL (unlike
/// the PDF whitespace of the scanner).
pub fn is_js_space(byte: u8) -> bool {
matches!(byte, 0x09..=0x0D | 0x20 | 0xA0)
}
/// Length of the `STRING_TOKEN` starting at `at` in `text`, `None` when none starts there.
///
/// In the literal branch the alternatives of `(?:\\.|[^\\)])*` never overlap (one takes a
/// backslash, the other refuses it) and no item starts on `)`, so backtracking cannot find
/// another closing parenthesis: a backslash before CR, LF or the end fails the token.
pub fn string_token_len(text: &[u8], at: usize) -> Option<usize> {
let end = match text.get(at)? {
b'(' => literal_end(text, at + 1)?,
b'<' => hex_end(text, at + 1)?,
_ => return None,
};
Some(end - at)
}
/// Index just past the `)` closing a literal whose body starts at `from`.
fn literal_end(text: &[u8], from: usize) -> Option<usize> {
let mut i = from;
loop {
match *text.get(i)? {
b')' => return Some(i + 1),
b'\\' => match text.get(i + 1) {
Some(&next) if next != b'\n' && next != b'\r' => i += 2,
_ => return None,
},
_ => i += 1,
}
}
}
/// Index just past the `>` closing a hex string whose body starts at `from`.
fn hex_end(text: &[u8], from: usize) -> Option<usize> {
let mut i = from;
while text
.get(i)
.is_some_and(|&b| b.is_ascii_hexdigit() || is_js_space(b))
{
i += 1;
}
(text.get(i) == Some(&b'>')).then_some(i + 1)
}
/// A `(literal)` or `<hex>` string token as text; `None` for anything else. `token` holds
/// the raw bytes (the TypeScript took them as a Latin-1 string).
pub fn decode_pdf_string(token: &[u8]) -> Option<String> {
if let Some(hex) = enclosed(token, b'<', b'>') {
return Some(decode_text_bytes(&hex_bytes(hex)));
}
let literal = enclosed(token, b'(', b')')?;
Some(decode_text_bytes(&unescape_literal(literal)))
}
/// The body of `token` when it starts with `open` and ends with `close`.
fn enclosed(token: &[u8], open: u8, close: u8) -> Option<&[u8]> {
match token {
[first, body @ .., last] if *first == open && *last == close => Some(body),
_ => None,
}
}
/// Hex digits only, read in pairs; an odd trailing nibble is dropped.
fn hex_bytes(body: &[u8]) -> Vec<u8> {
let nibbles: Vec<u8> = body.iter().filter_map(|&b| nibble(b)).collect();
let (pairs, _odd) = nibbles.as_chunks::<2>();
pairs.iter().map(|&[high, low]| (high << 4) | low).collect()
}
fn nibble(byte: u8) -> Option<u8> {
match byte {
b'0'..=b'9' => Some(byte - b'0'),
b'a'..=b'f' => Some(byte - b'a' + 10),
b'A'..=b'F' => Some(byte - b'A' + 10),
_ => None,
}
}
/// `body.replace(/\\([()\\])/g, '$1')`: only the three escapes the writer produces are
/// undone; any other backslash stays as is (`\n`, `\157` are kept literally).
fn unescape_literal(body: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(body.len());
let mut i = 0;
while let Some(&byte) = body.get(i) {
match body.get(i + 1) {
Some(&next @ (b'(' | b')' | b'\\')) if byte == b'\\' => {
out.push(next);
i += 2;
}
_ => {
out.push(byte);
i += 1;
}
}
}
out
}
/// UTF-16BE after a `FE FF` mark (how pdf-lib writes text strings; an odd trailing byte is
/// dropped, a lone surrogate becomes U+FFFD), else Latin-1 bytes.
pub fn decode_text_bytes(raw: &[u8]) -> String {
match raw {
[0xFE, 0xFF, rest @ ..] => {
let (pairs, _odd) = rest.as_chunks::<2>();
let units: Vec<u16> = pairs.iter().map(|&pair| u16::from_be_bytes(pair)).collect();
String::from_utf16_lossy(&units)
}
_ => raw.iter().map(|&b| char::from(b)).collect(),
}
}
+10 -3
View File
@@ -39,11 +39,18 @@ pub fn sha256_hex(data: &[u8]) -> String {
pub fn r4_input(dir: &str, file: &str) -> Vec<u8> {
let base = repo_root().join("apps/app/tests/fixtures/pdf/r4").join(dir);
let stored = base.join(file);
if stored.exists() {
if stored.exists() && !file.ends_with(".gen.json") {
return std::fs::read(&stored).unwrap_or_else(|e| panic!("{stored:?}: {e}"));
}
let slug = file.rsplit_once('.').map_or(file, |(stem, _)| stem);
let description = base.join(format!("{slug}.gen.json"));
// A generator row names its description (`<slug>.gen.json`); a stored name may also be
// asked for by its would-be byte name (`<slug>.pdf`).
let description = match file.strip_suffix(".gen.json") {
Some(_) => base.join(file),
None => base.join(format!(
"{}.gen.json",
file.rsplit_once('.').map_or(file, |(stem, _)| stem)
)),
};
let text = std::fs::read_to_string(&description)
.unwrap_or_else(|e| panic!("{description:?}: {e} (no stored bytes for {file})"));
let spec: Value = serde_json::from_str(&text).unwrap();
+127 -5
View File
@@ -1,10 +1,132 @@
//! R4.2a placeholder: the O7 scan suite, written by its track.
//! O7 scan suite (R4.2a): `find_embedded_files`, `has_archive_stamp` and `decode_pdf_string`
//! against the outcomes the TypeScript recorded (`oracles/r4/read.json`), on every `pdf`
//! input of the oracle and every `decodePdfString` row.
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
mod common;
use common::r4::{latin1_bytes, r4_input, r4_oracle, sha256_hex};
use faunosteo_core::pdf::read::scan::find_embedded_files;
use faunosteo_core::pdf::read::stamp::has_archive_stamp;
use faunosteo_core::pdf::read::strings::decode_pdf_string;
use faunosteo_core::pdf::read::{EmbeddedFileCandidate, StreamFilter};
use serde_json::{Value, json};
/// The bytes of an oracle input, checked against the row's length and digest. A generator
/// row names its `<slug>.gen.json` description, which `r4_input` expands from any file name
/// of that slug (the description itself is stored under the row's name).
fn input_bytes(row: &Value) -> Vec<u8> {
let file = row["file"].as_str().unwrap();
if row["stored"] == "generator" {
let slug = file.strip_suffix(".gen.json").unwrap();
return r4_input("read", &format!("{slug}.pdf"));
}
let bytes = r4_input("read", file);
assert_eq!(
bytes.len() as u64,
row["length"].as_u64().unwrap(),
"{file}: length"
);
assert_eq!(
sha256_hex(&bytes),
row["sha256"].as_str().unwrap(),
"{file}: digest"
);
bytes
}
fn candidate_json(candidate: &EmbeddedFileCandidate) -> Value {
let filter = match candidate.filter {
StreamFilter::FlateDecode => json!("FlateDecode"),
StreamFilter::None => Value::Null,
};
json!({
"name": candidate.name,
"offset": candidate.offset,
"length": candidate.length,
"filter": filter,
})
}
#[derive(Default)]
struct Tally {
inputs: usize,
candidates: usize,
stamps: usize,
failures: Vec<String>,
}
impl Tally {
fn check(&mut self, file: &str, field: &str, actual: Value, expected: &Value) {
if actual != *expected {
self.failures
.push(format!("{file}: {field}: got {actual}, oracle {expected}"));
}
}
fn scan(&mut self, row: &Value) {
let file = row["file"].as_str().unwrap();
let bytes = input_bytes(row);
let outcomes = &row["outcomes"];
self.inputs += 1;
if let Some(expected) = outcomes.get("findEmbeddedFiles") {
let found: Vec<Value> = find_embedded_files(&bytes)
.iter()
.map(candidate_json)
.collect();
self.candidates += found.len();
self.check(file, "findEmbeddedFiles", Value::Array(found), expected);
}
if let Some(expected) = outcomes.get("hasArchiveStamp") {
self.stamps += 1;
self.check(
file,
"hasArchiveStamp",
json!(has_archive_stamp(&bytes)),
expected,
);
}
}
}
#[test]
fn placeholder_reads_an_oracle() {
let oracle = common::r4::r4_oracle("read.json");
fn every_pdf_input_scans_as_the_typescript_did() {
let oracle = r4_oracle("read.json");
assert_eq!(oracle["oracle"], "O7");
let bytes = common::r4::r4_input("read", "scan.demo-export.pdf");
assert_eq!(bytes.len(), 78463);
let mut tally = Tally::default();
let rows = oracle["inputs"].as_array().unwrap();
for row in rows.iter().filter(|row| row["kind"] == "pdf") {
tally.scan(row);
}
assert!(tally.failures.is_empty(), "{}", tally.failures.join("\n"));
assert_eq!(tally.inputs, 144, "pdf inputs");
assert!(tally.candidates > 0 && tally.stamps == tally.inputs);
println!(
"O7 scan: {} pdf inputs of {}, {} candidates, {} stamp outcomes",
tally.inputs,
rows.len(),
tally.candidates,
tally.stamps
);
}
#[test]
fn decode_pdf_string_rows_match() {
let oracle = r4_oracle("read.json");
let rows = oracle["decodePdfString"].as_array().unwrap();
let failures: Vec<String> = rows
.iter()
.filter_map(|row| {
let token = row["token"].as_str().unwrap();
let actual = json!(decode_pdf_string(&latin1_bytes(token)));
(actual != row["result"]).then(|| {
format!(
"decodePdfString({token:?}): got {actual}, oracle {}",
row["result"]
)
})
})
.collect();
assert!(failures.is_empty(), "{}", failures.join("\n"));
assert_eq!(rows.len(), 19, "decodePdfString rows");
}