feat(domain): R4.2a pdf::read scan, strings and stamp (port of scan.ts, JS regex semantics explicit)
find_embedded_files, decode_pdf_string, has_archive_stamp on the frozen O7 oracle: 144 PDF inputs, 178 candidates, 19 string rows, all equal to the TypeScript.
This commit is contained in:
1 parent
09ce18b6dc
commit
a3fc44fd3a
5 files changed
+819
-12
No files matched your search
@@ -1,5 +1,39 @@
|
||||
//! R4.2a: `findEmbeddedFiles` of scan.ts (header, typed dictionaries, stream bounds,
|
||||
//! Filespec names). Written by the R4.2a track; `index_of` is shared by the other files.
|
||||
//! Every `/Type /EmbeddedFile` stream in clear: `findEmbeddedFiles` of
|
||||
//! `packages/pdf/src/attachment/scan.ts` (typed dictionaries, `n 0 obj` headers, stream
|
||||
//! bounds, Filespec names), byte for byte.
|
||||
//!
|
||||
//! Where the TypeScript ran a JavaScript regular expression on a Latin-1 window, a matcher
|
||||
//! below walks the bytes with the same leftmost-match result: the start positions are tried
|
||||
//! in order, and the backtracking of each pattern is resolved by hand where its classes
|
||||
//! (JavaScript `\s`, digits, keywords) cannot overlap. The PDF helpers (`is_whitespace`,
|
||||
//! `skip_whitespace`) keep their own whitespace set, NUL included and VT, NBSP excluded.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::ops::Range;
|
||||
|
||||
use super::strings::{decode_pdf_string, is_js_space, string_token_len};
|
||||
use super::{
|
||||
ATTACHMENT_NAME_PREFIX, EmbeddedFileCandidate, MAX_ATTACHMENT_BYTES, MAX_CANDIDATES,
|
||||
MAX_DICT_BYTES, MAX_PDF_BYTES, StreamFilter,
|
||||
};
|
||||
|
||||
/// Farthest `endstream` worth finding: a longer candidate is refused anyway.
|
||||
pub const MEASURE_WINDOW: usize = MAX_ATTACHMENT_BYTES + 16;
|
||||
|
||||
/// Window read before a dictionary for its `n 0 obj` header.
|
||||
const HEADER_WINDOW: usize = 48;
|
||||
/// The longest digit run `\d{1,10}` takes.
|
||||
const MAX_DIGITS: usize = 10;
|
||||
|
||||
const TYPE: &[u8] = b"/Type";
|
||||
const EMBEDDED_FILE: &[u8] = b"/EmbeddedFile";
|
||||
const FILESPEC: &[u8] = b"/Filespec";
|
||||
const STREAM: &[u8] = b"stream";
|
||||
const ENDSTREAM: &[u8] = b"endstream";
|
||||
const FLATE_DECODE: &[u8] = b"/FlateDecode";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Byte helpers: every loop is bounded by `end` (at most MAX_PDF_BYTES)
|
||||
|
||||
/// Naive forward search of `needle` in `bytes[from..end]`; the needle's first byte is the
|
||||
/// fast reject (scan.ts `indexOf`). `None` when absent or when the window is shorter than
|
||||
@@ -20,3 +54,486 @@ pub fn index_of(bytes: &[u8], needle: &[u8], from: usize, end: usize) -> Option<
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// PDF whitespace as the scanner reads it: NUL included, VT and NBSP excluded.
|
||||
pub(super) fn is_whitespace(byte: u8) -> bool {
|
||||
matches!(byte, 0x20 | 0x0A | 0x0D | 0x09 | 0x0C | 0x00)
|
||||
}
|
||||
|
||||
/// PDF delimiters and whitespace end a name token.
|
||||
pub(super) fn is_delimiter(byte: u8) -> bool {
|
||||
is_whitespace(byte) || b"()<>[]{}/%".contains(&byte)
|
||||
}
|
||||
|
||||
pub(super) fn skip_whitespace(bytes: &[u8], from: usize, end: usize) -> usize {
|
||||
let mut i = from;
|
||||
while i < end && bytes.get(i).is_some_and(|&b| is_whitespace(b)) {
|
||||
i += 1;
|
||||
}
|
||||
i
|
||||
}
|
||||
|
||||
fn is_digit(byte: u8) -> bool {
|
||||
byte.is_ascii_digit()
|
||||
}
|
||||
|
||||
/// `\w` without the `u` flag.
|
||||
fn is_word(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || byte == b'_'
|
||||
}
|
||||
|
||||
/// End of the run of `class` bytes starting at `from` (`from` itself when the run is empty).
|
||||
fn run_end(text: &[u8], from: usize, class: fn(u8) -> bool) -> usize {
|
||||
let mut i = from;
|
||||
while text.get(i).is_some_and(|&b| class(b)) {
|
||||
i += 1;
|
||||
}
|
||||
i
|
||||
}
|
||||
|
||||
/// Start of the run of `class` bytes ending at `to`.
|
||||
fn run_start(text: &[u8], to: usize, class: fn(u8) -> bool) -> usize {
|
||||
let mut i = to;
|
||||
while i > 0 && text.get(i - 1).is_some_and(|&b| class(b)) {
|
||||
i -= 1;
|
||||
}
|
||||
i
|
||||
}
|
||||
|
||||
/// `class+` forwards from `from`: the run end, `None` when the run is empty.
|
||||
fn some_run_end(text: &[u8], from: usize, class: fn(u8) -> bool) -> Option<usize> {
|
||||
let end = run_end(text, from, class);
|
||||
(end > from).then_some(end)
|
||||
}
|
||||
|
||||
/// `class+` backwards from `to`: the run start, `None` when the run is empty.
|
||||
fn some_run_start(text: &[u8], to: usize, class: fn(u8) -> bool) -> Option<usize> {
|
||||
let start = run_start(text, to, class);
|
||||
(start < to).then_some(start)
|
||||
}
|
||||
|
||||
/// `\d{1,10}` that no digit follows: the end of a whole run of one to ten digits (a longer
|
||||
/// run fails, every backtracked length being followed by a digit).
|
||||
fn short_number_end(text: &[u8], from: usize) -> Option<usize> {
|
||||
let end = some_run_end(text, from, is_digit)?;
|
||||
(end - from <= MAX_DIGITS).then_some(end)
|
||||
}
|
||||
|
||||
/// `Number()` of at most ten ASCII digits.
|
||||
fn parse_digits(digits: &[u8]) -> u64 {
|
||||
digits
|
||||
.iter()
|
||||
.fold(0, |n, &d| n * 10 + u64::from(d.wrapping_sub(b'0')))
|
||||
}
|
||||
|
||||
/// Every start of `needle` in `text`, in order: the start positions a regular expression
|
||||
/// beginning with that literal can match at.
|
||||
fn occurrences<'a>(text: &'a [u8], needle: &'a [u8]) -> impl Iterator<Item = usize> + 'a {
|
||||
let mut from = 0;
|
||||
std::iter::from_fn(move || {
|
||||
let at = index_of(text, needle, from, text.len())?;
|
||||
from = at + 1;
|
||||
Some(at)
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Dictionaries: `/Type /Name` matches and their `<< … >>` bounds
|
||||
|
||||
/// Every `/Type /<name>` (space optional) whose name ends on a delimiter, at most
|
||||
/// [`MAX_CANDIDATES`]; the positions are those of `/Type`.
|
||||
fn find_typed_dicts(bytes: &[u8], name: &[u8], end: usize) -> Vec<usize> {
|
||||
let mut hits = Vec::new();
|
||||
let mut from = 0;
|
||||
while hits.len() < MAX_CANDIDATES {
|
||||
let Some(at) = index_of(bytes, name, from, end) else {
|
||||
break;
|
||||
};
|
||||
from = at + name.len();
|
||||
if from < end && bytes.get(from).is_some_and(|&b| !is_delimiter(b)) {
|
||||
continue;
|
||||
}
|
||||
if let Some(type_at) = type_key_before(bytes, at) {
|
||||
hits.push(type_at);
|
||||
}
|
||||
}
|
||||
hits
|
||||
}
|
||||
|
||||
/// The `/Type` key ending right before `at`, PDF whitespace between them skipped.
|
||||
fn type_key_before(bytes: &[u8], at: usize) -> Option<usize> {
|
||||
let before = run_start(bytes, at, is_whitespace);
|
||||
let type_at = before.checked_sub(TYPE.len())?;
|
||||
(bytes.get(type_at..before) == Some(TYPE)).then_some(type_at)
|
||||
}
|
||||
|
||||
/// Index of the `<<` opening the dictionary holding `pos` (nested dicts skipped).
|
||||
fn dict_start(bytes: &[u8], pos: usize) -> Option<usize> {
|
||||
let floor = pos.saturating_sub(MAX_DICT_BYTES);
|
||||
let mut depth = 0usize;
|
||||
let mut i = pos;
|
||||
while i >= floor + 2 {
|
||||
match bytes.get(i - 2..i)? {
|
||||
[b'>', b'>'] => {
|
||||
depth += 1;
|
||||
i -= 2;
|
||||
}
|
||||
[b'<', b'<'] => {
|
||||
if depth == 0 {
|
||||
return Some(i - 2);
|
||||
}
|
||||
depth -= 1;
|
||||
i -= 2;
|
||||
}
|
||||
_ => i -= 1,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Index just past the `>>` closing the dictionary opened at `start`.
|
||||
fn dict_end(bytes: &[u8], start: usize, end: usize) -> Option<usize> {
|
||||
let limit = end.min(start + MAX_DICT_BYTES);
|
||||
let mut depth = 0i64;
|
||||
let mut i = start;
|
||||
while i + 1 < limit {
|
||||
match bytes.get(i..i + 2)? {
|
||||
[b'<', b'<'] => {
|
||||
depth += 1;
|
||||
i += 2;
|
||||
}
|
||||
[b'>', b'>'] => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
return Some(i + 2);
|
||||
}
|
||||
i += 2;
|
||||
}
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// A dictionary's bounds and the object number of the `n 0 obj` header before it.
|
||||
struct DictSpan {
|
||||
start: usize,
|
||||
end: usize,
|
||||
object_number: Option<u64>,
|
||||
}
|
||||
|
||||
impl DictSpan {
|
||||
fn text<'a>(&self, bytes: &'a [u8]) -> &'a [u8] {
|
||||
bytes.get(self.start..self.end).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The dictionary around a `/Type` match and the header before it.
|
||||
fn dict_around(bytes: &[u8], pos: usize, end: usize) -> Option<DictSpan> {
|
||||
let start = dict_start(bytes, pos)?;
|
||||
let close = dict_end(bytes, start, end)?;
|
||||
let window = bytes.get(start.saturating_sub(HEADER_WINDOW)..start)?;
|
||||
Some(DictSpan {
|
||||
start,
|
||||
end: close,
|
||||
object_number: object_number(window),
|
||||
})
|
||||
}
|
||||
|
||||
/// `/(\d{1,10})\s+\d{1,10}\s+obj\s*$/` on the window before a dictionary. The end anchor
|
||||
/// fixes every part but the first: the leftmost match starts the object number up to ten
|
||||
/// digits before its end, so a longer run keeps its last ten digits.
|
||||
fn object_number(window: &[u8]) -> Option<u64> {
|
||||
let keyword_end = run_start(window, window.len(), is_js_space);
|
||||
let keyword = keyword_end.checked_sub(3)?;
|
||||
if window.get(keyword..keyword_end) != Some(b"obj".as_slice()) {
|
||||
return None;
|
||||
}
|
||||
let generation_end = some_run_start(window, keyword, is_js_space)?;
|
||||
let generation = some_run_start(window, generation_end, is_digit)?;
|
||||
if generation_end - generation > MAX_DIGITS {
|
||||
return None;
|
||||
}
|
||||
let number_end = some_run_start(window, generation, is_js_space)?;
|
||||
let run = some_run_start(window, number_end, is_digit)?;
|
||||
let number = number_end - (number_end - run).min(MAX_DIGITS);
|
||||
Some(parse_digits(window.get(number..number_end)?))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stream data bounds
|
||||
|
||||
/// Data offset after `stream` + `\r\n` or `\n`. The EOL bytes are read from the whole file,
|
||||
/// as `bytes[i]` did; an offset past `end` never yields a candidate.
|
||||
fn stream_data_offset(bytes: &[u8], dict_close: usize, end: usize) -> Option<usize> {
|
||||
let at = skip_whitespace(bytes, dict_close, end);
|
||||
if index_of(bytes, STREAM, at, end.min(at + STREAM.len())) != Some(at) {
|
||||
return None;
|
||||
}
|
||||
let mut i = at + STREAM.len();
|
||||
if bytes.get(i) == Some(&b'\r') {
|
||||
i += 1;
|
||||
}
|
||||
(bytes.get(i) == Some(&b'\n')).then_some(i + 1)
|
||||
}
|
||||
|
||||
/// A direct `/Length N` (never an `N 0 R` reference), when it lands on `endstream`.
|
||||
fn declared_length(bytes: &[u8], dict: &[u8], offset: usize, end: usize) -> Option<usize> {
|
||||
let length = occurrences(dict, b"/Length").find_map(|at| direct_length_at(dict, at + 7))?;
|
||||
let stop = offset.checked_add(usize::try_from(length).ok()?)?;
|
||||
if stop > end {
|
||||
return None;
|
||||
}
|
||||
let tail = skip_whitespace(bytes, stop, end);
|
||||
let found = index_of(bytes, ENDSTREAM, tail, end.min(tail + ENDSTREAM.len()));
|
||||
(found == Some(tail)).then_some(stop - offset)
|
||||
}
|
||||
|
||||
/// `\s+(\d{1,10})(?!\d)(?!\s+\d+\s+R)` right after a `/Length` key.
|
||||
fn direct_length_at(dict: &[u8], from: usize) -> Option<u64> {
|
||||
let digits = some_run_end(dict, from, is_js_space)?;
|
||||
let after = short_number_end(dict, digits)?;
|
||||
if reference_tail_at(dict, after) {
|
||||
return None;
|
||||
}
|
||||
Some(parse_digits(dict.get(digits..after)?))
|
||||
}
|
||||
|
||||
/// `\s+\d+\s+R` at `from` (every run whole: each class is followed by another).
|
||||
fn reference_tail_at(text: &[u8], from: usize) -> bool {
|
||||
let key = some_run_end(text, from, is_js_space)
|
||||
.and_then(|digits| some_run_end(text, digits, is_digit))
|
||||
.and_then(|gap| some_run_end(text, gap, is_js_space));
|
||||
key.is_some_and(|key| text.get(key) == Some(&b'R'))
|
||||
}
|
||||
|
||||
/// Finds `endstream` once over the file instead of once per candidate (scan.ts
|
||||
/// `endstreamFinder`): candidates come in offset order, so the last hit serves every offset
|
||||
/// before it and a miss clears the bytes it covered; an offset out of order restarts.
|
||||
struct EndstreamFinder<'a> {
|
||||
bytes: &'a [u8],
|
||||
end: usize,
|
||||
last_hit: Option<usize>,
|
||||
cleared_to: usize,
|
||||
last_offset: usize,
|
||||
}
|
||||
|
||||
impl<'a> EndstreamFinder<'a> {
|
||||
fn new(bytes: &'a [u8], end: usize) -> Self {
|
||||
Self {
|
||||
bytes,
|
||||
end,
|
||||
last_hit: None,
|
||||
cleared_to: 0,
|
||||
last_offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Index of the first `endstream` at or after `offset`, within [`MEASURE_WINDOW`].
|
||||
fn find(&mut self, offset: usize) -> Option<usize> {
|
||||
if offset < self.last_offset {
|
||||
self.last_hit = None;
|
||||
self.cleared_to = 0;
|
||||
}
|
||||
self.last_offset = offset;
|
||||
if let Some(hit) = self.last_hit.filter(|&hit| hit >= offset) {
|
||||
return Some(hit);
|
||||
}
|
||||
let limit = self.end.min(offset.saturating_add(MEASURE_WINDOW));
|
||||
let at = index_of(self.bytes, ENDSTREAM, offset.max(self.cleared_to), limit);
|
||||
match at {
|
||||
Some(hit) => self.last_hit = Some(hit),
|
||||
None => {
|
||||
let covered = (limit + 1).saturating_sub(ENDSTREAM.len());
|
||||
self.cleared_to = self.cleared_to.max(covered);
|
||||
}
|
||||
}
|
||||
at
|
||||
}
|
||||
|
||||
/// Fallback for an indirect or wrong `/Length`: up to `endstream`, one `\n` then one
|
||||
/// `\r` before it dropped.
|
||||
fn measured_length(&mut self, offset: usize) -> Option<usize> {
|
||||
let mut stop = self.find(offset)?;
|
||||
if stop > offset && self.bytes.get(stop - 1) == Some(&b'\n') {
|
||||
stop -= 1;
|
||||
}
|
||||
if stop > offset && self.bytes.get(stop - 1) == Some(&b'\r') {
|
||||
stop -= 1;
|
||||
}
|
||||
Some(stop - offset)
|
||||
}
|
||||
}
|
||||
|
||||
/// `/\/Filter\s*(\[\s*)?\/FlateDecode\b/` anywhere in the dictionary.
|
||||
fn filter_of(dict: &[u8]) -> StreamFilter {
|
||||
if occurrences(dict, b"/Filter").any(|at| flate_at(dict, at + 7)) {
|
||||
StreamFilter::FlateDecode
|
||||
} else {
|
||||
StreamFilter::None
|
||||
}
|
||||
}
|
||||
|
||||
/// `\s*(\[\s*)?\/FlateDecode\b` at `from`; without the group a `[` cannot start the name.
|
||||
fn flate_at(text: &[u8], from: usize) -> bool {
|
||||
let mut name = run_end(text, from, is_js_space);
|
||||
if text.get(name) == Some(&b'[') {
|
||||
name = run_end(text, name + 1, is_js_space);
|
||||
}
|
||||
let rest = text.get(name..).unwrap_or_default();
|
||||
rest.starts_with(FLATE_DECODE) && !rest.get(FLATE_DECODE.len()).is_some_and(|&b| is_word(b))
|
||||
}
|
||||
|
||||
/// An `EmbeddedFile` dictionary followed by `stream`, before its data is measured.
|
||||
struct StreamHead {
|
||||
dict: DictSpan,
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
/// Every `/Type /EmbeddedFile` dictionary followed by `stream`, sorted (stable) by data
|
||||
/// offset: nested dictionaries list outermost first, and their descending offsets would
|
||||
/// restart the `endstream` finder each time.
|
||||
fn embedded_stream_heads(bytes: &[u8], end: usize) -> Vec<StreamHead> {
|
||||
let mut heads: Vec<StreamHead> = find_typed_dicts(bytes, EMBEDDED_FILE, end)
|
||||
.into_iter()
|
||||
.filter_map(|pos| {
|
||||
let dict = dict_around(bytes, pos, end)?;
|
||||
let offset = stream_data_offset(bytes, dict.end, end)?;
|
||||
Some(StreamHead { dict, offset })
|
||||
})
|
||||
.collect();
|
||||
heads.sort_by_key(|head| head.offset);
|
||||
heads
|
||||
}
|
||||
|
||||
/// The head's data bounds, unnamed; the finder runs only when no direct length holds.
|
||||
fn candidate_of(
|
||||
bytes: &[u8],
|
||||
head: &StreamHead,
|
||||
end: usize,
|
||||
finder: &mut EndstreamFinder<'_>,
|
||||
) -> Option<EmbeddedFileCandidate> {
|
||||
let dict = head.dict.text(bytes);
|
||||
let length = declared_length(bytes, dict, head.offset, end)
|
||||
.or_else(|| finder.measured_length(head.offset))?;
|
||||
Some(EmbeddedFileCandidate {
|
||||
offset: head.offset,
|
||||
length,
|
||||
filter: filter_of(dict),
|
||||
name: None,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Filespec names
|
||||
|
||||
/// The first `/\/EF\s*<<[\s\S]*?>>/` match: its bounds and the bytes between `<<` and `>>`.
|
||||
struct EfSpan {
|
||||
start: usize,
|
||||
inner: Range<usize>,
|
||||
end: usize,
|
||||
}
|
||||
|
||||
fn ef_span(text: &[u8]) -> Option<EfSpan> {
|
||||
let (start, open) = occurrences(text, b"/EF").find_map(|at| {
|
||||
let open = run_end(text, at + 3, is_js_space);
|
||||
let opens = text.get(open..).is_some_and(|rest| rest.starts_with(b"<<"));
|
||||
opens.then_some((at, open + 2))
|
||||
})?;
|
||||
// Lazy: the first `>>`. A later `/EF` starts past this `<<`, so none can find a `>>`
|
||||
// that this one missed.
|
||||
let close = index_of(text, b">>", open, text.len())?;
|
||||
Some(EfSpan {
|
||||
start,
|
||||
inner: open..close,
|
||||
end: close + 2,
|
||||
})
|
||||
}
|
||||
|
||||
/// `/F` (else `/UF`) of a Filespec dictionary, the first `/EF` sub-dictionary set aside.
|
||||
fn filespec_name(dict: &[u8]) -> Option<String> {
|
||||
let without_ef = match ef_span(dict) {
|
||||
Some(ef) => [dict.get(..ef.start)?, dict.get(ef.end..)?].concat(),
|
||||
None => dict.to_vec(),
|
||||
};
|
||||
let token = keyed_token(&without_ef, b"/F").or_else(|| keyed_token(&without_ef, b"/UF"))?;
|
||||
decode_pdf_string(token)
|
||||
}
|
||||
|
||||
/// `/<key>\s*(STRING_TOKEN)`, leftmost: the token of the first key occurrence that has one.
|
||||
fn keyed_token<'a>(text: &'a [u8], key: &[u8]) -> Option<&'a [u8]> {
|
||||
occurrences(text, key).find_map(|at| {
|
||||
let start = run_end(text, at + key.len(), is_js_space);
|
||||
let length = string_token_len(text, start)?;
|
||||
text.get(start..start + length)
|
||||
})
|
||||
}
|
||||
|
||||
/// Object number of the stream a Filespec's `/EF << /F n 0 R >>` points at.
|
||||
fn filespec_stream_object(dict: &[u8]) -> Option<u64> {
|
||||
let inner = dict.get(ef_span(dict)?.inner)?;
|
||||
occurrences(inner, b"/F").find_map(|at| reference_at(inner, at + 2))
|
||||
}
|
||||
|
||||
/// `\s+(\d{1,10})\s+\d{1,10}\s+R` at `from`: the referenced object number.
|
||||
fn reference_at(text: &[u8], from: usize) -> Option<u64> {
|
||||
let number = some_run_end(text, from, is_js_space)?;
|
||||
let number_end = short_number_end(text, number)?;
|
||||
let generation = some_run_end(text, number_end, is_js_space)?;
|
||||
let generation_end = short_number_end(text, generation)?;
|
||||
let key = some_run_end(text, generation_end, is_js_space)?;
|
||||
if text.get(key) != Some(&b'R') {
|
||||
return None;
|
||||
}
|
||||
Some(parse_digits(text.get(number..number_end)?))
|
||||
}
|
||||
|
||||
/// Stream object number → file name, from every Filespec dictionary in clear; the first
|
||||
/// Filespec naming an object wins.
|
||||
fn filespec_names(bytes: &[u8], end: usize) -> BTreeMap<u64, String> {
|
||||
let mut names = BTreeMap::new();
|
||||
for pos in find_typed_dicts(bytes, FILESPEC, end) {
|
||||
let Some(dict) = dict_around(bytes, pos, end) else {
|
||||
continue;
|
||||
};
|
||||
let text = dict.text(bytes);
|
||||
let target = filespec_stream_object(text);
|
||||
let name = filespec_name(text);
|
||||
if let (Some(target), Some(name)) = (target, name) {
|
||||
names.entry(target).or_insert(name);
|
||||
}
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public scan
|
||||
|
||||
/// Every `/Type /EmbeddedFile` stream of the file, named when its Filespec is in clear
|
||||
/// (export without object streams); `osteo-report*` names first, then data offset order.
|
||||
/// At most [`MAX_CANDIDATES`], within [`MAX_PDF_BYTES`].
|
||||
pub fn find_embedded_files(bytes: &[u8]) -> Vec<EmbeddedFileCandidate> {
|
||||
let end = bytes.len().min(MAX_PDF_BYTES);
|
||||
let names = filespec_names(bytes, end);
|
||||
let mut finder = EndstreamFinder::new(bytes, end);
|
||||
let mut found: Vec<EmbeddedFileCandidate> = embedded_stream_heads(bytes, end)
|
||||
.iter()
|
||||
.filter_map(|head| {
|
||||
let mut candidate = candidate_of(bytes, head, end, &mut finder)?;
|
||||
candidate.name = head
|
||||
.dict
|
||||
.object_number
|
||||
.and_then(|number| names.get(&number).cloned());
|
||||
Some(candidate)
|
||||
})
|
||||
.collect();
|
||||
found.sort_by_key(|candidate| !is_attachment_name(candidate));
|
||||
found
|
||||
}
|
||||
|
||||
fn is_attachment_name(candidate: &EmbeddedFileCandidate) -> bool {
|
||||
candidate
|
||||
.name
|
||||
.as_deref()
|
||||
.is_some_and(|name| name.starts_with(ATTACHMENT_NAME_PREFIX))
|
||||
}
|
||||
@@ -1 +1,42 @@
|
||||
//! R4.2a: `hasArchiveStamp` of scan.ts. Written by the R4.2a track.
|
||||
//! The archive stamp: `hasArchiveStamp` and `keywordsAt` of
|
||||
//! `packages/pdf/src/attachment/scan.ts`. Every archive carries `faunosteo-archive/<n>` in
|
||||
//! its `/Keywords`; a stamped file without a readable attachment was stripped by another tool.
|
||||
|
||||
use super::scan::{index_of, is_delimiter, skip_whitespace};
|
||||
use super::strings::{decode_pdf_string, string_token_len};
|
||||
use super::{ARCHIVE_STAMP, MAX_CANDIDATES, MAX_DICT_BYTES, MAX_PDF_BYTES};
|
||||
|
||||
const KEYWORDS: &[u8] = b"/Keywords";
|
||||
|
||||
/// True when a visible `/Keywords` string carries the archive stamp. At most
|
||||
/// [`MAX_CANDIDATES`] occurrences of the key are looked at, delimited or not.
|
||||
pub fn has_archive_stamp(bytes: &[u8]) -> bool {
|
||||
let end = bytes.len().min(MAX_PDF_BYTES);
|
||||
let mut from = 0;
|
||||
for _ in 0..MAX_CANDIDATES {
|
||||
let Some(at) = index_of(bytes, KEYWORDS, from, end) else {
|
||||
return false;
|
||||
};
|
||||
from = at + KEYWORDS.len();
|
||||
// `bytes[from] ?? 0x20`: the file's own next byte, even past the size cap.
|
||||
let next = bytes.get(from).copied().unwrap_or(b' ');
|
||||
if is_delimiter(next) && keywords_at(bytes, at, end).is_some_and(stamped) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn stamped(text: String) -> bool {
|
||||
text.contains(ARCHIVE_STAMP)
|
||||
}
|
||||
|
||||
/// The string token right after the `/Keywords` key at `key_at`, as text; the token must
|
||||
/// start the window and fit in its [`MAX_DICT_BYTES`].
|
||||
fn keywords_at(bytes: &[u8], key_at: usize, end: usize) -> Option<String> {
|
||||
let start = skip_whitespace(bytes, key_at + KEYWORDS.len(), end);
|
||||
let limit = end.min(start + MAX_DICT_BYTES);
|
||||
let window = bytes.get(start..limit)?;
|
||||
let length = string_token_len(window, 0)?;
|
||||
decode_pdf_string(window.get(..length)?)
|
||||
}
|
||||
@@ -1 +1,121 @@
|
||||
//! R4.2a: `decodePdfString` of scan.ts. Written by the R4.2a track.
|
||||
//! PDF string tokens as text: `decodePdfString`, `decodeTextBytes` and the `STRING_TOKEN`
|
||||
//! pattern of `packages/pdf/src/attachment/scan.ts`.
|
||||
//!
|
||||
//! The TypeScript matched `STRING_TOKEN` (`\((?:\\.|[^\\)])*\)|<[0-9a-fA-F\s]*>`) with a
|
||||
//! JavaScript regular expression on a Latin-1 window, one character per byte. The matcher
|
||||
//! below takes the bytes and gives the same answer: `\s` is JavaScript's set restricted to
|
||||
//! Latin-1, and `.` refuses CR and LF only.
|
||||
|
||||
/// JavaScript's `\s` on Latin-1 text: tab, LF, VT, FF, CR, space and NBSP, never NUL (unlike
|
||||
/// the PDF whitespace of the scanner).
|
||||
pub fn is_js_space(byte: u8) -> bool {
|
||||
matches!(byte, 0x09..=0x0D | 0x20 | 0xA0)
|
||||
}
|
||||
|
||||
/// Length of the `STRING_TOKEN` starting at `at` in `text`, `None` when none starts there.
|
||||
///
|
||||
/// In the literal branch the alternatives of `(?:\\.|[^\\)])*` never overlap (one takes a
|
||||
/// backslash, the other refuses it) and no item starts on `)`, so backtracking cannot find
|
||||
/// another closing parenthesis: a backslash before CR, LF or the end fails the token.
|
||||
pub fn string_token_len(text: &[u8], at: usize) -> Option<usize> {
|
||||
let end = match text.get(at)? {
|
||||
b'(' => literal_end(text, at + 1)?,
|
||||
b'<' => hex_end(text, at + 1)?,
|
||||
_ => return None,
|
||||
};
|
||||
Some(end - at)
|
||||
}
|
||||
|
||||
/// Index just past the `)` closing a literal whose body starts at `from`.
|
||||
fn literal_end(text: &[u8], from: usize) -> Option<usize> {
|
||||
let mut i = from;
|
||||
loop {
|
||||
match *text.get(i)? {
|
||||
b')' => return Some(i + 1),
|
||||
b'\\' => match text.get(i + 1) {
|
||||
Some(&next) if next != b'\n' && next != b'\r' => i += 2,
|
||||
_ => return None,
|
||||
},
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Index just past the `>` closing a hex string whose body starts at `from`.
|
||||
fn hex_end(text: &[u8], from: usize) -> Option<usize> {
|
||||
let mut i = from;
|
||||
while text
|
||||
.get(i)
|
||||
.is_some_and(|&b| b.is_ascii_hexdigit() || is_js_space(b))
|
||||
{
|
||||
i += 1;
|
||||
}
|
||||
(text.get(i) == Some(&b'>')).then_some(i + 1)
|
||||
}
|
||||
|
||||
/// A `(literal)` or `<hex>` string token as text; `None` for anything else. `token` holds
|
||||
/// the raw bytes (the TypeScript took them as a Latin-1 string).
|
||||
pub fn decode_pdf_string(token: &[u8]) -> Option<String> {
|
||||
if let Some(hex) = enclosed(token, b'<', b'>') {
|
||||
return Some(decode_text_bytes(&hex_bytes(hex)));
|
||||
}
|
||||
let literal = enclosed(token, b'(', b')')?;
|
||||
Some(decode_text_bytes(&unescape_literal(literal)))
|
||||
}
|
||||
|
||||
/// The body of `token` when it starts with `open` and ends with `close`.
|
||||
fn enclosed(token: &[u8], open: u8, close: u8) -> Option<&[u8]> {
|
||||
match token {
|
||||
[first, body @ .., last] if *first == open && *last == close => Some(body),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Hex digits only, read in pairs; an odd trailing nibble is dropped.
|
||||
fn hex_bytes(body: &[u8]) -> Vec<u8> {
|
||||
let nibbles: Vec<u8> = body.iter().filter_map(|&b| nibble(b)).collect();
|
||||
let (pairs, _odd) = nibbles.as_chunks::<2>();
|
||||
pairs.iter().map(|&[high, low]| (high << 4) | low).collect()
|
||||
}
|
||||
|
||||
fn nibble(byte: u8) -> Option<u8> {
|
||||
match byte {
|
||||
b'0'..=b'9' => Some(byte - b'0'),
|
||||
b'a'..=b'f' => Some(byte - b'a' + 10),
|
||||
b'A'..=b'F' => Some(byte - b'A' + 10),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `body.replace(/\\([()\\])/g, '$1')`: only the three escapes the writer produces are
|
||||
/// undone; any other backslash stays as is (`\n`, `\157` are kept literally).
|
||||
fn unescape_literal(body: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(body.len());
|
||||
let mut i = 0;
|
||||
while let Some(&byte) = body.get(i) {
|
||||
match body.get(i + 1) {
|
||||
Some(&next @ (b'(' | b')' | b'\\')) if byte == b'\\' => {
|
||||
out.push(next);
|
||||
i += 2;
|
||||
}
|
||||
_ => {
|
||||
out.push(byte);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// UTF-16BE after a `FE FF` mark (how pdf-lib writes text strings; an odd trailing byte is
|
||||
/// dropped, a lone surrogate becomes U+FFFD), else Latin-1 bytes.
|
||||
pub fn decode_text_bytes(raw: &[u8]) -> String {
|
||||
match raw {
|
||||
[0xFE, 0xFF, rest @ ..] => {
|
||||
let (pairs, _odd) = rest.as_chunks::<2>();
|
||||
let units: Vec<u16> = pairs.iter().map(|&pair| u16::from_be_bytes(pair)).collect();
|
||||
String::from_utf16_lossy(&units)
|
||||
}
|
||||
_ => raw.iter().map(|&b| char::from(b)).collect(),
|
||||
}
|
||||
}
|
||||
@@ -39,11 +39,18 @@ pub fn sha256_hex(data: &[u8]) -> String {
|
||||
pub fn r4_input(dir: &str, file: &str) -> Vec<u8> {
|
||||
let base = repo_root().join("apps/app/tests/fixtures/pdf/r4").join(dir);
|
||||
let stored = base.join(file);
|
||||
if stored.exists() {
|
||||
if stored.exists() && !file.ends_with(".gen.json") {
|
||||
return std::fs::read(&stored).unwrap_or_else(|e| panic!("{stored:?}: {e}"));
|
||||
}
|
||||
let slug = file.rsplit_once('.').map_or(file, |(stem, _)| stem);
|
||||
let description = base.join(format!("{slug}.gen.json"));
|
||||
// A generator row names its description (`<slug>.gen.json`); a stored name may also be
|
||||
// asked for by its would-be byte name (`<slug>.pdf`).
|
||||
let description = match file.strip_suffix(".gen.json") {
|
||||
Some(_) => base.join(file),
|
||||
None => base.join(format!(
|
||||
"{}.gen.json",
|
||||
file.rsplit_once('.').map_or(file, |(stem, _)| stem)
|
||||
)),
|
||||
};
|
||||
let text = std::fs::read_to_string(&description)
|
||||
.unwrap_or_else(|e| panic!("{description:?}: {e} (no stored bytes for {file})"));
|
||||
let spec: Value = serde_json::from_str(&text).unwrap();
|
||||
|
||||
@@ -1,10 +1,132 @@
|
||||
//! R4.2a placeholder: the O7 scan suite, written by its track.
|
||||
//! O7 scan suite (R4.2a): `find_embedded_files`, `has_archive_stamp` and `decode_pdf_string`
|
||||
//! against the outcomes the TypeScript recorded (`oracles/r4/read.json`), on every `pdf`
|
||||
//! input of the oracle and every `decodePdfString` row.
|
||||
#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)]
|
||||
|
||||
mod common;
|
||||
|
||||
use common::r4::{latin1_bytes, r4_input, r4_oracle, sha256_hex};
|
||||
use faunosteo_core::pdf::read::scan::find_embedded_files;
|
||||
use faunosteo_core::pdf::read::stamp::has_archive_stamp;
|
||||
use faunosteo_core::pdf::read::strings::decode_pdf_string;
|
||||
use faunosteo_core::pdf::read::{EmbeddedFileCandidate, StreamFilter};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// The bytes of an oracle input, checked against the row's length and digest. A generator
|
||||
/// row names its `<slug>.gen.json` description, which `r4_input` expands from any file name
|
||||
/// of that slug (the description itself is stored under the row's name).
|
||||
fn input_bytes(row: &Value) -> Vec<u8> {
|
||||
let file = row["file"].as_str().unwrap();
|
||||
if row["stored"] == "generator" {
|
||||
let slug = file.strip_suffix(".gen.json").unwrap();
|
||||
return r4_input("read", &format!("{slug}.pdf"));
|
||||
}
|
||||
let bytes = r4_input("read", file);
|
||||
assert_eq!(
|
||||
bytes.len() as u64,
|
||||
row["length"].as_u64().unwrap(),
|
||||
"{file}: length"
|
||||
);
|
||||
assert_eq!(
|
||||
sha256_hex(&bytes),
|
||||
row["sha256"].as_str().unwrap(),
|
||||
"{file}: digest"
|
||||
);
|
||||
bytes
|
||||
}
|
||||
|
||||
fn candidate_json(candidate: &EmbeddedFileCandidate) -> Value {
|
||||
let filter = match candidate.filter {
|
||||
StreamFilter::FlateDecode => json!("FlateDecode"),
|
||||
StreamFilter::None => Value::Null,
|
||||
};
|
||||
json!({
|
||||
"name": candidate.name,
|
||||
"offset": candidate.offset,
|
||||
"length": candidate.length,
|
||||
"filter": filter,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Tally {
|
||||
inputs: usize,
|
||||
candidates: usize,
|
||||
stamps: usize,
|
||||
failures: Vec<String>,
|
||||
}
|
||||
|
||||
impl Tally {
|
||||
fn check(&mut self, file: &str, field: &str, actual: Value, expected: &Value) {
|
||||
if actual != *expected {
|
||||
self.failures
|
||||
.push(format!("{file}: {field}: got {actual}, oracle {expected}"));
|
||||
}
|
||||
}
|
||||
|
||||
fn scan(&mut self, row: &Value) {
|
||||
let file = row["file"].as_str().unwrap();
|
||||
let bytes = input_bytes(row);
|
||||
let outcomes = &row["outcomes"];
|
||||
self.inputs += 1;
|
||||
if let Some(expected) = outcomes.get("findEmbeddedFiles") {
|
||||
let found: Vec<Value> = find_embedded_files(&bytes)
|
||||
.iter()
|
||||
.map(candidate_json)
|
||||
.collect();
|
||||
self.candidates += found.len();
|
||||
self.check(file, "findEmbeddedFiles", Value::Array(found), expected);
|
||||
}
|
||||
if let Some(expected) = outcomes.get("hasArchiveStamp") {
|
||||
self.stamps += 1;
|
||||
self.check(
|
||||
file,
|
||||
"hasArchiveStamp",
|
||||
json!(has_archive_stamp(&bytes)),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placeholder_reads_an_oracle() {
|
||||
let oracle = common::r4::r4_oracle("read.json");
|
||||
fn every_pdf_input_scans_as_the_typescript_did() {
|
||||
let oracle = r4_oracle("read.json");
|
||||
assert_eq!(oracle["oracle"], "O7");
|
||||
let bytes = common::r4::r4_input("read", "scan.demo-export.pdf");
|
||||
assert_eq!(bytes.len(), 78463);
|
||||
let mut tally = Tally::default();
|
||||
let rows = oracle["inputs"].as_array().unwrap();
|
||||
for row in rows.iter().filter(|row| row["kind"] == "pdf") {
|
||||
tally.scan(row);
|
||||
}
|
||||
assert!(tally.failures.is_empty(), "{}", tally.failures.join("\n"));
|
||||
assert_eq!(tally.inputs, 144, "pdf inputs");
|
||||
assert!(tally.candidates > 0 && tally.stamps == tally.inputs);
|
||||
println!(
|
||||
"O7 scan: {} pdf inputs of {}, {} candidates, {} stamp outcomes",
|
||||
tally.inputs,
|
||||
rows.len(),
|
||||
tally.candidates,
|
||||
tally.stamps
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_pdf_string_rows_match() {
|
||||
let oracle = r4_oracle("read.json");
|
||||
let rows = oracle["decodePdfString"].as_array().unwrap();
|
||||
let failures: Vec<String> = rows
|
||||
.iter()
|
||||
.filter_map(|row| {
|
||||
let token = row["token"].as_str().unwrap();
|
||||
let actual = json!(decode_pdf_string(&latin1_bytes(token)));
|
||||
(actual != row["result"]).then(|| {
|
||||
format!(
|
||||
"decodePdfString({token:?}): got {actual}, oracle {}",
|
||||
row["result"]
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
assert!(failures.is_empty(), "{}", failures.join("\n"));
|
||||
assert_eq!(rows.len(), 19, "decodePdfString rows");
|
||||
}
|
||||
Reference in new issue
Block a user