// R4.2c UTF-8 lossy oracle (plan 2026-10-17 §B.1.2, LRN-059): what the // TypeScript import path's `new TextDecoder().decode(bytes)` (non-fatal, one // leading BOM stripped) gives for 400 byte strings of 1 to 48 bytes. The // strings come from a seeded xorshift32 (seed 20261018) mixing random bytes // with the edge sequences of UTF-8 decoding: truncated multi-byte sequences, // overlongs, UTF-8-encoded surrogates, code points above U+10FFFF, F5..FF, // lone continuation bytes, one and two leading BOMs, a BOM in the middle. // Writes crates/core/tests/fixtures/pdf/r4/utf8-lossy.json, read by // crates/core/tests/pdf_json_js.rs (`decode_lossy` must give the same text). // Deterministic: rerunning rewrites the same cases (the generator field // records the Node version that decoded them). // Usage: node scripts/r4-utf8-oracle.mjs import { mkdirSync, writeFileSync } from 'node:fs' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url))) const OUT = join(ROOT, 'crates/core/tests/fixtures/pdf/r4/utf8-lossy.json') const SEED = 20261018 const CASES = 400 const MAX_LENGTH = 48 const BOM = [0xef, 0xbb, 0xbf] const EDGES = [ BOM, // overlongs [0xc0, 0x80], [0xc1, 0xbf], [0xe0, 0x80, 0x80], [0xe0, 0x9f, 0xbf], [0xf0, 0x80, 0x80, 0x80], [0xf0, 0x8f, 0xbf, 0xbf], // UTF-8-encoded surrogates, alone and as a CESU-8 pair [0xed, 0xa0, 0x80], [0xed, 0xaf, 0xbf], [0xed, 0xb0, 0x80], [0xed, 0xbf, 0xbf], [0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80], // above U+10FFFF, bytes never valid [0xf4, 0x90, 0x80, 0x80], [0xf7, 0xbf, 0xbf, 0xbf], [0xf5], [0xf8, 0x88, 0x80, 0x80, 0x80], [0xfc, 0x84, 0x80, 0x80, 0x80, 0x80], [0xfe], [0xff], [0xfe, 0xff], // lone continuation bytes [0x80], [0xbf], [0x80, 0x80, 0x80], // truncated sequences [0xc3], [0xe2], [0xe2, 0x82], [0xf0], [0xf0, 0x9f], [0xf0, 0x9f, 0x98], [0xef, 0xbb], // valid, range boundaries included [0xc2, 0x80], [0xc3, 0xa9], [0xe2, 0x82, 0xac], [0xed, 0x9f, 0xbf], [0xee, 0x80, 0x80], [0xef, 0xbf, 0xbd], [0xf0, 0x9f, 0x98, 0x80], [0xf4, 0x8f, 0xbf, 0xbf], [0x00], [0x7f], ] const ASCII = (text) => Array.from(text, (c) => c.charCodeAt(0)) const FIXED = [ ...EDGES, [...BOM, ...BOM], [...BOM, ...ASCII('{"a":1}')], [...BOM, ...BOM, ...ASCII('{}')], [...ASCII('a'), ...BOM, ...ASCII('b')], [0xef, 0xbb, ...ASCII('{')], [...BOM, 0xef, 0xbb], [...BOM, 0x80], ] function xorshift32(seed) { let state = seed >>> 0 return (bound) => { state ^= state << 13 state >>>= 0 state ^= state >>> 17 state ^= state << 5 state >>>= 0 return state % bound } } function randomCase(next) { const length = 1 + next(MAX_LENGTH) const prefix = next(8) const bytes = prefix === 0 ? [...BOM] : prefix === 1 ? [...BOM, ...BOM] : [] while (bytes.length < length) { const pick = next(4) if (pick === 0) bytes.push(next(256)) else if (pick === 1) bytes.push(0x20 + next(0x5f)) else bytes.push(...EDGES[next(EDGES.length)]) } return bytes.slice(0, length) } const next = xorshift32(SEED) const inputs = [...FIXED] while (inputs.length < CASES) inputs.push(randomCase(next)) const decoder = new TextDecoder() const cases = inputs.map((bytes) => ({ bytes: Buffer.from(bytes).toString('hex'), text: decoder.decode(Uint8Array.from(bytes)), })) const generator = `scripts/r4-utf8-oracle.mjs, Node ${process.version}` // One case per line: the file stays readable and diffable. const rows = cases.map((c) => ` ${JSON.stringify(c)}`).join(',\n') const head = `{\n "generator": ${JSON.stringify(generator)},\n "cases": [\n` const text = `${head}${rows}\n ]\n}\n` mkdirSync(dirname(OUT), { recursive: true }) writeFileSync(OUT, text) const size = Buffer.byteLength(text) console.log(`${cases.length} cases, ${size} bytes -> ${OUT}`)