Files
faunosteo/scripts/r4-utf8-oracle.mjs
T
bchanot 9850a8341a feat(domain): R4.2c json_js: JS-path side effects of the attachment probe
starts_json_object, depth and value prescan (OD-3 proxy), TextDecoder-parity lossy decode
(400-case oracle generated by scripts/r4-utf8-oracle.mjs, 0 difference), overflow → null
rewrite with the JSON number grammar, lone-surrogate neutralised probe (OD-4).
2026-10-18 15:44:54 +02:00

126 lines
3.8 KiB
JavaScript

// R4.2c UTF-8 lossy oracle (plan 2026-10-17 §B.1.2, LRN-059): what the
// TypeScript import path's `new TextDecoder().decode(bytes)` (non-fatal, one
// leading BOM stripped) gives for 400 byte strings of 1 to 48 bytes. The
// strings come from a seeded xorshift32 (seed 20261018) mixing random bytes
// with the edge sequences of UTF-8 decoding: truncated multi-byte sequences,
// overlongs, UTF-8-encoded surrogates, code points above U+10FFFF, F5..FF,
// lone continuation bytes, one and two leading BOMs, a BOM in the middle.
// Writes crates/core/tests/fixtures/pdf/r4/utf8-lossy.json, read by
// crates/core/tests/pdf_json_js.rs (`decode_lossy` must give the same text).
// Deterministic: rerunning rewrites the same cases (the generator field
// records the Node version that decoded them).
// Usage: node scripts/r4-utf8-oracle.mjs
import { mkdirSync, writeFileSync } from 'node:fs'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
const OUT = join(ROOT, 'crates/core/tests/fixtures/pdf/r4/utf8-lossy.json')
const SEED = 20261018
const CASES = 400
const MAX_LENGTH = 48
const BOM = [0xef, 0xbb, 0xbf]
const EDGES = [
BOM,
// overlongs
[0xc0, 0x80],
[0xc1, 0xbf],
[0xe0, 0x80, 0x80],
[0xe0, 0x9f, 0xbf],
[0xf0, 0x80, 0x80, 0x80],
[0xf0, 0x8f, 0xbf, 0xbf],
// UTF-8-encoded surrogates, alone and as a CESU-8 pair
[0xed, 0xa0, 0x80],
[0xed, 0xaf, 0xbf],
[0xed, 0xb0, 0x80],
[0xed, 0xbf, 0xbf],
[0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80],
// above U+10FFFF, bytes never valid
[0xf4, 0x90, 0x80, 0x80],
[0xf7, 0xbf, 0xbf, 0xbf],
[0xf5],
[0xf8, 0x88, 0x80, 0x80, 0x80],
[0xfc, 0x84, 0x80, 0x80, 0x80, 0x80],
[0xfe],
[0xff],
[0xfe, 0xff],
// lone continuation bytes
[0x80],
[0xbf],
[0x80, 0x80, 0x80],
// truncated sequences
[0xc3],
[0xe2],
[0xe2, 0x82],
[0xf0],
[0xf0, 0x9f],
[0xf0, 0x9f, 0x98],
[0xef, 0xbb],
// valid, range boundaries included
[0xc2, 0x80],
[0xc3, 0xa9],
[0xe2, 0x82, 0xac],
[0xed, 0x9f, 0xbf],
[0xee, 0x80, 0x80],
[0xef, 0xbf, 0xbd],
[0xf0, 0x9f, 0x98, 0x80],
[0xf4, 0x8f, 0xbf, 0xbf],
[0x00],
[0x7f],
]
const ASCII = (text) => Array.from(text, (c) => c.charCodeAt(0))
const FIXED = [
...EDGES,
[...BOM, ...BOM],
[...BOM, ...ASCII('{"a":1}')],
[...BOM, ...BOM, ...ASCII('{}')],
[...ASCII('a'), ...BOM, ...ASCII('b')],
[0xef, 0xbb, ...ASCII('{')],
[...BOM, 0xef, 0xbb],
[...BOM, 0x80],
]
function xorshift32(seed) {
let state = seed >>> 0
return (bound) => {
state ^= state << 13
state >>>= 0
state ^= state >>> 17
state ^= state << 5
state >>>= 0
return state % bound
}
}
function randomCase(next) {
const length = 1 + next(MAX_LENGTH)
const prefix = next(8)
const bytes = prefix === 0 ? [...BOM] : prefix === 1 ? [...BOM, ...BOM] : []
while (bytes.length < length) {
const pick = next(4)
if (pick === 0) bytes.push(next(256))
else if (pick === 1) bytes.push(0x20 + next(0x5f))
else bytes.push(...EDGES[next(EDGES.length)])
}
return bytes.slice(0, length)
}
const next = xorshift32(SEED)
const inputs = [...FIXED]
while (inputs.length < CASES) inputs.push(randomCase(next))
const decoder = new TextDecoder()
const cases = inputs.map((bytes) => ({
bytes: Buffer.from(bytes).toString('hex'),
text: decoder.decode(Uint8Array.from(bytes)),
}))
const generator = `scripts/r4-utf8-oracle.mjs, Node ${process.version}`
// One case per line: the file stays readable and diffable.
const rows = cases.map((c) => ` ${JSON.stringify(c)}`).join(',\n')
const head = `{\n "generator": ${JSON.stringify(generator)},\n "cases": [\n`
const text = `${head}${rows}\n ]\n}\n`
mkdirSync(dirname(OUT), { recursive: true })
writeFileSync(OUT, text)
const size = Buffer.byteLength(text)
console.log(`${cases.length} cases, ${size} bytes -> ${OUT}`)