starts_json_object, depth and value prescan (OD-3 proxy), TextDecoder-parity lossy decode (400-case oracle generated by scripts/r4-utf8-oracle.mjs, 0 difference), overflow → null rewrite with the JSON number grammar, lone-surrogate neutralised probe (OD-4).
126 lines
3.8 KiB
JavaScript
126 lines
3.8 KiB
JavaScript
// R4.2c UTF-8 lossy oracle (plan 2026-10-17 §B.1.2, LRN-059): what the
|
|
// TypeScript import path's `new TextDecoder().decode(bytes)` (non-fatal, one
|
|
// leading BOM stripped) gives for 400 byte strings of 1 to 48 bytes. The
|
|
// strings come from a seeded xorshift32 (seed 20261018) mixing random bytes
|
|
// with the edge sequences of UTF-8 decoding: truncated multi-byte sequences,
|
|
// overlongs, UTF-8-encoded surrogates, code points above U+10FFFF, F5..FF,
|
|
// lone continuation bytes, one and two leading BOMs, a BOM in the middle.
|
|
// Writes crates/core/tests/fixtures/pdf/r4/utf8-lossy.json, read by
|
|
// crates/core/tests/pdf_json_js.rs (`decode_lossy` must give the same text).
|
|
// Deterministic: rerunning rewrites the same cases (the generator field
|
|
// records the Node version that decoded them).
|
|
// Usage: node scripts/r4-utf8-oracle.mjs
|
|
import { mkdirSync, writeFileSync } from 'node:fs'
|
|
import { dirname, join, resolve } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
|
|
const OUT = join(ROOT, 'crates/core/tests/fixtures/pdf/r4/utf8-lossy.json')
|
|
const SEED = 20261018
|
|
const CASES = 400
|
|
const MAX_LENGTH = 48
|
|
|
|
const BOM = [0xef, 0xbb, 0xbf]
|
|
const EDGES = [
|
|
BOM,
|
|
// overlongs
|
|
[0xc0, 0x80],
|
|
[0xc1, 0xbf],
|
|
[0xe0, 0x80, 0x80],
|
|
[0xe0, 0x9f, 0xbf],
|
|
[0xf0, 0x80, 0x80, 0x80],
|
|
[0xf0, 0x8f, 0xbf, 0xbf],
|
|
// UTF-8-encoded surrogates, alone and as a CESU-8 pair
|
|
[0xed, 0xa0, 0x80],
|
|
[0xed, 0xaf, 0xbf],
|
|
[0xed, 0xb0, 0x80],
|
|
[0xed, 0xbf, 0xbf],
|
|
[0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80],
|
|
// above U+10FFFF, bytes never valid
|
|
[0xf4, 0x90, 0x80, 0x80],
|
|
[0xf7, 0xbf, 0xbf, 0xbf],
|
|
[0xf5],
|
|
[0xf8, 0x88, 0x80, 0x80, 0x80],
|
|
[0xfc, 0x84, 0x80, 0x80, 0x80, 0x80],
|
|
[0xfe],
|
|
[0xff],
|
|
[0xfe, 0xff],
|
|
// lone continuation bytes
|
|
[0x80],
|
|
[0xbf],
|
|
[0x80, 0x80, 0x80],
|
|
// truncated sequences
|
|
[0xc3],
|
|
[0xe2],
|
|
[0xe2, 0x82],
|
|
[0xf0],
|
|
[0xf0, 0x9f],
|
|
[0xf0, 0x9f, 0x98],
|
|
[0xef, 0xbb],
|
|
// valid, range boundaries included
|
|
[0xc2, 0x80],
|
|
[0xc3, 0xa9],
|
|
[0xe2, 0x82, 0xac],
|
|
[0xed, 0x9f, 0xbf],
|
|
[0xee, 0x80, 0x80],
|
|
[0xef, 0xbf, 0xbd],
|
|
[0xf0, 0x9f, 0x98, 0x80],
|
|
[0xf4, 0x8f, 0xbf, 0xbf],
|
|
[0x00],
|
|
[0x7f],
|
|
]
|
|
const ASCII = (text) => Array.from(text, (c) => c.charCodeAt(0))
|
|
const FIXED = [
|
|
...EDGES,
|
|
[...BOM, ...BOM],
|
|
[...BOM, ...ASCII('{"a":1}')],
|
|
[...BOM, ...BOM, ...ASCII('{}')],
|
|
[...ASCII('a'), ...BOM, ...ASCII('b')],
|
|
[0xef, 0xbb, ...ASCII('{')],
|
|
[...BOM, 0xef, 0xbb],
|
|
[...BOM, 0x80],
|
|
]
|
|
|
|
function xorshift32(seed) {
|
|
let state = seed >>> 0
|
|
return (bound) => {
|
|
state ^= state << 13
|
|
state >>>= 0
|
|
state ^= state >>> 17
|
|
state ^= state << 5
|
|
state >>>= 0
|
|
return state % bound
|
|
}
|
|
}
|
|
|
|
function randomCase(next) {
|
|
const length = 1 + next(MAX_LENGTH)
|
|
const prefix = next(8)
|
|
const bytes = prefix === 0 ? [...BOM] : prefix === 1 ? [...BOM, ...BOM] : []
|
|
while (bytes.length < length) {
|
|
const pick = next(4)
|
|
if (pick === 0) bytes.push(next(256))
|
|
else if (pick === 1) bytes.push(0x20 + next(0x5f))
|
|
else bytes.push(...EDGES[next(EDGES.length)])
|
|
}
|
|
return bytes.slice(0, length)
|
|
}
|
|
|
|
const next = xorshift32(SEED)
|
|
const inputs = [...FIXED]
|
|
while (inputs.length < CASES) inputs.push(randomCase(next))
|
|
const decoder = new TextDecoder()
|
|
const cases = inputs.map((bytes) => ({
|
|
bytes: Buffer.from(bytes).toString('hex'),
|
|
text: decoder.decode(Uint8Array.from(bytes)),
|
|
}))
|
|
const generator = `scripts/r4-utf8-oracle.mjs, Node ${process.version}`
|
|
// One case per line: the file stays readable and diffable.
|
|
const rows = cases.map((c) => ` ${JSON.stringify(c)}`).join(',\n')
|
|
const head = `{\n "generator": ${JSON.stringify(generator)},\n "cases": [\n`
|
|
const text = `${head}${rows}\n ]\n}\n`
|
|
mkdirSync(dirname(OUT), { recursive: true })
|
|
writeFileSync(OUT, text)
|
|
const size = Buffer.byteLength(text)
|
|
console.log(`${cases.length} cases, ${size} bytes -> ${OUT}`)
|