feat(domain): R4.2c json_js: JS-path side effects of the attachment probe
starts_json_object, depth and value prescan (OD-3 proxy), TextDecoder-parity lossy decode (400-case oracle generated by scripts/r4-utf8-oracle.mjs, 0 difference), overflow → null rewrite with the JSON number grammar, lone-surrogate neutralised probe (OD-4).
This commit is contained in:
1 parent
55819fbd32
commit
9850a8341a
4 files changed
+1083
-2
No files matched your search
@@ -0,0 +1,125 @@
|
||||
// R4.2c UTF-8 lossy oracle (plan 2026-10-17 §B.1.2, LRN-059): what the
|
||||
// TypeScript import path's `new TextDecoder().decode(bytes)` (non-fatal, one
|
||||
// leading BOM stripped) gives for 400 byte strings of 1 to 48 bytes. The
|
||||
// strings come from a seeded xorshift32 (seed 20261018) mixing random bytes
|
||||
// with the edge sequences of UTF-8 decoding: truncated multi-byte sequences,
|
||||
// overlongs, UTF-8-encoded surrogates, code points above U+10FFFF, F5..FF,
|
||||
// lone continuation bytes, one and two leading BOMs, a BOM in the middle.
|
||||
// Writes crates/core/tests/fixtures/pdf/r4/utf8-lossy.json, read by
|
||||
// crates/core/tests/pdf_json_js.rs (`decode_lossy` must give the same text).
|
||||
// Deterministic: rerunning rewrites the same cases (the generator field
|
||||
// records the Node version that decoded them).
|
||||
// Usage: node scripts/r4-utf8-oracle.mjs
|
||||
import { mkdirSync, writeFileSync } from 'node:fs'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)))
|
||||
const OUT = join(ROOT, 'crates/core/tests/fixtures/pdf/r4/utf8-lossy.json')
|
||||
const SEED = 20261018
|
||||
const CASES = 400
|
||||
const MAX_LENGTH = 48
|
||||
|
||||
const BOM = [0xef, 0xbb, 0xbf]
|
||||
const EDGES = [
|
||||
BOM,
|
||||
// overlongs
|
||||
[0xc0, 0x80],
|
||||
[0xc1, 0xbf],
|
||||
[0xe0, 0x80, 0x80],
|
||||
[0xe0, 0x9f, 0xbf],
|
||||
[0xf0, 0x80, 0x80, 0x80],
|
||||
[0xf0, 0x8f, 0xbf, 0xbf],
|
||||
// UTF-8-encoded surrogates, alone and as a CESU-8 pair
|
||||
[0xed, 0xa0, 0x80],
|
||||
[0xed, 0xaf, 0xbf],
|
||||
[0xed, 0xb0, 0x80],
|
||||
[0xed, 0xbf, 0xbf],
|
||||
[0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80],
|
||||
// above U+10FFFF, bytes never valid
|
||||
[0xf4, 0x90, 0x80, 0x80],
|
||||
[0xf7, 0xbf, 0xbf, 0xbf],
|
||||
[0xf5],
|
||||
[0xf8, 0x88, 0x80, 0x80, 0x80],
|
||||
[0xfc, 0x84, 0x80, 0x80, 0x80, 0x80],
|
||||
[0xfe],
|
||||
[0xff],
|
||||
[0xfe, 0xff],
|
||||
// lone continuation bytes
|
||||
[0x80],
|
||||
[0xbf],
|
||||
[0x80, 0x80, 0x80],
|
||||
// truncated sequences
|
||||
[0xc3],
|
||||
[0xe2],
|
||||
[0xe2, 0x82],
|
||||
[0xf0],
|
||||
[0xf0, 0x9f],
|
||||
[0xf0, 0x9f, 0x98],
|
||||
[0xef, 0xbb],
|
||||
// valid, range boundaries included
|
||||
[0xc2, 0x80],
|
||||
[0xc3, 0xa9],
|
||||
[0xe2, 0x82, 0xac],
|
||||
[0xed, 0x9f, 0xbf],
|
||||
[0xee, 0x80, 0x80],
|
||||
[0xef, 0xbf, 0xbd],
|
||||
[0xf0, 0x9f, 0x98, 0x80],
|
||||
[0xf4, 0x8f, 0xbf, 0xbf],
|
||||
[0x00],
|
||||
[0x7f],
|
||||
]
|
||||
const ASCII = (text) => Array.from(text, (c) => c.charCodeAt(0))
|
||||
const FIXED = [
|
||||
...EDGES,
|
||||
[...BOM, ...BOM],
|
||||
[...BOM, ...ASCII('{"a":1}')],
|
||||
[...BOM, ...BOM, ...ASCII('{}')],
|
||||
[...ASCII('a'), ...BOM, ...ASCII('b')],
|
||||
[0xef, 0xbb, ...ASCII('{')],
|
||||
[...BOM, 0xef, 0xbb],
|
||||
[...BOM, 0x80],
|
||||
]
|
||||
|
||||
function xorshift32(seed) {
|
||||
let state = seed >>> 0
|
||||
return (bound) => {
|
||||
state ^= state << 13
|
||||
state >>>= 0
|
||||
state ^= state >>> 17
|
||||
state ^= state << 5
|
||||
state >>>= 0
|
||||
return state % bound
|
||||
}
|
||||
}
|
||||
|
||||
function randomCase(next) {
|
||||
const length = 1 + next(MAX_LENGTH)
|
||||
const prefix = next(8)
|
||||
const bytes = prefix === 0 ? [...BOM] : prefix === 1 ? [...BOM, ...BOM] : []
|
||||
while (bytes.length < length) {
|
||||
const pick = next(4)
|
||||
if (pick === 0) bytes.push(next(256))
|
||||
else if (pick === 1) bytes.push(0x20 + next(0x5f))
|
||||
else bytes.push(...EDGES[next(EDGES.length)])
|
||||
}
|
||||
return bytes.slice(0, length)
|
||||
}
|
||||
|
||||
const next = xorshift32(SEED)
|
||||
const inputs = [...FIXED]
|
||||
while (inputs.length < CASES) inputs.push(randomCase(next))
|
||||
const decoder = new TextDecoder()
|
||||
const cases = inputs.map((bytes) => ({
|
||||
bytes: Buffer.from(bytes).toString('hex'),
|
||||
text: decoder.decode(Uint8Array.from(bytes)),
|
||||
}))
|
||||
const generator = `scripts/r4-utf8-oracle.mjs, Node ${process.version}`
|
||||
// One case per line: the file stays readable and diffable.
|
||||
const rows = cases.map((c) => ` ${JSON.stringify(c)}`).join(',\n')
|
||||
const head = `{\n "generator": ${JSON.stringify(generator)},\n "cases": [\n`
|
||||
const text = `${head}${rows}\n ]\n}\n`
|
||||
mkdirSync(dirname(OUT), { recursive: true })
|
||||
writeFileSync(OUT, text)
|
||||
const size = Buffer.byteLength(text)
|
||||
console.log(`${cases.length} cases, ${size} bytes -> ${OUT}`)
|
||||
Reference in new issue
Block a user