216 lines
6.7 KiB
TypeScript
216 lines
6.7 KiB
TypeScript
import { createHash } from 'node:crypto'
|
|
import {
|
|
existsSync,
|
|
mkdirSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
writeFileSync,
|
|
} from 'node:fs'
|
|
import { dirname, resolve } from 'node:path'
|
|
import { expect } from 'vitest'
|
|
import { inflateFlate } from '../../src/attachment/inflate'
|
|
import { FIXTURES_DIR, REPO_ROOT } from './draft'
|
|
import { ascii, concat, latin1, MiB } from './r4-reader-craft'
|
|
|
|
/**
|
|
* Write-or-check plumbing of the R4.1 reader oracles (O7, O9). With
|
|
* `MAKE_R4_ORACLES=1` the generators write their inputs and oracles; without
|
|
* it they rebuild everything and require the committed files to be the same
|
|
* (bytes for inputs, deep equality for JSON).
|
|
*/
|
|
|
|
export const MAKE = process.env.MAKE_R4_ORACLES === '1'
|
|
export const R4_INPUTS_DIR = resolve(FIXTURES_DIR, 'pdf/r4')
|
|
export const R4_ORACLES_DIR = resolve(REPO_ROOT, 'crates/core/tests/oracles/r4')
|
|
|
|
/** Inputs above this size are committed as a generator description, never as bytes. */
|
|
export const MAX_STORED_BYTES = 2 * MiB
|
|
|
|
export const sha256 = (data: Uint8Array | string): string =>
|
|
createHash('sha256').update(data).digest('hex')
|
|
|
|
export const jsonText = (value: unknown): string =>
|
|
`${JSON.stringify(value, null, 2)}\n`
|
|
|
|
export function relativeToRepo(path: string): string {
|
|
return path.slice(REPO_ROOT.length + 1)
|
|
}
|
|
|
|
export function storeFile(path: string, bytes: Uint8Array): void {
|
|
if (MAKE) {
|
|
mkdirSync(dirname(path), { recursive: true })
|
|
writeFileSync(path, bytes)
|
|
return
|
|
}
|
|
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
|
|
expect(sha256(readFileSync(path)), relativeToRepo(path)).toBe(sha256(bytes))
|
|
}
|
|
|
|
export function readJsonFile<T>(path: string): T {
|
|
return JSON.parse(readFileSync(path, 'utf8')) as T
|
|
}
|
|
|
|
/**
|
|
* `normalise` maps the committed JSON before the comparison, for the parts
|
|
* this generator does not own (the Node version that wrote it, the browser
|
|
* columns of O9).
|
|
*/
|
|
export function storeJson(
|
|
path: string,
|
|
value: unknown,
|
|
normalise: (committed: unknown) => unknown = (committed) => committed,
|
|
): void {
|
|
if (MAKE) {
|
|
storeFile(path, new TextEncoder().encode(jsonText(value)))
|
|
return
|
|
}
|
|
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
|
|
expect(normalise(readJsonFile(path)), relativeToRepo(path)).toEqual(
|
|
JSON.parse(jsonText(value)),
|
|
)
|
|
}
|
|
|
|
/** The Node build that produced an oracle: recorded, never compared (inputs are compared byte for byte). */
|
|
export const generatedWith = () => ({
|
|
node: process.version,
|
|
zlib: process.versions.zlib,
|
|
})
|
|
|
|
/** The input directory holds exactly the listed files: no stale input survives a rename. */
|
|
export function expectOnlyFiles(dir: string, files: string[]): void {
|
|
expect(readdirSync(dir).sort(), relativeToRepo(dir)).toEqual(
|
|
[...new Set(files)].sort(),
|
|
)
|
|
}
|
|
|
|
/** inflateFlate's verdict as the oracles record it: null, or the output length and digest. */
|
|
export async function inflateVerdict(input: Uint8Array, limit: number) {
|
|
const out = await inflateFlate(input, limit)
|
|
return out === null ? null : { length: out.byteLength, sha256: sha256(out) }
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Generator descriptions (`<slug>.gen.json`) for the inputs over 2 MiB
|
|
|
|
/**
|
|
* Concatenated in order: `latin1` one byte per character, `fill` the byte
|
|
* value repeated `count` times, `slice` the bytes [start, end) of another
|
|
* input file of the same directory.
|
|
*/
|
|
export type Part =
|
|
| { latin1: string }
|
|
| { fill: number; count: number }
|
|
| { slice: string; start: number; end: number }
|
|
|
|
export interface GeneratorDescription {
|
|
generator: 'concat/1'
|
|
length: number
|
|
sha256: string
|
|
parts: Part[]
|
|
}
|
|
|
|
/** Bytes a description may reference instead of spelling them out. */
|
|
export interface SliceRef {
|
|
file: string
|
|
start: number
|
|
end: number
|
|
bytes: Uint8Array
|
|
}
|
|
|
|
export function expandParts(parts: Part[], dir: string): Uint8Array {
|
|
return concat(
|
|
parts.map((part) => {
|
|
if ('latin1' in part) return ascii(part.latin1)
|
|
if ('fill' in part) return new Uint8Array(part.count).fill(part.fill)
|
|
const source = new Uint8Array(readFileSync(resolve(dir, part.slice)))
|
|
return source.subarray(part.start, part.end)
|
|
}),
|
|
)
|
|
}
|
|
|
|
/** Runs this long or longer of one byte value become a `fill` part. */
|
|
const MIN_RUN = 4096
|
|
|
|
function runParts(bytes: Uint8Array, start: number, end: number): Part[] {
|
|
const parts: Part[] = []
|
|
let textFrom = start
|
|
let i = start
|
|
while (i < end) {
|
|
let j = i + 1
|
|
while (j < end && bytes[j] === bytes[i]) j += 1
|
|
if (j - i >= MIN_RUN) {
|
|
if (textFrom < i) {
|
|
parts.push({ latin1: latin1(bytes.subarray(textFrom, i)) })
|
|
}
|
|
parts.push({ fill: bytes[i]!, count: j - i })
|
|
textFrom = j
|
|
}
|
|
i = j
|
|
}
|
|
if (textFrom < end)
|
|
parts.push({ latin1: latin1(bytes.subarray(textFrom, end)) })
|
|
return parts
|
|
}
|
|
|
|
interface RefHit {
|
|
at: number
|
|
ref: SliceRef
|
|
}
|
|
|
|
function refHits(bytes: Uint8Array, refs: SliceRef[]): RefHit[] {
|
|
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
const hits: RefHit[] = []
|
|
for (const ref of refs) {
|
|
let at = buffer.indexOf(ref.bytes)
|
|
while (at >= 0) {
|
|
hits.push({ at, ref })
|
|
at = buffer.indexOf(ref.bytes, at + ref.bytes.byteLength)
|
|
}
|
|
}
|
|
return hits.sort((a, b) => a.at - b.at)
|
|
}
|
|
|
|
/** Derives the description from the bytes the original test builds, so both stay the same input. */
|
|
export function partsOf(bytes: Uint8Array, refs: SliceRef[]): Part[] {
|
|
const parts: Part[] = []
|
|
let at = 0
|
|
for (const hit of refHits(bytes, refs)) {
|
|
parts.push(...runParts(bytes, at, hit.at))
|
|
parts.push({ slice: hit.ref.file, start: hit.ref.start, end: hit.ref.end })
|
|
at = hit.at + hit.ref.bytes.byteLength
|
|
}
|
|
parts.push(...runParts(bytes, at, bytes.byteLength))
|
|
return parts
|
|
}
|
|
|
|
export interface StoredInput {
|
|
file: string
|
|
stored: 'bytes' | 'generator'
|
|
length: number
|
|
sha256: string
|
|
}
|
|
|
|
/** Commits an input as bytes, or above MAX_STORED_BYTES as a description checked to expand to the same bytes. */
|
|
export function storeInput(
|
|
dir: string,
|
|
name: { slug: string; ext: string },
|
|
bytes: Uint8Array,
|
|
refs: SliceRef[] = [],
|
|
): StoredInput {
|
|
const digest = { length: bytes.byteLength, sha256: sha256(bytes) }
|
|
if (bytes.byteLength <= MAX_STORED_BYTES) {
|
|
const file = `${name.slug}${name.ext}`
|
|
storeFile(resolve(dir, file), bytes)
|
|
return { file, stored: 'bytes', ...digest }
|
|
}
|
|
const file = `${name.slug}.gen.json`
|
|
const description: GeneratorDescription = {
|
|
generator: 'concat/1',
|
|
...digest,
|
|
parts: partsOf(bytes, refs),
|
|
}
|
|
storeJson(resolve(dir, file), description)
|
|
expect(sha256(expandParts(description.parts, dir)), file).toBe(digest.sha256)
|
|
return { file, stored: 'generator', ...digest }
|
|
}
|