Files
faunosteo/packages/pdf/tests/helpers/r4-reader-io.ts
T

216 lines
6.7 KiB
TypeScript

import { createHash } from 'node:crypto'
import {
existsSync,
mkdirSync,
readdirSync,
readFileSync,
writeFileSync,
} from 'node:fs'
import { dirname, resolve } from 'node:path'
import { expect } from 'vitest'
import { inflateFlate } from '../../src/attachment/inflate'
import { FIXTURES_DIR, REPO_ROOT } from './draft'
import { ascii, concat, latin1, MiB } from './r4-reader-craft'
/**
* Write-or-check plumbing of the R4.1 reader oracles (O7, O9). With
* `MAKE_R4_ORACLES=1` the generators write their inputs and oracles; without
* it they rebuild everything and require the committed files to be the same
* (bytes for inputs, deep equality for JSON).
*/
export const MAKE = process.env.MAKE_R4_ORACLES === '1'
export const R4_INPUTS_DIR = resolve(FIXTURES_DIR, 'pdf/r4')
export const R4_ORACLES_DIR = resolve(REPO_ROOT, 'crates/core/tests/oracles/r4')
/** Inputs above this size are committed as a generator description, never as bytes. */
export const MAX_STORED_BYTES = 2 * MiB
export const sha256 = (data: Uint8Array | string): string =>
createHash('sha256').update(data).digest('hex')
export const jsonText = (value: unknown): string =>
`${JSON.stringify(value, null, 2)}\n`
export function relativeToRepo(path: string): string {
return path.slice(REPO_ROOT.length + 1)
}
export function storeFile(path: string, bytes: Uint8Array): void {
if (MAKE) {
mkdirSync(dirname(path), { recursive: true })
writeFileSync(path, bytes)
return
}
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
expect(sha256(readFileSync(path)), relativeToRepo(path)).toBe(sha256(bytes))
}
export function readJsonFile<T>(path: string): T {
return JSON.parse(readFileSync(path, 'utf8')) as T
}
/**
* `normalise` maps the committed JSON before the comparison, for the parts
* this generator does not own (the Node version that wrote it, the browser
* columns of O9).
*/
export function storeJson(
path: string,
value: unknown,
normalise: (committed: unknown) => unknown = (committed) => committed,
): void {
if (MAKE) {
storeFile(path, new TextEncoder().encode(jsonText(value)))
return
}
expect(existsSync(path), `missing ${relativeToRepo(path)}`).toBe(true)
expect(normalise(readJsonFile(path)), relativeToRepo(path)).toEqual(
JSON.parse(jsonText(value)),
)
}
/** The Node build that produced an oracle: recorded, never compared (inputs are compared byte for byte). */
export const generatedWith = () => ({
node: process.version,
zlib: process.versions.zlib,
})
/** The input directory holds exactly the listed files: no stale input survives a rename. */
export function expectOnlyFiles(dir: string, files: string[]): void {
expect(readdirSync(dir).sort(), relativeToRepo(dir)).toEqual(
[...new Set(files)].sort(),
)
}
/** inflateFlate's verdict as the oracles record it: null, or the output length and digest. */
export async function inflateVerdict(input: Uint8Array, limit: number) {
const out = await inflateFlate(input, limit)
return out === null ? null : { length: out.byteLength, sha256: sha256(out) }
}
// ---------------------------------------------------------------------------
// Generator descriptions (`<slug>.gen.json`) for the inputs over 2 MiB
/**
* Concatenated in order: `latin1` one byte per character, `fill` the byte
* value repeated `count` times, `slice` the bytes [start, end) of another
* input file of the same directory.
*/
export type Part =
| { latin1: string }
| { fill: number; count: number }
| { slice: string; start: number; end: number }
export interface GeneratorDescription {
generator: 'concat/1'
length: number
sha256: string
parts: Part[]
}
/** Bytes a description may reference instead of spelling them out. */
export interface SliceRef {
file: string
start: number
end: number
bytes: Uint8Array
}
export function expandParts(parts: Part[], dir: string): Uint8Array {
return concat(
parts.map((part) => {
if ('latin1' in part) return ascii(part.latin1)
if ('fill' in part) return new Uint8Array(part.count).fill(part.fill)
const source = new Uint8Array(readFileSync(resolve(dir, part.slice)))
return source.subarray(part.start, part.end)
}),
)
}
/** Runs this long or longer of one byte value become a `fill` part. */
const MIN_RUN = 4096
function runParts(bytes: Uint8Array, start: number, end: number): Part[] {
const parts: Part[] = []
let textFrom = start
let i = start
while (i < end) {
let j = i + 1
while (j < end && bytes[j] === bytes[i]) j += 1
if (j - i >= MIN_RUN) {
if (textFrom < i) {
parts.push({ latin1: latin1(bytes.subarray(textFrom, i)) })
}
parts.push({ fill: bytes[i]!, count: j - i })
textFrom = j
}
i = j
}
if (textFrom < end)
parts.push({ latin1: latin1(bytes.subarray(textFrom, end)) })
return parts
}
interface RefHit {
at: number
ref: SliceRef
}
function refHits(bytes: Uint8Array, refs: SliceRef[]): RefHit[] {
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
const hits: RefHit[] = []
for (const ref of refs) {
let at = buffer.indexOf(ref.bytes)
while (at >= 0) {
hits.push({ at, ref })
at = buffer.indexOf(ref.bytes, at + ref.bytes.byteLength)
}
}
return hits.sort((a, b) => a.at - b.at)
}
/** Derives the description from the bytes the original test builds, so both stay the same input. */
export function partsOf(bytes: Uint8Array, refs: SliceRef[]): Part[] {
const parts: Part[] = []
let at = 0
for (const hit of refHits(bytes, refs)) {
parts.push(...runParts(bytes, at, hit.at))
parts.push({ slice: hit.ref.file, start: hit.ref.start, end: hit.ref.end })
at = hit.at + hit.ref.bytes.byteLength
}
parts.push(...runParts(bytes, at, bytes.byteLength))
return parts
}
export interface StoredInput {
file: string
stored: 'bytes' | 'generator'
length: number
sha256: string
}
/** Commits an input as bytes, or above MAX_STORED_BYTES as a description checked to expand to the same bytes. */
export function storeInput(
dir: string,
name: { slug: string; ext: string },
bytes: Uint8Array,
refs: SliceRef[] = [],
): StoredInput {
const digest = { length: bytes.byteLength, sha256: sha256(bytes) }
if (bytes.byteLength <= MAX_STORED_BYTES) {
const file = `${name.slug}${name.ext}`
storeFile(resolve(dir, file), bytes)
return { file, stored: 'bytes', ...digest }
}
const file = `${name.slug}.gen.json`
const description: GeneratorDescription = {
generator: 'concat/1',
...digest,
parts: partsOf(bytes, refs),
}
storeJson(resolve(dir, file), description)
expect(sha256(expandParts(description.parts, dir)), file).toBe(digest.sha256)
return { file, stored: 'generator', ...digest }
}