Skip to content

Commit 0be1448

Browse files
fix(cli): read file input by position instead of openAsBlob (DSPX-5160)
Node's fs.openAsBlob reports a file-backed Blob's size mod 2^32 (nodejs/node#66207), so the zip reader's tail read of any TDF >= 4 GiB landed mid-payload and decrypt/inspect failed with 'Unable to retrieve CD manifest'. Open input files with a FileHandle-backed Chunker that reads by explicit 64-bit-safe position instead, and use it for both encrypt and decrypt. Small JSON option files now use readFile. Signed-off-by: Dave Mihalcik <dmihalcik@virtru.com>
1 parent ebc317a commit 0be1448

3 files changed

Lines changed: 291 additions & 36 deletions

File tree

‎cli/src/cli.ts‎

Lines changed: 52 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { createWriteStream, openAsBlob } from 'node:fs';
2-
import { stat } from 'node:fs/promises';
1+
import { createWriteStream } from 'node:fs';
2+
import { readFile } from 'node:fs/promises';
33
import { Writable } from 'node:stream';
44
import yargs from 'yargs';
55
import { hideBin } from 'yargs/helpers';
@@ -19,6 +19,7 @@ import {
1919
PUBLIC_KEY_ALGORITHMS,
2020
} from '@opentdf/sdk';
2121
import { CLIError, Level, log } from './logger.js';
22+
import { type FileChunker, openFileChunker } from './file-chunker.js';
2223
import * as assertions from '@opentdf/sdk/assertions';
2324
import { base64 } from '@opentdf/sdk/encodings';
2425
import { type KeyPair } from '@opentdf/sdk/singlecontainer';
@@ -135,8 +136,7 @@ async function parseAssertionVerificationKeys(
135136
} catch (err) {
136137
// try as file name:
137138
try {
138-
const jsonFile = await openAsBlob(s);
139-
u = JSON.parse(await jsonFile.text()) as assertions.AssertionVerificationKeys;
139+
u = JSON.parse(await readFile(s, 'utf8')) as assertions.AssertionVerificationKeys;
140140
} catch (err2) {
141141
const cause = asError(err2);
142142
throw new CLIError(
@@ -234,8 +234,7 @@ async function parseAssertionConfig(s: string): Promise<assertions.AssertionConf
234234
} catch (err) {
235235
// try as file name:
236236
try {
237-
const jsonFile = await openAsBlob(s);
238-
u = JSON.parse(await jsonFile.text()) as unknown;
237+
u = JSON.parse(await readFile(s, 'utf8')) as unknown;
239238
} catch (err2) {
240239
const cause = asError(err2);
241240
throw new CLIError(
@@ -347,7 +346,7 @@ async function parseCreateTDFOptions(
347346
): Promise<CreateTDFOptions> {
348347
const c: CreateTDFOptions = { ...(await parseCreateOptions(argv)), ...flags };
349348
if (argv.assertions?.length) {
350-
// Reads from disk (parseAssertionConfig falls back to openAsBlob), so it
349+
// Reads from disk (parseAssertionConfig falls back to reading a file), so it
351350
// stays here rather than in the pure pass.
352351
c.assertionConfigs = await parseAssertionConfig(argv.assertions);
353352
}
@@ -359,16 +358,23 @@ async function fileAsSource(file: string): Promise<Source> {
359358
if (!file) {
360359
throw new CLIError('CRITICAL', 'Must specify file or pipe');
361360
}
361+
let location: FileChunker;
362362
try {
363-
const stats = await stat(file);
364-
if (!stats?.isFile()) {
365-
throw new CLIError('CRITICAL', `File does not exist [${file}]`);
366-
}
363+
// Not fs.openAsBlob: Node truncates a file-backed Blob's size to 32 bits,
364+
// which breaks every read of a file >= 4 GiB (DSPX-5160).
365+
location = await openFileChunker(file);
367366
} catch (e) {
368367
throw new CLIError('CRITICAL', `File is not accessable [${file}]`, asError(e));
369368
}
370-
log('DEBUG', `Using input from file [${file}]`);
371-
return { type: 'file-browser', location: await openAsBlob(file) };
369+
log('DEBUG', `Using input from file [${file}] (${location.size} bytes)`);
370+
return { type: 'chunker', location };
371+
}
372+
373+
/** Releases the file handle, if any, opened by {@link fileAsSource}. */
374+
async function closeSource(source?: Source): Promise<void> {
375+
if (source?.type === 'chunker' && 'close' in source.location) {
376+
await (source.location as FileChunker).close();
377+
}
372378
}
373379

374380
export const handleArgs = (args: string[]) => {
@@ -606,8 +612,10 @@ export const handleArgs = (args: string[]) => {
606612
platformUrl: argv.platformUrl || guessPolicyUrl(argv),
607613
authProvider: new InvalidAuthProvider(),
608614
});
615+
let readOptions: ReadOptions | undefined;
609616
try {
610-
const reader = ct.open(await parseReadOptions(argv));
617+
readOptions = await parseReadOptions(argv);
618+
const reader = ct.open(readOptions);
611619
const manifest = await reader.manifest();
612620
try {
613621
const dataAttributes = await reader.attributes();
@@ -618,6 +626,7 @@ export const handleArgs = (args: string[]) => {
618626
}
619627
} finally {
620628
ct.close();
629+
await closeSource(readOptions?.source);
621630
}
622631
}
623632
)
@@ -656,10 +665,12 @@ export const handleArgs = (args: string[]) => {
656665
policyEndpoint: guessedPolicyEndpoint,
657666
platformUrl: argv.platformUrl || guessedPolicyEndpoint,
658667
});
668+
let readOptions: ReadOptions | undefined;
659669
try {
660670
log('SILLY', `Initialized client`);
661671
log('DEBUG', `About to TDF3 decrypt [${String(argv.file)}]`);
662-
const ct = await client.read(await parseReadOptions(argv));
672+
readOptions = await parseReadOptions(argv);
673+
const ct = await client.read(readOptions);
663674
const destination = argv.output ? createWriteStream(argv.output) : process.stdout;
664675
await ct.pipeTo(Writable.toWeb(destination) as WritableStream<Uint8Array>);
665676

@@ -696,6 +707,7 @@ export const handleArgs = (args: string[]) => {
696707
console.assert(!argv.dpop || dpopToken, 'DPoP requested but absent');
697708
} finally {
698709
client.close();
710+
await closeSource(readOptions?.source);
699711
}
700712
}
701713
)
@@ -720,30 +732,34 @@ export const handleArgs = (args: string[]) => {
720732
throw new CLIError('CRITICAL', 'oidcEndpoint must be specified');
721733
}
722734
const createOptions = await parseCreateTDFOptions(argv, createFlags);
723-
const authProvider = await processAuth(argv);
724-
log('DEBUG', `Initialized auth provider ${JSON.stringify(authProvider)}`);
725-
const guessedPolicyEndpoint = guessPolicyUrl(argv);
726-
727-
const client = new OpenTDF({
728-
authProvider,
729-
defaultCreateOptions: {
730-
defaultKASEndpoint: argv.kasEndpoint,
731-
},
732-
disableDPoP: !argv.dpop,
733-
policyEndpoint: guessedPolicyEndpoint,
734-
platformUrl: argv.platformUrl || guessedPolicyEndpoint,
735-
});
736735
try {
737-
log('SILLY', `Initialized client`);
738-
log('DEBUG', `TDF Create`);
739-
const ct: DecoratedStream = await client.createTDF(createOptions);
740-
if (!ct) {
741-
throw new CLIError('CRITICAL', 'Encrypt configuration error: No output?');
736+
const authProvider = await processAuth(argv);
737+
log('DEBUG', `Initialized auth provider ${JSON.stringify(authProvider)}`);
738+
const guessedPolicyEndpoint = guessPolicyUrl(argv);
739+
740+
const client = new OpenTDF({
741+
authProvider,
742+
defaultCreateOptions: {
743+
defaultKASEndpoint: argv.kasEndpoint,
744+
},
745+
disableDPoP: !argv.dpop,
746+
policyEndpoint: guessedPolicyEndpoint,
747+
platformUrl: argv.platformUrl || guessedPolicyEndpoint,
748+
});
749+
try {
750+
log('SILLY', `Initialized client`);
751+
log('DEBUG', `TDF Create`);
752+
const ct: DecoratedStream = await client.createTDF(createOptions);
753+
if (!ct) {
754+
throw new CLIError('CRITICAL', 'Encrypt configuration error: No output?');
755+
}
756+
const destination = argv.output ? createWriteStream(argv.output) : process.stdout;
757+
await ct.pipeTo(Writable.toWeb(destination) as WritableStream<Uint8Array>);
758+
} finally {
759+
client.close();
742760
}
743-
const destination = argv.output ? createWriteStream(argv.output) : process.stdout;
744-
await ct.pipeTo(Writable.toWeb(destination) as WritableStream<Uint8Array>);
745761
} finally {
746-
client.close();
762+
await closeSource(createOptions.source);
747763
}
748764
}
749765
)

‎cli/src/file-chunker.ts‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { type FileHandle, open } from 'node:fs/promises';
2+
import { type Chunker } from '@opentdf/sdk';
3+
4+
/**
5+
* Largest single `FileHandle.read` we issue. Node rejects reads longer than
6+
* 2^31-1 bytes, so larger ranges are read in a loop.
7+
*/
8+
const MAX_READ = 1 << 30;
9+
10+
/**
11+
* A {@link Chunker} over a local file, plus the means to release it.
12+
*/
13+
export type FileChunker = Chunker & {
14+
/** File size in bytes, taken from `fstat` when the file was opened. */
15+
readonly size: number;
16+
/** Closes the underlying file handle. Safe to call more than once. */
17+
close(): Promise<void>;
18+
};
19+
20+
/**
21+
* Resolve chunker-style `[byteStart, byteEnd)` arguments against a known size,
22+
* using the same rules as `Uint8Array.prototype.slice`: negative values count
23+
* back from the end, and results are clamped to `[0, size]`.
24+
*/
25+
export function resolveRange(
26+
size: number,
27+
byteStart?: number,
28+
byteEnd?: number
29+
): [start: number, end: number] {
30+
const clamp = (n: number) => Math.min(Math.max(n < 0 ? size + n : n, 0), size);
31+
const start = byteStart === undefined ? 0 : clamp(byteStart);
32+
const end = byteEnd === undefined ? size : clamp(byteEnd);
33+
return [start, Math.max(start, end)];
34+
}
35+
36+
/**
37+
* Opens a local file for random access by explicit byte position.
38+
*
39+
* This exists because `fs.openAsBlob()` cannot be used for files of 4 GiB or
40+
* more: Node reports a file-backed Blob's `size` modulo 2^32, so `slice()`
41+
* with a negative offset (which the zip reader uses to find the central
42+
* directory) reads from the wrong place. See nodejs/node#66207 and DSPX-5160.
43+
*
44+
* `FileHandle.read` takes a numeric position, which is exact up to 2^53, so
45+
* this works for any file size the SDK can otherwise handle.
46+
*
47+
* The returned function can be passed as `{ type: 'chunker', location }` to
48+
* both decrypt (random access) and encrypt (read sequentially in chunks until
49+
* an empty chunk is returned). Call `close()` when finished.
50+
*/
51+
export async function openFileChunker(path: string): Promise<FileChunker> {
52+
const handle: FileHandle = await open(path, 'r');
53+
let size: number;
54+
try {
55+
const stats = await handle.stat();
56+
if (!stats.isFile()) {
57+
throw new Error(`not a regular file [${path}]`);
58+
}
59+
size = stats.size;
60+
} catch (e) {
61+
await handle.close();
62+
throw e;
63+
}
64+
let closed: Promise<void> | undefined;
65+
66+
const chunker = async (byteStart?: number, byteEnd?: number): Promise<Uint8Array> => {
67+
if (closed) {
68+
throw new Error(`file is closed [${path}]`);
69+
}
70+
const [start, end] = resolveRange(size, byteStart, byteEnd);
71+
const buf = new Uint8Array(end - start);
72+
let filled = 0;
73+
while (filled < buf.length) {
74+
const length = Math.min(buf.length - filled, MAX_READ);
75+
const { bytesRead } = await handle.read(buf, filled, length, start + filled);
76+
if (bytesRead === 0) {
77+
// File shrank since it was opened; return what we have.
78+
return buf.subarray(0, filled);
79+
}
80+
filled += bytesRead;
81+
}
82+
return buf;
83+
};
84+
85+
return Object.assign(chunker, {
86+
size,
87+
close: () => (closed ??= handle.close()),
88+
});
89+
}

0 commit comments

Comments
 (0)