forked from heygen-com/hyperframes
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathhost-registry-assets.ts
More file actions
273 lines (247 loc) · 9.18 KB
/
Copy pathhost-registry-assets.ts
File metadata and controls
273 lines (247 loc) · 9.18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
#!/usr/bin/env tsx
/**
* Move a registry item's binary assets onto the public CDN.
*
* A registry item is served straight out of this repository
* (`DEFAULT_REGISTRY_URL` points at raw.githubusercontent.com), so every byte a
* block ships is a byte in the history forever. Compositions are text and cost
* nothing; images are not. One carousel family arrived with 396 JPEGs.
*
* This rewrites those files as `url` entries in the manifest and takes the
* bytes out of the tree. `path` is untouched — it still says where the file
* lands relative to the item, so the composition HTML, the installer's target
* mirroring and the catalog preview renderer all keep working unchanged.
*
* Keys are content-addressed: the same image referenced by twenty-five blocks
* uploads once and every manifest points at the same URL. It also means a
* changed image gets a new key, which is the only safe way to publish behind
* `immutable, max-age=31536000` — a re-upload under the old name would keep
* serving the old bytes at the edge until the TTL expired.
*
* Usage:
* bun run scripts/host-registry-assets.ts --all
* bun run scripts/host-registry-assets.ts carousel-orbit-1 carousel-orbit-2
* bun run scripts/host-registry-assets.ts --all --dry-run
* bun run scripts/host-registry-assets.ts --all --no-upload # stage only
*
* Requires AWS credentials for the heygen engineering account
* (profile: engineering-767398024897) with s3:PutObject, unless --no-upload.
* Contributors without AWS access: open the PR with the assets committed and a
* maintainer will run this before merging.
*/
import { createHash } from "node:crypto";
import {
copyFileSync,
constants,
closeSync,
fstatSync,
openSync,
existsSync,
mkdirSync,
readdirSync,
readFileSync,
rmSync,
unlinkSync,
writeFileSync,
} from "node:fs";
import { execFileSync } from "node:child_process";
import { dirname, extname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { runAsCommand } from "./entrypoint.ts";
// Import from source, like every other script here: bun workspace linking does
// not resolve for scripts outside packages/.
import { ITEM_TYPE_DIRS, type FileTarget, type RegistryItem } from "../packages/core/src/index.js";
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const CDN_BASE = "https://static.heygen.ai/hyperframes-oss/registry-assets";
const S3_DEST = "s3://heygen-public/hyperframes-oss/registry-assets";
const PROFILE = process.env.AWS_PROFILE ?? "engineering-767398024897";
const STAGING_DIR = resolve(repoRoot, ".registry-assets-staging");
/**
* Extensions worth hosting. Text files stay in the repository: they diff, they
* review, and they cost a few hundred bytes. Only the opaque blobs leave.
*/
const HOSTED_EXTENSIONS = new Set([
".png",
".jpg",
".jpeg",
".webp",
".gif",
".avif",
".mp4",
".webm",
".mov",
".mp3",
".wav",
".m4a",
".woff",
".woff2",
".ttf",
".otf",
".glb",
".gltf",
]);
interface Plan {
/** Absolute path of the file to remove from the tree. */
source: string;
/** Content-addressed object key, without the bucket prefix. */
key: string;
/** Full digest of the bytes used to derive the key. */
digest: string;
/** Public URL the manifest will carry. */
url: string;
}
function itemDirsUnder(typeDir: string): string[] {
const root = join(repoRoot, "registry", typeDir);
if (!existsSync(root)) return [];
return readdirSync(root, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => join(root, entry.name))
.filter((dir) => existsSync(join(dir, "registry-item.json")));
}
/** Every item directory under registry/ that has a manifest. */
function allItemDirs(): string[] {
const typeDirs = [...new Set(Object.values(ITEM_TYPE_DIRS))];
return typeDirs.flatMap(itemDirsUnder).sort();
}
function itemDirFor(name: string): string {
const match = allItemDirs().find((dir) => dir.endsWith(`/${name}`));
if (match === undefined) throw new Error(`No registry item named "${name}".`);
return match;
}
/**
* A file that already carries `url` is left exactly as it is: its bytes are
* gone from the tree, so re-deriving a key from a file that is not there would
* either crash or, worse, silently drop the entry.
*/
function needsHosting(file: FileTarget): boolean {
return file.url === undefined && HOSTED_EXTENSIONS.has(extname(file.path).toLowerCase());
}
/** Point one file at the CDN, and say which bytes have to get there. */
function hostFile(itemName: string, itemDir: string, file: FileTarget): Plan {
const source = resolve(itemDir, file.path);
const fd = openSync(source, constants.O_RDONLY | constants.O_NONBLOCK);
let digest: string;
try {
if (!fstatSync(fd).isFile()) {
throw new Error(`${itemName}: files[] declares "${file.path}", which is not a file.`);
}
digest = createHash("sha256").update(readFileSync(fd)).digest("hex");
} finally {
closeSync(fd);
}
const key = `${digest.slice(0, 16)}${extname(file.path).toLowerCase()}`;
file.url = `${CDN_BASE}/${key}`;
return { source, key, digest, url: file.url };
}
/** Prepare a manifest update and identify the bytes that must reach the CDN. */
function planItem(itemDir: string): {
plans: Plan[];
manifestPath: string;
manifest: RegistryItem;
} {
const manifestPath = join(itemDir, "registry-item.json");
const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as RegistryItem;
const plans = (manifest.files ?? [])
.filter(needsHosting)
.map((file) => hostFile(manifest.name, itemDir, file));
return { plans, manifestPath, manifest };
}
function writeManifests(items: ReturnType<typeof planItem>[]): void {
for (const { manifestPath, manifest } of items) {
writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf-8");
}
}
/** Collect every planned object into one flat directory, deduplicated by key. */
function stage(plans: Plan[]): number {
rmSync(STAGING_DIR, { recursive: true, force: true });
mkdirSync(STAGING_DIR, { recursive: true });
const seen = new Set<string>();
for (const plan of plans) {
if (seen.has(plan.key)) continue;
seen.add(plan.key);
const stagedPath = join(STAGING_DIR, plan.key);
copyFileSync(plan.source, stagedPath);
const digest = createHash("sha256").update(readFileSync(stagedPath)).digest("hex");
if (digest !== plan.digest) {
unlinkSync(stagedPath);
throw new Error(`Asset changed while staging: ${plan.source}. No upload was attempted.`);
}
}
return seen.size;
}
function upload(): void {
// `s3 sync` rather than `cp --recursive`: keys are content-addressed, so an
// object already up there is byte-identical and re-sending it is waste.
//
// No CloudFront invalidation, and that is not an omission — a new key has
// never been requested, so no edge holds a stale copy of it. That is the
// whole point of hashing the name.
execFileSync(
"aws",
[
"--profile",
PROFILE,
"s3",
"sync",
`${STAGING_DIR}/`,
`${S3_DEST}/`,
"--cache-control",
"public, max-age=31536000, immutable",
"--metadata-directive",
"REPLACE",
],
{ stdio: "inherit" },
);
}
/** Report what would move, changing nothing. */
function reportDryRun(itemDirs: string[]): void {
for (const dir of itemDirs) {
const { plans } = planItem(dir);
if (plans.length > 0) {
console.log(`${dir.replace(`${repoRoot}/`, "")}: ${plans.length} file(s) would be hosted`);
}
}
}
/** Drop the local copies. Only ever called once the bytes are somewhere else. */
function discardStagedSources(plans: Plan[]): void {
for (const plan of plans) {
if (existsSync(plan.source)) unlinkSync(plan.source);
}
rmSync(STAGING_DIR, { recursive: true, force: true });
}
function resolveItemDirs(argv: string[]): string[] {
if (argv.includes("--all")) return allItemDirs();
const names = argv.filter((arg) => !arg.startsWith("--"));
if (names.length === 0) throw new Error("Nothing to do: pass item names or --all.");
return names.map(itemDirFor);
}
export async function main(argv: string[]): Promise<void> {
const itemDirs = resolveItemDirs(argv);
if (argv.includes("--dry-run")) {
reportDryRun(itemDirs);
return;
}
const items = itemDirs.map(planItem).filter((item) => item.plans.length > 0);
const plans = items.flatMap((item) => item.plans);
if (plans.length === 0) {
console.log("No unhosted binary assets found — nothing to do.");
return;
}
const objects = stage(plans);
writeManifests(items);
console.log(
`Staged ${objects} object(s) from ${plans.length} manifest entr(ies) → ${STAGING_DIR}`,
);
if (argv.includes("--no-upload")) {
console.log(
`Skipping upload. Publish with:\n aws --profile ${PROFILE} s3 sync ${STAGING_DIR}/ ${S3_DEST}/ --cache-control "public, max-age=31536000, immutable" --metadata-directive REPLACE`,
);
return;
}
// Discarding before the upload succeeded would leave manifests pointing at
// URLs that 404, with no local copy to retry from.
upload();
discardStagedSources(plans);
console.log(`Published ${objects} object(s). Live under ${CDN_BASE}/`);
}
runAsCommand(import.meta.url, () => main(process.argv.slice(2)));