|
| 1 | +import type { NormalizedRun, PollOptions, RunInputValue, WaitForRunSnapshotOptions } from '@deepnote/cloud' |
| 2 | +import { |
| 3 | + describeRunError, |
| 4 | + isSuccessStatus, |
| 5 | + pollRunUntilComplete, |
| 6 | + triggerNotebookRun, |
| 7 | + waitForRunSnapshot, |
| 8 | +} from '@deepnote/cloud' |
| 9 | +import { extractOutputs } from './extract-outputs' |
| 10 | +import { finishResult, type OrchestrationStepExecutor } from './orchestrate' |
| 11 | +import { parseSnapshot } from './snapshot-view' |
| 12 | + |
| 13 | +/** |
| 14 | + * The step executor every pipeline uses by default: three plain HTTP calls per step — start a run, |
| 15 | + * poll it, read its snapshot. |
| 16 | + * |
| 17 | + * It is `fetch` and nothing else, which is what makes a pipeline portable. The same executor runs |
| 18 | + * in a Node script, in CI, and in a browser page; there is no server in the middle and no local |
| 19 | + * kernel involved. |
| 20 | + */ |
| 21 | + |
| 22 | +/** The API origin Deepnote Cloud serves from, used when a caller does not name one. */ |
| 23 | +export const DEFAULT_CLOUD_API_URL = 'https://api.deepnote.com' |
| 24 | + |
| 25 | +export interface CloudExecutorOptions { |
| 26 | + /** |
| 27 | + * Deepnote API token. |
| 28 | + * |
| 29 | + * In a published app this is the short-lived, viewer-scoped token the Deepnote shell issues, and |
| 30 | + * it arrives with the origin to send it to. In a script it is an ordinary API token. |
| 31 | + */ |
| 32 | + token: string |
| 33 | + /** API origin. Pair this with the token that was issued for it. */ |
| 34 | + baseUrl?: string |
| 35 | + /** Poll tuning shared by every step. */ |
| 36 | + poll?: Omit<PollOptions, 'onStatus'> |
| 37 | + /** Snapshot-settling tuning shared by every step. */ |
| 38 | + snapshot?: WaitForRunSnapshotOptions |
| 39 | + /** Abort every in-flight request for this pipeline. */ |
| 40 | + signal?: AbortSignal |
| 41 | +} |
| 42 | + |
| 43 | +/** |
| 44 | + * Coerce a pipeline's input value to what `POST /v2/runs` accepts. |
| 45 | + * |
| 46 | + * The API takes exactly `string | boolean | string[]`, so numbers are stringified here rather than |
| 47 | + * rejected — a pipeline computing `monthly_target_k` should not have to remember to call |
| 48 | + * `String()`. Anything without an unambiguous textual form is refused instead of guessed at. |
| 49 | + */ |
| 50 | +export function toRunInputs(inputs: Record<string, unknown>): Record<string, RunInputValue> { |
| 51 | + const coerced: Record<string, RunInputValue> = {} |
| 52 | + for (const [name, value] of Object.entries(inputs)) { |
| 53 | + if (typeof value === 'string' || typeof value === 'boolean') { |
| 54 | + coerced[name] = value |
| 55 | + } else if (typeof value === 'number') { |
| 56 | + if (!Number.isFinite(value)) { |
| 57 | + throw new Error(`Input "${name}" is ${String(value)}, which Deepnote cannot accept.`) |
| 58 | + } |
| 59 | + coerced[name] = String(value) |
| 60 | + } else if (Array.isArray(value) && value.every(part => typeof part === 'string')) { |
| 61 | + coerced[name] = value as string[] |
| 62 | + } else if (value !== undefined && value !== null) { |
| 63 | + throw new Error( |
| 64 | + `Input "${name}" is a ${Array.isArray(value) ? 'mixed array' : typeof value}. Deepnote inputs accept a string, boolean, or array of strings.` |
| 65 | + ) |
| 66 | + } |
| 67 | + } |
| 68 | + return coerced |
| 69 | +} |
| 70 | + |
| 71 | +/** Build the executor {@link orchestrate} uses. Exported for callers composing their own engine. */ |
| 72 | +export function createCloudStepExecutor(options: CloudExecutorOptions): OrchestrationStepExecutor { |
| 73 | + const baseUrl = options.baseUrl ?? DEFAULT_CLOUD_API_URL |
| 74 | + const { token } = options |
| 75 | + |
| 76 | + return async ({ id, step, startedMs, startedAt, emit }) => { |
| 77 | + if (!step.notebookId) { |
| 78 | + throw new Error(`Orchestration step "${id}" has no notebookId to run.`) |
| 79 | + } |
| 80 | + |
| 81 | + const started = await triggerNotebookRun( |
| 82 | + baseUrl, |
| 83 | + token, |
| 84 | + { notebookId: step.notebookId, inputs: toRunInputs(step.inputs ?? {}) }, |
| 85 | + { signal: options.signal } |
| 86 | + ) |
| 87 | + emit({ type: 'step_status', stepId: id, status: started.status }) |
| 88 | + |
| 89 | + const completed: NormalizedRun = await pollRunUntilComplete(baseUrl, token, started.runId, { |
| 90 | + ...options.poll, |
| 91 | + signal: options.signal, |
| 92 | + onStatus: status => { |
| 93 | + emit({ type: 'step_status', stepId: id, status }) |
| 94 | + }, |
| 95 | + } as PollOptions) |
| 96 | + |
| 97 | + // Read the snapshot even when the run failed: it is usually the only place the failing block's |
| 98 | + // error is recorded, and a page that shows nothing is worse than one that shows why. |
| 99 | + const settled = await waitForRunSnapshot(baseUrl, token, completed, { |
| 100 | + ...options.snapshot, |
| 101 | + signal: options.signal, |
| 102 | + }) |
| 103 | + const snapshotYaml = settled.content |
| 104 | + const success = isSuccessStatus(completed.status) |
| 105 | + |
| 106 | + return finishResult( |
| 107 | + { |
| 108 | + id, |
| 109 | + target: 'cloud', |
| 110 | + success, |
| 111 | + status: completed.status, |
| 112 | + outputs: snapshotYaml ? extractOutputs(snapshotYaml) : [], |
| 113 | + snapshotYaml, |
| 114 | + snapshot: snapshotYaml ? parseSnapshot(snapshotYaml) : null, |
| 115 | + runId: completed.runId, |
| 116 | + error: success |
| 117 | + ? undefined |
| 118 | + : (describeRunError(completed) ?? `the run finished with status "${completed.status}"`), |
| 119 | + }, |
| 120 | + startedMs, |
| 121 | + startedAt |
| 122 | + ) |
| 123 | + } |
| 124 | +} |
0 commit comments