Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions examples/local-runner/orchestration/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# orchestration

A pipeline as a script: fan out across regional notebooks, gate on their structured results, and
report. About 40 lines.

```bash
DEEPNOTE_TOKEN=… NA_NOTEBOOK_ID=… EU_NOTEBOOK_ID=… APAC_NOTEBOOK_ID=… pnpm example:orchestration
```

The point of the example is that none of it is Node-specific. `orchestrate` runs every step as an
HTTP call to Deepnote, so the same pipeline runs unchanged in a browser page with no server behind
it — see [`run-app`](../run-app). What Node adds here is a shell, not a capability.

Notebooks are named by id and must already exist: running a pipeline needs permission to run a
notebook, not to create one.

Each regional notebook should end by emitting a JSON object the gate can read:

```python
print(json.dumps({"region": region, "revenueK": 812.4, "qualityScore": 0.97}))
```

`outputs.lastJson(step)` reads that back without depending on block ids, which Deepnote reassigns
when it creates a notebook.
70 changes: 70 additions & 0 deletions examples/local-runner/orchestration/run.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// A pipeline in ~40 lines: fan out, gate on the results, then decide.
//
// Nothing here is browser- or Node-specific. The same code runs in a page — every step is an HTTP
// call to Deepnote, so there is no server, no local kernel, and no orchestration daemon.
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
//
// In a real project, import from '@deepnote/local-runner' after installing it.
import { orchestrate } from '../../../packages/local-runner/dist/index.js'

try {
process.loadEnvFile()
} catch {}

const token = process.env.DEEPNOTE_TOKEN
if (!token) {
console.error('Set DEEPNOTE_TOKEN (or put it in a .env) to run this example.')
process.exit(1)
}

// Notebooks already in Deepnote. Pass their ids in rather than editing this file.
const REGIONS = [
{ name: 'North America', notebookId: process.env.NA_NOTEBOOK_ID },
{ name: 'Europe', notebookId: process.env.EU_NOTEBOOK_ID },
{ name: 'Asia Pacific', notebookId: process.env.APAC_NOTEBOOK_ID },
].filter(region => region.notebookId)

if (REGIONS.length === 0) {
console.error('Set NA_NOTEBOOK_ID / EU_NOTEBOOK_ID / APAC_NOTEBOOK_ID to the notebooks to run.')
process.exit(1)
}

const QUALITY_THRESHOLD = 0.95

const { value, graph, durationMs } = await orchestrate(
async ({ run, control, outputs }) => {
// Ordinary control flow is the pipeline: these fan out concurrently because Promise.all does.
const analyses = await Promise.all(
REGIONS.map(region =>
run({
id: `analyze-${slug(region.name)}`,
label: `${region.name} analysis`,
notebookId: region.notebookId,
inputs: { region: region.name, trailing_months: 6 },
})
)
)
const readings = analyses.map(step => outputs.lastJson(step))

// A control node is a local decision that should still show up in the graph.
const belowThreshold = await control(
{
id: 'quality-gate',
kind: 'gate',
label: `${QUALITY_THRESHOLD * 100}% quality gate`,
dependsOn: analyses.map(step => step.id),
},
() => readings.filter(reading => reading.qualityScore < QUALITY_THRESHOLD).map(reading => reading.region)
)

return { checked: readings.length, belowThreshold }
},
{ token, onEvent: event => event.type === 'step_started' && console.log(` → ${event.stepId}`) }
)

console.log(`\n ${value.checked} regions in ${(durationMs / 1000).toFixed(1)}s`)
console.log(` below threshold: ${value.belowThreshold.join(', ') || 'none'}`)
console.log(` graph: ${graph.nodes.length} nodes, ${graph.edges.length} edges\n`)

function slug(value) {
return value.toLowerCase().replace(/[^a-z0-9]+/g, '-')
}
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"build": "pnpm -r run build",
"example:gallery": "pnpm --filter @deepnote/local-runner... build && node examples/local-runner/gallery/serve.mjs",
"example:local-runner": "pnpm --filter @deepnote/local-runner... build && node examples/local-runner/run-app/serve.mjs",
"example:orchestration": "pnpm --filter @deepnote/local-runner... build && node examples/local-runner/orchestration/run.mjs",
"example:schedule-cloud": "pnpm --filter @deepnote/local-runner... build && node examples/local-runner/schedule-cloud.mjs",
"example:snapshot-viewer": "pnpm --filter @deepnote/local-runner... build && node examples/local-runner/snapshot-viewer/serve.mjs",
"license-check": "license-checker-rseidelsohn --json --onlyAllow \"MIT;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC\" --excludePackages \"deepnote\"",
Expand Down
59 changes: 59 additions & 0 deletions packages/local-runner/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,65 @@ The server binds to `127.0.0.1` and provides no WebSocket, watch, or rendering.
— or, to _view_ an existing snapshot rather than run one, read it directly (below); that needs no
server at all.

### Orchestrate notebook pipelines

Run several notebooks as one pipeline — fan out, gate on the results, decide:

```ts
import { orchestrate } from "@deepnote/local-runner";

const { value, graph } = await orchestrate(
async ({ run, control, outputs }) => {
const analyses = await Promise.all(
REGIONS.map((region) =>
run({
id: region.name,
notebookId: region.notebookId,
inputs: { region: region.name },
}),
),
);
const readings = analyses.map((step) => outputs.lastJson(step));

const failing = await control(
{
id: "quality-gate",
kind: "gate",
dependsOn: analyses.map((s) => s.id),
},
() => readings.filter((r) => r.qualityScore < 0.95).map((r) => r.region),
);

return { checked: readings.length, failing };
},
{ token, onEvent: (event) => render(event) },
);
```

This is deliberately an imperative API, not a workflow language: `await`, `Promise.all`, loops and
branches in the callback provide sequencing, concurrency, and conditionals. The library records what
happened — the graph, the events, the normalized results.

**It needs no server and no local kernel.** Every step is an HTTP call to Deepnote, so the same
pipeline runs in a script, in CI, and in a browser page. Nothing reachable from `orchestrate`
imports `node:*`.

Notebooks are addressed by id and must already exist. Running a pipeline needs permission to run a
notebook, not to create one — which is what lets a page do it with a viewer's short-lived token.

`control` records a local decision as a node, so a gate or an aggregation shows up in the graph
instead of happening invisibly between steps. `outputs.lastJson(step)` and
`outputs.lastAgentText(step)` read a step's results without depending on block ids, which Deepnote
reassigns when it creates a notebook.

A failed notebook throws `OrchestrationStepError` carrying the result, so a caller can still show
how far the run got; `allowFailure: true` returns it instead.

`runOrchestration(workflow, options, executor)` is the same engine with the runner left open, for
callers that want to run steps somewhere else.

See [`examples/local-runner/orchestration`](../../examples/local-runner/orchestration).

### Read a snapshot — no Python, no kernel

A snapshot is a `.deepnote` file with the outputs stored inline, so reading one is parsing, not
Expand Down
33 changes: 2 additions & 31 deletions packages/local-runner/src/cloud-common.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import type { DeepnoteFile } from '@deepnote/blocks'
import { deepnoteFileSchema, deepnoteSnapshotSchema, parseYaml } from '@deepnote/blocks'
import { describeRunError, findNotebook, getWorkspace, type NormalizedRun, notebookUrl } from '@deepnote/cloud'
import type { RunBlockOutput } from './run-with-inputs'
import type { SnapshotView } from './snapshot-view'
import { parseSnapshot } from './snapshot-view'

export { extractOutputs } from './extract-outputs'

/**
* The plumbing every cloud entry point needs — `run-in-cloud.ts` and `cloud-runs.ts` both reach for
* all of it. Internal: none of this is exported from `index.ts`.
Expand Down Expand Up @@ -150,32 +150,3 @@ function describeFailedBlocks(snapshotYaml: string): string | undefined {
}
return undefined
}

/**
* Parse the per-block outputs out of a cloud snapshot's YAML, in document order. Any executable
* block type carries outputs — code, SQL, visualization, big-number — so read them off whatever
* block has them (via {@link parseSnapshot}) rather than special-casing `code`.
*
* A snapshot that won't parse throws rather than returning nothing: the run succeeded, so "no
* outputs" is a claim about the notebook, and it would be a false one. The caller still has the raw
* YAML to inspect.
*/
export function extractOutputs(snapshotYaml: string): RunBlockOutput[] {
let view: SnapshotView
try {
view = parseSnapshot(snapshotYaml)
} catch (error) {
throw new Error(
`Deepnote returned a snapshot that could not be parsed: ${error instanceof Error ? error.message : String(error)}`
)
}
const outputs: RunBlockOutput[] = []
for (const notebook of view.notebooks) {
for (const block of notebook.blocks) {
if (block.outputs.length > 0) {
outputs.push({ blockId: block.id, outputs: block.outputs, executionCount: block.executionCount })
}
}
}
return outputs
}
124 changes: 124 additions & 0 deletions packages/local-runner/src/cloud-executor.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import type { NormalizedRun, PollOptions, RunInputValue, WaitForRunSnapshotOptions } from '@deepnote/cloud'
import {
describeRunError,
isSuccessStatus,
pollRunUntilComplete,
triggerNotebookRun,
waitForRunSnapshot,
} from '@deepnote/cloud'
import { extractOutputs } from './extract-outputs'
import { finishResult, type OrchestrationStepExecutor } from './orchestrate'
import { parseSnapshot } from './snapshot-view'

/**
* The step executor every pipeline uses by default: three plain HTTP calls per step — start a run,
* poll it, read its snapshot.
*
* It is `fetch` and nothing else, which is what makes a pipeline portable. The same executor runs
* in a Node script, in CI, and in a browser page; there is no server in the middle and no local
* kernel involved.
*/

/** The API origin Deepnote Cloud serves from, used when a caller does not name one. */
export const DEFAULT_CLOUD_API_URL = 'https://api.deepnote.com'

export interface CloudExecutorOptions {
/**
* Deepnote API token.
*
* In a published app this is the short-lived, viewer-scoped token the Deepnote shell issues, and
* it arrives with the origin to send it to. In a script it is an ordinary API token.
*/
token: string
/** API origin. Pair this with the token that was issued for it. */
baseUrl?: string
/** Poll tuning shared by every step. */
poll?: Omit<PollOptions, 'onStatus'>
/** Snapshot-settling tuning shared by every step. */
snapshot?: WaitForRunSnapshotOptions
/** Abort every in-flight request for this pipeline. */
signal?: AbortSignal
}

/**
* Coerce a pipeline's input value to what `POST /v2/runs` accepts.
*
* The API takes exactly `string | boolean | string[]`, so numbers are stringified here rather than
* rejected — a pipeline computing `monthly_target_k` should not have to remember to call
* `String()`. Anything without an unambiguous textual form is refused instead of guessed at.
*/
export function toRunInputs(inputs: Record<string, unknown>): Record<string, RunInputValue> {
const coerced: Record<string, RunInputValue> = {}
for (const [name, value] of Object.entries(inputs)) {
if (typeof value === 'string' || typeof value === 'boolean') {
coerced[name] = value
} else if (typeof value === 'number') {
if (!Number.isFinite(value)) {
throw new Error(`Input "${name}" is ${String(value)}, which Deepnote cannot accept.`)
}
coerced[name] = String(value)
} else if (Array.isArray(value) && value.every(part => typeof part === 'string')) {
coerced[name] = value as string[]
} else if (value !== undefined && value !== null) {
throw new Error(
`Input "${name}" is a ${Array.isArray(value) ? 'mixed array' : typeof value}. Deepnote inputs accept a string, boolean, or array of strings.`
)
}
}
return coerced
}

/** Build the executor {@link orchestrate} uses. Exported for callers composing their own engine. */
export function createCloudStepExecutor(options: CloudExecutorOptions): OrchestrationStepExecutor {
const baseUrl = options.baseUrl ?? DEFAULT_CLOUD_API_URL
const { token } = options

return async ({ id, step, startedMs, startedAt, emit }) => {
if (!step.notebookId) {
throw new Error(`Orchestration step "${id}" has no notebookId to run.`)
}

const started = await triggerNotebookRun(
baseUrl,
token,
{ notebookId: step.notebookId, inputs: toRunInputs(step.inputs ?? {}) },
{ signal: options.signal }
)
emit({ type: 'step_status', stepId: id, status: started.status })

const completed: NormalizedRun = await pollRunUntilComplete(baseUrl, token, started.runId, {
...options.poll,
signal: options.signal,
onStatus: status => {
emit({ type: 'step_status', stepId: id, status })
},
} as PollOptions)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

// Read the snapshot even when the run failed: it is usually the only place the failing block's
// error is recorded, and a page that shows nothing is worse than one that shows why.
const settled = await waitForRunSnapshot(baseUrl, token, completed, {
...options.snapshot,
signal: options.signal,
})
const snapshotYaml = settled.content
const success = isSuccessStatus(completed.status)

return finishResult(
{
id,
target: 'cloud',
success,
status: completed.status,
outputs: snapshotYaml ? extractOutputs(snapshotYaml) : [],
snapshotYaml,
snapshot: snapshotYaml ? parseSnapshot(snapshotYaml) : null,
runId: completed.runId,
error: success
? undefined
: (describeRunError(completed) ?? `the run finished with status "${completed.status}"`),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
},
startedMs,
startedAt
)
}
}
37 changes: 37 additions & 0 deletions packages/local-runner/src/extract-outputs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import type { RunBlockOutput } from './run-with-inputs'
import type { SnapshotView } from './snapshot-view'
import { parseSnapshot } from './snapshot-view'

/**
* Read the per-block outputs out of a cloud snapshot's YAML, in document order.
*
* Split out of `cloud-common.ts` so it can bundle for the browser: everything else there resolves
* tokens from `process.env`, which a page has no business carrying. Only `parseSnapshot` and the
* block schemas are reachable from here.
*
* Any executable block type carries outputs — code, SQL, visualization, big-number — so read them
* off whatever block has them rather than special-casing `code`.
*
* A snapshot that won't parse throws rather than returning nothing: the run succeeded, so "no
* outputs" is a claim about the notebook, and it would be a false one. The caller still has the raw
* YAML to inspect.
*/
export function extractOutputs(snapshotYaml: string): RunBlockOutput[] {
let view: SnapshotView
try {
view = parseSnapshot(snapshotYaml)
} catch (error) {
throw new Error(
`Deepnote returned a snapshot that could not be parsed: ${error instanceof Error ? error.message : String(error)}`
)
}
const outputs: RunBlockOutput[] = []
for (const notebook of view.notebooks) {
for (const block of notebook.blocks) {
if (block.outputs.length > 0) {
outputs.push({ blockId: block.id, outputs: block.outputs, executionCount: block.executionCount })
}
}
}
return outputs
}
Loading
Loading