|
| 1 | +#!/usr/bin/env node |
| 2 | +// newsletter-abtest — one email a day while a newsletter A/B test is running. |
| 3 | +// |
| 4 | +// newsletter-abtest snapshot every issue, mail the digest |
| 5 | +// newsletter-abtest --dry-run build it, print it, send nothing |
| 6 | +// newsletter-abtest --print snapshot and print, no mail |
| 7 | +// newsletter-abtest --prefix X issues whose id starts with X (default profullstack-) |
| 8 | +// newsletter-abtest --to a@b.com someone else |
| 9 | +// |
| 10 | +// Why a daily snapshot and not a single reading at the end: opens and clicks |
| 11 | +// arrive for days after a send, so the only way to tell a real winner from an |
| 12 | +// early lead is to watch the same numbers move. Each run appends one line per |
| 13 | +// issue to snapshots.jsonl and reports the change since the run before. |
| 14 | +// |
| 15 | +// The verdict is a two-proportion z test on clicks, pooled across every issue |
| 16 | +// of the campaign, because one issue's arms are far too small on their own: |
| 17 | +// at a 1% click rate you need thousands per arm to separate them. When the |
| 18 | +// arms are not separable the digest says so and prints the n it would take. |
| 19 | +// |
| 20 | +// Secrets: RESEND_API_KEY from the environment, else the team vault, else |
| 21 | +// ~/.config/logicsrc/shell.env. Cron gets almost no environment, so PATH is |
| 22 | +// fixed below and a failure is mailed too: a silent nightly job is worse than |
| 23 | +// none. |
| 24 | +import { execFileSync } from 'node:child_process'; |
| 25 | +import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs'; |
| 26 | +import { homedir, hostname, tmpdir } from 'node:os'; |
| 27 | +import { join } from 'node:path'; |
| 28 | + |
| 29 | +process.env.PATH = [ |
| 30 | + join(homedir(), '.local/bin'), |
| 31 | + join(homedir(), '.local/share/mise/shims'), |
| 32 | + '/usr/local/bin', '/usr/bin', '/bin', |
| 33 | +].join(':'); |
| 34 | + |
| 35 | +const args = process.argv.slice(2); |
| 36 | +const has = (f) => args.includes(f); |
| 37 | +const val = (f, d) => { const i = args.indexOf(f); return i >= 0 && args[i + 1] ? args[i + 1] : d; }; |
| 38 | + |
| 39 | +const PREFIX = val('--prefix', 'profullstack-'); |
| 40 | +const TO = val('--to', 'anthony@profullstack.com'); |
| 41 | +const FROM = 'Newsletter A/B <anthony@profullstack.com>'; |
| 42 | +const DATA = join(homedir(), '.local/share/newsletter-abtest'); |
| 43 | +const SNAPSHOTS = join(DATA, 'snapshots.jsonl'); |
| 44 | +const PLAN = join(homedir(), '.config/newsletter-abtest/plan.json'); |
| 45 | + |
| 46 | +// ---------------------------------------------------------------- myna |
| 47 | + |
| 48 | +const myna = (...a) => execFileSync('myna', a, { encoding: 'utf8', timeout: 120_000, maxBuffer: 16 << 20 }); |
| 49 | + |
| 50 | +function issues() { |
| 51 | + const all = JSON.parse(myna('newsletter', '--json')); |
| 52 | + return all.filter((n) => n.id.startsWith(PREFIX) && n.status !== 'draft'); |
| 53 | +} |
| 54 | + |
| 55 | +function stats(id) { |
| 56 | + return JSON.parse(myna('newsletter', 'stats', id, '--json')); |
| 57 | +} |
| 58 | + |
| 59 | +// ---------------------------------------------------------------- statistics |
| 60 | + |
| 61 | +/** Standard normal tail, Abramowitz and Stegun 26.2.17; good to 7 decimals. */ |
| 62 | +function phi(z) { |
| 63 | + const t = 1 / (1 + 0.2316419 * Math.abs(z)); |
| 64 | + const d = 0.3989422804014327 * Math.exp((-z * z) / 2); |
| 65 | + const p = d * t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429)))); |
| 66 | + return z > 0 ? 1 - p : p; |
| 67 | +} |
| 68 | + |
| 69 | +/** Two-sided p for two proportions: successes a of na against b of nb. */ |
| 70 | +function twoProportion(a, na, b, nb) { |
| 71 | + if (!na || !nb) return { p: 1, z: 0 }; |
| 72 | + const p1 = a / na, p2 = b / nb, pooled = (a + b) / (na + nb); |
| 73 | + const se = Math.sqrt(pooled * (1 - pooled) * (1 / na + 1 / nb)); |
| 74 | + if (!se) return { p: 1, z: 0 }; |
| 75 | + const z = (p1 - p2) / se; |
| 76 | + return { p: 2 * (1 - phi(Math.abs(z))), z }; |
| 77 | +} |
| 78 | + |
| 79 | +/** Recipients per arm for 80% power at alpha 0.05, given two observed rates. */ |
| 80 | +function needPerArm(p1, p2) { |
| 81 | + if (p1 === p2) return Infinity; |
| 82 | + const n = (7.849 * (p1 * (1 - p1) + p2 * (1 - p2))) / ((p1 - p2) ** 2); |
| 83 | + return Math.ceil(n); |
| 84 | +} |
| 85 | + |
| 86 | +const pct = (x) => `${(x * 100).toFixed(2)}%`; |
| 87 | +const plus = (n) => (n > 0 ? `+${n}` : `${n}`); |
| 88 | + |
| 89 | +/** Sum rows into one arm. */ |
| 90 | +function fold(rows) { |
| 91 | + return rows.reduce((t, r) => ({ |
| 92 | + sent: t.sent + r.sent, opens: t.opens + r.opens, opensTotal: t.opensTotal + r.opensTotal, |
| 93 | + clicks: t.clicks + r.clicks, unsubscribes: t.unsubscribes + r.unsubscribes, |
| 94 | + }), { sent: 0, opens: 0, opensTotal: 0, clicks: 0, unsubscribes: 0 }); |
| 95 | +} |
| 96 | + |
| 97 | +/** Group pooled rows by a key, best click rate first. */ |
| 98 | +function arms(rows, key) { |
| 99 | + const by = new Map(); |
| 100 | + for (const r of rows) { |
| 101 | + const k = r[key]; |
| 102 | + if (!by.has(k)) by.set(k, []); |
| 103 | + by.get(k).push(r); |
| 104 | + } |
| 105 | + return [...by.entries()] |
| 106 | + .map(([name, rs]) => ({ name, ...fold(rs) })) |
| 107 | + .map((a) => ({ ...a, ctr: a.sent ? a.clicks / a.sent : 0, openRate: a.sent ? a.opens / a.sent : 0 })) |
| 108 | + .sort((x, y) => y.ctr - x.ctr); |
| 109 | +} |
| 110 | + |
| 111 | +/** The verdict line for a set of arms: a winner, or what it would take. */ |
| 112 | +function verdict(list, label) { |
| 113 | + if (list.length < 2) return `${label}: only one arm so far.`; |
| 114 | + const [top, next] = list; |
| 115 | + const { p } = twoProportion(top.clicks, top.sent, next.clicks, next.sent); |
| 116 | + if (p < 0.05) return `${label}: ${top.name} beats ${next.name} on clicks, ${pct(top.ctr)} against ${pct(next.ctr)} (p = ${p.toFixed(3)}). Call it.`; |
| 117 | + const need = needPerArm(top.ctr, next.ctr); |
| 118 | + const short = Math.max(0, need - top.sent); |
| 119 | + const more = Number.isFinite(need) |
| 120 | + ? `about ${need.toLocaleString()} per arm would settle it, ${short ? `${short.toLocaleString()} more than ${top.name} has` : 'which it now has'}` |
| 121 | + : 'the two are level'; |
| 122 | + return `${label}: ${top.name} leads at ${pct(top.ctr)} against ${pct(next.ctr)}, but that gap is noise so far (p = ${p.toFixed(2)}); ${more}.`; |
| 123 | +} |
| 124 | + |
| 125 | +// ---------------------------------------------------------------- the plan |
| 126 | + |
| 127 | +/** The month's sends, so the digest says what is due. Absent: no schedule block. */ |
| 128 | +function plan() { |
| 129 | + if (!existsSync(PLAN)) return null; |
| 130 | + try { return JSON.parse(readFileSync(PLAN, 'utf8')); } catch { return null; } |
| 131 | +} |
| 132 | + |
| 133 | +function scheduleLines(p) { |
| 134 | + if (!p?.sends?.length) return []; |
| 135 | + const today = new Date().toISOString().slice(0, 10); |
| 136 | + const lines = []; |
| 137 | + for (const s of p.sends) { |
| 138 | + const days = Math.round((Date.parse(`${s.date}T00:00:00Z`) - Date.parse(`${today}T00:00:00Z`)) / 86400000); |
| 139 | + const when = s.sent ? 'sent' : days === 0 ? 'DUE TODAY' : days < 0 ? `${-days} days overdue` : `in ${days} days`; |
| 140 | + lines.push(` ${s.date} ${s.id.padEnd(18)} ${when}`); |
| 141 | + } |
| 142 | + return lines; |
| 143 | +} |
| 144 | + |
| 145 | +// ---------------------------------------------------------------- the report |
| 146 | + |
| 147 | +function build({ record = true } = {}) { |
| 148 | + const list = issues(); |
| 149 | + const now = new Date().toISOString(); |
| 150 | + const previous = readSnapshots(); |
| 151 | + const sections = []; |
| 152 | + const pooled = []; |
| 153 | + |
| 154 | + for (const n of list) { |
| 155 | + const s = stats(n.id); |
| 156 | + const total = fold(s.rows); |
| 157 | + const before = previous.filter((x) => x.id === n.id).pop(); |
| 158 | + const delta = before |
| 159 | + ? { |
| 160 | + opens: total.opens - before.total.opens, |
| 161 | + clicks: total.clicks - before.total.clicks, |
| 162 | + unsubscribes: total.unsubscribes - before.total.unsubscribes, |
| 163 | + hours: Math.max(1, Math.round((Date.parse(now) - Date.parse(before.at)) / 3_600_000)), |
| 164 | + } |
| 165 | + : null; |
| 166 | + if (record) appendFileSync(SNAPSHOTS, `${JSON.stringify({ at: now, id: n.id, total, rows: s.rows })}\n`); |
| 167 | + pooled.push(...s.rows); |
| 168 | + sections.push({ id: n.id, subject: n.subject, rows: s.rows, total, delta }); |
| 169 | + } |
| 170 | + |
| 171 | + const lines = []; |
| 172 | + lines.push(`Newsletter A/B, ${now.slice(0, 10)}`); |
| 173 | + lines.push(''); |
| 174 | + |
| 175 | + for (const sec of sections) { |
| 176 | + const t = sec.total; |
| 177 | + lines.push(`${sec.id} ${sec.subject}`); |
| 178 | + lines.push(` ${t.sent.toLocaleString()} sent, ${t.opens} opens (${pct(t.sent ? t.opens / t.sent : 0)}), ${t.clicks} clicks (${pct(t.sent ? t.clicks / t.sent : 0)}), ${t.unsubscribes} unsubscribed`); |
| 179 | + if (sec.delta) lines.push(` in the last ${sec.delta.hours}h: ${plus(sec.delta.opens)} opens, ${plus(sec.delta.clicks)} clicks, ${plus(sec.delta.unsubscribes)} unsubscribes`); |
| 180 | + else lines.push(' first reading, so no change to show yet'); |
| 181 | + lines.push(''); |
| 182 | + } |
| 183 | + |
| 184 | + if (pooled.length) { |
| 185 | + const bySubject = arms(pooled, 'subjectKey'); |
| 186 | + const byCta = arms(pooled, 'cta'); |
| 187 | + lines.push('Pooled across every issue'); |
| 188 | + lines.push(''); |
| 189 | + lines.push(' Subject line'); |
| 190 | + for (const a of bySubject) lines.push(` ${a.name} ${a.sent.toLocaleString()} sent ${a.opens} opens ${a.clicks} clicks ${pct(a.ctr)} CTR ${a.unsubscribes} unsubs`); |
| 191 | + lines.push(` ${verdict(bySubject, 'Verdict')}`); |
| 192 | + lines.push(''); |
| 193 | + lines.push(' Call to action'); |
| 194 | + for (const a of byCta) lines.push(` ${a.name.padEnd(18)} ${a.sent.toLocaleString()} sent ${a.opens} opens ${a.clicks} clicks ${pct(a.ctr)} CTR ${a.unsubscribes} unsubs`); |
| 195 | + lines.push(` ${verdict(byCta, 'Verdict')}`); |
| 196 | + lines.push(''); |
| 197 | + } |
| 198 | + |
| 199 | + const p = plan(); |
| 200 | + const sched = scheduleLines(p); |
| 201 | + if (sched.length) { |
| 202 | + lines.push('This month'); |
| 203 | + lines.push(...sched); |
| 204 | + const due = p.sends.find((s) => !s.sent); |
| 205 | + if (due) { |
| 206 | + lines.push(''); |
| 207 | + lines.push(` ${due.id}: ${due.test ?? ''}`); |
| 208 | + lines.push(` myna newsletter blast <draft.md> --id ${due.id} --list ${due.list ?? 'profullstack-users'} \\`); |
| 209 | + lines.push(` --subject "..." --cta-set ${due.ctaSet ?? 'default'} --via resend --service "a Profullstack, Inc. product"`); |
| 210 | + lines.push(` then, once the test copy looks right: myna newsletter blast --go ${due.id}`); |
| 211 | + } |
| 212 | + lines.push(''); |
| 213 | + } |
| 214 | + |
| 215 | + lines.push(`Opens undercount: Apple Mail and Gmail hide the pixel for many readers, so clicks are the firmer number.`); |
| 216 | + lines.push(`From ${hostname()}. Stop these with: crontab -e, delete the newsletter-abtest line.`); |
| 217 | + return lines.join('\n'); |
| 218 | +} |
| 219 | + |
| 220 | +function readSnapshots() { |
| 221 | + if (!existsSync(SNAPSHOTS)) return []; |
| 222 | + return readFileSync(SNAPSHOTS, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean); |
| 223 | +} |
| 224 | + |
| 225 | +// ---------------------------------------------------------------- mail |
| 226 | + |
| 227 | +function parseEnv(text) { |
| 228 | + const out = {}; |
| 229 | + for (const line of text.split('\n')) { |
| 230 | + const m = line.match(/^\s*(?:export\s+)?([A-Z0-9_]+)\s*=\s*(.*)$/i); |
| 231 | + if (m) out[m[1]] = m[2].replace(/^['"]|['"]$/g, '').trim(); |
| 232 | + } |
| 233 | + return out; |
| 234 | +} |
| 235 | + |
| 236 | +function resendKey() { |
| 237 | + if (process.env.RESEND_API_KEY) return process.env.RESEND_API_KEY; |
| 238 | + const dir = mkdtempSync(join(tmpdir(), 'newsletter-abtest-')); |
| 239 | + try { |
| 240 | + execFileSync('logicsrc', ['teams', 'pull', 'profullstack', 'fleet-nightly', 'prod', '--env', join(dir, 'env')], { stdio: ['ignore', 'ignore', 'ignore'], timeout: 60_000 }); |
| 241 | + const k = parseEnv(readFileSync(join(dir, 'env'), 'utf8')).RESEND_API_KEY; |
| 242 | + if (k) return k; |
| 243 | + } catch {} finally { rmSync(dir, { recursive: true, force: true }); } |
| 244 | + const f = join(homedir(), '.config/logicsrc/shell.env'); |
| 245 | + if (existsSync(f)) return parseEnv(readFileSync(f, 'utf8')).RESEND_API_KEY; |
| 246 | + return null; |
| 247 | +} |
| 248 | + |
| 249 | +async function mail(subject, text) { |
| 250 | + const key = resendKey(); |
| 251 | + if (!key) throw new Error('No RESEND_API_KEY in the environment, the vault or shell.env'); |
| 252 | + const r = await fetch('https://api.resend.com/emails', { |
| 253 | + method: 'POST', |
| 254 | + headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' }, |
| 255 | + body: JSON.stringify({ from: FROM, to: [TO], subject, text, html: `<pre style="font:14px ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap">${text.replace(/[&<>]/g, (c) => ({ '&': '&', '<': '<', '>': '>' })[c])}</pre>` }), |
| 256 | + }); |
| 257 | + if (!r.ok) throw new Error(`resend ${r.status}: ${(await r.text()).slice(0, 300)}`); |
| 258 | +} |
| 259 | + |
| 260 | +// ---------------------------------------------------------------- main |
| 261 | + |
| 262 | +async function main() { |
| 263 | + mkdirSync(DATA, { recursive: true }); |
| 264 | + let report; |
| 265 | + try { |
| 266 | + report = build({ record: !has('--dry-run') }); |
| 267 | + } catch (e) { |
| 268 | + report = `newsletter-abtest failed on ${hostname()}:\n\n${e?.stack || e}`; |
| 269 | + if (!has('--dry-run') && !has('--print')) await mail('Newsletter A/B: the digest failed', report); |
| 270 | + console.error(report); |
| 271 | + process.exit(1); |
| 272 | + } |
| 273 | + if (has('--dry-run') || has('--print')) { console.log(report); return; } |
| 274 | + await mail(`Newsletter A/B, ${new Date().toISOString().slice(0, 10)}`, report); |
| 275 | + console.log(report); |
| 276 | +} |
| 277 | + |
| 278 | +main().catch((e) => { console.error(e?.stack || e); process.exit(1); }); |
0 commit comments