/** * The before/after instrument. * * Turns "recall should save tokens" into a number that can be re-run. It reads * two independent stores, because they answer different questions: * * - the cost ledger knows **money and per-request volume**: requests made, * cache-read tokens, and therefore the average context each request carried. * That average is the figure this whole workstream is measured against. * - the session projection cache knows **context shape**: the surface each * session reached, and whether that session used recall at all. * * Run it before the plugin sees real use to capture the baseline, then run it * again after. Sessions that used recall are the "after" group; everything else * is the baseline. Nothing is inferred — when there is no "after" group it says * so instead of printing a table of zeroes. * * node scripts/baseline.mjs [--ledger ] [--cache ] [--json] */ import { readFile } from 'node:fs/promises' import { homedir } from 'node:os' import { join } from 'node:path' const DSH_HOME = process.env.DSH_HOME ?? join(homedir(), '.dsh') function argValue(flag, fallback) { const at = process.argv.indexOf(flag) return at === -1 ? fallback : process.argv[at + 1] } const LEDGER = argValue('--ledger', join(DSH_HOME, 'storages', 'cost-meter', 'ledger.json')) const CACHE = argValue('--cache', join(DSH_HOME, 'storages', 'session_projcache.json')) const AS_JSON = process.argv.includes('--json') /** Format an integer with thousands separators. */ const n = (value) => Math.round(value).toLocaleString('en-GB') /** A compact human figure. */ function short(value) { if (value >= 1e9) return (value / 1e9).toFixed(1) + 'B' if (value >= 1e6) return (value / 1e6).toFixed(1) + 'M' if (value >= 1e3) return Math.round(value / 1e3) + 'k' return String(Math.round(value)) } async function readJson(path, fallback) { try { return JSON.parse(await readFile(path, 'utf8')) } catch { return fallback } } /** * Per-session request volume and cost, from the ledger. * * The ledger groups by day, and a session that spans midnight appears under both * — so sessions are merged by id rather than listed. */ function sessionsFromLedger(ledger) { const byId = new Map() for (const day of Object.values(ledger?.days ?? {})) { for (const session of day.sessions ?? []) { const id = session.id if (typeof id !== 'string') continue const entry = byId.get(id) ?? { id, calls: 0, cacheRead: 0, input: 0, output: 0, cost: 0 } entry.calls += session.calls ?? 0 entry.cacheRead += session.cacheRead ?? 0 entry.input += session.input ?? 0 entry.output += session.output ?? 0 entry.cost += session.cost ?? 0 byId.set(id, entry) } } for (const entry of byId.values()) { // What each request carried: the cached prefix plus the new input. entry.contextPerRequest = entry.calls > 0 ? (entry.cacheRead + entry.input) / entry.calls : 0 } return [...byId.values()] } /** Context shape and recall usage, from the projection cache. */ function sessionsFromCache(cache) { const rows = cache?.tables?.sessions ?? {} const out = [] for (const [id, entry] of Object.entries(rows)) { const units = entry?.rows ?? {} const pressure = units.contextPressure?.val const stats = units.sessionStats?.val const usage = units.recallUsage?.val out.push({ id, cwd: entry?.identity?.cwd ?? '(unknown)', surfaceTokens: pressure?.surfaceTokens ?? 0, contextWindow: pressure?.contextWindow ?? 0, steps: stats?.steps ?? 0, turns: stats?.turns ?? 0, recalls: usage?.searches ?? 0, usedRecall: (usage?.searches ?? 0) > 0, }) } return out } /** Mean and median of a numeric list. */ function describe(values) { if (values.length === 0) return { count: 0, mean: 0, median: 0, max: 0 } const sorted = [...values].sort((a, b) => a - b) const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length return { count: sorted.length, mean, median: sorted[Math.floor(sorted.length / 2)], max: sorted[sorted.length - 1], } } async function main() { const ledger = await readJson(LEDGER, undefined) const cache = await readJson(CACHE, undefined) if (ledger === undefined && cache === undefined) { console.error(`no data. Looked for:\n ${LEDGER}\n ${CACHE}`) process.exitCode = 1 return } const money = sessionsFromLedger(ledger) const shape = sessionsFromCache(cache) const shapeById = new Map(shape.map((row) => [row.id, row])) const totalCalls = money.reduce((sum, row) => sum + row.calls, 0) const totalCacheRead = money.reduce((sum, row) => sum + row.cacheRead, 0) const totalInput = money.reduce((sum, row) => sum + row.input, 0) const totalCost = money.reduce((sum, row) => sum + row.cost, 0) const averageContext = totalCalls > 0 ? (totalCacheRead + totalInput) / totalCalls : 0 // Workspaces, so a comparison is not an average over unrelated projects. const byWorkspace = new Map() for (const row of money) { const cwd = shapeById.get(row.id)?.cwd ?? '(not in projection cache)' const entry = byWorkspace.get(cwd) ?? { cwd, sessions: 0, calls: 0, cacheRead: 0, input: 0, cost: 0 } entry.sessions += 1 entry.calls += row.calls entry.cacheRead += row.cacheRead entry.input += row.input entry.cost += row.cost byWorkspace.set(cwd, entry) } // The comparison the objective names: average context PER REQUEST, which only // the ledger knows, split by whether that session used recall. Peak surface is // reported separately because it answers a different question — what a session // grew to, not what each request carried. const withRecall = money.filter((row) => shapeById.get(row.id)?.usedRecall === true) const without = money.filter((row) => shapeById.get(row.id)?.usedRecall !== true) const pressure = shape.filter((row) => row.surfaceTokens > 0) const window = pressure[0]?.contextWindow ?? 0 const trigger = Math.round(window * 0.8) const report = { baseline: { sessions: money.length, requests: totalCalls, cacheReadTokens: totalCacheRead, inputTokens: totalInput, costUsd: Number(totalCost.toFixed(2)), // Cost-weighted: every request counts once. This is the figure that maps // to money, and the one the objective names. pooledAverageContextPerRequest: Math.round(averageContext), // Typical: every session counts once. Differs from the pooled figure by // whatever the largest sessions do, which here is a factor of two — so // quoting only one of them would mislead in a different direction // depending on which was chosen. meanSessionContextPerRequest: Math.round(describe(money.map((row) => row.contextPerRequest)).mean), }, workspaces: [...byWorkspace.values()] .sort((a, b) => b.calls - a.calls) .map((entry) => ({ cwd: entry.cwd, sessions: entry.sessions, requests: entry.calls, averageContextPerRequest: entry.calls > 0 ? Math.round((entry.cacheRead + entry.input) / entry.calls) : 0, costUsd: Number(entry.cost.toFixed(2)), })), contextShape: { sessionsMeasured: pressure.length, contextWindow: window, compactionTrigger: trigger, surface: describe(pressure.map((row) => row.surfaceTokens)), reachedCompaction: pressure.filter((row) => row.surfaceTokens >= trigger && trigger > 0).length, }, recall: { sessionsWithRecallUsage: withRecall.length, sessionsWithout: without.length, comparable: withRecall.length > 0 && without.length > 0, after: withRecall.length > 0 ? describe(withRecall.map((row) => row.contextPerRequest)) : null, before: without.length > 0 ? describe(without.map((row) => row.contextPerRequest)) : null, }, sources: { ledger: LEDGER, cache: CACHE }, } if (AS_JSON) { console.log(JSON.stringify(report, null, 2)) return } const B = report.baseline console.log('BASELINE — every session on this machine') console.log(` sessions ${n(B.sessions)}`) console.log(` requests ${n(B.requests)}`) console.log(` cache-read tokens ${n(B.cacheReadTokens)}`) console.log(` avg context/request (pooled) ${n(B.pooledAverageContextPerRequest)}`) console.log(` avg context/request (typical) ${n(B.meanSessionContextPerRequest)} mean of per-session averages`) console.log(` cost $${B.costUsd.toFixed(2)}`) console.log('') console.log('BY WORKSPACE') for (const entry of report.workspaces.slice(0, 6)) { console.log( ` ${String(entry.requests).padStart(7)} req ${String(entry.averageContextPerRequest).padStart(8)} ctx/req ` + `$${entry.costUsd.toFixed(2).padStart(8)} ${entry.cwd}`, ) } if (report.workspaces.length > 6) console.log(` … and ${report.workspaces.length - 6} more`) console.log('') const S = report.contextShape console.log('CONTEXT SHAPE — what sessions reach') console.log(` sessions with a reading ${n(S.sessionsMeasured)} of ${n(shape.length)}`) console.log(` peak surface ${n(S.surface.max)}`) console.log(` median surface ${n(S.surface.median)}`) console.log(` compaction trigger ${n(S.compactionTrigger)}`) console.log(` sessions that reached it ${n(S.reachedCompaction)} of ${n(S.sessionsMeasured)}`) console.log('') console.log('BEFORE / AFTER — recall usage') const R = report.recall console.log(` sessions with recall usage ${n(R.sessionsWithRecallUsage)}`) console.log(` sessions without ${n(R.sessionsWithout)}`) console.log(' metric: each session\'s own average context per request (ledger),') console.log(' not peak surface — the two answer different questions') if (!R.comparable) { // The honest output while the plugin has not run. A table here would be a // comparison of nothing against nothing. console.log('') console.log(' Nothing to compare yet: no session has used recall. Run this again') console.log(' after a week of use and the two groups will report side by side.') } else { console.log('') console.log(` before ${n(R.before.mean)} ctx/req over ${n(R.before.count)} sessions (median ${n(R.before.median)})`) console.log(` after ${n(R.after.mean)} ctx/req over ${n(R.after.count)} sessions (median ${n(R.after.median)})`) const delta = R.before.mean > 0 ? ((R.before.mean - R.after.mean) / R.before.mean) * 100 : 0 console.log(` change ${delta >= 0 ? '-' : '+'}${Math.abs(delta).toFixed(1)}%`) } } await main()