init

scripts/baseline.mjs

@@ -0,0 +1,259 @@
+/**
+ * The before/after instrument.
+ *
+ * Turns "recall should save tokens" into a number that can be re-run. It reads
+ * two independent stores, because they answer different questions:
+ *
+ *   - the cost ledger knows **money and per-request volume**: requests made,
+ *     cache-read tokens, and therefore the average context each request carried.
+ *     That average is the figure this whole workstream is measured against.
+ *   - the session projection cache knows **context shape**: the surface each
+ *     session reached, and whether that session used recall at all.
+ *
+ * Run it before the plugin sees real use to capture the baseline, then run it
+ * again after. Sessions that used recall are the "after" group; everything else
+ * is the baseline. Nothing is inferred — when there is no "after" group it says
+ * so instead of printing a table of zeroes.
+ *
+ *   node scripts/baseline.mjs [--ledger <path>] [--cache <path>] [--json]
+ */
+
+import { readFile } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+const DSH_HOME = process.env.DSH_HOME ?? join(homedir(), '.dsh')
+
+function argValue(flag, fallback) {
+  const at = process.argv.indexOf(flag)
+  return at === -1 ? fallback : process.argv[at + 1]
+}
+
+const LEDGER = argValue('--ledger', join(DSH_HOME, 'storages', 'cost-meter', 'ledger.json'))
+const CACHE = argValue('--cache', join(DSH_HOME, 'storages', 'session_projcache.json'))
+const AS_JSON = process.argv.includes('--json')
+
+/** Format an integer with thousands separators. */
+const n = (value) => Math.round(value).toLocaleString('en-GB')
+
+/** A compact human figure. */
+function short(value) {
+  if (value >= 1e9) return (value / 1e9).toFixed(1) + 'B'
+  if (value >= 1e6) return (value / 1e6).toFixed(1) + 'M'
+  if (value >= 1e3) return Math.round(value / 1e3) + 'k'
+  return String(Math.round(value))
+}
+
+async function readJson(path, fallback) {
+  try {
+    return JSON.parse(await readFile(path, 'utf8'))
+  } catch {
+    return fallback
+  }
+}
+
+/**
+ * Per-session request volume and cost, from the ledger.
+ *
+ * The ledger groups by day, and a session that spans midnight appears under both
+ * — so sessions are merged by id rather than listed.
+ */
+function sessionsFromLedger(ledger) {
+  const byId = new Map()
+  for (const day of Object.values(ledger?.days ?? {})) {
+    for (const session of day.sessions ?? []) {
+      const id = session.id
+      if (typeof id !== 'string') continue
+      const entry = byId.get(id) ?? { id, calls: 0, cacheRead: 0, input: 0, output: 0, cost: 0 }
+      entry.calls += session.calls ?? 0
+      entry.cacheRead += session.cacheRead ?? 0
+      entry.input += session.input ?? 0
+      entry.output += session.output ?? 0
+      entry.cost += session.cost ?? 0
+      byId.set(id, entry)
+    }
+  }
+  for (const entry of byId.values()) {
+    // What each request carried: the cached prefix plus the new input.
+    entry.contextPerRequest = entry.calls > 0 ? (entry.cacheRead + entry.input) / entry.calls : 0
+  }
+  return [...byId.values()]
+}
+
+/** Context shape and recall usage, from the projection cache. */
+function sessionsFromCache(cache) {
+  const rows = cache?.tables?.sessions ?? {}
+  const out = []
+  for (const [id, entry] of Object.entries(rows)) {
+    const units = entry?.rows ?? {}
+    const pressure = units.contextPressure?.val
+    const stats = units.sessionStats?.val
+    const usage = units.recallUsage?.val
+    out.push({
+      id,
+      cwd: entry?.identity?.cwd ?? '(unknown)',
+      surfaceTokens: pressure?.surfaceTokens ?? 0,
+      contextWindow: pressure?.contextWindow ?? 0,
+      steps: stats?.steps ?? 0,
+      turns: stats?.turns ?? 0,
+      recalls: usage?.searches ?? 0,
+      usedRecall: (usage?.searches ?? 0) > 0,
+    })
+  }
+  return out
+}
+
+/** Mean and median of a numeric list. */
+function describe(values) {
+  if (values.length === 0) return { count: 0, mean: 0, median: 0, max: 0 }
+  const sorted = [...values].sort((a, b) => a - b)
+  const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length
+  return {
+    count: sorted.length,
+    mean,
+    median: sorted[Math.floor(sorted.length / 2)],
+    max: sorted[sorted.length - 1],
+  }
+}
+
+async function main() {
+  const ledger = await readJson(LEDGER, undefined)
+  const cache = await readJson(CACHE, undefined)
+
+  if (ledger === undefined && cache === undefined) {
+    console.error(`no data. Looked for:\n  ${LEDGER}\n  ${CACHE}`)
+    process.exitCode = 1
+    return
+  }
+
+  const money = sessionsFromLedger(ledger)
+  const shape = sessionsFromCache(cache)
+  const shapeById = new Map(shape.map((row) => [row.id, row]))
+
+  const totalCalls = money.reduce((sum, row) => sum + row.calls, 0)
+  const totalCacheRead = money.reduce((sum, row) => sum + row.cacheRead, 0)
+  const totalInput = money.reduce((sum, row) => sum + row.input, 0)
+  const totalCost = money.reduce((sum, row) => sum + row.cost, 0)
+  const averageContext = totalCalls > 0 ? (totalCacheRead + totalInput) / totalCalls : 0
+
+  // Workspaces, so a comparison is not an average over unrelated projects.
+  const byWorkspace = new Map()
+  for (const row of money) {
+    const cwd = shapeById.get(row.id)?.cwd ?? '(not in projection cache)'
+    const entry = byWorkspace.get(cwd) ?? { cwd, sessions: 0, calls: 0, cacheRead: 0, input: 0, cost: 0 }
+    entry.sessions += 1
+    entry.calls += row.calls
+    entry.cacheRead += row.cacheRead
+    entry.input += row.input
+    entry.cost += row.cost
+    byWorkspace.set(cwd, entry)
+  }
+
+  // The comparison the objective names: average context PER REQUEST, which only
+  // the ledger knows, split by whether that session used recall. Peak surface is
+  // reported separately because it answers a different question — what a session
+  // grew to, not what each request carried.
+  const withRecall = money.filter((row) => shapeById.get(row.id)?.usedRecall === true)
+  const without = money.filter((row) => shapeById.get(row.id)?.usedRecall !== true)
+  const pressure = shape.filter((row) => row.surfaceTokens > 0)
+  const window = pressure[0]?.contextWindow ?? 0
+  const trigger = Math.round(window * 0.8)
+
+  const report = {
+    baseline: {
+      sessions: money.length,
+      requests: totalCalls,
+      cacheReadTokens: totalCacheRead,
+      inputTokens: totalInput,
+      costUsd: Number(totalCost.toFixed(2)),
+      // Cost-weighted: every request counts once. This is the figure that maps
+      // to money, and the one the objective names.
+      pooledAverageContextPerRequest: Math.round(averageContext),
+      // Typical: every session counts once. Differs from the pooled figure by
+      // whatever the largest sessions do, which here is a factor of two — so
+      // quoting only one of them would mislead in a different direction
+      // depending on which was chosen.
+      meanSessionContextPerRequest: Math.round(describe(money.map((row) => row.contextPerRequest)).mean),
+    },
+    workspaces: [...byWorkspace.values()]
+      .sort((a, b) => b.calls - a.calls)
+      .map((entry) => ({
+        cwd: entry.cwd,
+        sessions: entry.sessions,
+        requests: entry.calls,
+        averageContextPerRequest: entry.calls > 0 ? Math.round((entry.cacheRead + entry.input) / entry.calls) : 0,
+        costUsd: Number(entry.cost.toFixed(2)),
+      })),
+    contextShape: {
+      sessionsMeasured: pressure.length,
+      contextWindow: window,
+      compactionTrigger: trigger,
+      surface: describe(pressure.map((row) => row.surfaceTokens)),
+      reachedCompaction: pressure.filter((row) => row.surfaceTokens >= trigger && trigger > 0).length,
+    },
+    recall: {
+      sessionsWithRecallUsage: withRecall.length,
+      sessionsWithout: without.length,
+      comparable: withRecall.length > 0 && without.length > 0,
+      after: withRecall.length > 0 ? describe(withRecall.map((row) => row.contextPerRequest)) : null,
+      before: without.length > 0 ? describe(without.map((row) => row.contextPerRequest)) : null,
+    },
+    sources: { ledger: LEDGER, cache: CACHE },
+  }
+
+  if (AS_JSON) {
+    console.log(JSON.stringify(report, null, 2))
+    return
+  }
+
+  const B = report.baseline
+  console.log('BASELINE — every session on this machine')
+  console.log(`  sessions                      ${n(B.sessions)}`)
+  console.log(`  requests                      ${n(B.requests)}`)
+  console.log(`  cache-read tokens             ${n(B.cacheReadTokens)}`)
+  console.log(`  avg context/request (pooled)  ${n(B.pooledAverageContextPerRequest)}`)
+  console.log(`  avg context/request (typical) ${n(B.meanSessionContextPerRequest)}   mean of per-session averages`)
+  console.log(`  cost                          $${B.costUsd.toFixed(2)}`)
+  console.log('')
+
+  console.log('BY WORKSPACE')
+  for (const entry of report.workspaces.slice(0, 6)) {
+    console.log(
+      `  ${String(entry.requests).padStart(7)} req  ${String(entry.averageContextPerRequest).padStart(8)} ctx/req  ` +
+        `$${entry.costUsd.toFixed(2).padStart(8)}  ${entry.cwd}`,
+    )
+  }
+  if (report.workspaces.length > 6) console.log(`  … and ${report.workspaces.length - 6} more`)
+  console.log('')
+
+  const S = report.contextShape
+  console.log('CONTEXT SHAPE — what sessions reach')
+  console.log(`  sessions with a reading       ${n(S.sessionsMeasured)} of ${n(shape.length)}`)
+  console.log(`  peak surface                  ${n(S.surface.max)}`)
+  console.log(`  median surface                ${n(S.surface.median)}`)
+  console.log(`  compaction trigger            ${n(S.compactionTrigger)}`)
+  console.log(`  sessions that reached it      ${n(S.reachedCompaction)} of ${n(S.sessionsMeasured)}`)
+  console.log('')
+
+  console.log('BEFORE / AFTER — recall usage')
+  const R = report.recall
+  console.log(`  sessions with recall usage    ${n(R.sessionsWithRecallUsage)}`)
+  console.log(`  sessions without              ${n(R.sessionsWithout)}`)
+  console.log('  metric: each session\'s own average context per request (ledger),')
+  console.log('          not peak surface — the two answer different questions')
+  if (!R.comparable) {
+    // The honest output while the plugin has not run. A table here would be a
+    // comparison of nothing against nothing.
+    console.log('')
+    console.log('  Nothing to compare yet: no session has used recall. Run this again')
+    console.log('  after a week of use and the two groups will report side by side.')
+  } else {
+    console.log('')
+    console.log(`  before  ${n(R.before.mean)} ctx/req over ${n(R.before.count)} sessions (median ${n(R.before.median)})`)
+    console.log(`  after   ${n(R.after.mean)} ctx/req over ${n(R.after.count)} sessions (median ${n(R.after.median)})`)
+    const delta = R.before.mean > 0 ? ((R.before.mean - R.after.mean) / R.before.mean) * 100 : 0
+    console.log(`  change  ${delta >= 0 ? '-' : '+'}${Math.abs(delta).toFixed(1)}%`)
+  }
+}
+
+await main()