260 lines
11 KiB
JavaScript
260 lines
11 KiB
JavaScript
/**
|
|
* The before/after instrument.
|
|
*
|
|
* Turns "recall should save tokens" into a number that can be re-run. It reads
|
|
* two independent stores, because they answer different questions:
|
|
*
|
|
* - the cost ledger knows **money and per-request volume**: requests made,
|
|
* cache-read tokens, and therefore the average context each request carried.
|
|
* That average is the figure this whole workstream is measured against.
|
|
* - the session projection cache knows **context shape**: the surface each
|
|
* session reached, and whether that session used recall at all.
|
|
*
|
|
* Run it before the plugin sees real use to capture the baseline, then run it
|
|
* again after. Sessions that used recall are the "after" group; everything else
|
|
* is the baseline. Nothing is inferred — when there is no "after" group it says
|
|
* so instead of printing a table of zeroes.
|
|
*
|
|
* node scripts/baseline.mjs [--ledger <path>] [--cache <path>] [--json]
|
|
*/
|
|
|
|
import { readFile } from 'node:fs/promises'
|
|
import { homedir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
|
|
const DSH_HOME = process.env.DSH_HOME ?? join(homedir(), '.dsh')
|
|
|
|
function argValue(flag, fallback) {
|
|
const at = process.argv.indexOf(flag)
|
|
return at === -1 ? fallback : process.argv[at + 1]
|
|
}
|
|
|
|
const LEDGER = argValue('--ledger', join(DSH_HOME, 'storages', 'cost-meter', 'ledger.json'))
|
|
const CACHE = argValue('--cache', join(DSH_HOME, 'storages', 'session_projcache.json'))
|
|
const AS_JSON = process.argv.includes('--json')
|
|
|
|
/** Format an integer with thousands separators. */
|
|
const n = (value) => Math.round(value).toLocaleString('en-GB')
|
|
|
|
/** A compact human figure. */
|
|
function short(value) {
|
|
if (value >= 1e9) return (value / 1e9).toFixed(1) + 'B'
|
|
if (value >= 1e6) return (value / 1e6).toFixed(1) + 'M'
|
|
if (value >= 1e3) return Math.round(value / 1e3) + 'k'
|
|
return String(Math.round(value))
|
|
}
|
|
|
|
async function readJson(path, fallback) {
|
|
try {
|
|
return JSON.parse(await readFile(path, 'utf8'))
|
|
} catch {
|
|
return fallback
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Per-session request volume and cost, from the ledger.
|
|
*
|
|
* The ledger groups by day, and a session that spans midnight appears under both
|
|
* — so sessions are merged by id rather than listed.
|
|
*/
|
|
function sessionsFromLedger(ledger) {
|
|
const byId = new Map()
|
|
for (const day of Object.values(ledger?.days ?? {})) {
|
|
for (const session of day.sessions ?? []) {
|
|
const id = session.id
|
|
if (typeof id !== 'string') continue
|
|
const entry = byId.get(id) ?? { id, calls: 0, cacheRead: 0, input: 0, output: 0, cost: 0 }
|
|
entry.calls += session.calls ?? 0
|
|
entry.cacheRead += session.cacheRead ?? 0
|
|
entry.input += session.input ?? 0
|
|
entry.output += session.output ?? 0
|
|
entry.cost += session.cost ?? 0
|
|
byId.set(id, entry)
|
|
}
|
|
}
|
|
for (const entry of byId.values()) {
|
|
// What each request carried: the cached prefix plus the new input.
|
|
entry.contextPerRequest = entry.calls > 0 ? (entry.cacheRead + entry.input) / entry.calls : 0
|
|
}
|
|
return [...byId.values()]
|
|
}
|
|
|
|
/** Context shape and recall usage, from the projection cache. */
|
|
function sessionsFromCache(cache) {
|
|
const rows = cache?.tables?.sessions ?? {}
|
|
const out = []
|
|
for (const [id, entry] of Object.entries(rows)) {
|
|
const units = entry?.rows ?? {}
|
|
const pressure = units.contextPressure?.val
|
|
const stats = units.sessionStats?.val
|
|
const usage = units.recallUsage?.val
|
|
out.push({
|
|
id,
|
|
cwd: entry?.identity?.cwd ?? '(unknown)',
|
|
surfaceTokens: pressure?.surfaceTokens ?? 0,
|
|
contextWindow: pressure?.contextWindow ?? 0,
|
|
steps: stats?.steps ?? 0,
|
|
turns: stats?.turns ?? 0,
|
|
recalls: usage?.searches ?? 0,
|
|
usedRecall: (usage?.searches ?? 0) > 0,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
/** Mean and median of a numeric list. */
|
|
function describe(values) {
|
|
if (values.length === 0) return { count: 0, mean: 0, median: 0, max: 0 }
|
|
const sorted = [...values].sort((a, b) => a - b)
|
|
const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length
|
|
return {
|
|
count: sorted.length,
|
|
mean,
|
|
median: sorted[Math.floor(sorted.length / 2)],
|
|
max: sorted[sorted.length - 1],
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const ledger = await readJson(LEDGER, undefined)
|
|
const cache = await readJson(CACHE, undefined)
|
|
|
|
if (ledger === undefined && cache === undefined) {
|
|
console.error(`no data. Looked for:\n ${LEDGER}\n ${CACHE}`)
|
|
process.exitCode = 1
|
|
return
|
|
}
|
|
|
|
const money = sessionsFromLedger(ledger)
|
|
const shape = sessionsFromCache(cache)
|
|
const shapeById = new Map(shape.map((row) => [row.id, row]))
|
|
|
|
const totalCalls = money.reduce((sum, row) => sum + row.calls, 0)
|
|
const totalCacheRead = money.reduce((sum, row) => sum + row.cacheRead, 0)
|
|
const totalInput = money.reduce((sum, row) => sum + row.input, 0)
|
|
const totalCost = money.reduce((sum, row) => sum + row.cost, 0)
|
|
const averageContext = totalCalls > 0 ? (totalCacheRead + totalInput) / totalCalls : 0
|
|
|
|
// Workspaces, so a comparison is not an average over unrelated projects.
|
|
const byWorkspace = new Map()
|
|
for (const row of money) {
|
|
const cwd = shapeById.get(row.id)?.cwd ?? '(not in projection cache)'
|
|
const entry = byWorkspace.get(cwd) ?? { cwd, sessions: 0, calls: 0, cacheRead: 0, input: 0, cost: 0 }
|
|
entry.sessions += 1
|
|
entry.calls += row.calls
|
|
entry.cacheRead += row.cacheRead
|
|
entry.input += row.input
|
|
entry.cost += row.cost
|
|
byWorkspace.set(cwd, entry)
|
|
}
|
|
|
|
// The comparison the objective names: average context PER REQUEST, which only
|
|
// the ledger knows, split by whether that session used recall. Peak surface is
|
|
// reported separately because it answers a different question — what a session
|
|
// grew to, not what each request carried.
|
|
const withRecall = money.filter((row) => shapeById.get(row.id)?.usedRecall === true)
|
|
const without = money.filter((row) => shapeById.get(row.id)?.usedRecall !== true)
|
|
const pressure = shape.filter((row) => row.surfaceTokens > 0)
|
|
const window = pressure[0]?.contextWindow ?? 0
|
|
const trigger = Math.round(window * 0.8)
|
|
|
|
const report = {
|
|
baseline: {
|
|
sessions: money.length,
|
|
requests: totalCalls,
|
|
cacheReadTokens: totalCacheRead,
|
|
inputTokens: totalInput,
|
|
costUsd: Number(totalCost.toFixed(2)),
|
|
// Cost-weighted: every request counts once. This is the figure that maps
|
|
// to money, and the one the objective names.
|
|
pooledAverageContextPerRequest: Math.round(averageContext),
|
|
// Typical: every session counts once. Differs from the pooled figure by
|
|
// whatever the largest sessions do, which here is a factor of two — so
|
|
// quoting only one of them would mislead in a different direction
|
|
// depending on which was chosen.
|
|
meanSessionContextPerRequest: Math.round(describe(money.map((row) => row.contextPerRequest)).mean),
|
|
},
|
|
workspaces: [...byWorkspace.values()]
|
|
.sort((a, b) => b.calls - a.calls)
|
|
.map((entry) => ({
|
|
cwd: entry.cwd,
|
|
sessions: entry.sessions,
|
|
requests: entry.calls,
|
|
averageContextPerRequest: entry.calls > 0 ? Math.round((entry.cacheRead + entry.input) / entry.calls) : 0,
|
|
costUsd: Number(entry.cost.toFixed(2)),
|
|
})),
|
|
contextShape: {
|
|
sessionsMeasured: pressure.length,
|
|
contextWindow: window,
|
|
compactionTrigger: trigger,
|
|
surface: describe(pressure.map((row) => row.surfaceTokens)),
|
|
reachedCompaction: pressure.filter((row) => row.surfaceTokens >= trigger && trigger > 0).length,
|
|
},
|
|
recall: {
|
|
sessionsWithRecallUsage: withRecall.length,
|
|
sessionsWithout: without.length,
|
|
comparable: withRecall.length > 0 && without.length > 0,
|
|
after: withRecall.length > 0 ? describe(withRecall.map((row) => row.contextPerRequest)) : null,
|
|
before: without.length > 0 ? describe(without.map((row) => row.contextPerRequest)) : null,
|
|
},
|
|
sources: { ledger: LEDGER, cache: CACHE },
|
|
}
|
|
|
|
if (AS_JSON) {
|
|
console.log(JSON.stringify(report, null, 2))
|
|
return
|
|
}
|
|
|
|
const B = report.baseline
|
|
console.log('BASELINE — every session on this machine')
|
|
console.log(` sessions ${n(B.sessions)}`)
|
|
console.log(` requests ${n(B.requests)}`)
|
|
console.log(` cache-read tokens ${n(B.cacheReadTokens)}`)
|
|
console.log(` avg context/request (pooled) ${n(B.pooledAverageContextPerRequest)}`)
|
|
console.log(` avg context/request (typical) ${n(B.meanSessionContextPerRequest)} mean of per-session averages`)
|
|
console.log(` cost $${B.costUsd.toFixed(2)}`)
|
|
console.log('')
|
|
|
|
console.log('BY WORKSPACE')
|
|
for (const entry of report.workspaces.slice(0, 6)) {
|
|
console.log(
|
|
` ${String(entry.requests).padStart(7)} req ${String(entry.averageContextPerRequest).padStart(8)} ctx/req ` +
|
|
`$${entry.costUsd.toFixed(2).padStart(8)} ${entry.cwd}`,
|
|
)
|
|
}
|
|
if (report.workspaces.length > 6) console.log(` … and ${report.workspaces.length - 6} more`)
|
|
console.log('')
|
|
|
|
const S = report.contextShape
|
|
console.log('CONTEXT SHAPE — what sessions reach')
|
|
console.log(` sessions with a reading ${n(S.sessionsMeasured)} of ${n(shape.length)}`)
|
|
console.log(` peak surface ${n(S.surface.max)}`)
|
|
console.log(` median surface ${n(S.surface.median)}`)
|
|
console.log(` compaction trigger ${n(S.compactionTrigger)}`)
|
|
console.log(` sessions that reached it ${n(S.reachedCompaction)} of ${n(S.sessionsMeasured)}`)
|
|
console.log('')
|
|
|
|
console.log('BEFORE / AFTER — recall usage')
|
|
const R = report.recall
|
|
console.log(` sessions with recall usage ${n(R.sessionsWithRecallUsage)}`)
|
|
console.log(` sessions without ${n(R.sessionsWithout)}`)
|
|
console.log(' metric: each session\'s own average context per request (ledger),')
|
|
console.log(' not peak surface — the two answer different questions')
|
|
if (!R.comparable) {
|
|
// The honest output while the plugin has not run. A table here would be a
|
|
// comparison of nothing against nothing.
|
|
console.log('')
|
|
console.log(' Nothing to compare yet: no session has used recall. Run this again')
|
|
console.log(' after a week of use and the two groups will report side by side.')
|
|
} else {
|
|
console.log('')
|
|
console.log(` before ${n(R.before.mean)} ctx/req over ${n(R.before.count)} sessions (median ${n(R.before.median)})`)
|
|
console.log(` after ${n(R.after.mean)} ctx/req over ${n(R.after.count)} sessions (median ${n(R.after.median)})`)
|
|
const delta = R.before.mean > 0 ? ((R.before.mean - R.after.mean) / R.before.mean) * 100 : 0
|
|
console.log(` change ${delta >= 0 ? '-' : '+'}${Math.abs(delta).toFixed(1)}%`)
|
|
}
|
|
}
|
|
|
|
await main()
|