import { strict as assert } from 'node:assert' import { test } from 'node:test' import { deriveUsage, emptyUsage, foldUsage, usageSummary } from '../lib/usage.js' /** A tool result shaped like the ones the tool actually returns. */ function searchValue(overrides = {}) { return { action: 'search', ok: true, focus: { path: '.', source: 'workspace' }, weak: false, results: [ { path: 'src/auth/session.ts', excerpt: 'x'.repeat(1000), similarity: 0.6 }, { path: 'src/weather/forecast.ts', excerpt: 'y'.repeat(500), similarity: 0.4 }, ], measure: { excerpts: 2, returnedBytes: 1500, sourceBytes: 60_000, files: 2 }, ...overrides, } } /** Wrap a record the way the harness wraps the tool's metadata. */ function asEvent(meta) { return { type: 'tool/result', turn: 1, step: 1, message: {}, meta } } test('emptyUsage is a plain zeroed record', () => { const state = emptyUsage() assert.equal(state.searches, 0) assert.equal(state.sourceBytes, 0) // The projection stores this as plain JSON, so nothing exotic may appear. assert.deepEqual(JSON.parse(JSON.stringify(state)), state) }) test('deriveUsage records a search without carrying any content', () => { const { recall } = deriveUsage({ action: 'search' }, searchValue()) assert.equal(recall.action, 'search') assert.equal(recall.excerpts, 2) assert.equal(recall.returnedBytes, 1500) assert.equal(recall.sourceBytes, 60_000) assert.equal(recall.files, 2) assert.equal(recall.focus, '.') // The whole point of the record is that it is tiny: no excerpts, no query. const serialised = JSON.stringify(recall) assert.ok(!serialised.includes('x'.repeat(50)), 'excerpt text must not reach the log') assert.ok(serialised.length < 300, `record was ${serialised.length} bytes`) }) test('deriveUsage marks a failure and a weak result', () => { assert.equal(deriveUsage({ action: 'search' }, { action: 'search', ok: false }).recall.failed, 1) assert.equal(deriveUsage({ action: 'search' }, searchValue({ weak: true })).recall.weak, 1) }) test('deriveUsage records an index without pretending it returned excerpts', () => { const { recall } = deriveUsage({ action: 'index' }, { action: 'index', ok: true, index: { embedded: 4 } }) assert.equal(recall.action, 'index') assert.equal(recall.embedded, 4) assert.equal(recall.excerpts, undefined) }) test('foldUsage returns the SAME reference for events it does not care about', () => { const state = emptyUsage() // A fresh object here would republish the projection on every step of every // session, which is why the framework states this requirement explicitly. assert.equal(foldUsage(state, { type: 'user/message' }), state) assert.equal(foldUsage(state, { type: 'tool/result', meta: {} }), state) assert.equal(foldUsage(state, { type: 'tool/result' }), state) assert.equal(foldUsage(state, { type: 'tool/result', meta: { other: 1 } }), state) assert.equal(foldUsage(state, { type: 'tool/result', meta: { recall: null } }), state) }) test('foldUsage accumulates one conversation', () => { let state = emptyUsage() state = foldUsage(state, asEvent(deriveUsage({}, searchValue()))) state = foldUsage(state, asEvent(deriveUsage({}, searchValue()))) state = foldUsage(state, asEvent(deriveUsage({ action: 'index' }, { action: 'index', index: { embedded: 3 } }))) assert.equal(state.searches, 2) assert.equal(state.indexes, 1) assert.equal(state.excerpts, 4) assert.equal(state.returnedBytes, 3000) assert.equal(state.sourceBytes, 120_000) assert.equal(state.files, 4) }) test('a fold and a direct derive agree on the same numbers', () => { // Guards against the two paths drifting: whatever the tool writes must be // what the running total adds up to. const value = searchValue() const folded = foldUsage(emptyUsage(), asEvent(deriveUsage({}, value))) const summary = usageSummary(folded) assert.equal(summary.searches, 1) assert.equal(summary.sourceBytes, value.measure.sourceBytes) assert.equal(summary.returnedBytes, value.measure.returnedBytes) }) test('usageSummary reports the comparison, and calls it an upper bound', () => { const state = foldUsage(emptyUsage(), asEvent(deriveUsage({}, searchValue()))) const summary = usageSummary(state) assert.equal(summary.avoidedBytes, 58_500) assert.equal(summary.avoidedTokens, Math.round(58_500 / 4)) assert.equal(summary.compression, 40) // The dollars are real rather than rounded to nothing: the direct saving is // small, which is exactly why the surface must not lead with it. assert.ok(summary.avoidedUsd > 0) }) test('each step re-reads the avoided context, so the fold compounds it', () => { let state = foldUsage(emptyUsage(), asEvent(deriveUsage({}, searchValue()))) const after = usageSummary(state) assert.equal(after.rereadTokens, 0, 'nothing has been re-read yet') // Every step after the search re-reads what the conversation is carrying. for (let step = 0; step < 100; step += 1) state = foldUsage(state, { type: 'step/start' }) const later = usageSummary(state) assert.equal(later.rereadTokens, after.avoidedTokens * 100) assert.ok(later.avoidedUsd > after.avoidedUsd) assert.equal(later.avoidedUsd, Number((((after.avoidedTokens + later.rereadTokens) / 1e6) * 0.007).toFixed(4))) }) test('a step before anything was avoided adds nothing', () => { const state = emptyUsage() assert.equal(foldUsage(state, { type: 'step/start' }), state, 'and keeps the same reference') }) test('usageSummary survives a conversation with no calls', () => { const summary = usageSummary(emptyUsage()) assert.equal(summary.searches, 0) assert.equal(summary.avoidedTokens, 0) assert.equal(summary.compression, 0, 'no division by zero') assert.equal(summary.avoidedUsd, 0) }) test('an excerpt larger than its source cannot produce a negative saving', () => { const state = foldUsage(emptyUsage(), asEvent({ recall: { action: 'search', excerpts: 1, returnedBytes: 900, sourceBytes: 100 }, })) assert.equal(usageSummary(state).avoidedBytes, 0) })