init

test/usage.test.js

@@ -0,0 +1,139 @@
+import { strict as assert } from 'node:assert'
+import { test } from 'node:test'
+
+import { deriveUsage, emptyUsage, foldUsage, usageSummary } from '../lib/usage.js'
+
+/** A tool result shaped like the ones the tool actually returns. */
+function searchValue(overrides = {}) {
+  return {
+    action: 'search',
+    ok: true,
+    focus: { path: '.', source: 'workspace' },
+    weak: false,
+    results: [
+      { path: 'src/auth/session.ts', excerpt: 'x'.repeat(1000), similarity: 0.6 },
+      { path: 'src/weather/forecast.ts', excerpt: 'y'.repeat(500), similarity: 0.4 },
+    ],
+    measure: { excerpts: 2, returnedBytes: 1500, sourceBytes: 60_000, files: 2 },
+    ...overrides,
+  }
+}
+
+/** Wrap a record the way the harness wraps the tool's metadata. */
+function asEvent(meta) {
+  return { type: 'tool/result', turn: 1, step: 1, message: {}, meta }
+}
+
+test('emptyUsage is a plain zeroed record', () => {
+  const state = emptyUsage()
+  assert.equal(state.searches, 0)
+  assert.equal(state.sourceBytes, 0)
+  // The projection stores this as plain JSON, so nothing exotic may appear.
+  assert.deepEqual(JSON.parse(JSON.stringify(state)), state)
+})
+
+test('deriveUsage records a search without carrying any content', () => {
+  const { recall } = deriveUsage({ action: 'search' }, searchValue())
+  assert.equal(recall.action, 'search')
+  assert.equal(recall.excerpts, 2)
+  assert.equal(recall.returnedBytes, 1500)
+  assert.equal(recall.sourceBytes, 60_000)
+  assert.equal(recall.files, 2)
+  assert.equal(recall.focus, '.')
+  // The whole point of the record is that it is tiny: no excerpts, no query.
+  const serialised = JSON.stringify(recall)
+  assert.ok(!serialised.includes('x'.repeat(50)), 'excerpt text must not reach the log')
+  assert.ok(serialised.length < 300, `record was ${serialised.length} bytes`)
+})
+
+test('deriveUsage marks a failure and a weak result', () => {
+  assert.equal(deriveUsage({ action: 'search' }, { action: 'search', ok: false }).recall.failed, 1)
+  assert.equal(deriveUsage({ action: 'search' }, searchValue({ weak: true })).recall.weak, 1)
+})
+
+test('deriveUsage records an index without pretending it returned excerpts', () => {
+  const { recall } = deriveUsage({ action: 'index' }, { action: 'index', ok: true, index: { embedded: 4 } })
+  assert.equal(recall.action, 'index')
+  assert.equal(recall.embedded, 4)
+  assert.equal(recall.excerpts, undefined)
+})
+
+test('foldUsage returns the SAME reference for events it does not care about', () => {
+  const state = emptyUsage()
+  // A fresh object here would republish the projection on every step of every
+  // session, which is why the framework states this requirement explicitly.
+  assert.equal(foldUsage(state, { type: 'user/message' }), state)
+  assert.equal(foldUsage(state, { type: 'tool/result', meta: {} }), state)
+  assert.equal(foldUsage(state, { type: 'tool/result' }), state)
+  assert.equal(foldUsage(state, { type: 'tool/result', meta: { other: 1 } }), state)
+  assert.equal(foldUsage(state, { type: 'tool/result', meta: { recall: null } }), state)
+})
+
+test('foldUsage accumulates one conversation', () => {
+  let state = emptyUsage()
+  state = foldUsage(state, asEvent(deriveUsage({}, searchValue())))
+  state = foldUsage(state, asEvent(deriveUsage({}, searchValue())))
+  state = foldUsage(state, asEvent(deriveUsage({ action: 'index' }, { action: 'index', index: { embedded: 3 } })))
+
+  assert.equal(state.searches, 2)
+  assert.equal(state.indexes, 1)
+  assert.equal(state.excerpts, 4)
+  assert.equal(state.returnedBytes, 3000)
+  assert.equal(state.sourceBytes, 120_000)
+  assert.equal(state.files, 4)
+})
+
+test('a fold and a direct derive agree on the same numbers', () => {
+  // Guards against the two paths drifting: whatever the tool writes must be
+  // what the running total adds up to.
+  const value = searchValue()
+  const folded = foldUsage(emptyUsage(), asEvent(deriveUsage({}, value)))
+  const summary = usageSummary(folded)
+  assert.equal(summary.searches, 1)
+  assert.equal(summary.sourceBytes, value.measure.sourceBytes)
+  assert.equal(summary.returnedBytes, value.measure.returnedBytes)
+})
+
+test('usageSummary reports the comparison, and calls it an upper bound', () => {
+  const state = foldUsage(emptyUsage(), asEvent(deriveUsage({}, searchValue())))
+  const summary = usageSummary(state)
+  assert.equal(summary.avoidedBytes, 58_500)
+  assert.equal(summary.avoidedTokens, Math.round(58_500 / 4))
+  assert.equal(summary.compression, 40)
+  // The dollars are real rather than rounded to nothing: the direct saving is
+  // small, which is exactly why the surface must not lead with it.
+  assert.ok(summary.avoidedUsd > 0)
+})
+
+test('each step re-reads the avoided context, so the fold compounds it', () => {
+  let state = foldUsage(emptyUsage(), asEvent(deriveUsage({}, searchValue())))
+  const after = usageSummary(state)
+  assert.equal(after.rereadTokens, 0, 'nothing has been re-read yet')
+
+  // Every step after the search re-reads what the conversation is carrying.
+  for (let step = 0; step < 100; step += 1) state = foldUsage(state, { type: 'step/start' })
+  const later = usageSummary(state)
+  assert.equal(later.rereadTokens, after.avoidedTokens * 100)
+  assert.ok(later.avoidedUsd > after.avoidedUsd)
+  assert.equal(later.avoidedUsd, Number((((after.avoidedTokens + later.rereadTokens) / 1e6) * 0.007).toFixed(4)))
+})
+
+test('a step before anything was avoided adds nothing', () => {
+  const state = emptyUsage()
+  assert.equal(foldUsage(state, { type: 'step/start' }), state, 'and keeps the same reference')
+})
+
+test('usageSummary survives a conversation with no calls', () => {
+  const summary = usageSummary(emptyUsage())
+  assert.equal(summary.searches, 0)
+  assert.equal(summary.avoidedTokens, 0)
+  assert.equal(summary.compression, 0, 'no division by zero')
+  assert.equal(summary.avoidedUsd, 0)
+})
+
+test('an excerpt larger than its source cannot produce a negative saving', () => {
+  const state = foldUsage(emptyUsage(), asEvent({
+    recall: { action: 'search', excerpts: 1, returnedBytes: 900, sourceBytes: 100 },
+  }))
+  assert.equal(usageSummary(state).avoidedBytes, 0)
+})