dsh-recall/test/usage.test.js

import { strict as assert } from 'node:assert'
import { test } from 'node:test'

import { deriveUsage, emptyUsage, foldUsage, usageSummary } from '../lib/usage.js'

/** A tool result shaped like the ones the tool actually returns. */
function searchValue(overrides = {}) {
  return {
    action: 'search',
    ok: true,
    focus: { path: '.', source: 'workspace' },
    weak: false,
    results: [
      { path: 'src/auth/session.ts', excerpt: 'x'.repeat(1000), similarity: 0.6 },
      { path: 'src/weather/forecast.ts', excerpt: 'y'.repeat(500), similarity: 0.4 },
    ],
    measure: { excerpts: 2, returnedBytes: 1500, sourceBytes: 60_000, files: 2 },
    ...overrides,
  }
}

/** Wrap a record the way the harness wraps the tool's metadata. */
function asEvent(meta) {
  return { type: 'tool/result', turn: 1, step: 1, message: {}, meta }
}

test('emptyUsage is a plain zeroed record', () => {
  const state = emptyUsage()
  assert.equal(state.searches, 0)
  assert.equal(state.sourceBytes, 0)
  // The projection stores this as plain JSON, so nothing exotic may appear.
  assert.deepEqual(JSON.parse(JSON.stringify(state)), state)
})

test('deriveUsage records a search without carrying any content', () => {
  const { recall } = deriveUsage({ action: 'search' }, searchValue())
  assert.equal(recall.action, 'search')
  assert.equal(recall.excerpts, 2)
  assert.equal(recall.returnedBytes, 1500)
  assert.equal(recall.sourceBytes, 60_000)
  assert.equal(recall.files, 2)
  assert.equal(recall.focus, '.')
  // The whole point of the record is that it is tiny: no excerpts, no query.
  const serialised = JSON.stringify(recall)
  assert.ok(!serialised.includes('x'.repeat(50)), 'excerpt text must not reach the log')
  assert.ok(serialised.length < 300, `record was ${serialised.length} bytes`)
})

test('deriveUsage marks a failure and a weak result', () => {
  assert.equal(deriveUsage({ action: 'search' }, { action: 'search', ok: false }).recall.failed, 1)
  assert.equal(deriveUsage({ action: 'search' }, searchValue({ weak: true })).recall.weak, 1)
})

test('deriveUsage records an index without pretending it returned excerpts', () => {
  const { recall } = deriveUsage({ action: 'index' }, { action: 'index', ok: true, index: { embedded: 4 } })
  assert.equal(recall.action, 'index')
  assert.equal(recall.embedded, 4)
  assert.equal(recall.excerpts, undefined)
})

test('foldUsage returns the SAME reference for events it does not care about', () => {
  const state = emptyUsage()
  // A fresh object here would republish the projection on every step of every
  // session, which is why the framework states this requirement explicitly.
  assert.equal(foldUsage(state, { type: 'user/message' }), state)
  assert.equal(foldUsage(state, { type: 'tool/result', meta: {} }), state)
  assert.equal(foldUsage(state, { type: 'tool/result' }), state)
  assert.equal(foldUsage(state, { type: 'tool/result', meta: { other: 1 } }), state)
  assert.equal(foldUsage(state, { type: 'tool/result', meta: { recall: null } }), state)
})

test('foldUsage accumulates one conversation', () => {
  let state = emptyUsage()
  state = foldUsage(state, asEvent(deriveUsage({}, searchValue())))
  state = foldUsage(state, asEvent(deriveUsage({}, searchValue())))
  state = foldUsage(state, asEvent(deriveUsage({ action: 'index' }, { action: 'index', index: { embedded: 3 } })))

  assert.equal(state.searches, 2)
  assert.equal(state.indexes, 1)
  assert.equal(state.excerpts, 4)
  assert.equal(state.returnedBytes, 3000)
  assert.equal(state.sourceBytes, 120_000)
  assert.equal(state.files, 4)
})

test('a fold and a direct derive agree on the same numbers', () => {
  // Guards against the two paths drifting: whatever the tool writes must be
  // what the running total adds up to.
  const value = searchValue()
  const folded = foldUsage(emptyUsage(), asEvent(deriveUsage({}, value)))
  const summary = usageSummary(folded)
  assert.equal(summary.searches, 1)
  assert.equal(summary.sourceBytes, value.measure.sourceBytes)
  assert.equal(summary.returnedBytes, value.measure.returnedBytes)
})

test('usageSummary reports the comparison, and calls it an upper bound', () => {
  const state = foldUsage(emptyUsage(), asEvent(deriveUsage({}, searchValue())))
  const summary = usageSummary(state)
  assert.equal(summary.avoidedBytes, 58_500)
  assert.equal(summary.avoidedTokens, Math.round(58_500 / 4))
  assert.equal(summary.compression, 40)
  // The dollars are real rather than rounded to nothing: the direct saving is
  // small, which is exactly why the surface must not lead with it.
  assert.ok(summary.avoidedUsd > 0)
})

test('each step re-reads the avoided context, so the fold compounds it', () => {
  let state = foldUsage(emptyUsage(), asEvent(deriveUsage({}, searchValue())))
  const after = usageSummary(state)
  assert.equal(after.rereadTokens, 0, 'nothing has been re-read yet')

  // Every step after the search re-reads what the conversation is carrying.
  for (let step = 0; step < 100; step += 1) state = foldUsage(state, { type: 'step/start' })
  const later = usageSummary(state)
  assert.equal(later.rereadTokens, after.avoidedTokens * 100)
  assert.ok(later.avoidedUsd > after.avoidedUsd)
  assert.equal(later.avoidedUsd, Number((((after.avoidedTokens + later.rereadTokens) / 1e6) * 0.007).toFixed(4)))
})

test('a step before anything was avoided adds nothing', () => {
  const state = emptyUsage()
  assert.equal(foldUsage(state, { type: 'step/start' }), state, 'and keeps the same reference')
})

test('usageSummary survives a conversation with no calls', () => {
  const summary = usageSummary(emptyUsage())
  assert.equal(summary.searches, 0)
  assert.equal(summary.avoidedTokens, 0)
  assert.equal(summary.compression, 0, 'no division by zero')
  assert.equal(summary.avoidedUsd, 0)
})

test('an excerpt larger than its source cannot produce a negative saving', () => {
  const state = foldUsage(emptyUsage(), asEvent({
    recall: { action: 'search', excerpts: 1, returnedBytes: 900, sourceBytes: 100 },
  }))
  assert.equal(usageSummary(state).avoidedBytes, 0)
})