dsh-recall/scripts/measure.mjs

/**
 * Real-scale measurement.
 *
 * Runs the actual plugin — `apply` plus the registered tool — against a real
 * workspace and a real embedding endpoint, then reports what the tool costs the
 * context window versus what reading the files would have cost.
 *
 * The comparison is the whole point of the plugin, so it is measured rather than
 * asserted: the excerpt characters are counted from the tool's own output, and
 * the "read instead" figure is the character count of the files the results came
 * from. Token counts are `characters / 4`, which is an approximation and is
 * labelled as one everywhere it appears.
 *
 *   node scripts/measure.mjs [workspace] [query] [budgetMs]
 */

import { readFile } from 'node:fs/promises'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'

import { apply, Config, settingsSnapshot } from '../index.js'

// Whatever repository you run this against, and whatever you want to ask it.
// The defaults are the working directory and a query that names a behaviour
// rather than a symbol — which is the kind of question the plugin exists for.
const WORKSPACE = process.argv[2] ?? process.cwd()
const QUERY = process.argv[3] ?? 'where is the retry policy applied'
const BUDGET_MS = Number(process.argv[4] ?? 90_000)

// The endpoint to measure against: the Ollama convention unless RECALL_API_BASE
// says otherwise. Nothing here is specific to the machine this was written on.
const API_BASE = process.env.RECALL_API_BASE ?? 'http://127.0.0.1:11434'
const MODEL = process.env.RECALL_MODEL ?? 'unclemusclez/jina-embeddings-v2-base-code:latest'

// The measurement's configuration, stated as the few values it cares about and
// resolved through the plugin's own schema — so this script can never drift
// from the Config the plugin validates against. A hand-written literal used to
// stand here, and it was missing ten fields: harmless while nothing checked,
// and a refused call the moment `execute` began validating the section it was
// handed.
const config = settingsSnapshot(
  Config({
    embeddingApiBase: API_BASE,
    embeddingModel: MODEL,
    embeddingBatchSize: 32,
    embeddingConcurrency: 4,
    embeddingTimeoutMs: 120_000,
    excerptLines: 40,
    syncBudgetMs: BUDGET_MS,
  }),
)

/** The smallest context `apply` touches, with no settings provider mounted. */
function context() {
  const tools = []
  const cleanups = []
  return {
    tools,
    ctx: {
      logger: () => ({ warn() {}, info() {}, error() {} }),
      effect: (fn) => {
        const dispose = fn()
        if (typeof dispose === 'function') cleanups.push(dispose)
      },
      get: () => undefined,
      inject: () => {},
      tools: { register: (tool) => tools.push(tool) },
    },
    dispose: () => cleanups.forEach((fn) => fn()),
  }
}

const tokens = (text) => Math.round(text.length / 4)

async function main() {
  const stateDir = await mkdtemp(join(tmpdir(), 'recall-measure-'))
  const { ctx, tools, dispose } = context()
  apply(ctx, { ...config, stateDir })
  const tool = tools[0]
  const exec = { signal: new AbortController().signal, agent: { session: { header: { cwd: WORKSPACE } } } }

  console.log(`workspace : ${WORKSPACE}`)
  console.log(`endpoint  : ${API_BASE}  model: ${MODEL}`)
  console.log(`query     : ${QUERY}`)
  console.log('')

  // --- cold start, bounded -------------------------------------------------
  const coldStart = Date.now()
  const first = await tool.execute({ query: QUERY }, exec)
  const coldMs = Date.now() - coldStart
  console.log('--- FIRST SEARCH (bounded cold start) ---')
  console.log(`elapsed        : ${(coldMs / 1000).toFixed(1)}s  (budget ${BUDGET_MS / 1000}s)`)

  // A search that did not run has no index or results to report, and the tool
  // already says why. Without this the first `first.index.*` line throws a
  // TypeError that buries the actual cause — an endpoint that is not there is
  // the ordinary case now that the default is the loopback convention.
  if (first.ok !== true) {
    console.log(`failed         : ${first.errorCode ?? 'UNKNOWN'}`)
    console.log(`note           : ${first.note ?? '(none)'}`)
    if (first.remedy !== undefined) console.log(`remedy         : ${first.remedy}`)
    console.log('')
    console.log(`nothing to measure: point RECALL_API_BASE at a reachable endpoint (currently ${API_BASE}).`)
    process.exitCode = 1
    return
  }

  console.log(`scanned files  : ${first.index.scanned}`)
  console.log(`indexed files  : ${first.index.files}`)
  console.log(`chunks         : ${first.index.chunks}`)
  console.log(`embedded now   : ${first.index.embedded}`)
  console.log(`still pending  : ${first.index.pending}`)
  console.log(`skipped (big)  : ${first.index.oversized}`)
  console.log(`results        : ${first.results.length}  best similarity ${first.best}`)
  console.log('')

  // --- the token comparison ------------------------------------------------
  const rendered = tool.output.render({ query: QUERY }, first)[0].text
  let wholeFileChars = 0
  const seen = new Set()
  for (const result of first.results) {
    if (seen.has(result.path)) continue
    seen.add(result.path)
    try {
      wholeFileChars += (await readFile(join(WORKSPACE, result.path), 'utf8')).length
    } catch {
      // A result for a file that has since moved does not change the argument.
    }
  }

  console.log('--- WHAT THE MODEL RECEIVED ---')
  console.log(`recall output  : ${rendered.length} chars  ~${tokens(rendered)} tokens`)
  console.log(`whole files    : ${wholeFileChars} chars  ~${Math.round(wholeFileChars / 4)} tokens across ${seen.size} file(s)`)
  if (wholeFileChars > 0) {
    console.log(`ratio          : ${(wholeFileChars / rendered.length).toFixed(1)}x smaller`)
  }
  console.log('')
  for (const [at, result] of first.results.entries()) {
    console.log(`${at + 1}. ${result.path}:${result.startLine}-${result.endLine}  sim ${result.similarity}  ${result.truncated ? '(truncated)' : ''}`)
  }
  console.log('')

  // --- incremental ---------------------------------------------------------
  const secondStart = Date.now()
  const second = await tool.execute({ query: 'what happens when the SSE stream reconnects' }, exec)
  const secondMs = Date.now() - secondStart
  console.log('--- SECOND SEARCH (warm index, different query) ---')
  console.log(`elapsed        : ${secondMs}ms`)
  console.log(`embedded       : ${second.index.embedded} file(s)`)
  console.log(`pending        : ${second.index.pending}`)
  console.log(`results        : ${second.results.length}`)
  console.log('')
  console.log('(token figures are characters/4 — an approximation, not a measurement)')

  dispose()
  await rm(stateDir, { recursive: true, force: true })
}

await main()