dsh-recall/scripts/measure.mjs

/**
 * Real-scale measurement.
 *
 * Runs the actual plugin — `apply` plus the registered tool — against a real
 * workspace and a real embedding endpoint, then reports what the tool costs the
 * context window versus what reading the files would have cost.
 *
 * The comparison is the whole point of the plugin, so it is measured rather than
 * asserted: the excerpt characters are counted from the tool's own output, and
 * the "read instead" figure is the character count of the files the results came
 * from. Token counts are `characters / 4`, which is an approximation and is
 * labelled as one everywhere it appears.
 *
 *   node scripts/measure.mjs [workspace] [query] [budgetMs]
 */

import { readFile } from 'node:fs/promises'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'

import { apply } from '../index.js'

const WORKSPACE = process.argv[2] ?? '/root/workspaces/ha-runtime'
const QUERY = process.argv[3] ?? 'how does the occupancy state machine decide a room is occupied'
const BUDGET_MS = Number(process.argv[4] ?? 90_000)

const API_BASE = process.env.RECALL_API_BASE ?? 'http://10.50.1.2:11434'
const MODEL = process.env.RECALL_MODEL ?? 'unclemusclez/jina-embeddings-v2-base-code:latest'

const config = {
  embeddingApiBase: API_BASE,
  embeddingModel: MODEL,
  embeddingDimensions: 768,
  embeddingApiKey: '',
  embeddingBatchSize: 32,
  embeddingConcurrency: 4,
  embeddingTimeoutMs: 120_000,
  extensions: [],
  exclude: [],
  maxFiles: 20_000,
  maxFileBytes: 1_048_576,
  chunkLines: 48,
  maxChunkLines: 80,
  overlapLines: 8,
  defaultLimit: 8,
  maxLimit: 25,
  perFileLimit: 2,
  excerptLines: 40,
  minSimilarity: 0.2,
  syncBudgetMs: BUDGET_MS,
  stateDir: '',
}

/** The smallest context `apply` touches, with no settings provider mounted. */
function context() {
  const tools = []
  const cleanups = []
  return {
    tools,
    ctx: {
      logger: () => ({ warn() {}, info() {}, error() {} }),
      effect: (fn) => {
        const dispose = fn()
        if (typeof dispose === 'function') cleanups.push(dispose)
      },
      get: () => undefined,
      inject: () => {},
      tools: { register: (tool) => tools.push(tool) },
    },
    dispose: () => cleanups.forEach((fn) => fn()),
  }
}

const tokens = (text) => Math.round(text.length / 4)

async function main() {
  const stateDir = await mkdtemp(join(tmpdir(), 'recall-measure-'))
  const { ctx, tools, dispose } = context()
  apply(ctx, { ...config, stateDir })
  const tool = tools[0]
  const exec = { signal: new AbortController().signal, agent: { session: { header: { cwd: WORKSPACE } } } }

  console.log(`workspace : ${WORKSPACE}`)
  console.log(`endpoint  : ${API_BASE}  model: ${MODEL}`)
  console.log(`query     : ${QUERY}`)
  console.log('')

  // --- cold start, bounded -------------------------------------------------
  const coldStart = Date.now()
  const first = await tool.execute({ query: QUERY }, exec)
  const coldMs = Date.now() - coldStart
  console.log('--- FIRST SEARCH (bounded cold start) ---')
  console.log(`elapsed        : ${(coldMs / 1000).toFixed(1)}s  (budget ${BUDGET_MS / 1000}s)`)
  console.log(`scanned files  : ${first.index.scanned}`)
  console.log(`indexed files  : ${first.index.files}`)
  console.log(`chunks         : ${first.index.chunks}`)
  console.log(`embedded now   : ${first.index.embedded}`)
  console.log(`still pending  : ${first.index.pending}`)
  console.log(`skipped (big)  : ${first.index.oversized}`)
  console.log(`results        : ${first.results.length}  best similarity ${first.best}`)
  console.log('')

  // --- the token comparison ------------------------------------------------
  const rendered = tool.output.render({ query: QUERY }, first)[0].text
  let wholeFileChars = 0
  const seen = new Set()
  for (const result of first.results) {
    if (seen.has(result.path)) continue
    seen.add(result.path)
    try {
      wholeFileChars += (await readFile(join(WORKSPACE, result.path), 'utf8')).length
    } catch {
      // A result for a file that has since moved does not change the argument.
    }
  }

  console.log('--- WHAT THE MODEL RECEIVED ---')
  console.log(`recall output  : ${rendered.length} chars  ~${tokens(rendered)} tokens`)
  console.log(`whole files    : ${wholeFileChars} chars  ~${Math.round(wholeFileChars / 4)} tokens across ${seen.size} file(s)`)
  if (wholeFileChars > 0) {
    console.log(`ratio          : ${(wholeFileChars / rendered.length).toFixed(1)}x smaller`)
  }
  console.log('')
  for (const [at, result] of first.results.entries()) {
    console.log(`${at + 1}. ${result.path}:${result.startLine}-${result.endLine}  sim ${result.similarity}  ${result.truncated ? '(truncated)' : ''}`)
  }
  console.log('')

  // --- incremental ---------------------------------------------------------
  const secondStart = Date.now()
  const second = await tool.execute({ query: 'what happens when the SSE stream reconnects' }, exec)
  const secondMs = Date.now() - secondStart
  console.log('--- SECOND SEARCH (warm index, different query) ---')
  console.log(`elapsed        : ${secondMs}ms`)
  console.log(`embedded       : ${second.index.embedded} file(s)`)
  console.log(`pending        : ${second.index.pending}`)
  console.log(`results        : ${second.results.length}`)
  console.log('')
  console.log('(token figures are characters/4 — an approximation, not a measurement)')

  dispose()
  await rm(stateDir, { recursive: true, force: true })
}

await main()