init

scripts/measure.mjs

@@ -0,0 +1,146 @@
+/**
+ * Real-scale measurement.
+ *
+ * Runs the actual plugin — `apply` plus the registered tool — against a real
+ * workspace and a real embedding endpoint, then reports what the tool costs the
+ * context window versus what reading the files would have cost.
+ *
+ * The comparison is the whole point of the plugin, so it is measured rather than
+ * asserted: the excerpt characters are counted from the tool's own output, and
+ * the "read instead" figure is the character count of the files the results came
+ * from. Token counts are `characters / 4`, which is an approximation and is
+ * labelled as one everywhere it appears.
+ *
+ *   node scripts/measure.mjs [workspace] [query] [budgetMs]
+ */
+
+import { readFile } from 'node:fs/promises'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { apply } from '../index.js'
+
+const WORKSPACE = process.argv[2] ?? '/root/workspaces/ha-runtime'
+const QUERY = process.argv[3] ?? 'how does the occupancy state machine decide a room is occupied'
+const BUDGET_MS = Number(process.argv[4] ?? 90_000)
+
+const API_BASE = process.env.RECALL_API_BASE ?? 'http://10.50.1.2:11434'
+const MODEL = process.env.RECALL_MODEL ?? 'unclemusclez/jina-embeddings-v2-base-code:latest'
+
+const config = {
+  embeddingApiBase: API_BASE,
+  embeddingModel: MODEL,
+  embeddingDimensions: 768,
+  embeddingApiKey: '',
+  embeddingBatchSize: 32,
+  embeddingConcurrency: 4,
+  embeddingTimeoutMs: 120_000,
+  extensions: [],
+  exclude: [],
+  maxFiles: 20_000,
+  maxFileBytes: 1_048_576,
+  chunkLines: 48,
+  maxChunkLines: 80,
+  overlapLines: 8,
+  defaultLimit: 8,
+  maxLimit: 25,
+  perFileLimit: 2,
+  excerptLines: 40,
+  minSimilarity: 0.2,
+  syncBudgetMs: BUDGET_MS,
+  stateDir: '',
+}
+
+/** The smallest context `apply` touches, with no settings provider mounted. */
+function context() {
+  const tools = []
+  const cleanups = []
+  return {
+    tools,
+    ctx: {
+      logger: () => ({ warn() {}, info() {}, error() {} }),
+      effect: (fn) => {
+        const dispose = fn()
+        if (typeof dispose === 'function') cleanups.push(dispose)
+      },
+      get: () => undefined,
+      inject: () => {},
+      tools: { register: (tool) => tools.push(tool) },
+    },
+    dispose: () => cleanups.forEach((fn) => fn()),
+  }
+}
+
+const tokens = (text) => Math.round(text.length / 4)
+
+async function main() {
+  const stateDir = await mkdtemp(join(tmpdir(), 'recall-measure-'))
+  const { ctx, tools, dispose } = context()
+  apply(ctx, { ...config, stateDir })
+  const tool = tools[0]
+  const exec = { signal: new AbortController().signal, agent: { session: { header: { cwd: WORKSPACE } } } }
+
+  console.log(`workspace : ${WORKSPACE}`)
+  console.log(`endpoint  : ${API_BASE}  model: ${MODEL}`)
+  console.log(`query     : ${QUERY}`)
+  console.log('')
+
+  // --- cold start, bounded -------------------------------------------------
+  const coldStart = Date.now()
+  const first = await tool.execute({ query: QUERY }, exec)
+  const coldMs = Date.now() - coldStart
+  console.log('--- FIRST SEARCH (bounded cold start) ---')
+  console.log(`elapsed        : ${(coldMs / 1000).toFixed(1)}s  (budget ${BUDGET_MS / 1000}s)`)
+  console.log(`scanned files  : ${first.index.scanned}`)
+  console.log(`indexed files  : ${first.index.files}`)
+  console.log(`chunks         : ${first.index.chunks}`)
+  console.log(`embedded now   : ${first.index.embedded}`)
+  console.log(`still pending  : ${first.index.pending}`)
+  console.log(`skipped (big)  : ${first.index.oversized}`)
+  console.log(`results        : ${first.results.length}  best similarity ${first.best}`)
+  console.log('')
+
+  // --- the token comparison ------------------------------------------------
+  const rendered = tool.output.render({ query: QUERY }, first)[0].text
+  let wholeFileChars = 0
+  const seen = new Set()
+  for (const result of first.results) {
+    if (seen.has(result.path)) continue
+    seen.add(result.path)
+    try {
+      wholeFileChars += (await readFile(join(WORKSPACE, result.path), 'utf8')).length
+    } catch {
+      // A result for a file that has since moved does not change the argument.
+    }
+  }
+
+  console.log('--- WHAT THE MODEL RECEIVED ---')
+  console.log(`recall output  : ${rendered.length} chars  ~${tokens(rendered)} tokens`)
+  console.log(`whole files    : ${wholeFileChars} chars  ~${Math.round(wholeFileChars / 4)} tokens across ${seen.size} file(s)`)
+  if (wholeFileChars > 0) {
+    console.log(`ratio          : ${(wholeFileChars / rendered.length).toFixed(1)}x smaller`)
+  }
+  console.log('')
+  for (const [at, result] of first.results.entries()) {
+    console.log(`${at + 1}. ${result.path}:${result.startLine}-${result.endLine}  sim ${result.similarity}  ${result.truncated ? '(truncated)' : ''}`)
+  }
+  console.log('')
+
+  // --- incremental ---------------------------------------------------------
+  const secondStart = Date.now()
+  const second = await tool.execute({ query: 'what happens when the SSE stream reconnects' }, exec)
+  const secondMs = Date.now() - secondStart
+  console.log('--- SECOND SEARCH (warm index, different query) ---')
+  console.log(`elapsed        : ${secondMs}ms`)
+  console.log(`embedded       : ${second.index.embedded} file(s)`)
+  console.log(`pending        : ${second.index.pending}`)
+  console.log(`results        : ${second.results.length}`)
+  console.log('')
+  console.log('(token figures are characters/4 — an approximation, not a measurement)')
+
+  dispose()
+  await rm(stateDir, { recursive: true, force: true })
+}
+
+await main()