/** * Real-scale measurement. * * Runs the actual plugin — `apply` plus the registered tool — against a real * workspace and a real embedding endpoint, then reports what the tool costs the * context window versus what reading the files would have cost. * * The comparison is the whole point of the plugin, so it is measured rather than * asserted: the excerpt characters are counted from the tool's own output, and * the "read instead" figure is the character count of the files the results came * from. Token counts are `characters / 4`, which is an approximation and is * labelled as one everywhere it appears. * * node scripts/measure.mjs [workspace] [query] [budgetMs] */ import { readFile } from 'node:fs/promises' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { apply, Config, settingsSnapshot } from '../index.js' // Whatever repository you run this against, and whatever you want to ask it. // The defaults are the working directory and a query that names a behaviour // rather than a symbol — which is the kind of question the plugin exists for. const WORKSPACE = process.argv[2] ?? process.cwd() const QUERY = process.argv[3] ?? 'where is the retry policy applied' const BUDGET_MS = Number(process.argv[4] ?? 90_000) // The endpoint to measure against: the Ollama convention unless RECALL_API_BASE // says otherwise. Nothing here is specific to the machine this was written on. const API_BASE = process.env.RECALL_API_BASE ?? 'http://127.0.0.1:11434' const MODEL = process.env.RECALL_MODEL ?? 'unclemusclez/jina-embeddings-v2-base-code:latest' // The measurement's configuration, stated as the few values it cares about and // resolved through the plugin's own schema — so this script can never drift // from the Config the plugin validates against. A hand-written literal used to // stand here, and it was missing ten fields: harmless while nothing checked, // and a refused call the moment `execute` began validating the section it was // handed. const config = settingsSnapshot( Config({ embeddingApiBase: API_BASE, embeddingModel: MODEL, embeddingBatchSize: 32, embeddingConcurrency: 4, embeddingTimeoutMs: 120_000, excerptLines: 40, syncBudgetMs: BUDGET_MS, }), ) /** The smallest context `apply` touches, with no settings provider mounted. */ function context() { const tools = [] const cleanups = [] return { tools, ctx: { logger: () => ({ warn() {}, info() {}, error() {} }), effect: (fn) => { const dispose = fn() if (typeof dispose === 'function') cleanups.push(dispose) }, get: () => undefined, inject: () => {}, tools: { register: (tool) => tools.push(tool) }, }, dispose: () => cleanups.forEach((fn) => fn()), } } const tokens = (text) => Math.round(text.length / 4) async function main() { const stateDir = await mkdtemp(join(tmpdir(), 'recall-measure-')) const { ctx, tools, dispose } = context() apply(ctx, { ...config, stateDir }) const tool = tools[0] const exec = { signal: new AbortController().signal, agent: { session: { header: { cwd: WORKSPACE } } } } console.log(`workspace : ${WORKSPACE}`) console.log(`endpoint : ${API_BASE} model: ${MODEL}`) console.log(`query : ${QUERY}`) console.log('') // --- cold start, bounded ------------------------------------------------- const coldStart = Date.now() const first = await tool.execute({ query: QUERY }, exec) const coldMs = Date.now() - coldStart console.log('--- FIRST SEARCH (bounded cold start) ---') console.log(`elapsed : ${(coldMs / 1000).toFixed(1)}s (budget ${BUDGET_MS / 1000}s)`) // A search that did not run has no index or results to report, and the tool // already says why. Without this the first `first.index.*` line throws a // TypeError that buries the actual cause — an endpoint that is not there is // the ordinary case now that the default is the loopback convention. if (first.ok !== true) { console.log(`failed : ${first.errorCode ?? 'UNKNOWN'}`) console.log(`note : ${first.note ?? '(none)'}`) if (first.remedy !== undefined) console.log(`remedy : ${first.remedy}`) console.log('') console.log(`nothing to measure: point RECALL_API_BASE at a reachable endpoint (currently ${API_BASE}).`) process.exitCode = 1 return } console.log(`scanned files : ${first.index.scanned}`) console.log(`indexed files : ${first.index.files}`) console.log(`chunks : ${first.index.chunks}`) console.log(`embedded now : ${first.index.embedded}`) console.log(`still pending : ${first.index.pending}`) console.log(`skipped (big) : ${first.index.oversized}`) console.log(`results : ${first.results.length} best similarity ${first.best}`) console.log('') // --- the token comparison ------------------------------------------------ const rendered = tool.output.render({ query: QUERY }, first)[0].text let wholeFileChars = 0 const seen = new Set() for (const result of first.results) { if (seen.has(result.path)) continue seen.add(result.path) try { wholeFileChars += (await readFile(join(WORKSPACE, result.path), 'utf8')).length } catch { // A result for a file that has since moved does not change the argument. } } console.log('--- WHAT THE MODEL RECEIVED ---') console.log(`recall output : ${rendered.length} chars ~${tokens(rendered)} tokens`) console.log(`whole files : ${wholeFileChars} chars ~${Math.round(wholeFileChars / 4)} tokens across ${seen.size} file(s)`) if (wholeFileChars > 0) { console.log(`ratio : ${(wholeFileChars / rendered.length).toFixed(1)}x smaller`) } console.log('') for (const [at, result] of first.results.entries()) { console.log(`${at + 1}. ${result.path}:${result.startLine}-${result.endLine} sim ${result.similarity} ${result.truncated ? '(truncated)' : ''}`) } console.log('') // --- incremental --------------------------------------------------------- const secondStart = Date.now() const second = await tool.execute({ query: 'what happens when the SSE stream reconnects' }, exec) const secondMs = Date.now() - secondStart console.log('--- SECOND SEARCH (warm index, different query) ---') console.log(`elapsed : ${secondMs}ms`) console.log(`embedded : ${second.index.embedded} file(s)`) console.log(`pending : ${second.index.pending}`) console.log(`results : ${second.results.length}`) console.log('') console.log('(token figures are characters/4 — an approximation, not a measurement)') dispose() await rm(stateDir, { recursive: true, force: true }) } await main()