/** * dsh-recall — local-embedding code recall for DeepSeek Harness. * * One tool (`recall`) over a per-workspace vector index built by a local * embedding model, plus a `Config` schema whose every field is live-editable — * the endpoint is configuration, not a constant. * * **How the settings surface works (harness 0.1.7).** A plugin no longer * registers a settings section. It exports a `Config` schema; the Harness keys * a form for the profile entry by that row's `id`, and a field is offered to * the UI only when its schema node is declared `volatile()` — the marker that * means "editable without remounting the plugin". So this file's schema carries * two jobs at once: validation, and being the whole of the settings page's * description. `apply` therefore receives volatile fields as live references * and reads them through {@link settingsSnapshot} per call, which is what makes * a saved edit take effect without a restart. * * **The economics this exists for.** In a long agent session the dominant cost * is not the model's reasoning; it is re-reading the accumulated transcript on * every step. Measured on this harness over 32 days, 65% of spend was re-reading * cached context and 96% of that context was message history — mostly file * contents the agent had already read. A tool that answers "where is this * handled" with a ranked 40-line excerpt instead of four whole files keeps that * history small, and the saving compounds on every subsequent step because the * context it did not add is never re-sent again. * * **Why the index is warmed lazily rather than at load.** `apply` must not block * on embedding a repository; a plugin that took minutes to activate would make * every profile start slow, including the ones that never call the tool. Instead * a search does a *bounded* sync — read and hash everything, embed only what * fits in a time budget — and reports honestly what is still pending. A full * warm-up is available as an explicit background action. * * **Why the endpoint is configuration and not a constant.** The Ollama default * of `127.0.0.1:11434` is a convention, not a fact — a deployment often runs * Ollama on another host. Baking one machine's address into a bundle layer that * other machines apply is exactly the mistake the composition docs warn about, * so the endpoint is a schema default the profile can override, at rest or * live. */ import { readFile, stat } from 'node:fs/promises' import { createHash } from 'node:crypto' import { basename, join, resolve } from 'node:path' import { defineTool } from '@deepseek-ai/dsh-tools' // A real dependency rather than a duck-typed `{ parse }`: the projection service // calls these schemas, and `dsh-session-projection` declares zod itself, so it is // present wherever projections are. import { z } from 'zod' import Schema from '@deepseek-ai/schemastery' import { chunkFile, embedText } from './lib/chunk.js' import { EmbeddingError, createEmbedder } from './lib/embed.js' import { DEFAULT_FOCUS_MARKERS, discoverScopes, focusKey, resolveFocus } from './lib/focus.js' import { indexFile, resolveDshHome, workspaceStateDir } from './lib/paths.js' import { fuseRankings, rankOrder, selectFrom } from './lib/rank.js' import { RecallIndex } from './lib/store.js' import { createExpander } from './lib/expand.js' import { createReranker } from './lib/rerank.js' import { deriveUsage, emptyUsage, foldUsage, usageSummary } from './lib/usage.js' import { DEFAULT_EXTENSIONS, DEFAULT_SKIP_DIRS, listCodeFiles } from './lib/walk.js' /** * This plugin's settings namespace, which in 0.1.7 IS its profile entry id. * * `ctx.settings.describe()` keys one form per composed entry by the `id` the * patch layer gives that row — `dsh-recall` in cordis.patch.yml — and writes go * back to the same key. It is not a name the plugin registers; the schema * exported as `Config` is the registration, and this constant is the id those * settings surfaces address it by. Renaming the row without renaming this * would leave the `configure` action writing to a row that does not exist. */ const SETTINGS_NAMESPACE = 'dsh-recall' /** Every configuration field this plugin exposes. */ export const SETTING_KEYS = [ 'embeddingApiBase', 'embeddingModel', 'embeddingDimensions', 'embeddingApiKey', 'embeddingBatchSize', 'embeddingConcurrency', 'embeddingTimeoutMs', 'embeddingKeepAlive', 'extensions', 'exclude', 'maxFiles', 'maxFileBytes', 'chunkLines', 'maxChunkLines', 'overlapLines', 'defaultLimit', 'maxLimit', 'perFileLimit', 'excerptLines', 'totalExcerptLines', 'minSimilarity', 'syncBudgetMs', 'stateDir', 'focus', 'focusMarkers', 'expansionEnabled', 'expansionModel', 'expansionCount', 'expansionTimeoutMs', 'rerankEnabled', 'rerankModel', 'rerankCandidates', 'rerankTimeoutMs', ] /** * Every field is `.volatile()`. * * A volatile field is one the configuration document may change without * remounting the plugin: the Loader re-parses the raw config, writes the new * value into the running instance's existing reference, and emits * `loader/volatile-update`. Two consequences decide the shape below: * * - `ctx.settings.describe` SKIPS an entry with no volatile field, so a schema * that marks none is invisible to every settings surface — there is no page * to draw rather than an empty one. This is why the plugin cannot expose a * settings page at all without these markers. * - a volatile field arrives in `apply`'s config as a reference (`{ get() }`) * rather than a value, which is what {@link settingsSnapshot} unwraps. * * Every field qualifies: all of them are read afresh per tool call, so a live * edit takes effect on the next call with nothing to re-register. * * The `.min`/`.max` bounds duplicate what {@link validateSettings} enforces. * That is deliberate rather than redundant: those are the constraints a schema * can express, and expressing them here is what makes the generated form refuse * the invalid draft in front of the person typing it. `validateSettings` keeps * the cross-field rules no single field's schema can state. */ export const Config = Schema.object({ /** Embedding endpoint, without a trailing slash. */ embeddingApiBase: Schema.string().default('http://127.0.0.1:11434').volatile(), /** Model id as the endpoint knows it. */ embeddingModel: Schema.string().default('unclemusclez/jina-embeddings-v2-base-code:latest').volatile(), /** Vector width. Must match the model; a mismatch is refused, not truncated. */ embeddingDimensions: Schema.number().min(1).default(768).volatile(), /** Bearer token for a gateway that wants one. */ // `role('secret')` keeps this out of every settings response: the wire // surfaces strip schema-declared secret positions, so a page can report that a // key is set without ever receiving it. embeddingApiKey: Schema.string().default('').role('secret').volatile(), /** Inputs per request on the batch endpoint. */ embeddingBatchSize: Schema.number().min(1).default(16).volatile(), /** Requests in flight. */ embeddingConcurrency: Schema.number().min(1).default(2).volatile(), /** * Per-request timeout. * * Deliberately short. A local endpoint that has wedged does not refuse the * connection — it accepts it and never answers, so a long timeout turns a * broken backend into a tool call that appears to hang. Thirty seconds is far * more than a batch of a few thousand characters needs on any CPU, and it * fails in bounded time instead of leaving the caller guessing. */ embeddingTimeoutMs: Schema.number().min(1).default(30_000).volatile(), /** Ollama `keep_alive`. Keeps the model resident across an index run. */ embeddingKeepAlive: Schema.string().default('5m').volatile(), /** Extensions to index. Empty means the built-in code set. */ extensions: Schema.array(String).default([]).volatile(), /** Extra glob patterns to skip, on top of the built-in directory list. */ exclude: Schema.array(String).default([]).volatile(), /** Most files a workspace walk will consider. */ maxFiles: Schema.number().min(1).default(20_000).volatile(), /** Files larger than this are skipped as generated. */ maxFileBytes: Schema.number().min(1).default(1_048_576).volatile(), /** Target lines per chunk. */ chunkLines: Schema.number().min(1).default(48).volatile(), /** Hard cap before a chunk is split. */ maxChunkLines: Schema.number().min(1).default(80).volatile(), /** Lines shared between the halves of a split chunk. */ overlapLines: Schema.number().min(0).default(8).volatile(), /** Results returned when the caller does not say. */ defaultLimit: Schema.number().min(1).default(8).volatile(), /** Ceiling on the caller's `limit`. */ maxLimit: Schema.number().min(1).default(25).volatile(), /** Most chunks one file may contribute to one result set. */ perFileLimit: Schema.number().min(1).default(2).volatile(), /** * Lines shown per excerpt. * * Defaults to the chunk cap, so a chunk — already a bounded semantic unit — * is returned whole. Cutting one in half is how a search turns into a * follow-up read that costs more than the lines it saved. */ excerptLines: Schema.number().min(1).default(80).volatile(), /** Total lines across one result set, allocated by rank. */ totalExcerptLines: Schema.number().min(1).default(400).volatile(), /** Below this best-similarity, the result set is reported as weak. */ minSimilarity: Schema.number().min(0).max(1).default(0.2).volatile(), /** Embedding time one `search` call may spend warming the index. */ syncBudgetMs: Schema.number().min(0).default(20_000).volatile(), /** Index directory. Empty means `$DSH_HOME/dsh-recall/workspaces/`. */ stateDir: Schema.string().default('').volatile(), /** * Narrows the index and every search to one directory. * * Empty resolves by marker: the nearest ancestor of the session's directory * carrying a `focusMarkers` file becomes the scope, and the workspace root is * the fallback — which is the behaviour that existed before focus did, so * adopting this cannot regress a deployment that never configures it. */ focus: Schema.string().default('').volatile(), /** Filenames that mark a directory as its own scope. Empty means the built-ins. */ focusMarkers: Schema.array(String).default([]).volatile(), /** * Ask the local model for alternative phrasings and fuse the rankings. * * Off by default. It costs one local generation plus an embedding per variant, * and its benefit has not been measured against a real model — a default that * spends latency on an unquantified gain is not a default worth having. */ expansionEnabled: Schema.boolean().default(false).volatile(), /** Model that writes the phrasings. Empty uses `embeddingModel`'s endpoint, not its model. */ expansionModel: Schema.string().default('').volatile(), /** Most variants to ask for. */ expansionCount: Schema.number().min(1).default(4).volatile(), /** Budget for the whole expansion, in ms. Expiry just means search without it. */ expansionTimeoutMs: Schema.number().min(1).default(8_000).volatile(), /** * Have the local model score the top candidates against the query. * * Off by default for the same reason, and more so: it is one more model call * on the critical path of every search. */ rerankEnabled: Schema.boolean().default(false).volatile(), /** Model that scores candidates. */ rerankModel: Schema.string().default('').volatile(), /** Candidates scored per search. The reranker caps this at 12. */ rerankCandidates: Schema.number().min(1).default(8).volatile(), /** Budget for the scoring call, in ms. */ rerankTimeoutMs: Schema.number().min(1).default(15_000).volatile(), }) /** * Read one configuration field. * * A volatile field is a reference in the running config, not a plain value; a * non-volatile one (or a config object a test handed straight to `apply`) is * the value itself. Both are accepted so the same path serves a real profile * and a direct call. * * @param {unknown} value a field of the resolved config * @returns {unknown} the current value */ export function configValue(value) { return value !== null && typeof value === 'object' && typeof value.get === 'function' ? value.get() : value } /** * A plain snapshot of the whole settings section. * * Taken per tool call rather than cached, because that is exactly what a * volatile field buys: the running instance's references are updated in place * when the document changes, so the next read sees the new value without the * plugin being remounted. * * @param {Record} config the resolved composition config * @returns {Record} every declared field, unwrapped */ export function settingsSnapshot(config) { const snapshot = {} for (const key of Object.keys(config ?? {})) snapshot[key] = configValue(config[key]) return snapshot } /** Resolve the harness home, preferring the harness's own resolver. */ function dshHomeOf(ctx) { try { const homePath = ctx.get('dshHomePath') if (typeof homePath === 'function') { const resolved = homePath() if (typeof resolved === 'string' && resolved !== '') return resolved } } catch { // Declared optional; absent outside dsh-app-boot, which is expected. } return resolveDshHome() } /** * Constraints a single field's schema cannot express. * * Called on the section a call is about to act on, and on the section a * `configure` patch would leave behind. The Host validates every field against * `Config` — including the `.min`/`.max` bounds — but the rules here relate two * fields to each other (`defaultLimit` against `maxLimit`, `overlapLines` * against `maxChunkLines`), and 0.1.7 has no write-time validator hook for * those. So the check moves to the two moments that can still refuse: before a * write this plugin performs, and before a configured value is put to work. * * @param {Record} value a schema-valid settings section * @throws {Error} when the section cannot be acted on */ export function validateSettings(value) { const base = String(value.embeddingApiBase ?? '').trim() let url try { url = new URL(base) } catch { throw new Error(`embeddingApiBase is not a URL: ${JSON.stringify(base)}`) } if (url.protocol !== 'http:' && url.protocol !== 'https:') { throw new Error(`embeddingApiBase must be http or https, not ${url.protocol}`) } if (!(Number(value.embeddingDimensions) > 0)) { throw new Error('embeddingDimensions must be a positive number') } for (const key of [ 'embeddingBatchSize', 'embeddingConcurrency', 'embeddingTimeoutMs', 'maxLimit', 'defaultLimit', 'perFileLimit', 'excerptLines', 'totalExcerptLines', 'chunkLines', 'maxChunkLines', 'maxFiles', 'maxFileBytes', 'expansionCount', 'expansionTimeoutMs', 'rerankCandidates', 'rerankTimeoutMs', ]) { if (!(Number(value[key]) >= 1)) throw new Error(`${key} must be at least 1`) } if (Number(value.syncBudgetMs) < 0) throw new Error('syncBudgetMs cannot be negative') if (Number(value.overlapLines) < 0) throw new Error('overlapLines cannot be negative') if (Number(value.maxChunkLines) < Number(value.chunkLines)) { throw new Error('maxChunkLines cannot be below chunkLines') } if (Number(value.overlapLines) >= Number(value.maxChunkLines)) { throw new Error('overlapLines must be smaller than maxChunkLines, or a split never advances') } if (Number(value.defaultLimit) > Number(value.maxLimit)) { throw new Error(`defaultLimit (${value.defaultLimit}) cannot exceed maxLimit (${value.maxLimit})`) } if (Number(value.minSimilarity) < 0 || Number(value.minSimilarity) > 1) { throw new Error('minSimilarity is a cosine similarity and must be between 0 and 1') } } /** SHA-256 of file content, used as the change key. */ function hashOf(content) { return createHash('sha256').update(content).digest('hex') } /** * Serialise work per key. * * Two tool calls can arrive while the first is still warming the index, and two * concurrent syncs of the same workspace would interleave writes to one SQLite * handle. Chaining on a per-workspace promise makes the second wait rather than * race, which is what lets `search` stay marked concurrency-safe. */ function createLock() { const chains = new Map() return async function withLock(key, work) { const previous = chains.get(key) ?? Promise.resolve() let release const gate = new Promise((resolveGate) => { release = resolveGate }) chains.set( key, previous.then(() => gate), ) await previous try { return await work() } finally { release() if (chains.get(key) === gate) chains.delete(key) } } } /** * Schema of the persisted fold. * * The projection stores state as plain JSON and validates it with `.parse()` * before seeding a fold from a checkpoint, so this must accept exactly what * `emptyUsage()` produces and what `foldUsage` returns. */ const usageStateSchema = z.object({ version: z.number(), searches: z.number(), indexes: z.number(), failed: z.number(), excerpts: z.number(), returnedBytes: z.number(), sourceBytes: z.number(), files: z.number(), weak: z.number(), rereadBytes: z.number(), lastFocus: z.string(), }) /** Schema of the wire view — what the browser receives. */ const usageViewSchema = z.object({ searches: z.number(), indexes: z.number(), failed: z.number(), excerpts: z.number(), files: z.number(), weak: z.number(), returnedBytes: z.number(), sourceBytes: z.number(), avoidedBytes: z.number(), avoidedTokens: z.number(), compression: z.number(), rereadTokens: z.number(), avoidedUsd: z.number(), lastFocus: z.string(), }) /** * Per-conversation recall usage, folded from the tool metadata in the log. * * A projection rather than a client-side fold over conversation nodes: nodes * carry only the loaded window, so a chip built on them would undercount the * moment a session outgrew what the browser had scrolled through. This folds * the whole log and is replayed from it. */ const USAGE_KEY = 'recallUsage' /** Views are compared by reference, so one state must yield one view object. */ const usageViews = new WeakMap() function usageViewFor(state) { let view = usageViews.get(state) if (view === undefined) { view = usageSummary(state) usageViews.set(state, view) } return view } export const usageProjection = { key: USAGE_KEY, stateSchema: usageStateSchema, init: () => emptyUsage(), apply: (state, event) => foldUsage(state, event), stateVersion: 1, wire: { viewSchema: usageViewSchema, view: usageViewFor }, } export const name = 'dsh-recall' export const inject = ['tools'] /** * Register the tool and the settings namespace. * * @param {import('@deepseek-ai/cordis').Context} ctx * @param {Record} config validated composition config */ export function apply(ctx, config) { const logger = ctx.logger('dsh-recall') const dshHome = dshHomeOf(ctx) const withLock = createLock() // Discovered scopes per workspace, cached for a short window. Discovery is a // directory walk on the hot path of every tool call, and re-walking a large // repository to learn something that changes only when a marker file is added // is work with no benefit. const SCOPE_CACHE_MS = 30_000 const scopeCache = new Map() // Open indexes are filesystem handles, so they are an effect: Cordis closes // them on unload and on a config change, and a reload never leaks a handle. const openIndexes = new Map() ctx.effect(() => () => { for (const index of openIndexes.values()) index.close() openIndexes.clear() }) // The authoritative settings source: a plain snapshot of the running config. // // No settings provider is involved in READING. The `Config` schema above // declares every field volatile, so a document edit updates this instance's // config references in place and the very next snapshot sees it. That is the // 0.1.7 model — there is no `installSection` to attach to, and nothing to // subscribe to for a value that is already live. const readSettings = () => settingsSnapshot(config) let writeSettings // Per-conversation usage, folded from the tool metadata the log already // carries. Optional: a profile without projections still gets the tool, it // just does not get a surface that reports what it saved. ctx.inject(['sessionProjections'], (projectionCtx) => { try { projectionCtx.sessionProjections.register(usageProjection) } catch (error) { logger.warn('dsh-recall: usage projection unavailable: %s', error?.message ?? error) } }) // Writing is still the settings provider's job, because the value has to // reach the profile's own patch layer rather than live only in this process. // // `ns` is the profile ENTRY id — the `id` of this plugin's row in the composed // tree, which is `dsh-recall` in cordis.patch.yml — not a namespace this // plugin invents. The revision is read back from `describe()` so a // configuration dialog that saved in the meantime refuses this write instead // of being silently overwritten. ctx.inject(['settings'], (settingsCtx) => { writeSettings = async (patch) => { const revision = settingsCtx.settings .describe() .find((descriptor) => descriptor.ns === SETTINGS_NAMESPACE)?.revision await settingsCtx.settings.update(SETTINGS_NAMESPACE, patch, revision) } }) /** The embedder for a settings snapshot. */ const embedderFor = (settings) => createEmbedder({ apiBase: String(settings.embeddingApiBase), model: String(settings.embeddingModel), dimensions: Number(settings.embeddingDimensions), apiKey: String(settings.embeddingApiKey ?? ''), batchSize: Number(settings.embeddingBatchSize), concurrency: Number(settings.embeddingConcurrency), timeoutMs: Number(settings.embeddingTimeoutMs), keepAlive: String(settings.embeddingKeepAlive ?? ''), }) /** * A one-shot embedder for probing the endpoint. * * No retries and a short timeout, because a probe exists to answer a question * quickly: `status` should report a wedged backend, not wait for one. Three * retries against a socket that accepts and never replies would take longer * than the tool call that provoked it. */ const proberFor = (settings) => createEmbedder({ apiBase: String(settings.embeddingApiBase), model: String(settings.embeddingModel), dimensions: Number(settings.embeddingDimensions), apiKey: String(settings.embeddingApiKey ?? ''), batchSize: 1, concurrency: 1, timeoutMs: Math.min(Number(settings.embeddingTimeoutMs), 10_000), maxAttempts: 1, keepAlive: String(settings.embeddingKeepAlive ?? ''), }) /** * The index fields the tool's output schema declares. * * Built by picking rather than by spreading the sync result, because that * result also carries internal fields (`stale`, `chunksWritten`) which are not * part of the model-facing contract — and the schema is * `additionalProperties: false`, so spreading one in is a validation failure * rather than a harmless extra. */ function indexView(current, extra = {}) { const out = { ...current } const fields = [ 'files', 'chunks', 'pending', 'scanned', 'embedded', 'removed', 'oversized', 'unreadable', 'truncated', ] for (const key of fields) if (extra[key] !== undefined) out[key] = extra[key] return out } /** * Discovered scopes for a workspace, cached briefly. * * Discovery is a directory walk, not a file read, so it is cheap — but it runs * on every tool call and a large repository has a lot of directories. A short * cache keeps the cost off the hot path while still noticing a marker file * added a moment ago. * * @param {string} workspace absolute workspace root * @param {readonly string[]} markers marker filenames */ async function scopesFor(workspace, markers) { const key = `${workspace}\u0000${markers.join(',')}` const hit = scopeCache.get(key) if (hit !== undefined && Date.now() - hit.at < SCOPE_CACHE_MS) return hit.scopes let scopes = [] try { scopes = await discoverScopes({ workspace, markers }) } catch { // A workspace that cannot be walked simply has no discoverable scopes; // resolution still falls back to markers and then the workspace. scopes = [] } cacheScopes(key, scopes) return scopes } /** Replace the scope cache, dropping entries for workspaces that are gone. */ function cacheScopes(key, scopes) { scopeCache.set(key, { at: Date.now(), scopes }) if (scopeCache.size > 32) { for (const [existing, entry] of scopeCache) { if (Date.now() - entry.at > SCOPE_CACHE_MS * 4) scopeCache.delete(existing) } } } /** * Open (or reuse) the index for one workspace and focus. * * @param {Record} settings resolved settings * @param {string} cwd workspace root * @param {string} focus focus key, `.` for the whole workspace */ function indexFor(settings, cwd, focus = '.') { const absolute = (path) => join(resolve(path), focus === '.' ? 'index.sqlite' : join('focus', focus, 'index.sqlite')) const file = typeof settings.stateDir === 'string' && settings.stateDir.trim() !== '' ? absolute(settings.stateDir.trim()) : indexFile(dshHome, cwd, focus) let index = openIndexes.get(file) if (index === undefined) { index = new RecallIndex(file) openIndexes.set(file, index) } const identity = { model: String(settings.embeddingModel), dimensions: String(settings.embeddingDimensions), apiBase: String(settings.embeddingApiBase), } // `open` is idempotent for an already-open handle; the identity check is the // part that must run every time, because the settings can change under it. const outcome = index.db === undefined ? index.open(identity) : index.recheck(identity) return { index, file, discarded: outcome.discarded } } /** * Bring one workspace's index up to date. * * Phase one reads and hashes every candidate — fast, bounded by disk. Phase * two embeds what changed, and is the only part given a deadline. Splitting * them is what makes the reported state honest: after phase one the plugin * knows exactly what is stale, so a partial index is described rather than * guessed at. */ async function syncWorkspace(input) { const { root, prefix = '', index, settings, budgetMs, signal, onProgress } = input const walk = await listCodeFiles(root, { ...(settings.extensions?.length ? { extensions: settings.extensions } : {}), skipDirs: DEFAULT_SKIP_DIRS, exclude: settings.exclude ?? [], maxFiles: Number(settings.maxFiles), signal, }) // `listCodeFiles` reports paths relative to the walk root, which under a // focus is the focus directory. They are rewritten to be relative to the // WORKSPACE, because that is the path the model will hand to `read` — a // focus-relative result would silently read the wrong file. if (prefix !== '') { for (const file of walk.files) file.path = `${prefix}/${file.path}` } const known = index.fileHashes() const present = new Set(walk.files.map((file) => file.path)) const removed = index.prune([...known.keys()].filter((path) => !present.has(path))) const maxBytes = Number(settings.maxFileBytes) const pending = [] let oversized = 0 let unreadable = 0 for (const file of walk.files) { if (signal?.aborted) throw signal.reason ?? new Error('aborted') let content try { const info = await stat(file.absolute) if (info.size > maxBytes) { oversized += 1 continue } content = await readFile(file.absolute, 'utf8') } catch { unreadable += 1 continue } const hash = hashOf(content) if (known.get(file.path) === hash) continue pending.push({ ...file, content, hash, bytes: Buffer.byteLength(content) }) } // Anything still holding an old hash is stale until it is re-embedded. const stale = new Set(pending.map((file) => file.path)) const deadline = Date.now() + budgetMs let embedded = 0 let chunksWritten = 0 for (const file of pending) { if (signal?.aborted) throw signal.reason ?? new Error('aborted') // Always embed at least one file, or a zero budget would make the feature // indistinguishable from a broken index. if (embedded > 0 && Date.now() >= deadline) break const { chunks } = chunkFile({ path: file.path, content: file.content, chunkLines: Number(settings.chunkLines), maxChunkLines: Number(settings.maxChunkLines), overlapLines: Number(settings.overlapLines), }) if (chunks.length === 0) { // An empty or generated file still gets a row, so it is not re-read and // re-chunked on every subsequent search. index.replaceFile(file.path, file.hash, file.bytes, []) embedded += 1 stale.delete(file.path) onProgress?.(embedded, pending.length) continue } const vectors = await input.embedder.embed( chunks.map((chunk) => embedText(file.path, chunk)), signal, ) index.replaceFile( file.path, file.hash, file.bytes, chunks.map((chunk, at) => ({ ...chunk, vec: vectors[at] })), ) chunksWritten += chunks.length embedded += 1 stale.delete(file.path) onProgress?.(embedded, pending.length) } const stats = index.stats() return { ...stats, scanned: walk.files.length, truncated: walk.truncated, removed, embedded, chunksWritten, pending: pending.length - embedded, stale: [...stale], oversized, unreadable, } } const TOOL_OUTPUT = { type: 'object', additionalProperties: false, properties: { action: { type: 'string', required: true }, ok: { type: 'boolean', required: true }, query: { type: 'string' }, workspace: { type: 'string' }, focus: { type: 'object', additionalProperties: false, properties: { path: { type: 'string', required: true }, source: { type: 'string', required: true }, marker: { type: 'string' }, problem: { type: 'string' }, }, }, results: { type: 'array', items: { type: 'object', additionalProperties: false, properties: { path: { type: 'string', required: true }, startLine: { type: 'integer', required: true }, endLine: { type: 'integer', required: true }, symbol: { type: 'string' }, similarity: { type: 'number', required: true }, excerpt: { type: 'string', required: true }, truncated: { type: 'boolean', required: true }, stale: { type: 'boolean', required: true }, }, }, }, index: { type: 'object', additionalProperties: false, properties: { files: { type: 'integer' }, chunks: { type: 'integer' }, pending: { type: 'integer' }, scanned: { type: 'integer' }, embedded: { type: 'integer' }, removed: { type: 'integer' }, oversized: { type: 'integer' }, unreadable: { type: 'integer' }, truncated: { type: 'boolean' }, database: { type: 'string' }, stateDir: { type: 'string' }, }, }, best: { type: 'number' }, endpoint: { type: 'object', additionalProperties: false, properties: { ok: { type: 'boolean', required: true }, url: { type: 'string', required: true }, model: { type: 'string', required: true }, errorCode: { type: 'string' }, message: { type: 'string' }, }, }, weak: { type: 'boolean' }, /** * What this call returned, against what reading those files would have * cost. `sourceBytes` is an upper bound and is labelled as one wherever * it is shown — a careful reader might have opened one file, not four. */ measure: { type: 'object', additionalProperties: false, properties: { excerpts: { type: 'integer', required: true }, returnedBytes: { type: 'integer', required: true }, sourceBytes: { type: 'integer', required: true }, files: { type: 'integer', required: true }, }, }, jobId: { type: 'string' }, /** Every scope discovered in this workspace, so a caller can pick one. */ scopes: { type: 'array', items: { type: 'object', additionalProperties: false, properties: { path: { type: 'string', required: true }, marker: { type: 'string', required: true }, }, }, }, note: { type: 'string' }, errorCode: { type: 'string' }, remedy: { type: 'string' }, }, } /** Render one search result set for the model. */ function renderResults(value) { if (value.jobId !== undefined) { return [ { type: 'text', text: `Indexing ${value.workspace} as background job ${value.jobId}. You will be notified when it settles; ` + `collect it with job_output for "${value.jobId}" (add wait: true if you are blocked on it). ` + 'Searches work meanwhile against whatever is already indexed.', }, ] } if (value.ok !== true) { const parts = [value.note ?? `recall ${value.action} failed.`] if (value.remedy !== undefined) parts.push(`Next: ${value.remedy}`) return [{ type: 'text', text: parts.join('\n\n') }] } if (value.action === 'configure') { return [{ type: 'text', text: value.note ?? 'Configuration updated.' }] } if (value.action !== 'search') { const { files = 0, chunks = 0, pending = 0, scanned = 0, embedded = 0 } = value.index ?? {} return [ { type: 'text', text: `Index for ${value.workspace}: ${files} files, ${chunks} chunks. ` + `Scanned ${scanned}, embedded ${embedded} this call, ${pending} still pending.`, }, ] } const results = value.results ?? [] if (results.length === 0) { const suffix = (value.index?.pending ?? 0) > 0 ? ` ${value.index.pending} file(s) are still unindexed.` : '' return [ { type: 'text', text: `No match for ${JSON.stringify(value.query)}.${suffix} ` + 'If you know the exact identifier, grep is the better tool for this query.', }, ] } const scope = value.focus?.path !== undefined && value.focus.path !== '.' ? ` in ${value.focus.path}` : '' const header = `${results.length} excerpt(s) for ${JSON.stringify(value.query)}${scope} — ` + `from ${value.index?.chunks ?? 0} indexed chunks` + ((value.index?.pending ?? 0) > 0 ? `, ${value.index.pending} file(s) still unindexed` : '') const body = results.map((result, at) => { const symbol = result.symbol === undefined ? '' : ` (${result.symbol})` const stale = result.stale ? ' [stale — file changed since indexing]' : '' const cut = result.truncated ? '\n… excerpt truncated; read the file for the rest' : '' return ( `--- ${at + 1}. ${result.path}:${result.startLine}-${result.endLine}${symbol} ` + `similarity ${result.similarity}${stale}\n${result.excerpt}${cut}` ) }) // A scope is a window, not a filter on relevance. When it holds nothing // strong, the most likely reason is that the answer is shared code the scope // excludes — 41.6% of the workspace this was measured on — so say that // rather than leaving the caller to conclude the code does not exist. const outside = value.focus?.path !== undefined && value.focus.path !== '.' ? ` Only ${value.focus.path} is indexed here; shared code outside it is not. Retry without focus to search the whole workspace.` : '' const advice = value.weak ? `\n\nNo strong match. Prefer grep for a known identifier, or rephrase with the name of the thing.${outside}` : '' return [{ type: 'text', text: `${header}\n\n${body.join('\n\n')}${advice}` }] } ctx.tools.register( defineTool({ name: 'recall', description: 'Search this workspace semantically with a local embedding model and return ranked code excerpts ' + 'with file paths and line ranges — without reading whole files into the conversation. Use it when ' + 'you do not know the exact identifier: "where is X handled", "which module does Y", "how does Z ' + 'recover from failure". Prefer grep when you already know the exact symbol, and read when you need ' + 'the whole file. Because excerpts are returned inline, do not follow up with a read of the same ' + 'range. Actions: "search" (default) answers a query and warms a bounded slice of the index; ' + '"index" builds or refreshes the whole index, as a background job by default; "status" reports the ' + 'index and endpoint state. The index is incremental and lives outside your repository.', parameters: { query: { type: 'string', description: 'What to look for, in words. A description of behaviour works as well as a name.', }, action: { type: 'string', enum: ['search', 'index', 'status', 'configure'], description: 'What to do. Defaults to "search".', }, limit: { type: 'integer', description: 'Most excerpts to return. Defaults to the configured limit (8).', }, path: { type: 'string', description: 'Restrict the search to files under this workspace-relative directory, for example "src/services".', }, focus: { type: 'string', description: 'Scope this whole call — index and search — to one workspace-relative directory, for example ' + '"src/services/billing". Use it when the session is working on one module: the index ' + 'then holds only that module, so its near-identical neighbours cannot outrank it and the first ' + 'search does not wait on the rest of the repository. Empty uses the configured focus, then the ' + 'nearest enclosing directory that carries a CONTROLLER.md, DEBUG_UI.md or AGENTS.md.', }, run_in_background: { type: 'boolean', description: 'For action "index" only: return a job id instead of the result. Defaults to true, because a ' + 'cold index of a large repository takes minutes.', }, config: { type: 'json', description: 'Settings patch for action "configure", using this plugin\'s own camelCase field names, for ' + 'example {"embeddingApiBase":"http://127.0.0.1:11434"}.', }, }, output: { schema: TOOL_OUTPUT, render: (_args, value) => renderResults(value), // The harness persists this verbatim as the `meta` of the `tool/result` // event, which is what puts per-call usage in the session log — and // therefore in reach of a projection the browser can read. Pure, and it // carries totals rather than content: it is appended on every call and // replayed on every projection rebuild. presentationMeta: (args, value) => deriveUsage(args, value), }, isConcurrencySafe: (args) => (args.action ?? 'search') !== 'index', async execute(args, exec) { const settings = readSettings() const cwd = exec.agent?.session.header.cwd ?? process.cwd() const action = args.action ?? 'search' const workspace = basename(cwd) || cwd // Validate the live section before using it, every call. // // The Host parses a write against the schema, so anything a single field // can express is refused at the document; the cross-field rules — a cap // below its target, an overlap that never advances a split — are not // expressible that way, and 0.1.7 has no write-time validator hook to // hang them on. Checking here is what keeps a bad combination from // surfacing later as an unexplained empty search. try { validateSettings(settings) } catch (error) { return { action, ok: false, workspace, errorCode: 'INVALID_SETTINGS', note: `The dsh-recall configuration is not usable as it stands: ${error?.message ?? error}`, remedy: 'Fix the conflicting values in Settings → Plugins → dsh-recall, or in the dsh-recall ' + 'row of your cordis layer.', } } const embedder = embedderFor(settings) // Resolve the scope before anything else, because it decides which index // is opened. A caller's `focus` outranks the configured one, so a session // can name a controller without changing the deployment's default. const markers = settings.focusMarkers?.length ? settings.focusMarkers : DEFAULT_FOCUS_MARKERS const scopes = await scopesFor(cwd, markers) const focus = resolveFocus({ workspace: cwd, cwd, explicit: args.focus ?? settings.focus ?? '', markers, scopes, }) const focusPath = focusKey(cwd, focus.root) let opened try { opened = indexFor(settings, cwd, focusPath) } catch (error) { return { action, ok: false, workspace, errorCode: 'INDEX_UNAVAILABLE', note: `Could not open the index: ${error?.message ?? error}`, remedy: `Check that ${dshHome} is writable.`, } } const { index, file, discarded } = opened const stateDir = workspaceStateDir(dshHome, cwd) const focusView = { path: focusPath, source: focus.source, ...(focus.marker === undefined ? {} : { marker: focus.marker }), ...(focus.problem === undefined ? {} : { problem: focus.problem }), } const base = { action, workspace, focus: focusView, index: { database: file, stateDir } } if (action === 'status') { const stats = index.stats() // Probe the endpoint rather than merely naming it. A wedged backend // accepts the connection and never answers, so "configured" and // "working" are different claims and status should make the second // one. One attempt, short timeout — a probe that waits is not a probe. let endpoint try { await proberFor(settings).embedOne('dsh-recall endpoint probe', exec.signal) endpoint = { ok: true, url: String(settings.embeddingApiBase), model: String(settings.embeddingModel), } } catch (error) { endpoint = { ok: false, url: String(settings.embeddingApiBase), model: String(settings.embeddingModel), errorCode: error instanceof EmbeddingError ? error.code : 'PROBE_FAILED', message: error?.message ?? String(error), } } return { ...base, ok: true, index: indexView(base.index, stats), endpoint, scopes: scopes.map((entry) => ({ path: entry.path, marker: entry.marker })), note: `Index ${file} holds ${stats.files} files and ${stats.chunks} chunks, scoped to ` + `${focusPath === '.' ? 'the whole workspace' : focusPath} (${focus.source}` + `${focus.marker === undefined ? '' : `: ${focus.marker}`}). ` + (focus.problem === undefined ? '' : `Requested focus was ignored — ${focus.problem}. `) + // A workspace with discoverable scopes can be scoped by naming one, // so listing them is the difference between a feature that exists // and one a caller can actually use. (scopes.length === 0 ? 'No focusable scopes were found; pass focus to scope a call explicitly. ' : `Available scopes: ${scopes .slice(0, 12) .map((entry) => entry.path) .join(', ')}${scopes.length > 12 ? ` (+${scopes.length - 12} more)` : ''}. `) + `Embeddings come from "${settings.embeddingModel}" (${settings.embeddingDimensions} dims) at ` + `${settings.embeddingApiBase} — ` + (endpoint.ok === true ? 'endpoint answered.' : `endpoint did NOT answer (${endpoint.errorCode}). Run \`ollama ps\` on that host; a ` + 'model can be evicted, and an endpoint holding another model resident forever can ' + 'leave no room to load this one.') + (discarded === undefined ? '' : ` The index was rebuilt because ${discarded}.`), } } if (action === 'configure') { const patch = args.config if (patch === undefined || patch === null || typeof patch !== 'object' || Array.isArray(patch)) { return { ...base, ok: false, errorCode: 'BAD_PATCH', note: 'The `config` parameter must be an object.', } } const unknown = Object.keys(patch).filter((key) => !SETTING_KEYS.includes(key)) if (unknown.length > 0) { return { ...base, ok: false, errorCode: 'UNKNOWN_SETTING', note: `Unknown setting(s): ${unknown.join(', ')}.`, remedy: `Known settings: ${SETTING_KEYS.join(', ')}.`, } } // Judge the SECTION the write would leave behind, not the patch on its // own: `maxLimit` alone is always fine, and `defaultLimit` alone is // always fine, but raising one past the other is the mistake worth // catching — and it is invisible to any single value's schema. try { validateSettings({ ...settings, ...patch }) } catch (error) { return { ...base, ok: false, errorCode: 'INVALID_SETTING', note: `Refused: ${error?.message ?? error}`, remedy: 'Nothing was written. Send a patch whose resulting section is consistent.', } } if (writeSettings === undefined) { return { ...base, ok: false, errorCode: 'NO_SETTINGS_PROVIDER', note: 'No settings provider is mounted, so configuration cannot be persisted at runtime.', remedy: `Set these values in the dsh-recall row of your cordis layer instead: ${JSON.stringify(patch)}`, } } try { await writeSettings(patch) } catch (error) { return { ...base, ok: false, errorCode: 'SETTINGS_WRITE_FAILED', note: `The settings document refused the write: ${error?.message ?? error}`, remedy: 'Re-read the current values and retry; a concurrent edit is the usual cause.', } } return { ...base, ok: true, note: `Updated ${Object.keys(patch).join(', ')}.` } } // Full warm-up. Deliberately a background job by default: on a large // repository this is minutes of work, and blocking a turn on it is the // behaviour that makes a useful tool feel broken. if (action === 'index') { const jobs = ctx.get('jobs') const wantsBackground = args.run_in_background ?? true const runIndex = (signal) => withLock(file, () => syncWorkspace({ root: focus.root, prefix: focusPath === '.' ? '' : focusPath, index, embedder, settings, budgetMs: Number.POSITIVE_INFINITY, signal, }), ) if (wantsBackground && jobs !== undefined) { const jobId = jobs.start({ kind: 'recall-index', label: `recall index ${workspace}`, ...(exec.agent === undefined ? {} : { owner: exec.agent }), run: () => { const controller = new AbortController() const done = (async () => { const result = await runIndex(controller.signal) return { status: 'completed', detail: `${result.files} files, ${result.chunks} chunks`, output: `Indexed ${result.embedded} file(s) into ${result.chunks} chunks (${result.files} files ` + `total, ${result.removed} removed). Database: ${file}`, } })() return { cancel: () => controller.abort(), done } }, }) return { ...base, ok: true, jobId } } const result = await runIndex(exec.signal) return { ...base, ok: true, index: indexView(base.index, result) } } // search const query = typeof args.query === 'string' ? args.query.trim() : '' if (query === '') { return { ...base, ok: false, errorCode: 'NO_QUERY', note: 'A search needs a query.', remedy: 'Call recall with a `query` describing what you are looking for.', } } const limit = Math.max( 1, Math.min(Number(args.limit ?? settings.defaultLimit), Number(settings.maxLimit)), ) try { const sync = await withLock(file, () => syncWorkspace({ root: focus.root, prefix: focusPath === '.' ? '' : focusPath, index, embedder, settings, budgetMs: Math.max(0, Number(settings.syncBudgetMs)), signal: exec.signal, }), ) const matrix = index.matrix() if (matrix.rows.length === 0) { return { ...base, ok: true, query, results: [], weak: true, index: indexView(base.index, sync), note: 'Nothing is indexed yet, so there is nothing to search. ' + (sync.pending > 0 ? `${sync.pending} file(s) were found but the per-call embedding budget was reached.` : 'No indexable code was found in this workspace.'), ...(sync.pending > 0 ? { remedy: 'Run recall with action "index" to build the whole index in the background.', } : {}), } } // Query expansion: rank every phrasing and fuse the orders. The // expander never throws — on a slow or absent local model it returns // the original query alone, and this path is unchanged. const expansionModel = String(settings.expansionModel ?? '').trim() const queries = settings.expansionEnabled === true ? await createExpander({ apiBase: String(settings.embeddingApiBase), model: expansionModel === '' ? String(settings.embeddingModel) : expansionModel, timeoutMs: Number(settings.expansionTimeoutMs), count: Number(settings.expansionCount), }).expand(query, exec.signal) : [query] const orders = [] let similarity let vectorBest = 0 for (const variant of queries) { const vector = await embedder.embedOne(variant, exec.signal) const ranked = rankOrder({ matrix, queryVector: vector, query: variant }) orders.push(ranked.order) // The first variant is the caller's own query, so its similarities // are the honest ones to report against each result. if (similarity === undefined) { similarity = ranked.similarity vectorBest = ranked.vectorBest } } let results = selectFrom(fuseRankings(orders), { matrix, similarity, limit, perFile: Number(settings.perFileLimit), excerptLines: Number(settings.excerptLines), totalLines: Number(settings.totalExcerptLines), }) // Local re-ranking: one call scores the head of the list. `undefined` // means the model had no opinion, which is not the same as "all // irrelevant" — the fused order stands unchanged. const rerankModel = String(settings.rerankModel ?? '').trim() const rerankCount = Math.min(results.length, Number(settings.rerankCandidates), 12) if (settings.rerankEnabled === true && rerankCount > 1) { const head = results.slice(0, rerankCount) const localScores = await createReranker({ apiBase: String(settings.embeddingApiBase), model: rerankModel === '' ? String(settings.embeddingModel) : rerankModel, timeoutMs: Number(settings.rerankTimeoutMs), }).score(query, head.map((result) => result.excerpt), exec.signal) if (localScores !== undefined && localScores.length === head.length) { const blended = head .map((result, at) => ({ result, score: 0.4 * result.similarity + 0.6 * localScores[at] })) .sort((left, right) => right.score - left.score) .map((entry) => entry.result) results = [...blended, ...results.slice(rerankCount)] } } if (typeof args.path === 'string' && args.path.trim() !== '') { const prefix = args.path.trim().replace(/^\.\//, '').replace(/\/+$/, '') results = results.filter( (result) => result.path === prefix || result.path.startsWith(`${prefix}/`), ) } const stale = new Set(sync.stale ?? []) const best = results.length === 0 ? 0 : Math.max(...results.map((result) => result.similarity)) // The alternative to these excerpts: the whole of each file they came // from. Sizes come from the index, which recorded them at embed time, // so this costs nothing here — and it is an upper bound, because a // careful reader might have opened one of the four rather than all. const sourcePaths = [...new Set(results.map((result) => result.path))] const sourceSizes = index.bytesFor(sourcePaths) const sourceBytes = [...sourceSizes.values()].reduce((sum, size) => sum + size, 0) const returnedBytes = results.reduce( (sum, result) => sum + Buffer.byteLength(result.excerpt, 'utf8'), 0, ) const measure = { excerpts: results.length, returnedBytes, sourceBytes, files: sourceSizes.size, } return { ...base, ok: true, query, // Built field by field rather than spread: the ranker also returns // a fused RRF `score`, which is an internal ordering artefact and not // part of the declared result shape. results: results.map((result) => ({ path: result.path, startLine: result.startLine, endLine: result.endLine, ...(result.symbol === undefined ? {} : { symbol: result.symbol }), similarity: result.similarity, excerpt: result.excerpt, truncated: result.truncated, stale: stale.has(result.path), })), index: indexView(base.index, sync), best, weak: results.length === 0 || best < Number(settings.minSimilarity), measure, } } catch (error) { const code = error instanceof EmbeddingError ? error.code : 'RECALL_FAILED' const remedy = code === 'UNREACHABLE' || error?.cause?.code === 'ECONNREFUSED' ? `Is the embedding endpoint at ${settings.embeddingApiBase} running and reachable? Set ` + '`embeddingApiBase` through the plugin settings if it moved.' : code === 'DIMENSION_MISMATCH' ? `Set \`embeddingDimensions\` to match "${settings.embeddingModel}".` : undefined logger.warn('recall search failed: %s', error?.message ?? error) return { ...base, ok: false, query, errorCode: code, note: error?.message ?? String(error), ...(remedy === undefined ? {} : { remedy }), } } }, }), ) }