NotExpectedYet
de4d020dcb
The workspace index cannot answer questions about an installed framework or SDK, and in plugin development that is where most of the reading goes: 64 of 80 tool calls in one real session went to the harness checkout under /usr/local/lib/node_modules to look up the API being implemented. A root is entered whatever it is called, and its dependencies are not: a path ending in node_modules/... is the normal way to name a package tree, while a node_modules below the root is somebody else's code. A result carries the label that says which tree it came from and an absolute sourcePath, the only form a follow-up read can resolve. The tool description now also claims the move this replaces - searching before reading files to find out what a module contains.
1593 lines
67 KiB
JavaScript
1593 lines
67 KiB
JavaScript
/**
|
|
* dsh-recall — local-embedding code recall for DeepSeek Harness.
|
|
*
|
|
* One tool (`recall`) over a per-workspace vector index built by a local
|
|
* embedding model, plus a `Config` schema whose every field is live-editable —
|
|
* the endpoint is configuration, not a constant.
|
|
*
|
|
* **How the settings surface works (harness 0.1.7).** A plugin no longer
|
|
* registers a settings section. It exports a `Config` schema; the Harness keys
|
|
* a form for the profile entry by that row's `id`, and a field is offered to
|
|
* the UI only when its schema node is declared `volatile()` — the marker that
|
|
* means "editable without remounting the plugin". So this file's schema carries
|
|
* two jobs at once: validation, and being the whole of the settings page's
|
|
* description. `apply` therefore receives volatile fields as live references
|
|
* and reads them through {@link settingsSnapshot} per call, which is what makes
|
|
* a saved edit take effect without a restart.
|
|
*
|
|
* **The economics this exists for.** In a long agent session the dominant cost
|
|
* is not the model's reasoning; it is re-reading the accumulated transcript on
|
|
* every step. Measured on this harness over 32 days, 65% of spend was re-reading
|
|
* cached context and 96% of that context was message history — mostly file
|
|
* contents the agent had already read. A tool that answers "where is this
|
|
* handled" with a ranked 40-line excerpt instead of four whole files keeps that
|
|
* history small, and the saving compounds on every subsequent step because the
|
|
* context it did not add is never re-sent again.
|
|
*
|
|
* **Why the index is warmed lazily rather than at load.** `apply` must not block
|
|
* on embedding a repository; a plugin that took minutes to activate would make
|
|
* every profile start slow, including the ones that never call the tool. Instead
|
|
* a search does a *bounded* sync — read and hash everything, embed only what
|
|
* fits in a time budget — and reports honestly what is still pending. A full
|
|
* warm-up is available as an explicit background action.
|
|
*
|
|
* **Why the endpoint is configuration and not a constant.** The Ollama default
|
|
* of `127.0.0.1:11434` is a convention, not a fact — a deployment often runs
|
|
* Ollama on another host. Baking one machine's address into a bundle layer that
|
|
* other machines apply is exactly the mistake the composition docs warn about,
|
|
* so the endpoint is a schema default the profile can override, at rest or
|
|
* live.
|
|
*/
|
|
|
|
import { readFile, stat } from 'node:fs/promises'
|
|
import { createHash } from 'node:crypto'
|
|
import { basename, join, resolve, sep } from 'node:path'
|
|
|
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
// A real dependency rather than a duck-typed `{ parse }`: the projection service
|
|
// calls these schemas, and `dsh-session-projection` declares zod itself, so it is
|
|
// present wherever projections are.
|
|
import { z } from 'zod'
|
|
import Schema from '@deepseek-ai/schemastery'
|
|
|
|
import { chunkFile, embedText } from './lib/chunk.js'
|
|
import { EmbeddingError, createEmbedder } from './lib/embed.js'
|
|
import { DEFAULT_FOCUS_MARKERS, discoverScopes, focusKey, resolveFocus } from './lib/focus.js'
|
|
import { indexFile, resolveDshHome, workspaceStateDir } from './lib/paths.js'
|
|
import { fuseRankings, rankOrder, selectFrom } from './lib/rank.js'
|
|
import { listRootFiles, resolveSourceRoots } from './lib/roots.js'
|
|
import { RecallIndex } from './lib/store.js'
|
|
import { createExpander } from './lib/expand.js'
|
|
import { createReranker } from './lib/rerank.js'
|
|
import { deriveUsage, emptyUsage, foldUsage, usageSummary } from './lib/usage.js'
|
|
import { DEFAULT_EXTENSIONS, DEFAULT_SKIP_DIRS, listCodeFiles } from './lib/walk.js'
|
|
|
|
/**
|
|
* This plugin's settings namespace, which in 0.1.7 IS its profile entry id.
|
|
*
|
|
* `ctx.settings.describe()` keys one form per composed entry by the `id` the
|
|
* patch layer gives that row — `dsh-recall` in cordis.patch.yml — and writes go
|
|
* back to the same key. It is not a name the plugin registers; the schema
|
|
* exported as `Config` is the registration, and this constant is the id those
|
|
* settings surfaces address it by. Renaming the row without renaming this
|
|
* would leave the `configure` action writing to a row that does not exist.
|
|
*/
|
|
const SETTINGS_NAMESPACE = 'dsh-recall'
|
|
|
|
/** Every configuration field this plugin exposes. */
|
|
export const SETTING_KEYS = [
|
|
'embeddingApiBase',
|
|
'embeddingModel',
|
|
'embeddingDimensions',
|
|
'embeddingApiKey',
|
|
'embeddingBatchSize',
|
|
'embeddingConcurrency',
|
|
'embeddingTimeoutMs',
|
|
'embeddingKeepAlive',
|
|
'extensions',
|
|
'exclude',
|
|
'maxFiles',
|
|
'maxFileBytes',
|
|
'chunkLines',
|
|
'maxChunkLines',
|
|
'overlapLines',
|
|
'defaultLimit',
|
|
'maxLimit',
|
|
'perFileLimit',
|
|
'excerptLines',
|
|
'totalExcerptLines',
|
|
'minSimilarity',
|
|
'syncBudgetMs',
|
|
'stateDir',
|
|
'focus',
|
|
'focusMarkers',
|
|
'sourceRoots',
|
|
'expansionEnabled',
|
|
'expansionModel',
|
|
'expansionCount',
|
|
'expansionTimeoutMs',
|
|
'rerankEnabled',
|
|
'rerankModel',
|
|
'rerankCandidates',
|
|
'rerankTimeoutMs',
|
|
]
|
|
|
|
/**
|
|
* Every field is `.volatile()`.
|
|
*
|
|
* A volatile field is one the configuration document may change without
|
|
* remounting the plugin: the Loader re-parses the raw config, writes the new
|
|
* value into the running instance's existing reference, and emits
|
|
* `loader/volatile-update`. Two consequences decide the shape below:
|
|
*
|
|
* - `ctx.settings.describe` SKIPS an entry with no volatile field, so a schema
|
|
* that marks none is invisible to every settings surface — there is no page
|
|
* to draw rather than an empty one. This is why the plugin cannot expose a
|
|
* settings page at all without these markers.
|
|
* - a volatile field arrives in `apply`'s config as a reference (`{ get() }`)
|
|
* rather than a value, which is what {@link settingsSnapshot} unwraps.
|
|
*
|
|
* Every field qualifies: all of them are read afresh per tool call, so a live
|
|
* edit takes effect on the next call with nothing to re-register.
|
|
*
|
|
* The `.min`/`.max` bounds duplicate what {@link validateSettings} enforces.
|
|
* That is deliberate rather than redundant: those are the constraints a schema
|
|
* can express, and expressing them here is what makes the generated form refuse
|
|
* the invalid draft in front of the person typing it. `validateSettings` keeps
|
|
* the cross-field rules no single field's schema can state.
|
|
*/
|
|
export const Config = Schema.object({
|
|
/** Embedding endpoint, without a trailing slash. */
|
|
embeddingApiBase: Schema.string().default('http://127.0.0.1:11434').volatile(),
|
|
/** Model id as the endpoint knows it. */
|
|
embeddingModel: Schema.string().default('unclemusclez/jina-embeddings-v2-base-code:latest').volatile(),
|
|
/** Vector width. Must match the model; a mismatch is refused, not truncated. */
|
|
embeddingDimensions: Schema.number().min(1).default(768).volatile(),
|
|
/** Bearer token for a gateway that wants one. */
|
|
// `role('secret')` keeps this out of every settings response: the wire
|
|
// surfaces strip schema-declared secret positions, so a page can report that a
|
|
// key is set without ever receiving it.
|
|
embeddingApiKey: Schema.string().default('').role('secret').volatile(),
|
|
/** Inputs per request on the batch endpoint. */
|
|
embeddingBatchSize: Schema.number().min(1).default(16).volatile(),
|
|
/** Requests in flight. */
|
|
embeddingConcurrency: Schema.number().min(1).default(2).volatile(),
|
|
/**
|
|
* Per-request timeout.
|
|
*
|
|
* Deliberately short. A local endpoint that has wedged does not refuse the
|
|
* connection — it accepts it and never answers, so a long timeout turns a
|
|
* broken backend into a tool call that appears to hang. Thirty seconds is far
|
|
* more than a batch of a few thousand characters needs on any CPU, and it
|
|
* fails in bounded time instead of leaving the caller guessing.
|
|
*/
|
|
embeddingTimeoutMs: Schema.number().min(1).default(30_000).volatile(),
|
|
/** Ollama `keep_alive`. Keeps the model resident across an index run. */
|
|
embeddingKeepAlive: Schema.string().default('5m').volatile(),
|
|
/** Extensions to index. Empty means the built-in code set. */
|
|
extensions: Schema.array(String).default([]).volatile(),
|
|
/** Extra glob patterns to skip, on top of the built-in directory list. */
|
|
exclude: Schema.array(String).default([]).volatile(),
|
|
/** Most files a workspace walk will consider. */
|
|
maxFiles: Schema.number().min(1).default(20_000).volatile(),
|
|
/** Files larger than this are skipped as generated. */
|
|
maxFileBytes: Schema.number().min(1).default(1_048_576).volatile(),
|
|
/** Target lines per chunk. */
|
|
chunkLines: Schema.number().min(1).default(48).volatile(),
|
|
/** Hard cap before a chunk is split. */
|
|
maxChunkLines: Schema.number().min(1).default(80).volatile(),
|
|
/** Lines shared between the halves of a split chunk. */
|
|
overlapLines: Schema.number().min(0).default(8).volatile(),
|
|
/** Results returned when the caller does not say. */
|
|
defaultLimit: Schema.number().min(1).default(8).volatile(),
|
|
/** Ceiling on the caller's `limit`. */
|
|
maxLimit: Schema.number().min(1).default(25).volatile(),
|
|
/** Most chunks one file may contribute to one result set. */
|
|
perFileLimit: Schema.number().min(1).default(2).volatile(),
|
|
/**
|
|
* Lines shown per excerpt.
|
|
*
|
|
* Defaults to the chunk cap, so a chunk — already a bounded semantic unit —
|
|
* is returned whole. Cutting one in half is how a search turns into a
|
|
* follow-up read that costs more than the lines it saved.
|
|
*/
|
|
excerptLines: Schema.number().min(1).default(80).volatile(),
|
|
/** Total lines across one result set, allocated by rank. */
|
|
totalExcerptLines: Schema.number().min(1).default(400).volatile(),
|
|
/** Below this best-similarity, the result set is reported as weak. */
|
|
minSimilarity: Schema.number().min(0).max(1).default(0.2).volatile(),
|
|
/** Embedding time one `search` call may spend warming the index. */
|
|
syncBudgetMs: Schema.number().min(0).default(20_000).volatile(),
|
|
/** Index directory. Empty means `$DSH_HOME/dsh-recall/workspaces/<key>`. */
|
|
stateDir: Schema.string().default('').volatile(),
|
|
/**
|
|
* Narrows the index and every search to one directory.
|
|
*
|
|
* Empty resolves by marker: the nearest ancestor of the session's directory
|
|
* carrying a `focusMarkers` file becomes the scope, and the workspace root is
|
|
* the fallback — which is the behaviour that existed before focus did, so
|
|
* adopting this cannot regress a deployment that never configures it.
|
|
*/
|
|
focus: Schema.string().default('').volatile(),
|
|
/** Filenames that mark a directory as its own scope. Empty means the built-ins. */
|
|
focusMarkers: Schema.array(String).default([]).volatile(),
|
|
/**
|
|
* Extra trees to index beside the workspace, for code that is not in it.
|
|
*
|
|
* The workspace index cannot answer questions about the code you are writing
|
|
* *against* — an installed framework, a platform SDK, a sibling checkout. In
|
|
* plugin development that is where most of the reading goes: the harness's own
|
|
* packages are the only authoritative copy of the API being implemented, and
|
|
* they live outside the workspace.
|
|
*
|
|
* Each entry is a path, optionally prefixed with `name=`:
|
|
*
|
|
* - an **absolute** path is taken as given — what a deployment's installed
|
|
* tree needs;
|
|
* - a **relative** path resolves against the workspace, which is what makes a
|
|
* checked-in configuration portable;
|
|
* - the **name** defaults to the directory's basename and becomes the first
|
|
* segment of every result path from that root (`@deepseek-ai/dsh-agent/…`),
|
|
* so a result says which tree it came from. Use `name=` when the basename is
|
|
* long or ambiguous.
|
|
*
|
|
* The root's *contents* are indexed, never its dependencies: `node_modules`,
|
|
* build output and VCS directories are skipped below the root while the root
|
|
* itself is always entered — which is what lets a root be the scoped-packages
|
|
* directory of a package manager. An unusable entry is reported in the result
|
|
* rather than being fatal, so one typo cannot cost the other roots.
|
|
*
|
|
* A result from a root carries an absolute `sourcePath` for the follow-up read,
|
|
* because the labelled path is relative to the root rather than the workspace.
|
|
* Reading it may still need a scope grant from the caller's own policy; this
|
|
* plugin cannot grant one.
|
|
*/
|
|
sourceRoots: Schema.array(String).default([]).volatile(),
|
|
/**
|
|
* Ask the local model for alternative phrasings and fuse the rankings.
|
|
*
|
|
* Off by default. It costs one local generation plus an embedding per variant,
|
|
* and its benefit has not been measured against a real model — a default that
|
|
* spends latency on an unquantified gain is not a default worth having.
|
|
*/
|
|
expansionEnabled: Schema.boolean().default(false).volatile(),
|
|
/** Model that writes the phrasings. Empty uses `embeddingModel`'s endpoint, not its model. */
|
|
expansionModel: Schema.string().default('').volatile(),
|
|
/** Most variants to ask for. */
|
|
expansionCount: Schema.number().min(1).default(4).volatile(),
|
|
/** Budget for the whole expansion, in ms. Expiry just means search without it. */
|
|
expansionTimeoutMs: Schema.number().min(1).default(8_000).volatile(),
|
|
/**
|
|
* Have the local model score the top candidates against the query.
|
|
*
|
|
* Off by default for the same reason, and more so: it is one more model call
|
|
* on the critical path of every search.
|
|
*/
|
|
rerankEnabled: Schema.boolean().default(false).volatile(),
|
|
/** Model that scores candidates. */
|
|
rerankModel: Schema.string().default('').volatile(),
|
|
/** Candidates scored per search. The reranker caps this at 12. */
|
|
rerankCandidates: Schema.number().min(1).default(8).volatile(),
|
|
/** Budget for the scoring call, in ms. */
|
|
rerankTimeoutMs: Schema.number().min(1).default(15_000).volatile(),
|
|
})
|
|
|
|
/**
|
|
* Read one configuration field.
|
|
*
|
|
* A volatile field is a reference in the running config, not a plain value; a
|
|
* non-volatile one (or a config object a test handed straight to `apply`) is
|
|
* the value itself. Both are accepted so the same path serves a real profile
|
|
* and a direct call.
|
|
*
|
|
* @param {unknown} value a field of the resolved config
|
|
* @returns {unknown} the current value
|
|
*/
|
|
export function configValue(value) {
|
|
return value !== null && typeof value === 'object' && typeof value.get === 'function' ? value.get() : value
|
|
}
|
|
|
|
/**
|
|
* A plain snapshot of the whole settings section.
|
|
*
|
|
* Taken per tool call rather than cached, because that is exactly what a
|
|
* volatile field buys: the running instance's references are updated in place
|
|
* when the document changes, so the next read sees the new value without the
|
|
* plugin being remounted.
|
|
*
|
|
* @param {Record<string, any>} config the resolved composition config
|
|
* @returns {Record<string, any>} every declared field, unwrapped
|
|
*/
|
|
export function settingsSnapshot(config) {
|
|
const snapshot = {}
|
|
for (const key of Object.keys(config ?? {})) snapshot[key] = configValue(config[key])
|
|
return snapshot
|
|
}
|
|
|
|
/** Resolve the harness home, preferring the harness's own resolver. */
|
|
function dshHomeOf(ctx) {
|
|
try {
|
|
const homePath = ctx.get('dshHomePath')
|
|
if (typeof homePath === 'function') {
|
|
const resolved = homePath()
|
|
if (typeof resolved === 'string' && resolved !== '') return resolved
|
|
}
|
|
} catch {
|
|
// Declared optional; absent outside dsh-app-boot, which is expected.
|
|
}
|
|
return resolveDshHome()
|
|
}
|
|
|
|
/**
|
|
* Constraints a single field's schema cannot express.
|
|
*
|
|
* Called on the section a call is about to act on, and on the section a
|
|
* `configure` patch would leave behind. The Host validates every field against
|
|
* `Config` — including the `.min`/`.max` bounds — but the rules here relate two
|
|
* fields to each other (`defaultLimit` against `maxLimit`, `overlapLines`
|
|
* against `maxChunkLines`), and 0.1.7 has no write-time validator hook for
|
|
* those. So the check moves to the two moments that can still refuse: before a
|
|
* write this plugin performs, and before a configured value is put to work.
|
|
*
|
|
* @param {Record<string, any>} value a schema-valid settings section
|
|
* @throws {Error} when the section cannot be acted on
|
|
*/
|
|
export function validateSettings(value) {
|
|
const base = String(value.embeddingApiBase ?? '').trim()
|
|
let url
|
|
try {
|
|
url = new URL(base)
|
|
} catch {
|
|
throw new Error(`embeddingApiBase is not a URL: ${JSON.stringify(base)}`)
|
|
}
|
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
throw new Error(`embeddingApiBase must be http or https, not ${url.protocol}`)
|
|
}
|
|
if (!(Number(value.embeddingDimensions) > 0)) {
|
|
throw new Error('embeddingDimensions must be a positive number')
|
|
}
|
|
for (const key of [
|
|
'embeddingBatchSize',
|
|
'embeddingConcurrency',
|
|
'embeddingTimeoutMs',
|
|
'maxLimit',
|
|
'defaultLimit',
|
|
'perFileLimit',
|
|
'excerptLines',
|
|
'totalExcerptLines',
|
|
'chunkLines',
|
|
'maxChunkLines',
|
|
'maxFiles',
|
|
'maxFileBytes',
|
|
'expansionCount',
|
|
'expansionTimeoutMs',
|
|
'rerankCandidates',
|
|
'rerankTimeoutMs',
|
|
]) {
|
|
if (!(Number(value[key]) >= 1)) throw new Error(`${key} must be at least 1`)
|
|
}
|
|
if (Number(value.syncBudgetMs) < 0) throw new Error('syncBudgetMs cannot be negative')
|
|
if (Number(value.overlapLines) < 0) throw new Error('overlapLines cannot be negative')
|
|
if (Number(value.maxChunkLines) < Number(value.chunkLines)) {
|
|
throw new Error('maxChunkLines cannot be below chunkLines')
|
|
}
|
|
if (Number(value.overlapLines) >= Number(value.maxChunkLines)) {
|
|
throw new Error('overlapLines must be smaller than maxChunkLines, or a split never advances')
|
|
}
|
|
if (Number(value.defaultLimit) > Number(value.maxLimit)) {
|
|
throw new Error(`defaultLimit (${value.defaultLimit}) cannot exceed maxLimit (${value.maxLimit})`)
|
|
}
|
|
if (Number(value.minSimilarity) < 0 || Number(value.minSimilarity) > 1) {
|
|
throw new Error('minSimilarity is a cosine similarity and must be between 0 and 1')
|
|
}
|
|
}
|
|
|
|
/** SHA-256 of file content, used as the change key. */
|
|
function hashOf(content) {
|
|
return createHash('sha256').update(content).digest('hex')
|
|
}
|
|
|
|
/**
|
|
* Serialise work per key.
|
|
*
|
|
* Two tool calls can arrive while the first is still warming the index, and two
|
|
* concurrent syncs of the same workspace would interleave writes to one SQLite
|
|
* handle. Chaining on a per-workspace promise makes the second wait rather than
|
|
* race, which is what lets `search` stay marked concurrency-safe.
|
|
*/
|
|
function createLock() {
|
|
const chains = new Map()
|
|
return async function withLock(key, work) {
|
|
const previous = chains.get(key) ?? Promise.resolve()
|
|
let release
|
|
const gate = new Promise((resolveGate) => {
|
|
release = resolveGate
|
|
})
|
|
chains.set(
|
|
key,
|
|
previous.then(() => gate),
|
|
)
|
|
await previous
|
|
try {
|
|
return await work()
|
|
} finally {
|
|
release()
|
|
if (chains.get(key) === gate) chains.delete(key)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Schema of the persisted fold.
|
|
*
|
|
* The projection stores state as plain JSON and validates it with `.parse()`
|
|
* before seeding a fold from a checkpoint, so this must accept exactly what
|
|
* `emptyUsage()` produces and what `foldUsage` returns.
|
|
*/
|
|
const usageStateSchema = z.object({
|
|
version: z.number(),
|
|
searches: z.number(),
|
|
indexes: z.number(),
|
|
failed: z.number(),
|
|
excerpts: z.number(),
|
|
returnedBytes: z.number(),
|
|
sourceBytes: z.number(),
|
|
files: z.number(),
|
|
weak: z.number(),
|
|
rereadBytes: z.number(),
|
|
lastFocus: z.string(),
|
|
})
|
|
|
|
/** Schema of the wire view — what the browser receives. */
|
|
const usageViewSchema = z.object({
|
|
searches: z.number(),
|
|
indexes: z.number(),
|
|
failed: z.number(),
|
|
excerpts: z.number(),
|
|
files: z.number(),
|
|
weak: z.number(),
|
|
returnedBytes: z.number(),
|
|
sourceBytes: z.number(),
|
|
avoidedBytes: z.number(),
|
|
avoidedTokens: z.number(),
|
|
compression: z.number(),
|
|
rereadTokens: z.number(),
|
|
avoidedUsd: z.number(),
|
|
lastFocus: z.string(),
|
|
})
|
|
|
|
/**
|
|
* Per-conversation recall usage, folded from the tool metadata in the log.
|
|
*
|
|
* A projection rather than a client-side fold over conversation nodes: nodes
|
|
* carry only the loaded window, so a chip built on them would undercount the
|
|
* moment a session outgrew what the browser had scrolled through. This folds
|
|
* the whole log and is replayed from it.
|
|
*/
|
|
const USAGE_KEY = 'recallUsage'
|
|
|
|
/** Views are compared by reference, so one state must yield one view object. */
|
|
const usageViews = new WeakMap()
|
|
|
|
function usageViewFor(state) {
|
|
let view = usageViews.get(state)
|
|
if (view === undefined) {
|
|
view = usageSummary(state)
|
|
usageViews.set(state, view)
|
|
}
|
|
return view
|
|
}
|
|
|
|
export const usageProjection = {
|
|
key: USAGE_KEY,
|
|
stateSchema: usageStateSchema,
|
|
init: () => emptyUsage(),
|
|
apply: (state, event) => foldUsage(state, event),
|
|
stateVersion: 1,
|
|
wire: { viewSchema: usageViewSchema, view: usageViewFor },
|
|
}
|
|
|
|
export const name = 'dsh-recall'
|
|
export const inject = ['tools']
|
|
|
|
/**
|
|
* Register the tool and the settings namespace.
|
|
*
|
|
* @param {import('@deepseek-ai/cordis').Context} ctx
|
|
* @param {Record<string, any>} config validated composition config
|
|
*/
|
|
export function apply(ctx, config) {
|
|
const logger = ctx.logger('dsh-recall')
|
|
const dshHome = dshHomeOf(ctx)
|
|
const withLock = createLock()
|
|
|
|
// Discovered scopes per workspace, cached for a short window. Discovery is a
|
|
// directory walk on the hot path of every tool call, and re-walking a large
|
|
// repository to learn something that changes only when a marker file is added
|
|
// is work with no benefit.
|
|
const SCOPE_CACHE_MS = 30_000
|
|
const scopeCache = new Map()
|
|
|
|
// Open indexes are filesystem handles, so they are an effect: Cordis closes
|
|
// them on unload and on a config change, and a reload never leaks a handle.
|
|
const openIndexes = new Map()
|
|
ctx.effect(() => () => {
|
|
for (const index of openIndexes.values()) index.close()
|
|
openIndexes.clear()
|
|
})
|
|
|
|
// The authoritative settings source: a plain snapshot of the running config.
|
|
//
|
|
// No settings provider is involved in READING. The `Config` schema above
|
|
// declares every field volatile, so a document edit updates this instance's
|
|
// config references in place and the very next snapshot sees it. That is the
|
|
// 0.1.7 model — there is no `installSection` to attach to, and nothing to
|
|
// subscribe to for a value that is already live.
|
|
const readSettings = () => settingsSnapshot(config)
|
|
let writeSettings
|
|
|
|
// Per-conversation usage, folded from the tool metadata the log already
|
|
// carries. Optional: a profile without projections still gets the tool, it
|
|
// just does not get a surface that reports what it saved.
|
|
ctx.inject(['sessionProjections'], (projectionCtx) => {
|
|
try {
|
|
projectionCtx.sessionProjections.register(usageProjection)
|
|
} catch (error) {
|
|
logger.warn('dsh-recall: usage projection unavailable: %s', error?.message ?? error)
|
|
}
|
|
})
|
|
|
|
// Writing is still the settings provider's job, because the value has to
|
|
// reach the profile's own patch layer rather than live only in this process.
|
|
//
|
|
// `ns` is the profile ENTRY id — the `id` of this plugin's row in the composed
|
|
// tree, which is `dsh-recall` in cordis.patch.yml — not a namespace this
|
|
// plugin invents. The revision is read back from `describe()` so a
|
|
// configuration dialog that saved in the meantime refuses this write instead
|
|
// of being silently overwritten.
|
|
ctx.inject(['settings'], (settingsCtx) => {
|
|
writeSettings = async (patch) => {
|
|
const revision = settingsCtx.settings
|
|
.describe()
|
|
.find((descriptor) => descriptor.ns === SETTINGS_NAMESPACE)?.revision
|
|
await settingsCtx.settings.update(SETTINGS_NAMESPACE, patch, revision)
|
|
}
|
|
})
|
|
|
|
/** The embedder for a settings snapshot. */
|
|
const embedderFor = (settings) =>
|
|
createEmbedder({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
dimensions: Number(settings.embeddingDimensions),
|
|
apiKey: String(settings.embeddingApiKey ?? ''),
|
|
batchSize: Number(settings.embeddingBatchSize),
|
|
concurrency: Number(settings.embeddingConcurrency),
|
|
timeoutMs: Number(settings.embeddingTimeoutMs),
|
|
keepAlive: String(settings.embeddingKeepAlive ?? ''),
|
|
})
|
|
|
|
/**
|
|
* A one-shot embedder for probing the endpoint.
|
|
*
|
|
* No retries and a short timeout, because a probe exists to answer a question
|
|
* quickly: `status` should report a wedged backend, not wait for one. Three
|
|
* retries against a socket that accepts and never replies would take longer
|
|
* than the tool call that provoked it.
|
|
*/
|
|
const proberFor = (settings) =>
|
|
createEmbedder({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
dimensions: Number(settings.embeddingDimensions),
|
|
apiKey: String(settings.embeddingApiKey ?? ''),
|
|
batchSize: 1,
|
|
concurrency: 1,
|
|
timeoutMs: Math.min(Number(settings.embeddingTimeoutMs), 10_000),
|
|
maxAttempts: 1,
|
|
keepAlive: String(settings.embeddingKeepAlive ?? ''),
|
|
})
|
|
|
|
/**
|
|
* The index fields the tool's output schema declares.
|
|
*
|
|
* Built by picking rather than by spreading the sync result, because that
|
|
* result also carries internal fields (`stale`, `chunksWritten`) which are not
|
|
* part of the model-facing contract — and the schema is
|
|
* `additionalProperties: false`, so spreading one in is a validation failure
|
|
* rather than a harmless extra.
|
|
*/
|
|
function indexView(current, extra = {}) {
|
|
const out = { ...current }
|
|
const fields = [
|
|
'files',
|
|
'chunks',
|
|
'pending',
|
|
'scanned',
|
|
'embedded',
|
|
'removed',
|
|
'oversized',
|
|
'unreadable',
|
|
'truncated',
|
|
'sourceRoots',
|
|
]
|
|
for (const key of fields) if (extra[key] !== undefined) out[key] = extra[key]
|
|
return out
|
|
}
|
|
|
|
/**
|
|
* Discovered scopes for a workspace, cached briefly.
|
|
*
|
|
* Discovery is a directory walk, not a file read, so it is cheap — but it runs
|
|
* on every tool call and a large repository has a lot of directories. A short
|
|
* cache keeps the cost off the hot path while still noticing a marker file
|
|
* added a moment ago.
|
|
*
|
|
* @param {string} workspace absolute workspace root
|
|
* @param {readonly string[]} markers marker filenames
|
|
*/
|
|
async function scopesFor(workspace, markers) {
|
|
const key = `${workspace}\u0000${markers.join(',')}`
|
|
const hit = scopeCache.get(key)
|
|
if (hit !== undefined && Date.now() - hit.at < SCOPE_CACHE_MS) return hit.scopes
|
|
let scopes = []
|
|
try {
|
|
scopes = await discoverScopes({ workspace, markers })
|
|
} catch {
|
|
// A workspace that cannot be walked simply has no discoverable scopes;
|
|
// resolution still falls back to markers and then the workspace.
|
|
scopes = []
|
|
}
|
|
cacheScopes(key, scopes)
|
|
return scopes
|
|
}
|
|
|
|
/** Replace the scope cache, dropping entries for workspaces that are gone. */
|
|
function cacheScopes(key, scopes) {
|
|
scopeCache.set(key, { at: Date.now(), scopes })
|
|
if (scopeCache.size > 32) {
|
|
for (const [existing, entry] of scopeCache) {
|
|
if (Date.now() - entry.at > SCOPE_CACHE_MS * 4) scopeCache.delete(existing)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Open (or reuse) the index for one workspace and focus.
|
|
*
|
|
* @param {Record<string, any>} settings resolved settings
|
|
* @param {string} cwd workspace root
|
|
* @param {string} focus focus key, `.` for the whole workspace
|
|
*/
|
|
function indexFor(settings, cwd, focus = '.') {
|
|
const absolute = (path) => join(resolve(path), focus === '.' ? 'index.sqlite' : join('focus', focus, 'index.sqlite'))
|
|
const file =
|
|
typeof settings.stateDir === 'string' && settings.stateDir.trim() !== ''
|
|
? absolute(settings.stateDir.trim())
|
|
: indexFile(dshHome, cwd, focus)
|
|
let index = openIndexes.get(file)
|
|
if (index === undefined) {
|
|
index = new RecallIndex(file)
|
|
openIndexes.set(file, index)
|
|
}
|
|
const identity = {
|
|
model: String(settings.embeddingModel),
|
|
dimensions: String(settings.embeddingDimensions),
|
|
apiBase: String(settings.embeddingApiBase),
|
|
}
|
|
// `open` is idempotent for an already-open handle; the identity check is the
|
|
// part that must run every time, because the settings can change under it.
|
|
const outcome = index.db === undefined ? index.open(identity) : index.recheck(identity)
|
|
return { index, file, discarded: outcome.discarded }
|
|
}
|
|
|
|
/**
|
|
* Bring one workspace's index up to date.
|
|
*
|
|
* Phase one reads and hashes every candidate — fast, bounded by disk. Phase
|
|
* two embeds what changed, and is the only part given a deadline. Splitting
|
|
* them is what makes the reported state honest: after phase one the plugin
|
|
* knows exactly what is stale, so a partial index is described rather than
|
|
* guessed at.
|
|
*/
|
|
|
|
/**
|
|
* The candidate file list for one index: the workspace (or focus) tree plus
|
|
* every extra root.
|
|
*
|
|
* Split out from {@link syncWorkspace} because a search reports the absolute
|
|
* path of a result that came from an extra root, and the read phase is where
|
|
* that mapping is known. Both callers therefore see one list and one set of
|
|
* problems.
|
|
*/
|
|
async function walkIndex(input) {
|
|
const { root, prefix = '', settings, signal, sourceRoots = [] } = input
|
|
const walk = await listCodeFiles(root, {
|
|
...(settings.extensions?.length ? { extensions: settings.extensions } : {}),
|
|
skipDirs: DEFAULT_SKIP_DIRS,
|
|
exclude: settings.exclude ?? [],
|
|
maxFiles: Number(settings.maxFiles),
|
|
signal,
|
|
})
|
|
// `listCodeFiles` reports paths relative to the walk root, which under a
|
|
// focus is the focus directory. They are rewritten to be relative to the
|
|
// WORKSPACE, because that is the path the model will hand to `read` — a
|
|
// focus-relative result would silently read the wrong file.
|
|
if (prefix !== '') {
|
|
for (const file of walk.files) file.path = `${prefix}/${file.path}`
|
|
}
|
|
|
|
// Extra roots are walked as their own trees and merged into the same index,
|
|
// so one search ranks workspace code and library code together. Each root
|
|
// reports its own truncation: a root too large to index must say so rather
|
|
// than answer from its first few files.
|
|
const files = [...walk.files]
|
|
const problems = [...(input.sourceRootProblems ?? [])]
|
|
const summary = []
|
|
let truncated = walk.truncated
|
|
for (const sourceRoot of sourceRoots) {
|
|
const walkOfRoot = await listRootFiles({
|
|
name: sourceRoot.name,
|
|
path: sourceRoot.path,
|
|
...(settings.extensions?.length ? { extensions: settings.extensions } : {}),
|
|
exclude: settings.exclude ?? [],
|
|
maxFiles: Number(settings.maxFiles),
|
|
signal,
|
|
})
|
|
// A root labelled with the focus path itself would index two different
|
|
// files under one key — the second write silently replacing the first.
|
|
if (prefix !== '' && (prefix === sourceRoot.name || prefix.startsWith(`${sourceRoot.name}/`))) {
|
|
problems.push(
|
|
`the source root "${sourceRoot.name}" collides with the focus path "${prefix}"; ` +
|
|
'give it a different name= or search without that focus',
|
|
)
|
|
continue
|
|
}
|
|
files.push(...walkOfRoot.files)
|
|
summary.push({ name: sourceRoot.name, path: sourceRoot.path, files: walkOfRoot.files.length })
|
|
truncated = truncated || walkOfRoot.truncated
|
|
}
|
|
|
|
return { files, truncated, problems, summary }
|
|
}
|
|
|
|
async function syncWorkspace(input) {
|
|
const { index, settings, budgetMs, signal, onProgress } = input
|
|
const { files, truncated, problems: rootProblems, summary: rootSummary } = await walkIndex(input)
|
|
|
|
const known = index.fileHashes()
|
|
const present = new Set(files.map((file) => file.path))
|
|
const removed = index.prune([...known.keys()].filter((path) => !present.has(path)))
|
|
|
|
const maxBytes = Number(settings.maxFileBytes)
|
|
const pending = []
|
|
let oversized = 0
|
|
let unreadable = 0
|
|
|
|
for (const file of files) {
|
|
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
let content
|
|
try {
|
|
const info = await stat(file.absolute)
|
|
if (info.size > maxBytes) {
|
|
oversized += 1
|
|
continue
|
|
}
|
|
content = await readFile(file.absolute, 'utf8')
|
|
} catch {
|
|
unreadable += 1
|
|
continue
|
|
}
|
|
const hash = hashOf(content)
|
|
if (known.get(file.path) === hash) continue
|
|
pending.push({ ...file, content, hash, bytes: Buffer.byteLength(content) })
|
|
}
|
|
|
|
// Anything still holding an old hash is stale until it is re-embedded.
|
|
const stale = new Set(pending.map((file) => file.path))
|
|
const deadline = Date.now() + budgetMs
|
|
let embedded = 0
|
|
let chunksWritten = 0
|
|
|
|
for (const file of pending) {
|
|
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
// Always embed at least one file, or a zero budget would make the feature
|
|
// indistinguishable from a broken index.
|
|
if (embedded > 0 && Date.now() >= deadline) break
|
|
const { chunks } = chunkFile({
|
|
path: file.path,
|
|
content: file.content,
|
|
chunkLines: Number(settings.chunkLines),
|
|
maxChunkLines: Number(settings.maxChunkLines),
|
|
overlapLines: Number(settings.overlapLines),
|
|
})
|
|
if (chunks.length === 0) {
|
|
// An empty or generated file still gets a row, so it is not re-read and
|
|
// re-chunked on every subsequent search.
|
|
index.replaceFile(file.path, file.hash, file.bytes, [])
|
|
embedded += 1
|
|
stale.delete(file.path)
|
|
onProgress?.(embedded, pending.length)
|
|
continue
|
|
}
|
|
const vectors = await input.embedder.embed(
|
|
chunks.map((chunk) => embedText(file.path, chunk)),
|
|
signal,
|
|
)
|
|
index.replaceFile(
|
|
file.path,
|
|
file.hash,
|
|
file.bytes,
|
|
chunks.map((chunk, at) => ({ ...chunk, vec: vectors[at] })),
|
|
)
|
|
chunksWritten += chunks.length
|
|
embedded += 1
|
|
stale.delete(file.path)
|
|
onProgress?.(embedded, pending.length)
|
|
}
|
|
|
|
const stats = index.stats()
|
|
return {
|
|
...stats,
|
|
scanned: files.length,
|
|
truncated,
|
|
removed,
|
|
embedded,
|
|
chunksWritten,
|
|
pending: pending.length - embedded,
|
|
stale: [...stale],
|
|
oversized,
|
|
unreadable,
|
|
sourceRoots: rootSummary,
|
|
sourceRootProblems: rootProblems,
|
|
// Not part of the model-facing contract — the schema is closed — but a
|
|
// search needs it: a result from an extra root is labelled relative to
|
|
// that root, so the absolute path is the only thing `read` can use, and
|
|
// only the walk knows it.
|
|
fileAbsolutes: new Map(files.map((file) => [file.path, file.absolute])),
|
|
}
|
|
}
|
|
|
|
const TOOL_OUTPUT = {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
action: { type: 'string', required: true },
|
|
ok: { type: 'boolean', required: true },
|
|
query: { type: 'string' },
|
|
workspace: { type: 'string' },
|
|
focus: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
path: { type: 'string', required: true },
|
|
source: { type: 'string', required: true },
|
|
marker: { type: 'string' },
|
|
problem: { type: 'string' },
|
|
},
|
|
},
|
|
results: {
|
|
type: 'array',
|
|
items: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
path: { type: 'string', required: true },
|
|
startLine: { type: 'integer', required: true },
|
|
endLine: { type: 'integer', required: true },
|
|
symbol: { type: 'string' },
|
|
similarity: { type: 'number', required: true },
|
|
excerpt: { type: 'string', required: true },
|
|
truncated: { type: 'boolean', required: true },
|
|
stale: { type: 'boolean', required: true },
|
|
/**
|
|
* Absolute path, present only for a result that came from an extra
|
|
* index root. The labelled `path` of such a result is relative to
|
|
* its root rather than the workspace, so it is not what `read`
|
|
* resolves — this is.
|
|
*/
|
|
sourcePath: { type: 'string' },
|
|
},
|
|
},
|
|
},
|
|
index: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
files: { type: 'integer' },
|
|
chunks: { type: 'integer' },
|
|
pending: { type: 'integer' },
|
|
scanned: { type: 'integer' },
|
|
embedded: { type: 'integer' },
|
|
removed: { type: 'integer' },
|
|
oversized: { type: 'integer' },
|
|
unreadable: { type: 'integer' },
|
|
truncated: { type: 'boolean' },
|
|
database: { type: 'string' },
|
|
stateDir: { type: 'string' },
|
|
/** The extra roots this index holds, and how many files each contributed. */
|
|
sourceRoots: {
|
|
type: 'array',
|
|
items: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
name: { type: 'string', required: true },
|
|
path: { type: 'string', required: true },
|
|
files: { type: 'integer', required: true },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
/** Entries of `sourceRoots` that could not be used, one message each. */
|
|
sourceRootProblems: { type: 'array', items: { type: 'string' } },
|
|
best: { type: 'number' },
|
|
endpoint: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
ok: { type: 'boolean', required: true },
|
|
url: { type: 'string', required: true },
|
|
model: { type: 'string', required: true },
|
|
errorCode: { type: 'string' },
|
|
message: { type: 'string' },
|
|
},
|
|
},
|
|
weak: { type: 'boolean' },
|
|
/**
|
|
* What this call returned, against what reading those files would have
|
|
* cost. `sourceBytes` is an upper bound and is labelled as one wherever
|
|
* it is shown — a careful reader might have opened one file, not four.
|
|
*/
|
|
measure: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
excerpts: { type: 'integer', required: true },
|
|
returnedBytes: { type: 'integer', required: true },
|
|
sourceBytes: { type: 'integer', required: true },
|
|
files: { type: 'integer', required: true },
|
|
},
|
|
},
|
|
jobId: { type: 'string' },
|
|
/** Every scope discovered in this workspace, so a caller can pick one. */
|
|
scopes: {
|
|
type: 'array',
|
|
items: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
path: { type: 'string', required: true },
|
|
marker: { type: 'string', required: true },
|
|
},
|
|
},
|
|
},
|
|
note: { type: 'string' },
|
|
errorCode: { type: 'string' },
|
|
remedy: { type: 'string' },
|
|
},
|
|
}
|
|
|
|
/** Render one search result set for the model. */
|
|
function renderResults(value) {
|
|
if (value.jobId !== undefined) {
|
|
return [
|
|
{
|
|
type: 'text',
|
|
text:
|
|
`Indexing ${value.workspace} as background job ${value.jobId}. You will be notified when it settles; ` +
|
|
`collect it with job_output for "${value.jobId}" (add wait: true if you are blocked on it). ` +
|
|
'Searches work meanwhile against whatever is already indexed.',
|
|
},
|
|
]
|
|
}
|
|
if (value.ok !== true) {
|
|
const parts = [value.note ?? `recall ${value.action} failed.`]
|
|
if (value.remedy !== undefined) parts.push(`Next: ${value.remedy}`)
|
|
return [{ type: 'text', text: parts.join('\n\n') }]
|
|
}
|
|
if (value.action === 'configure') {
|
|
return [{ type: 'text', text: value.note ?? 'Configuration updated.' }]
|
|
}
|
|
if (value.action !== 'search') {
|
|
const { files = 0, chunks = 0, pending = 0, scanned = 0, embedded = 0 } = value.index ?? {}
|
|
const refused = (value.sourceRootProblems ?? []).length === 0
|
|
? ''
|
|
: `\n\nRefused source roots: ${value.sourceRootProblems.join('; ')}.`
|
|
return [
|
|
{
|
|
type: 'text',
|
|
text:
|
|
`Index for ${value.workspace}: ${files} files, ${chunks} chunks. ` +
|
|
`Scanned ${scanned}, embedded ${embedded} this call, ${pending} still pending.${refused}`,
|
|
},
|
|
]
|
|
}
|
|
|
|
const results = value.results ?? []
|
|
if (results.length === 0) {
|
|
const suffix = (value.index?.pending ?? 0) > 0 ? ` ${value.index.pending} file(s) are still unindexed.` : ''
|
|
const refused =
|
|
(value.sourceRootProblems ?? []).length === 0
|
|
? ''
|
|
: ` Refused source roots: ${value.sourceRootProblems.join('; ')}.`
|
|
return [
|
|
{
|
|
type: 'text',
|
|
text:
|
|
`No match for ${JSON.stringify(value.query)}.${suffix}${refused} ` +
|
|
'If you know the exact identifier, grep is the better tool for this query.',
|
|
},
|
|
]
|
|
}
|
|
|
|
const scope = value.focus?.path !== undefined && value.focus.path !== '.' ? ` in ${value.focus.path}` : ''
|
|
const header =
|
|
`${results.length} excerpt(s) for ${JSON.stringify(value.query)}${scope} — ` +
|
|
`from ${value.index?.chunks ?? 0} indexed chunks` +
|
|
((value.index?.pending ?? 0) > 0 ? `, ${value.index.pending} file(s) still unindexed` : '')
|
|
const body = results.map((result, at) => {
|
|
const symbol = result.symbol === undefined ? '' : ` (${result.symbol})`
|
|
const stale = result.stale ? ' [stale — file changed since indexing]' : ''
|
|
const cut = result.truncated ? '\n… excerpt truncated; read the file for the rest' : ''
|
|
// A result from an extra root is labelled relative to that root, so the
|
|
// workspace-relative `path` is not something `read` can resolve. The
|
|
// absolute path is stated when it is the one that differs — a workspace
|
|
// result already has a usable path and does not need the noise.
|
|
const source = result.sourcePath === undefined ? '' : `\n at ${result.sourcePath}`
|
|
return (
|
|
`--- ${at + 1}. ${result.path}:${result.startLine}-${result.endLine}${symbol} ` +
|
|
`similarity ${result.similarity}${stale}${source}\n${result.excerpt}${cut}`
|
|
)
|
|
})
|
|
// A scope is a window, not a filter on relevance. When it holds nothing
|
|
// strong, the most likely reason is that the answer is shared code the scope
|
|
// excludes — 41.6% of the workspace this was measured on — so say that
|
|
// rather than leaving the caller to conclude the code does not exist.
|
|
const outside =
|
|
value.focus?.path !== undefined && value.focus.path !== '.'
|
|
? ` Only ${value.focus.path} is indexed here; shared code outside it is not. Retry without focus to search the whole workspace.`
|
|
: ''
|
|
const advice = value.weak
|
|
? `\n\nNo strong match. Prefer grep for a known identifier, or rephrase with the name of the thing.${outside}`
|
|
: ''
|
|
return [{ type: 'text', text: `${header}\n\n${body.join('\n\n')}${advice}` }]
|
|
}
|
|
|
|
ctx.tools.register(
|
|
defineTool({
|
|
name: 'recall',
|
|
description:
|
|
'Search this workspace semantically with a local embedding model and return ranked code excerpts ' +
|
|
'with file paths and line ranges — without reading whole files into the conversation. Use it when ' +
|
|
'you do not know the exact identifier: "where is X handled", "which module does Y", "how does Z ' +
|
|
'recover from failure". ' +
|
|
// The move this exists to replace: reading a module's files one by one to
|
|
// find out what is in it. That is a search, and this is cheaper than the
|
|
// reads it avoids — including on the very first call, because a search
|
|
// warms a bounded slice of the index as it answers.
|
|
'Reach for it before reading files to find out what a module contains or ' +
|
|
'which of several alike modules does the job: one search returns the relevant lines from wherever ' +
|
|
'they are, where reading to find them costs the whole file for each wrong guess and every one of ' +
|
|
'those lines is re-sent on every later step. ' +
|
|
'Prefer grep when you already know the exact symbol, and read when you need the whole file. ' +
|
|
'Because excerpts are returned inline, do not follow up with a read of the same ' +
|
|
'range. A result labelled with a leading name (for example "@dsh/…") came from an extra source ' +
|
|
'root rather than the workspace; such a result also carries an absolute `sourcePath`, which is the ' +
|
|
'one `read` can resolve. Actions: "search" (default) answers a query and warms a bounded slice of ' +
|
|
'the index; "index" builds or refreshes the whole index, as a background job by default, which is ' +
|
|
'what makes the first search of a session cheap; "status" reports the ' +
|
|
'index and endpoint state. The index is incremental and lives outside your repository.',
|
|
parameters: {
|
|
query: {
|
|
type: 'string',
|
|
description: 'What to look for, in words. A description of behaviour works as well as a name.',
|
|
},
|
|
action: {
|
|
type: 'string',
|
|
enum: ['search', 'index', 'status', 'configure'],
|
|
description: 'What to do. Defaults to "search".',
|
|
},
|
|
limit: {
|
|
type: 'integer',
|
|
description: 'Most excerpts to return. Defaults to the configured limit (8).',
|
|
},
|
|
path: {
|
|
type: 'string',
|
|
description:
|
|
'Restrict the search to files under this workspace-relative directory, for example "src/services".',
|
|
},
|
|
focus: {
|
|
type: 'string',
|
|
description:
|
|
'Scope this whole call — index and search — to one workspace-relative directory, for example ' +
|
|
'"src/services/billing". Use it when the session is working on one module: the index ' +
|
|
'then holds only that module, so its near-identical neighbours cannot outrank it and the first ' +
|
|
'search does not wait on the rest of the repository. Empty uses the configured focus, then the ' +
|
|
'nearest enclosing directory that carries a CONTROLLER.md, DEBUG_UI.md or AGENTS.md.',
|
|
},
|
|
run_in_background: {
|
|
type: 'boolean',
|
|
description:
|
|
'For action "index" only: return a job id instead of the result. Defaults to true, because a ' +
|
|
'cold index of a large repository takes minutes.',
|
|
},
|
|
config: {
|
|
type: 'json',
|
|
description:
|
|
'Settings patch for action "configure", using this plugin\'s own camelCase field names, for ' +
|
|
'example {"embeddingApiBase":"http://127.0.0.1:11434"}.',
|
|
},
|
|
},
|
|
output: {
|
|
schema: TOOL_OUTPUT,
|
|
render: (_args, value) => renderResults(value),
|
|
// The harness persists this verbatim as the `meta` of the `tool/result`
|
|
// event, which is what puts per-call usage in the session log — and
|
|
// therefore in reach of a projection the browser can read. Pure, and it
|
|
// carries totals rather than content: it is appended on every call and
|
|
// replayed on every projection rebuild.
|
|
presentationMeta: (args, value) => deriveUsage(args, value),
|
|
},
|
|
isConcurrencySafe: (args) => (args.action ?? 'search') !== 'index',
|
|
async execute(args, exec) {
|
|
const settings = readSettings()
|
|
const cwd = exec.agent?.session.header.cwd ?? process.cwd()
|
|
const action = args.action ?? 'search'
|
|
const workspace = basename(cwd) || cwd
|
|
|
|
// Validate the live section before using it, every call.
|
|
//
|
|
// The Host parses a write against the schema, so anything a single field
|
|
// can express is refused at the document; the cross-field rules — a cap
|
|
// below its target, an overlap that never advances a split — are not
|
|
// expressible that way, and 0.1.7 has no write-time validator hook to
|
|
// hang them on. Checking here is what keeps a bad combination from
|
|
// surfacing later as an unexplained empty search.
|
|
try {
|
|
validateSettings(settings)
|
|
} catch (error) {
|
|
return {
|
|
action,
|
|
ok: false,
|
|
workspace,
|
|
errorCode: 'INVALID_SETTINGS',
|
|
note: `The dsh-recall configuration is not usable as it stands: ${error?.message ?? error}`,
|
|
remedy:
|
|
'Fix the conflicting values in Settings → Plugins → dsh-recall, or in the dsh-recall ' +
|
|
'row of your cordis layer.',
|
|
}
|
|
}
|
|
|
|
const embedder = embedderFor(settings)
|
|
|
|
// Resolve the scope before anything else, because it decides which index
|
|
// is opened. A caller's `focus` outranks the configured one, so a session
|
|
// can name a controller without changing the deployment's default.
|
|
const markers = settings.focusMarkers?.length ? settings.focusMarkers : DEFAULT_FOCUS_MARKERS
|
|
const scopes = await scopesFor(cwd, markers)
|
|
const focus = resolveFocus({
|
|
workspace: cwd,
|
|
cwd,
|
|
explicit: args.focus ?? settings.focus ?? '',
|
|
markers,
|
|
scopes,
|
|
})
|
|
const focusPath = focusKey(cwd, focus.root)
|
|
|
|
// Extra roots are resolved once per call against the live workspace, so
|
|
// an entry is judged where it is used rather than at load: a path that
|
|
// has not been installed yet is reported on the call that wanted it. A
|
|
// bad entry never aborts the call — it is reported beside the result, or
|
|
// the caller would lose the workspace index to one typo.
|
|
const resolvedRoots = resolveSourceRoots(settings.sourceRoots ?? [], { workspace: cwd })
|
|
const sourceRoots = resolvedRoots.roots
|
|
const sourceRootProblems = resolvedRoots.problems
|
|
|
|
let opened
|
|
try {
|
|
opened = indexFor(settings, cwd, focusPath)
|
|
} catch (error) {
|
|
return {
|
|
action,
|
|
ok: false,
|
|
workspace,
|
|
errorCode: 'INDEX_UNAVAILABLE',
|
|
note: `Could not open the index: ${error?.message ?? error}`,
|
|
remedy: `Check that ${dshHome} is writable.`,
|
|
}
|
|
}
|
|
const { index, file, discarded } = opened
|
|
const stateDir = workspaceStateDir(dshHome, cwd)
|
|
const focusView = {
|
|
path: focusPath,
|
|
source: focus.source,
|
|
...(focus.marker === undefined ? {} : { marker: focus.marker }),
|
|
...(focus.problem === undefined ? {} : { problem: focus.problem }),
|
|
}
|
|
const base = { action, workspace, focus: focusView, index: { database: file, stateDir } }
|
|
|
|
if (action === 'status') {
|
|
const stats = index.stats()
|
|
|
|
// Probe the endpoint rather than merely naming it. A wedged backend
|
|
// accepts the connection and never answers, so "configured" and
|
|
// "working" are different claims and status should make the second
|
|
// one. One attempt, short timeout — a probe that waits is not a probe.
|
|
let endpoint
|
|
try {
|
|
await proberFor(settings).embedOne('dsh-recall endpoint probe', exec.signal)
|
|
endpoint = {
|
|
ok: true,
|
|
url: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
}
|
|
} catch (error) {
|
|
endpoint = {
|
|
ok: false,
|
|
url: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
errorCode: error instanceof EmbeddingError ? error.code : 'PROBE_FAILED',
|
|
message: error?.message ?? String(error),
|
|
}
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
ok: true,
|
|
index: indexView(base.index, stats),
|
|
endpoint,
|
|
scopes: scopes.map((entry) => ({ path: entry.path, marker: entry.marker })),
|
|
...(sourceRootProblems.length === 0 ? {} : { sourceRootProblems }),
|
|
note:
|
|
`Index ${file} holds ${stats.files} files and ${stats.chunks} chunks, scoped to ` +
|
|
`${focusPath === '.' ? 'the whole workspace' : focusPath} (${focus.source}` +
|
|
`${focus.marker === undefined ? '' : `: ${focus.marker}`}). ` +
|
|
(focus.problem === undefined ? '' : `Requested focus was ignored — ${focus.problem}. `) +
|
|
// A workspace with discoverable scopes can be scoped by naming one,
|
|
// so listing them is the difference between a feature that exists
|
|
// and one a caller can actually use.
|
|
(scopes.length === 0
|
|
? 'No focusable scopes were found; pass focus to scope a call explicitly. '
|
|
: `Available scopes: ${scopes
|
|
.slice(0, 12)
|
|
.map((entry) => entry.path)
|
|
.join(', ')}${scopes.length > 12 ? ` (+${scopes.length - 12} more)` : ''}. `) +
|
|
// Extra roots are named even when the index has not been rebuilt
|
|
// since they were configured, so their file counts are deliberately
|
|
// absent here — `stats` describes the index, not the configuration.
|
|
(sourceRoots.length === 0
|
|
? ''
|
|
: `Extra roots indexed beside the workspace: ${sourceRoots
|
|
.map((entry) => `${entry.name} (${entry.path})`)
|
|
.join(', ')}. `) +
|
|
(sourceRootProblems.length === 0
|
|
? ''
|
|
: `Source roots refused: ${sourceRootProblems.join('; ')}. `) +
|
|
`Embeddings come from "${settings.embeddingModel}" (${settings.embeddingDimensions} dims) at ` +
|
|
`${settings.embeddingApiBase} — ` +
|
|
(endpoint.ok === true
|
|
? 'endpoint answered.'
|
|
: `endpoint did NOT answer (${endpoint.errorCode}). Run \`ollama ps\` on that host; a ` +
|
|
'model can be evicted, and an endpoint holding another model resident forever can ' +
|
|
'leave no room to load this one.') +
|
|
(discarded === undefined ? '' : ` The index was rebuilt because ${discarded}.`),
|
|
}
|
|
}
|
|
|
|
if (action === 'configure') {
|
|
const patch = args.config
|
|
if (patch === undefined || patch === null || typeof patch !== 'object' || Array.isArray(patch)) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'BAD_PATCH',
|
|
note: 'The `config` parameter must be an object.',
|
|
}
|
|
}
|
|
const unknown = Object.keys(patch).filter((key) => !SETTING_KEYS.includes(key))
|
|
if (unknown.length > 0) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'UNKNOWN_SETTING',
|
|
note: `Unknown setting(s): ${unknown.join(', ')}.`,
|
|
remedy: `Known settings: ${SETTING_KEYS.join(', ')}.`,
|
|
}
|
|
}
|
|
// Judge the SECTION the write would leave behind, not the patch on its
|
|
// own: `maxLimit` alone is always fine, and `defaultLimit` alone is
|
|
// always fine, but raising one past the other is the mistake worth
|
|
// catching — and it is invisible to any single value's schema.
|
|
try {
|
|
validateSettings({ ...settings, ...patch })
|
|
} catch (error) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'INVALID_SETTING',
|
|
note: `Refused: ${error?.message ?? error}`,
|
|
remedy: 'Nothing was written. Send a patch whose resulting section is consistent.',
|
|
}
|
|
}
|
|
if (writeSettings === undefined) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'NO_SETTINGS_PROVIDER',
|
|
note: 'No settings provider is mounted, so configuration cannot be persisted at runtime.',
|
|
remedy: `Set these values in the dsh-recall row of your cordis layer instead: ${JSON.stringify(patch)}`,
|
|
}
|
|
}
|
|
try {
|
|
await writeSettings(patch)
|
|
} catch (error) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'SETTINGS_WRITE_FAILED',
|
|
note: `The settings document refused the write: ${error?.message ?? error}`,
|
|
remedy: 'Re-read the current values and retry; a concurrent edit is the usual cause.',
|
|
}
|
|
}
|
|
return { ...base, ok: true, note: `Updated ${Object.keys(patch).join(', ')}.` }
|
|
}
|
|
|
|
// Full warm-up. Deliberately a background job by default: on a large
|
|
// repository this is minutes of work, and blocking a turn on it is the
|
|
// behaviour that makes a useful tool feel broken.
|
|
if (action === 'index') {
|
|
const jobs = ctx.get('jobs')
|
|
const wantsBackground = args.run_in_background ?? true
|
|
const runIndex = (signal) =>
|
|
withLock(file, () =>
|
|
syncWorkspace({
|
|
root: focus.root,
|
|
prefix: focusPath === '.' ? '' : focusPath,
|
|
index,
|
|
embedder,
|
|
settings,
|
|
budgetMs: Number.POSITIVE_INFINITY,
|
|
signal,
|
|
sourceRoots,
|
|
sourceRootProblems,
|
|
}),
|
|
)
|
|
|
|
if (wantsBackground && jobs !== undefined) {
|
|
const jobId = jobs.start({
|
|
kind: 'recall-index',
|
|
label: `recall index ${workspace}`,
|
|
...(exec.agent === undefined ? {} : { owner: exec.agent }),
|
|
run: () => {
|
|
const controller = new AbortController()
|
|
const done = (async () => {
|
|
const result = await runIndex(controller.signal)
|
|
return {
|
|
status: 'completed',
|
|
detail: `${result.files} files, ${result.chunks} chunks`,
|
|
output:
|
|
`Indexed ${result.embedded} file(s) into ${result.chunks} chunks (${result.files} files ` +
|
|
`total, ${result.removed} removed). Database: ${file}`,
|
|
}
|
|
})()
|
|
return { cancel: () => controller.abort(), done }
|
|
},
|
|
})
|
|
return { ...base, ok: true, jobId }
|
|
}
|
|
|
|
const result = await runIndex(exec.signal)
|
|
return {
|
|
...base,
|
|
ok: true,
|
|
index: indexView(base.index, result),
|
|
...(result.sourceRootProblems.length === 0
|
|
? {}
|
|
: { sourceRootProblems: result.sourceRootProblems }),
|
|
}
|
|
}
|
|
|
|
// search
|
|
const query = typeof args.query === 'string' ? args.query.trim() : ''
|
|
if (query === '') {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'NO_QUERY',
|
|
note: 'A search needs a query.',
|
|
remedy: 'Call recall with a `query` describing what you are looking for.',
|
|
}
|
|
}
|
|
|
|
const limit = Math.max(
|
|
1,
|
|
Math.min(Number(args.limit ?? settings.defaultLimit), Number(settings.maxLimit)),
|
|
)
|
|
|
|
try {
|
|
const sync = await withLock(file, () =>
|
|
syncWorkspace({
|
|
root: focus.root,
|
|
prefix: focusPath === '.' ? '' : focusPath,
|
|
index,
|
|
embedder,
|
|
settings,
|
|
budgetMs: Math.max(0, Number(settings.syncBudgetMs)),
|
|
signal: exec.signal,
|
|
sourceRoots,
|
|
sourceRootProblems,
|
|
}),
|
|
)
|
|
const matrix = index.matrix()
|
|
if (matrix.rows.length === 0) {
|
|
return {
|
|
...base,
|
|
ok: true,
|
|
query,
|
|
results: [],
|
|
weak: true,
|
|
index: indexView(base.index, sync),
|
|
note:
|
|
'Nothing is indexed yet, so there is nothing to search. ' +
|
|
(sync.pending > 0
|
|
? `${sync.pending} file(s) were found but the per-call embedding budget was reached.`
|
|
: 'No indexable code was found in this workspace.'),
|
|
...(sync.pending > 0
|
|
? {
|
|
remedy:
|
|
'Run recall with action "index" to build the whole index in the background.',
|
|
}
|
|
: {}),
|
|
}
|
|
}
|
|
|
|
// Query expansion: rank every phrasing and fuse the orders. The
|
|
// expander never throws — on a slow or absent local model it returns
|
|
// the original query alone, and this path is unchanged.
|
|
const expansionModel = String(settings.expansionModel ?? '').trim()
|
|
const queries = settings.expansionEnabled === true
|
|
? await createExpander({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: expansionModel === '' ? String(settings.embeddingModel) : expansionModel,
|
|
timeoutMs: Number(settings.expansionTimeoutMs),
|
|
count: Number(settings.expansionCount),
|
|
}).expand(query, exec.signal)
|
|
: [query]
|
|
|
|
const orders = []
|
|
let similarity
|
|
let vectorBest = 0
|
|
for (const variant of queries) {
|
|
const vector = await embedder.embedOne(variant, exec.signal)
|
|
const ranked = rankOrder({ matrix, queryVector: vector, query: variant })
|
|
orders.push(ranked.order)
|
|
// The first variant is the caller's own query, so its similarities
|
|
// are the honest ones to report against each result.
|
|
if (similarity === undefined) {
|
|
similarity = ranked.similarity
|
|
vectorBest = ranked.vectorBest
|
|
}
|
|
}
|
|
|
|
let results = selectFrom(fuseRankings(orders), {
|
|
matrix,
|
|
similarity,
|
|
limit,
|
|
perFile: Number(settings.perFileLimit),
|
|
excerptLines: Number(settings.excerptLines),
|
|
totalLines: Number(settings.totalExcerptLines),
|
|
})
|
|
|
|
// Local re-ranking: one call scores the head of the list. `undefined`
|
|
// means the model had no opinion, which is not the same as "all
|
|
// irrelevant" — the fused order stands unchanged.
|
|
const rerankModel = String(settings.rerankModel ?? '').trim()
|
|
const rerankCount = Math.min(results.length, Number(settings.rerankCandidates), 12)
|
|
if (settings.rerankEnabled === true && rerankCount > 1) {
|
|
const head = results.slice(0, rerankCount)
|
|
const localScores = await createReranker({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: rerankModel === '' ? String(settings.embeddingModel) : rerankModel,
|
|
timeoutMs: Number(settings.rerankTimeoutMs),
|
|
}).score(query, head.map((result) => result.excerpt), exec.signal)
|
|
if (localScores !== undefined && localScores.length === head.length) {
|
|
const blended = head
|
|
.map((result, at) => ({ result, score: 0.4 * result.similarity + 0.6 * localScores[at] }))
|
|
.sort((left, right) => right.score - left.score)
|
|
.map((entry) => entry.result)
|
|
results = [...blended, ...results.slice(rerankCount)]
|
|
}
|
|
}
|
|
|
|
if (typeof args.path === 'string' && args.path.trim() !== '') {
|
|
const prefix = args.path.trim().replace(/^\.\//, '').replace(/\/+$/, '')
|
|
results = results.filter(
|
|
(result) => result.path === prefix || result.path.startsWith(`${prefix}/`),
|
|
)
|
|
}
|
|
|
|
const stale = new Set(sync.stale ?? [])
|
|
const best = results.length === 0 ? 0 : Math.max(...results.map((result) => result.similarity))
|
|
// Where the result actually is, for the cases where the label is not a
|
|
// workspace-relative path. A workspace result keeps its plain path: the
|
|
// caller resolves it against the workspace and has no use for an
|
|
// absolute one. A result from an extra root cannot be resolved that
|
|
// way, so it carries the absolute path beside its label.
|
|
const absolutes = sync.fileAbsolutes ?? new Map()
|
|
const workspacePrefix = resolve(cwd).endsWith(sep) ? resolve(cwd) : `${resolve(cwd)}${sep}`
|
|
const sourcePathOf = (result) => {
|
|
const absolute = absolutes.get(result.path)
|
|
if (absolute === undefined) return undefined
|
|
return absolute.startsWith(workspacePrefix) ? undefined : absolute
|
|
}
|
|
|
|
// The alternative to these excerpts: the whole of each file they came
|
|
// from. Sizes come from the index, which recorded them at embed time,
|
|
// so this costs nothing here — and it is an upper bound, because a
|
|
// careful reader might have opened one of the four rather than all.
|
|
const sourcePaths = [...new Set(results.map((result) => result.path))]
|
|
const sourceSizes = index.bytesFor(sourcePaths)
|
|
const sourceBytes = [...sourceSizes.values()].reduce((sum, size) => sum + size, 0)
|
|
const returnedBytes = results.reduce(
|
|
(sum, result) => sum + Buffer.byteLength(result.excerpt, 'utf8'),
|
|
0,
|
|
)
|
|
const measure = {
|
|
excerpts: results.length,
|
|
returnedBytes,
|
|
sourceBytes,
|
|
files: sourceSizes.size,
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
ok: true,
|
|
query,
|
|
// Built field by field rather than spread: the ranker also returns
|
|
// a fused RRF `score`, which is an internal ordering artefact and not
|
|
// part of the declared result shape.
|
|
results: results.map((result) => ({
|
|
path: result.path,
|
|
startLine: result.startLine,
|
|
endLine: result.endLine,
|
|
...(result.symbol === undefined ? {} : { symbol: result.symbol }),
|
|
similarity: result.similarity,
|
|
excerpt: result.excerpt,
|
|
truncated: result.truncated,
|
|
stale: stale.has(result.path),
|
|
...(sourcePathOf(result) === undefined ? {} : { sourcePath: sourcePathOf(result) }),
|
|
})),
|
|
index: indexView(base.index, sync),
|
|
best,
|
|
weak: results.length === 0 || best < Number(settings.minSimilarity),
|
|
measure,
|
|
...(sync.sourceRootProblems.length === 0
|
|
? {}
|
|
: { sourceRootProblems: sync.sourceRootProblems }),
|
|
}
|
|
} catch (error) {
|
|
const code = error instanceof EmbeddingError ? error.code : 'RECALL_FAILED'
|
|
const remedy =
|
|
code === 'UNREACHABLE' || error?.cause?.code === 'ECONNREFUSED'
|
|
? `Is the embedding endpoint at ${settings.embeddingApiBase} running and reachable? Set ` +
|
|
'`embeddingApiBase` through the plugin settings if it moved.'
|
|
: code === 'DIMENSION_MISMATCH'
|
|
? `Set \`embeddingDimensions\` to match "${settings.embeddingModel}".`
|
|
: undefined
|
|
logger.warn('recall search failed: %s', error?.message ?? error)
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
query,
|
|
errorCode: code,
|
|
note: error?.message ?? String(error),
|
|
...(remedy === undefined ? {} : { remedy }),
|
|
}
|
|
}
|
|
},
|
|
}),
|
|
)
|
|
}
|