1280 lines
50 KiB
JavaScript
1280 lines
50 KiB
JavaScript
/**
|
|
* dsh-recall — local-embedding code recall for DeepSeek Harness.
|
|
*
|
|
* One tool (`recall`) over a per-workspace vector index built by a local
|
|
* embedding model, plus a settings namespace so the endpoint is editable at
|
|
* runtime.
|
|
*
|
|
* **The economics this exists for.** In a long agent session the dominant cost
|
|
* is not the model's reasoning; it is re-reading the accumulated transcript on
|
|
* every step. Measured on this harness over 32 days, 65% of spend was re-reading
|
|
* cached context and 96% of that context was message history — mostly file
|
|
* contents the agent had already read. A tool that answers "where is this
|
|
* handled" with a ranked 40-line excerpt instead of four whole files keeps that
|
|
* history small, and the saving compounds on every subsequent step because the
|
|
* context it did not add is never re-sent again.
|
|
*
|
|
* **Why the index is warmed lazily rather than at load.** `apply` must not block
|
|
* on embedding a repository; a plugin that took minutes to activate would make
|
|
* every profile start slow, including the ones that never call the tool. Instead
|
|
* a search does a *bounded* sync — read and hash everything, embed only what
|
|
* fits in a time budget — and reports honestly what is still pending. A full
|
|
* warm-up is available as an explicit background action.
|
|
*
|
|
* **Why there is a settings namespace and no baked endpoint.** The Ollama
|
|
* default of `127.0.0.1:11434` is a convention, not a fact — this deployment
|
|
* runs Ollama on another host. Baking one machine's address into a bundle layer
|
|
* that other machines apply is exactly the mistake the composition docs warn
|
|
* about, so the endpoint is a schema default the profile can override.
|
|
*/
|
|
|
|
import { readFile, stat } from 'node:fs/promises'
|
|
import { createHash } from 'node:crypto'
|
|
import { basename, join, resolve } from 'node:path'
|
|
|
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
// A real dependency rather than a duck-typed `{ parse }`: the projection service
|
|
// calls these schemas, and `dsh-session-projection` declares zod itself, so it is
|
|
// present wherever projections are.
|
|
import { z } from 'zod'
|
|
import Schema from '@deepseek-ai/schemastery'
|
|
|
|
import { chunkFile, embedText } from './lib/chunk.js'
|
|
import { EmbeddingError, createEmbedder } from './lib/embed.js'
|
|
import { DEFAULT_FOCUS_MARKERS, discoverScopes, focusKey, resolveFocus } from './lib/focus.js'
|
|
import { indexFile, resolveDshHome, workspaceStateDir } from './lib/paths.js'
|
|
import { fuseRankings, rankOrder, selectFrom } from './lib/rank.js'
|
|
import { RecallIndex } from './lib/store.js'
|
|
import { createExpander } from './lib/expand.js'
|
|
import { createReranker } from './lib/rerank.js'
|
|
import { deriveUsage, emptyUsage, foldUsage, usageSummary } from './lib/usage.js'
|
|
import { DEFAULT_EXTENSIONS, DEFAULT_SKIP_DIRS, listCodeFiles } from './lib/walk.js'
|
|
|
|
/** Settings namespace. Must match /^[a-z][a-z0-9-]*$/. */
|
|
const SETTINGS_NAMESPACE = 'dsh-recall'
|
|
|
|
/** Every configuration field this plugin exposes. */
|
|
export const SETTING_KEYS = [
|
|
'embeddingApiBase',
|
|
'embeddingModel',
|
|
'embeddingDimensions',
|
|
'embeddingApiKey',
|
|
'embeddingBatchSize',
|
|
'embeddingConcurrency',
|
|
'embeddingTimeoutMs',
|
|
'embeddingKeepAlive',
|
|
'extensions',
|
|
'exclude',
|
|
'maxFiles',
|
|
'maxFileBytes',
|
|
'chunkLines',
|
|
'maxChunkLines',
|
|
'overlapLines',
|
|
'defaultLimit',
|
|
'maxLimit',
|
|
'perFileLimit',
|
|
'excerptLines',
|
|
'totalExcerptLines',
|
|
'minSimilarity',
|
|
'syncBudgetMs',
|
|
'stateDir',
|
|
'focus',
|
|
'focusMarkers',
|
|
'expansionEnabled',
|
|
'expansionModel',
|
|
'expansionCount',
|
|
'expansionTimeoutMs',
|
|
'rerankEnabled',
|
|
'rerankModel',
|
|
'rerankCandidates',
|
|
'rerankTimeoutMs',
|
|
]
|
|
|
|
export const Config = Schema.object({
|
|
/** Embedding endpoint, without a trailing slash. */
|
|
embeddingApiBase: Schema.string().default('http://127.0.0.1:11434'),
|
|
/** Model id as the endpoint knows it. */
|
|
embeddingModel: Schema.string().default('unclemusclez/jina-embeddings-v2-base-code:latest'),
|
|
/** Vector width. Must match the model; a mismatch is refused, not truncated. */
|
|
embeddingDimensions: Schema.number().default(768),
|
|
/** Bearer token for a gateway that wants one. */
|
|
// `role('secret')` keeps this out of every settings response: the wire
|
|
// surfaces strip schema-declared secret positions, so a card can report that a
|
|
// key is set without ever receiving it.
|
|
embeddingApiKey: Schema.string().default('').role('secret'),
|
|
/** Inputs per request on the batch endpoint. */
|
|
embeddingBatchSize: Schema.number().default(16),
|
|
/** Requests in flight. */
|
|
embeddingConcurrency: Schema.number().default(2),
|
|
/**
|
|
* Per-request timeout.
|
|
*
|
|
* Deliberately short. A local endpoint that has wedged does not refuse the
|
|
* connection — it accepts it and never answers, so a long timeout turns a
|
|
* broken backend into a tool call that appears to hang. Thirty seconds is far
|
|
* more than a batch of a few thousand characters needs on any CPU, and it
|
|
* fails in bounded time instead of leaving the caller guessing.
|
|
*/
|
|
embeddingTimeoutMs: Schema.number().default(30_000),
|
|
/** Ollama `keep_alive`. Keeps the model resident across an index run. */
|
|
embeddingKeepAlive: Schema.string().default('5m'),
|
|
/** Extensions to index. Empty means the built-in code set. */
|
|
extensions: Schema.array(String).default([]),
|
|
/** Extra glob patterns to skip, on top of the built-in directory list. */
|
|
exclude: Schema.array(String).default([]),
|
|
/** Most files a workspace walk will consider. */
|
|
maxFiles: Schema.number().default(20_000),
|
|
/** Files larger than this are skipped as generated. */
|
|
maxFileBytes: Schema.number().default(1_048_576),
|
|
/** Target lines per chunk. */
|
|
chunkLines: Schema.number().default(48),
|
|
/** Hard cap before a chunk is split. */
|
|
maxChunkLines: Schema.number().default(80),
|
|
/** Lines shared between the halves of a split chunk. */
|
|
overlapLines: Schema.number().default(8),
|
|
/** Results returned when the caller does not say. */
|
|
defaultLimit: Schema.number().default(8),
|
|
/** Ceiling on the caller's `limit`. */
|
|
maxLimit: Schema.number().default(25),
|
|
/** Most chunks one file may contribute to one result set. */
|
|
perFileLimit: Schema.number().default(2),
|
|
/**
|
|
* Lines shown per excerpt.
|
|
*
|
|
* Defaults to the chunk cap, so a chunk — already a bounded semantic unit —
|
|
* is returned whole. Cutting one in half is how a search turns into a
|
|
* follow-up read that costs more than the lines it saved.
|
|
*/
|
|
excerptLines: Schema.number().default(80),
|
|
/** Total lines across one result set, allocated by rank. */
|
|
totalExcerptLines: Schema.number().default(400),
|
|
/** Below this best-similarity, the result set is reported as weak. */
|
|
minSimilarity: Schema.number().default(0.2),
|
|
/** Embedding time one `search` call may spend warming the index. */
|
|
syncBudgetMs: Schema.number().default(20_000),
|
|
/** Index directory. Empty means `$DSH_HOME/dsh-recall/workspaces/<key>`. */
|
|
stateDir: Schema.string().default(''),
|
|
/**
|
|
* Narrows the index and every search to one directory.
|
|
*
|
|
* Empty resolves by marker: the nearest ancestor of the session's directory
|
|
* carrying a `focusMarkers` file becomes the scope, and the workspace root is
|
|
* the fallback — which is the behaviour that existed before focus did, so
|
|
* adopting this cannot regress a deployment that never configures it.
|
|
*/
|
|
focus: Schema.string().default(''),
|
|
/** Filenames that mark a directory as its own scope. Empty means the built-ins. */
|
|
focusMarkers: Schema.array(String).default([]),
|
|
/**
|
|
* Ask the local model for alternative phrasings and fuse the rankings.
|
|
*
|
|
* Off by default. It costs one local generation plus an embedding per variant,
|
|
* and its benefit has not been measured against a real model — a default that
|
|
* spends latency on an unquantified gain is not a default worth having.
|
|
*/
|
|
expansionEnabled: Schema.boolean().default(false),
|
|
/** Model that writes the phrasings. Empty uses `embeddingModel`'s endpoint, not its model. */
|
|
expansionModel: Schema.string().default(''),
|
|
/** Most variants to ask for. */
|
|
expansionCount: Schema.number().default(4),
|
|
/** Budget for the whole expansion, in ms. Expiry just means search without it. */
|
|
expansionTimeoutMs: Schema.number().default(8_000),
|
|
/**
|
|
* Have the local model score the top candidates against the query.
|
|
*
|
|
* Off by default for the same reason, and more so: it is one more model call
|
|
* on the critical path of every search.
|
|
*/
|
|
rerankEnabled: Schema.boolean().default(false),
|
|
/** Model that scores candidates. */
|
|
rerankModel: Schema.string().default(''),
|
|
/** Candidates scored per search. The reranker caps this at 12. */
|
|
rerankCandidates: Schema.number().default(8),
|
|
/** Budget for the scoring call, in ms. */
|
|
rerankTimeoutMs: Schema.number().default(15_000),
|
|
})
|
|
|
|
/** Resolve the harness home, preferring the harness's own resolver. */
|
|
function dshHomeOf(ctx) {
|
|
try {
|
|
const homePath = ctx.get('dshHomePath')
|
|
if (typeof homePath === 'function') {
|
|
const resolved = homePath()
|
|
if (typeof resolved === 'string' && resolved !== '') return resolved
|
|
}
|
|
} catch {
|
|
// Declared optional; absent outside dsh-app-boot, which is expected.
|
|
}
|
|
return resolveDshHome()
|
|
}
|
|
|
|
/**
|
|
* Constraints the settings schema cannot express.
|
|
*
|
|
* Run by the settings provider when a value is WRITTEN, not when it is next
|
|
* used. That distinction is the point: a bad endpoint or an impossible limit
|
|
* should fail in front of the person who typed it, rather than surfacing later
|
|
* as a failed search with no obvious cause.
|
|
*
|
|
* @param {Record<string, any>} value a schema-valid settings section
|
|
* @throws {Error} when the section cannot be acted on
|
|
*/
|
|
export function validateSettings(value) {
|
|
const base = String(value.embeddingApiBase ?? '').trim()
|
|
let url
|
|
try {
|
|
url = new URL(base)
|
|
} catch {
|
|
throw new Error(`embeddingApiBase is not a URL: ${JSON.stringify(base)}`)
|
|
}
|
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
throw new Error(`embeddingApiBase must be http or https, not ${url.protocol}`)
|
|
}
|
|
if (!(Number(value.embeddingDimensions) > 0)) {
|
|
throw new Error('embeddingDimensions must be a positive number')
|
|
}
|
|
for (const key of [
|
|
'embeddingBatchSize',
|
|
'embeddingConcurrency',
|
|
'embeddingTimeoutMs',
|
|
'maxLimit',
|
|
'defaultLimit',
|
|
'perFileLimit',
|
|
'excerptLines',
|
|
'totalExcerptLines',
|
|
'chunkLines',
|
|
'maxChunkLines',
|
|
'maxFiles',
|
|
'maxFileBytes',
|
|
'expansionCount',
|
|
'expansionTimeoutMs',
|
|
'rerankCandidates',
|
|
'rerankTimeoutMs',
|
|
]) {
|
|
if (!(Number(value[key]) >= 1)) throw new Error(`${key} must be at least 1`)
|
|
}
|
|
if (Number(value.syncBudgetMs) < 0) throw new Error('syncBudgetMs cannot be negative')
|
|
if (Number(value.overlapLines) < 0) throw new Error('overlapLines cannot be negative')
|
|
if (Number(value.maxChunkLines) < Number(value.chunkLines)) {
|
|
throw new Error('maxChunkLines cannot be below chunkLines')
|
|
}
|
|
if (Number(value.overlapLines) >= Number(value.maxChunkLines)) {
|
|
throw new Error('overlapLines must be smaller than maxChunkLines, or a split never advances')
|
|
}
|
|
if (Number(value.defaultLimit) > Number(value.maxLimit)) {
|
|
throw new Error(`defaultLimit (${value.defaultLimit}) cannot exceed maxLimit (${value.maxLimit})`)
|
|
}
|
|
if (Number(value.minSimilarity) < 0 || Number(value.minSimilarity) > 1) {
|
|
throw new Error('minSimilarity is a cosine similarity and must be between 0 and 1')
|
|
}
|
|
}
|
|
|
|
/** SHA-256 of file content, used as the change key. */
|
|
function hashOf(content) {
|
|
return createHash('sha256').update(content).digest('hex')
|
|
}
|
|
|
|
/**
|
|
* Serialise work per key.
|
|
*
|
|
* Two tool calls can arrive while the first is still warming the index, and two
|
|
* concurrent syncs of the same workspace would interleave writes to one SQLite
|
|
* handle. Chaining on a per-workspace promise makes the second wait rather than
|
|
* race, which is what lets `search` stay marked concurrency-safe.
|
|
*/
|
|
function createLock() {
|
|
const chains = new Map()
|
|
return async function withLock(key, work) {
|
|
const previous = chains.get(key) ?? Promise.resolve()
|
|
let release
|
|
const gate = new Promise((resolveGate) => {
|
|
release = resolveGate
|
|
})
|
|
chains.set(
|
|
key,
|
|
previous.then(() => gate),
|
|
)
|
|
await previous
|
|
try {
|
|
return await work()
|
|
} finally {
|
|
release()
|
|
if (chains.get(key) === gate) chains.delete(key)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Schema of the persisted fold.
|
|
*
|
|
* The projection stores state as plain JSON and validates it with `.parse()`
|
|
* before seeding a fold from a checkpoint, so this must accept exactly what
|
|
* `emptyUsage()` produces and what `foldUsage` returns.
|
|
*/
|
|
const usageStateSchema = z.object({
|
|
version: z.number(),
|
|
searches: z.number(),
|
|
indexes: z.number(),
|
|
failed: z.number(),
|
|
excerpts: z.number(),
|
|
returnedBytes: z.number(),
|
|
sourceBytes: z.number(),
|
|
files: z.number(),
|
|
weak: z.number(),
|
|
rereadBytes: z.number(),
|
|
lastFocus: z.string(),
|
|
})
|
|
|
|
/** Schema of the wire view — what the browser receives. */
|
|
const usageViewSchema = z.object({
|
|
searches: z.number(),
|
|
indexes: z.number(),
|
|
failed: z.number(),
|
|
excerpts: z.number(),
|
|
files: z.number(),
|
|
weak: z.number(),
|
|
returnedBytes: z.number(),
|
|
sourceBytes: z.number(),
|
|
avoidedBytes: z.number(),
|
|
avoidedTokens: z.number(),
|
|
compression: z.number(),
|
|
rereadTokens: z.number(),
|
|
avoidedUsd: z.number(),
|
|
lastFocus: z.string(),
|
|
})
|
|
|
|
/**
|
|
* Per-conversation recall usage, folded from the tool metadata in the log.
|
|
*
|
|
* A projection rather than a client-side fold over conversation nodes: nodes
|
|
* carry only the loaded window, so a chip built on them would undercount the
|
|
* moment a session outgrew what the browser had scrolled through. This folds
|
|
* the whole log and is replayed from it.
|
|
*/
|
|
const USAGE_KEY = 'recallUsage'
|
|
|
|
/** Views are compared by reference, so one state must yield one view object. */
|
|
const usageViews = new WeakMap()
|
|
|
|
function usageViewFor(state) {
|
|
let view = usageViews.get(state)
|
|
if (view === undefined) {
|
|
view = usageSummary(state)
|
|
usageViews.set(state, view)
|
|
}
|
|
return view
|
|
}
|
|
|
|
export const usageProjection = {
|
|
key: USAGE_KEY,
|
|
stateSchema: usageStateSchema,
|
|
init: () => emptyUsage(),
|
|
apply: (state, event) => foldUsage(state, event),
|
|
stateVersion: 1,
|
|
wire: { viewSchema: usageViewSchema, view: usageViewFor },
|
|
}
|
|
|
|
export const name = 'dsh-recall'
|
|
export const inject = ['tools']
|
|
|
|
/**
|
|
* Register the tool and the settings namespace.
|
|
*
|
|
* @param {import('@deepseek-ai/cordis').Context} ctx
|
|
* @param {Record<string, any>} config validated composition config
|
|
*/
|
|
export function apply(ctx, config) {
|
|
const logger = ctx.logger('dsh-recall')
|
|
const dshHome = dshHomeOf(ctx)
|
|
const withLock = createLock()
|
|
|
|
// Discovered scopes per workspace, cached for a short window. Discovery is a
|
|
// directory walk on the hot path of every tool call, and re-walking a large
|
|
// repository to learn something that changes only when a marker file is added
|
|
// is work with no benefit.
|
|
const SCOPE_CACHE_MS = 30_000
|
|
const scopeCache = new Map()
|
|
|
|
// Open indexes are filesystem handles, so they are an effect: Cordis closes
|
|
// them on unload and on a config change, and a reload never leaks a handle.
|
|
const openIndexes = new Map()
|
|
ctx.effect(() => () => {
|
|
for (const index of openIndexes.values()) index.close()
|
|
openIndexes.clear()
|
|
})
|
|
|
|
// The authoritative settings source. Starts as the composition config and is
|
|
// replaced by the settings scope's thunk once a provider attaches.
|
|
let readSettings = () => config
|
|
let writeSettings
|
|
|
|
// Per-conversation usage, folded from the tool metadata the log already
|
|
// carries. Optional: a profile without projections still gets the tool, it
|
|
// just does not get a surface that reports what it saved.
|
|
ctx.inject(['sessionProjections'], (projectionCtx) => {
|
|
try {
|
|
projectionCtx.sessionProjections.register(usageProjection)
|
|
} catch (error) {
|
|
logger.warn('dsh-recall: usage projection unavailable: %s', error?.message ?? error)
|
|
}
|
|
})
|
|
|
|
ctx.inject(['settings'], (settingsCtx) => {
|
|
try {
|
|
settingsCtx.settings.installSection(ctx, SETTINGS_NAMESPACE, Config, config, {
|
|
setSource: (current) => {
|
|
readSettings = current
|
|
},
|
|
// Settings changes take effect on the next call: the embedder is built
|
|
// per call, and `recheck` re-validates the stored embedding identity
|
|
// against the new snapshot, so there are no registration-level facts to
|
|
// re-judge here.
|
|
onChange: () => {},
|
|
// Constraints the schema cannot express, refused at WRITE time rather
|
|
// than discovered on the next search.
|
|
validate: validateSettings,
|
|
})
|
|
writeSettings = (patch) => settingsCtx.settings.update(SETTINGS_NAMESPACE, patch)
|
|
} catch (error) {
|
|
logger.warn('dsh-recall: settings namespace unavailable: %s', error?.message ?? error)
|
|
}
|
|
})
|
|
|
|
/** The embedder for a settings snapshot. */
|
|
const embedderFor = (settings) =>
|
|
createEmbedder({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
dimensions: Number(settings.embeddingDimensions),
|
|
apiKey: String(settings.embeddingApiKey ?? ''),
|
|
batchSize: Number(settings.embeddingBatchSize),
|
|
concurrency: Number(settings.embeddingConcurrency),
|
|
timeoutMs: Number(settings.embeddingTimeoutMs),
|
|
keepAlive: String(settings.embeddingKeepAlive ?? ''),
|
|
})
|
|
|
|
/**
|
|
* A one-shot embedder for probing the endpoint.
|
|
*
|
|
* No retries and a short timeout, because a probe exists to answer a question
|
|
* quickly: `status` should report a wedged backend, not wait for one. Three
|
|
* retries against a socket that accepts and never replies would take longer
|
|
* than the tool call that provoked it.
|
|
*/
|
|
const proberFor = (settings) =>
|
|
createEmbedder({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
dimensions: Number(settings.embeddingDimensions),
|
|
apiKey: String(settings.embeddingApiKey ?? ''),
|
|
batchSize: 1,
|
|
concurrency: 1,
|
|
timeoutMs: Math.min(Number(settings.embeddingTimeoutMs), 10_000),
|
|
maxAttempts: 1,
|
|
keepAlive: String(settings.embeddingKeepAlive ?? ''),
|
|
})
|
|
|
|
/**
|
|
* The index fields the tool's output schema declares.
|
|
*
|
|
* Built by picking rather than by spreading the sync result, because that
|
|
* result also carries internal fields (`stale`, `chunksWritten`) which are not
|
|
* part of the model-facing contract — and the schema is
|
|
* `additionalProperties: false`, so spreading one in is a validation failure
|
|
* rather than a harmless extra.
|
|
*/
|
|
function indexView(current, extra = {}) {
|
|
const out = { ...current }
|
|
const fields = [
|
|
'files',
|
|
'chunks',
|
|
'pending',
|
|
'scanned',
|
|
'embedded',
|
|
'removed',
|
|
'oversized',
|
|
'unreadable',
|
|
'truncated',
|
|
]
|
|
for (const key of fields) if (extra[key] !== undefined) out[key] = extra[key]
|
|
return out
|
|
}
|
|
|
|
/**
|
|
* Discovered scopes for a workspace, cached briefly.
|
|
*
|
|
* Discovery is a directory walk, not a file read, so it is cheap — but it runs
|
|
* on every tool call and a large repository has a lot of directories. A short
|
|
* cache keeps the cost off the hot path while still noticing a marker file
|
|
* added a moment ago.
|
|
*
|
|
* @param {string} workspace absolute workspace root
|
|
* @param {readonly string[]} markers marker filenames
|
|
*/
|
|
async function scopesFor(workspace, markers) {
|
|
const key = `${workspace}\u0000${markers.join(',')}`
|
|
const hit = scopeCache.get(key)
|
|
if (hit !== undefined && Date.now() - hit.at < SCOPE_CACHE_MS) return hit.scopes
|
|
let scopes = []
|
|
try {
|
|
scopes = await discoverScopes({ workspace, markers })
|
|
} catch {
|
|
// A workspace that cannot be walked simply has no discoverable scopes;
|
|
// resolution still falls back to markers and then the workspace.
|
|
scopes = []
|
|
}
|
|
cacheScopes(key, scopes)
|
|
return scopes
|
|
}
|
|
|
|
/** Replace the scope cache, dropping entries for workspaces that are gone. */
|
|
function cacheScopes(key, scopes) {
|
|
scopeCache.set(key, { at: Date.now(), scopes })
|
|
if (scopeCache.size > 32) {
|
|
for (const [existing, entry] of scopeCache) {
|
|
if (Date.now() - entry.at > SCOPE_CACHE_MS * 4) scopeCache.delete(existing)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Open (or reuse) the index for one workspace and focus.
|
|
*
|
|
* @param {Record<string, any>} settings resolved settings
|
|
* @param {string} cwd workspace root
|
|
* @param {string} focus focus key, `.` for the whole workspace
|
|
*/
|
|
function indexFor(settings, cwd, focus = '.') {
|
|
const absolute = (path) => join(resolve(path), focus === '.' ? 'index.sqlite' : join('focus', focus, 'index.sqlite'))
|
|
const file =
|
|
typeof settings.stateDir === 'string' && settings.stateDir.trim() !== ''
|
|
? absolute(settings.stateDir.trim())
|
|
: indexFile(dshHome, cwd, focus)
|
|
let index = openIndexes.get(file)
|
|
if (index === undefined) {
|
|
index = new RecallIndex(file)
|
|
openIndexes.set(file, index)
|
|
}
|
|
const identity = {
|
|
model: String(settings.embeddingModel),
|
|
dimensions: String(settings.embeddingDimensions),
|
|
apiBase: String(settings.embeddingApiBase),
|
|
}
|
|
// `open` is idempotent for an already-open handle; the identity check is the
|
|
// part that must run every time, because the settings can change under it.
|
|
const outcome = index.db === undefined ? index.open(identity) : index.recheck(identity)
|
|
return { index, file, discarded: outcome.discarded }
|
|
}
|
|
|
|
/**
|
|
* Bring one workspace's index up to date.
|
|
*
|
|
* Phase one reads and hashes every candidate — fast, bounded by disk. Phase
|
|
* two embeds what changed, and is the only part given a deadline. Splitting
|
|
* them is what makes the reported state honest: after phase one the plugin
|
|
* knows exactly what is stale, so a partial index is described rather than
|
|
* guessed at.
|
|
*/
|
|
async function syncWorkspace(input) {
|
|
const { root, prefix = '', index, settings, budgetMs, signal, onProgress } = input
|
|
const walk = await listCodeFiles(root, {
|
|
...(settings.extensions?.length ? { extensions: settings.extensions } : {}),
|
|
skipDirs: DEFAULT_SKIP_DIRS,
|
|
exclude: settings.exclude ?? [],
|
|
maxFiles: Number(settings.maxFiles),
|
|
signal,
|
|
})
|
|
// `listCodeFiles` reports paths relative to the walk root, which under a
|
|
// focus is the focus directory. They are rewritten to be relative to the
|
|
// WORKSPACE, because that is the path the model will hand to `read` — a
|
|
// focus-relative result would silently read the wrong file.
|
|
if (prefix !== '') {
|
|
for (const file of walk.files) file.path = `${prefix}/${file.path}`
|
|
}
|
|
|
|
const known = index.fileHashes()
|
|
const present = new Set(walk.files.map((file) => file.path))
|
|
const removed = index.prune([...known.keys()].filter((path) => !present.has(path)))
|
|
|
|
const maxBytes = Number(settings.maxFileBytes)
|
|
const pending = []
|
|
let oversized = 0
|
|
let unreadable = 0
|
|
|
|
for (const file of walk.files) {
|
|
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
let content
|
|
try {
|
|
const info = await stat(file.absolute)
|
|
if (info.size > maxBytes) {
|
|
oversized += 1
|
|
continue
|
|
}
|
|
content = await readFile(file.absolute, 'utf8')
|
|
} catch {
|
|
unreadable += 1
|
|
continue
|
|
}
|
|
const hash = hashOf(content)
|
|
if (known.get(file.path) === hash) continue
|
|
pending.push({ ...file, content, hash, bytes: Buffer.byteLength(content) })
|
|
}
|
|
|
|
// Anything still holding an old hash is stale until it is re-embedded.
|
|
const stale = new Set(pending.map((file) => file.path))
|
|
const deadline = Date.now() + budgetMs
|
|
let embedded = 0
|
|
let chunksWritten = 0
|
|
|
|
for (const file of pending) {
|
|
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
// Always embed at least one file, or a zero budget would make the feature
|
|
// indistinguishable from a broken index.
|
|
if (embedded > 0 && Date.now() >= deadline) break
|
|
const { chunks } = chunkFile({
|
|
path: file.path,
|
|
content: file.content,
|
|
chunkLines: Number(settings.chunkLines),
|
|
maxChunkLines: Number(settings.maxChunkLines),
|
|
overlapLines: Number(settings.overlapLines),
|
|
})
|
|
if (chunks.length === 0) {
|
|
// An empty or generated file still gets a row, so it is not re-read and
|
|
// re-chunked on every subsequent search.
|
|
index.replaceFile(file.path, file.hash, file.bytes, [])
|
|
embedded += 1
|
|
stale.delete(file.path)
|
|
onProgress?.(embedded, pending.length)
|
|
continue
|
|
}
|
|
const vectors = await input.embedder.embed(
|
|
chunks.map((chunk) => embedText(file.path, chunk)),
|
|
signal,
|
|
)
|
|
index.replaceFile(
|
|
file.path,
|
|
file.hash,
|
|
file.bytes,
|
|
chunks.map((chunk, at) => ({ ...chunk, vec: vectors[at] })),
|
|
)
|
|
chunksWritten += chunks.length
|
|
embedded += 1
|
|
stale.delete(file.path)
|
|
onProgress?.(embedded, pending.length)
|
|
}
|
|
|
|
const stats = index.stats()
|
|
return {
|
|
...stats,
|
|
scanned: walk.files.length,
|
|
truncated: walk.truncated,
|
|
removed,
|
|
embedded,
|
|
chunksWritten,
|
|
pending: pending.length - embedded,
|
|
stale: [...stale],
|
|
oversized,
|
|
unreadable,
|
|
}
|
|
}
|
|
|
|
const TOOL_OUTPUT = {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
action: { type: 'string', required: true },
|
|
ok: { type: 'boolean', required: true },
|
|
query: { type: 'string' },
|
|
workspace: { type: 'string' },
|
|
focus: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
path: { type: 'string', required: true },
|
|
source: { type: 'string', required: true },
|
|
marker: { type: 'string' },
|
|
problem: { type: 'string' },
|
|
},
|
|
},
|
|
results: {
|
|
type: 'array',
|
|
items: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
path: { type: 'string', required: true },
|
|
startLine: { type: 'integer', required: true },
|
|
endLine: { type: 'integer', required: true },
|
|
symbol: { type: 'string' },
|
|
similarity: { type: 'number', required: true },
|
|
excerpt: { type: 'string', required: true },
|
|
truncated: { type: 'boolean', required: true },
|
|
stale: { type: 'boolean', required: true },
|
|
},
|
|
},
|
|
},
|
|
index: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
files: { type: 'integer' },
|
|
chunks: { type: 'integer' },
|
|
pending: { type: 'integer' },
|
|
scanned: { type: 'integer' },
|
|
embedded: { type: 'integer' },
|
|
removed: { type: 'integer' },
|
|
oversized: { type: 'integer' },
|
|
unreadable: { type: 'integer' },
|
|
truncated: { type: 'boolean' },
|
|
database: { type: 'string' },
|
|
stateDir: { type: 'string' },
|
|
},
|
|
},
|
|
best: { type: 'number' },
|
|
endpoint: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
ok: { type: 'boolean', required: true },
|
|
url: { type: 'string', required: true },
|
|
model: { type: 'string', required: true },
|
|
errorCode: { type: 'string' },
|
|
message: { type: 'string' },
|
|
},
|
|
},
|
|
weak: { type: 'boolean' },
|
|
/**
|
|
* What this call returned, against what reading those files would have
|
|
* cost. `sourceBytes` is an upper bound and is labelled as one wherever
|
|
* it is shown — a careful reader might have opened one file, not four.
|
|
*/
|
|
measure: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
excerpts: { type: 'integer', required: true },
|
|
returnedBytes: { type: 'integer', required: true },
|
|
sourceBytes: { type: 'integer', required: true },
|
|
files: { type: 'integer', required: true },
|
|
},
|
|
},
|
|
jobId: { type: 'string' },
|
|
/** Every scope discovered in this workspace, so a caller can pick one. */
|
|
scopes: {
|
|
type: 'array',
|
|
items: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
properties: {
|
|
path: { type: 'string', required: true },
|
|
marker: { type: 'string', required: true },
|
|
},
|
|
},
|
|
},
|
|
note: { type: 'string' },
|
|
errorCode: { type: 'string' },
|
|
remedy: { type: 'string' },
|
|
},
|
|
}
|
|
|
|
/** Render one search result set for the model. */
|
|
function renderResults(value) {
|
|
if (value.jobId !== undefined) {
|
|
return [
|
|
{
|
|
type: 'text',
|
|
text:
|
|
`Indexing ${value.workspace} as background job ${value.jobId}. You will be notified when it settles; ` +
|
|
`collect it with job_output for "${value.jobId}" (add wait: true if you are blocked on it). ` +
|
|
'Searches work meanwhile against whatever is already indexed.',
|
|
},
|
|
]
|
|
}
|
|
if (value.ok !== true) {
|
|
const parts = [value.note ?? `recall ${value.action} failed.`]
|
|
if (value.remedy !== undefined) parts.push(`Next: ${value.remedy}`)
|
|
return [{ type: 'text', text: parts.join('\n\n') }]
|
|
}
|
|
if (value.action === 'configure') {
|
|
return [{ type: 'text', text: value.note ?? 'Configuration updated.' }]
|
|
}
|
|
if (value.action !== 'search') {
|
|
const { files = 0, chunks = 0, pending = 0, scanned = 0, embedded = 0 } = value.index ?? {}
|
|
return [
|
|
{
|
|
type: 'text',
|
|
text:
|
|
`Index for ${value.workspace}: ${files} files, ${chunks} chunks. ` +
|
|
`Scanned ${scanned}, embedded ${embedded} this call, ${pending} still pending.`,
|
|
},
|
|
]
|
|
}
|
|
|
|
const results = value.results ?? []
|
|
if (results.length === 0) {
|
|
const suffix = (value.index?.pending ?? 0) > 0 ? ` ${value.index.pending} file(s) are still unindexed.` : ''
|
|
return [
|
|
{
|
|
type: 'text',
|
|
text:
|
|
`No match for ${JSON.stringify(value.query)}.${suffix} ` +
|
|
'If you know the exact identifier, grep is the better tool for this query.',
|
|
},
|
|
]
|
|
}
|
|
|
|
const scope = value.focus?.path !== undefined && value.focus.path !== '.' ? ` in ${value.focus.path}` : ''
|
|
const header =
|
|
`${results.length} excerpt(s) for ${JSON.stringify(value.query)}${scope} — ` +
|
|
`from ${value.index?.chunks ?? 0} indexed chunks` +
|
|
((value.index?.pending ?? 0) > 0 ? `, ${value.index.pending} file(s) still unindexed` : '')
|
|
const body = results.map((result, at) => {
|
|
const symbol = result.symbol === undefined ? '' : ` (${result.symbol})`
|
|
const stale = result.stale ? ' [stale — file changed since indexing]' : ''
|
|
const cut = result.truncated ? '\n… excerpt truncated; read the file for the rest' : ''
|
|
return (
|
|
`--- ${at + 1}. ${result.path}:${result.startLine}-${result.endLine}${symbol} ` +
|
|
`similarity ${result.similarity}${stale}\n${result.excerpt}${cut}`
|
|
)
|
|
})
|
|
// A scope is a window, not a filter on relevance. When it holds nothing
|
|
// strong, the most likely reason is that the answer is shared code the scope
|
|
// excludes — 41.6% of the workspace this was measured on — so say that
|
|
// rather than leaving the caller to conclude the code does not exist.
|
|
const outside =
|
|
value.focus?.path !== undefined && value.focus.path !== '.'
|
|
? ` Only ${value.focus.path} is indexed here; shared code outside it is not. Retry without focus to search the whole workspace.`
|
|
: ''
|
|
const advice = value.weak
|
|
? `\n\nNo strong match. Prefer grep for a known identifier, or rephrase with the name of the thing.${outside}`
|
|
: ''
|
|
return [{ type: 'text', text: `${header}\n\n${body.join('\n\n')}${advice}` }]
|
|
}
|
|
|
|
ctx.tools.register(
|
|
defineTool({
|
|
name: 'recall',
|
|
description:
|
|
'Search this workspace semantically with a local embedding model and return ranked code excerpts ' +
|
|
'with file paths and line ranges — without reading whole files into the conversation. Use it when ' +
|
|
'you do not know the exact identifier: "where is X handled", "which module does Y", "how does Z ' +
|
|
'recover from failure". Prefer grep when you already know the exact symbol, and read when you need ' +
|
|
'the whole file. Because excerpts are returned inline, do not follow up with a read of the same ' +
|
|
'range. Actions: "search" (default) answers a query and warms a bounded slice of the index; ' +
|
|
'"index" builds or refreshes the whole index, as a background job by default; "status" reports the ' +
|
|
'index and endpoint state. The index is incremental and lives outside your repository.',
|
|
parameters: {
|
|
query: {
|
|
type: 'string',
|
|
description: 'What to look for, in words. A description of behaviour works as well as a name.',
|
|
},
|
|
action: {
|
|
type: 'string',
|
|
enum: ['search', 'index', 'status', 'configure'],
|
|
description: 'What to do. Defaults to "search".',
|
|
},
|
|
limit: {
|
|
type: 'integer',
|
|
description: 'Most excerpts to return. Defaults to the configured limit (8).',
|
|
},
|
|
path: {
|
|
type: 'string',
|
|
description:
|
|
'Restrict the search to files under this workspace-relative directory, for example "src/services".',
|
|
},
|
|
focus: {
|
|
type: 'string',
|
|
description:
|
|
'Scope this whole call — index and search — to one workspace-relative directory, for example ' +
|
|
'"src/controllers/room/occupancy". Use it when the session is working on one module: the index ' +
|
|
'then holds only that module, so its near-identical neighbours cannot outrank it and the first ' +
|
|
'search does not wait on the rest of the repository. Empty uses the configured focus, then the ' +
|
|
'nearest enclosing directory that carries a CONTROLLER.md, DEBUG_UI.md or AGENTS.md.',
|
|
},
|
|
run_in_background: {
|
|
type: 'boolean',
|
|
description:
|
|
'For action "index" only: return a job id instead of the result. Defaults to true, because a ' +
|
|
'cold index of a large repository takes minutes.',
|
|
},
|
|
config: {
|
|
type: 'json',
|
|
description:
|
|
'Settings patch for action "configure", using this plugin\'s own camelCase field names, for ' +
|
|
'example {"embeddingApiBase":"http://10.50.1.2:11434"}.',
|
|
},
|
|
},
|
|
output: {
|
|
schema: TOOL_OUTPUT,
|
|
render: (_args, value) => renderResults(value),
|
|
// The harness persists this verbatim as the `meta` of the `tool/result`
|
|
// event, which is what puts per-call usage in the session log — and
|
|
// therefore in reach of a projection the browser can read. Pure, and it
|
|
// carries totals rather than content: it is appended on every call and
|
|
// replayed on every projection rebuild.
|
|
presentationMeta: (args, value) => deriveUsage(args, value),
|
|
},
|
|
isConcurrencySafe: (args) => (args.action ?? 'search') !== 'index',
|
|
async execute(args, exec) {
|
|
const settings = readSettings() ?? config
|
|
const cwd = exec.agent?.session.header.cwd ?? process.cwd()
|
|
const action = args.action ?? 'search'
|
|
const workspace = basename(cwd) || cwd
|
|
const embedder = embedderFor(settings)
|
|
|
|
// Resolve the scope before anything else, because it decides which index
|
|
// is opened. A caller's `focus` outranks the configured one, so a session
|
|
// can name a controller without changing the deployment's default.
|
|
const markers = settings.focusMarkers?.length ? settings.focusMarkers : DEFAULT_FOCUS_MARKERS
|
|
const scopes = await scopesFor(cwd, markers)
|
|
const focus = resolveFocus({
|
|
workspace: cwd,
|
|
cwd,
|
|
explicit: args.focus ?? settings.focus ?? '',
|
|
markers,
|
|
scopes,
|
|
})
|
|
const focusPath = focusKey(cwd, focus.root)
|
|
|
|
let opened
|
|
try {
|
|
opened = indexFor(settings, cwd, focusPath)
|
|
} catch (error) {
|
|
return {
|
|
action,
|
|
ok: false,
|
|
workspace,
|
|
errorCode: 'INDEX_UNAVAILABLE',
|
|
note: `Could not open the index: ${error?.message ?? error}`,
|
|
remedy: `Check that ${dshHome} is writable.`,
|
|
}
|
|
}
|
|
const { index, file, discarded } = opened
|
|
const stateDir = workspaceStateDir(dshHome, cwd)
|
|
const focusView = {
|
|
path: focusPath,
|
|
source: focus.source,
|
|
...(focus.marker === undefined ? {} : { marker: focus.marker }),
|
|
...(focus.problem === undefined ? {} : { problem: focus.problem }),
|
|
}
|
|
const base = { action, workspace, focus: focusView, index: { database: file, stateDir } }
|
|
|
|
if (action === 'status') {
|
|
const stats = index.stats()
|
|
|
|
// Probe the endpoint rather than merely naming it. A wedged backend
|
|
// accepts the connection and never answers, so "configured" and
|
|
// "working" are different claims and status should make the second
|
|
// one. One attempt, short timeout — a probe that waits is not a probe.
|
|
let endpoint
|
|
try {
|
|
await proberFor(settings).embedOne('dsh-recall endpoint probe', exec.signal)
|
|
endpoint = {
|
|
ok: true,
|
|
url: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
}
|
|
} catch (error) {
|
|
endpoint = {
|
|
ok: false,
|
|
url: String(settings.embeddingApiBase),
|
|
model: String(settings.embeddingModel),
|
|
errorCode: error instanceof EmbeddingError ? error.code : 'PROBE_FAILED',
|
|
message: error?.message ?? String(error),
|
|
}
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
ok: true,
|
|
index: indexView(base.index, stats),
|
|
endpoint,
|
|
scopes: scopes.map((entry) => ({ path: entry.path, marker: entry.marker })),
|
|
note:
|
|
`Index ${file} holds ${stats.files} files and ${stats.chunks} chunks, scoped to ` +
|
|
`${focusPath === '.' ? 'the whole workspace' : focusPath} (${focus.source}` +
|
|
`${focus.marker === undefined ? '' : `: ${focus.marker}`}). ` +
|
|
(focus.problem === undefined ? '' : `Requested focus was ignored — ${focus.problem}. `) +
|
|
// A workspace with discoverable scopes can be scoped by naming one,
|
|
// so listing them is the difference between a feature that exists
|
|
// and one a caller can actually use.
|
|
(scopes.length === 0
|
|
? 'No focusable scopes were found; pass focus to scope a call explicitly. '
|
|
: `Available scopes: ${scopes
|
|
.slice(0, 12)
|
|
.map((entry) => entry.path)
|
|
.join(', ')}${scopes.length > 12 ? ` (+${scopes.length - 12} more)` : ''}. `) +
|
|
`Embeddings come from "${settings.embeddingModel}" (${settings.embeddingDimensions} dims) at ` +
|
|
`${settings.embeddingApiBase} — ` +
|
|
(endpoint.ok === true
|
|
? 'endpoint answered.'
|
|
: `endpoint did NOT answer (${endpoint.errorCode}). Run \`ollama ps\` on that host; a ` +
|
|
'model can be evicted, and an endpoint holding another model resident forever can ' +
|
|
'leave no room to load this one.') +
|
|
(discarded === undefined ? '' : ` The index was rebuilt because ${discarded}.`),
|
|
}
|
|
}
|
|
|
|
if (action === 'configure') {
|
|
const patch = args.config
|
|
if (patch === undefined || patch === null || typeof patch !== 'object' || Array.isArray(patch)) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'BAD_PATCH',
|
|
note: 'The `config` parameter must be an object.',
|
|
}
|
|
}
|
|
const unknown = Object.keys(patch).filter((key) => !SETTING_KEYS.includes(key))
|
|
if (unknown.length > 0) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'UNKNOWN_SETTING',
|
|
note: `Unknown setting(s): ${unknown.join(', ')}.`,
|
|
remedy: `Known settings: ${SETTING_KEYS.join(', ')}.`,
|
|
}
|
|
}
|
|
if (writeSettings === undefined) {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'NO_SETTINGS_PROVIDER',
|
|
note: 'No settings provider is mounted, so configuration cannot be persisted at runtime.',
|
|
remedy: `Set these values in the dsh-recall row of your cordis layer instead: ${JSON.stringify(patch)}`,
|
|
}
|
|
}
|
|
await writeSettings(patch)
|
|
return { ...base, ok: true, note: `Updated ${Object.keys(patch).join(', ')}.` }
|
|
}
|
|
|
|
// Full warm-up. Deliberately a background job by default: on a large
|
|
// repository this is minutes of work, and blocking a turn on it is the
|
|
// behaviour that makes a useful tool feel broken.
|
|
if (action === 'index') {
|
|
const jobs = ctx.get('jobs')
|
|
const wantsBackground = args.run_in_background ?? true
|
|
const runIndex = (signal) =>
|
|
withLock(file, () =>
|
|
syncWorkspace({
|
|
root: focus.root,
|
|
prefix: focusPath === '.' ? '' : focusPath,
|
|
index,
|
|
embedder,
|
|
settings,
|
|
budgetMs: Number.POSITIVE_INFINITY,
|
|
signal,
|
|
}),
|
|
)
|
|
|
|
if (wantsBackground && jobs !== undefined) {
|
|
const jobId = jobs.start({
|
|
kind: 'recall-index',
|
|
label: `recall index ${workspace}`,
|
|
...(exec.agent === undefined ? {} : { owner: exec.agent }),
|
|
run: () => {
|
|
const controller = new AbortController()
|
|
const done = (async () => {
|
|
const result = await runIndex(controller.signal)
|
|
return {
|
|
status: 'completed',
|
|
detail: `${result.files} files, ${result.chunks} chunks`,
|
|
output:
|
|
`Indexed ${result.embedded} file(s) into ${result.chunks} chunks (${result.files} files ` +
|
|
`total, ${result.removed} removed). Database: ${file}`,
|
|
}
|
|
})()
|
|
return { cancel: () => controller.abort(), done }
|
|
},
|
|
})
|
|
return { ...base, ok: true, jobId }
|
|
}
|
|
|
|
const result = await runIndex(exec.signal)
|
|
return { ...base, ok: true, index: indexView(base.index, result) }
|
|
}
|
|
|
|
// search
|
|
const query = typeof args.query === 'string' ? args.query.trim() : ''
|
|
if (query === '') {
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
errorCode: 'NO_QUERY',
|
|
note: 'A search needs a query.',
|
|
remedy: 'Call recall with a `query` describing what you are looking for.',
|
|
}
|
|
}
|
|
|
|
const limit = Math.max(
|
|
1,
|
|
Math.min(Number(args.limit ?? settings.defaultLimit), Number(settings.maxLimit)),
|
|
)
|
|
|
|
try {
|
|
const sync = await withLock(file, () =>
|
|
syncWorkspace({
|
|
root: focus.root,
|
|
prefix: focusPath === '.' ? '' : focusPath,
|
|
index,
|
|
embedder,
|
|
settings,
|
|
budgetMs: Math.max(0, Number(settings.syncBudgetMs)),
|
|
signal: exec.signal,
|
|
}),
|
|
)
|
|
const matrix = index.matrix()
|
|
if (matrix.rows.length === 0) {
|
|
return {
|
|
...base,
|
|
ok: true,
|
|
query,
|
|
results: [],
|
|
weak: true,
|
|
index: indexView(base.index, sync),
|
|
note:
|
|
'Nothing is indexed yet, so there is nothing to search. ' +
|
|
(sync.pending > 0
|
|
? `${sync.pending} file(s) were found but the per-call embedding budget was reached.`
|
|
: 'No indexable code was found in this workspace.'),
|
|
...(sync.pending > 0
|
|
? {
|
|
remedy:
|
|
'Run recall with action "index" to build the whole index in the background.',
|
|
}
|
|
: {}),
|
|
}
|
|
}
|
|
|
|
// Query expansion: rank every phrasing and fuse the orders. The
|
|
// expander never throws — on a slow or absent local model it returns
|
|
// the original query alone, and this path is unchanged.
|
|
const expansionModel = String(settings.expansionModel ?? '').trim()
|
|
const queries = settings.expansionEnabled === true
|
|
? await createExpander({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: expansionModel === '' ? String(settings.embeddingModel) : expansionModel,
|
|
timeoutMs: Number(settings.expansionTimeoutMs),
|
|
count: Number(settings.expansionCount),
|
|
}).expand(query, exec.signal)
|
|
: [query]
|
|
|
|
const orders = []
|
|
let similarity
|
|
let vectorBest = 0
|
|
for (const variant of queries) {
|
|
const vector = await embedder.embedOne(variant, exec.signal)
|
|
const ranked = rankOrder({ matrix, queryVector: vector, query: variant })
|
|
orders.push(ranked.order)
|
|
// The first variant is the caller's own query, so its similarities
|
|
// are the honest ones to report against each result.
|
|
if (similarity === undefined) {
|
|
similarity = ranked.similarity
|
|
vectorBest = ranked.vectorBest
|
|
}
|
|
}
|
|
|
|
let results = selectFrom(fuseRankings(orders), {
|
|
matrix,
|
|
similarity,
|
|
limit,
|
|
perFile: Number(settings.perFileLimit),
|
|
excerptLines: Number(settings.excerptLines),
|
|
totalLines: Number(settings.totalExcerptLines),
|
|
})
|
|
|
|
// Local re-ranking: one call scores the head of the list. `undefined`
|
|
// means the model had no opinion, which is not the same as "all
|
|
// irrelevant" — the fused order stands unchanged.
|
|
const rerankModel = String(settings.rerankModel ?? '').trim()
|
|
const rerankCount = Math.min(results.length, Number(settings.rerankCandidates), 12)
|
|
if (settings.rerankEnabled === true && rerankCount > 1) {
|
|
const head = results.slice(0, rerankCount)
|
|
const localScores = await createReranker({
|
|
apiBase: String(settings.embeddingApiBase),
|
|
model: rerankModel === '' ? String(settings.embeddingModel) : rerankModel,
|
|
timeoutMs: Number(settings.rerankTimeoutMs),
|
|
}).score(query, head.map((result) => result.excerpt), exec.signal)
|
|
if (localScores !== undefined && localScores.length === head.length) {
|
|
const blended = head
|
|
.map((result, at) => ({ result, score: 0.4 * result.similarity + 0.6 * localScores[at] }))
|
|
.sort((left, right) => right.score - left.score)
|
|
.map((entry) => entry.result)
|
|
results = [...blended, ...results.slice(rerankCount)]
|
|
}
|
|
}
|
|
|
|
if (typeof args.path === 'string' && args.path.trim() !== '') {
|
|
const prefix = args.path.trim().replace(/^\.\//, '').replace(/\/+$/, '')
|
|
results = results.filter(
|
|
(result) => result.path === prefix || result.path.startsWith(`${prefix}/`),
|
|
)
|
|
}
|
|
|
|
const stale = new Set(sync.stale ?? [])
|
|
const best = results.length === 0 ? 0 : Math.max(...results.map((result) => result.similarity))
|
|
|
|
// The alternative to these excerpts: the whole of each file they came
|
|
// from. Sizes come from the index, which recorded them at embed time,
|
|
// so this costs nothing here — and it is an upper bound, because a
|
|
// careful reader might have opened one of the four rather than all.
|
|
const sourcePaths = [...new Set(results.map((result) => result.path))]
|
|
const sourceSizes = index.bytesFor(sourcePaths)
|
|
const sourceBytes = [...sourceSizes.values()].reduce((sum, size) => sum + size, 0)
|
|
const returnedBytes = results.reduce(
|
|
(sum, result) => sum + Buffer.byteLength(result.excerpt, 'utf8'),
|
|
0,
|
|
)
|
|
const measure = {
|
|
excerpts: results.length,
|
|
returnedBytes,
|
|
sourceBytes,
|
|
files: sourceSizes.size,
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
ok: true,
|
|
query,
|
|
// Built field by field rather than spread: the ranker also returns
|
|
// a fused RRF `score`, which is an internal ordering artefact and not
|
|
// part of the declared result shape.
|
|
results: results.map((result) => ({
|
|
path: result.path,
|
|
startLine: result.startLine,
|
|
endLine: result.endLine,
|
|
...(result.symbol === undefined ? {} : { symbol: result.symbol }),
|
|
similarity: result.similarity,
|
|
excerpt: result.excerpt,
|
|
truncated: result.truncated,
|
|
stale: stale.has(result.path),
|
|
})),
|
|
index: indexView(base.index, sync),
|
|
best,
|
|
weak: results.length === 0 || best < Number(settings.minSimilarity),
|
|
measure,
|
|
}
|
|
} catch (error) {
|
|
const code = error instanceof EmbeddingError ? error.code : 'RECALL_FAILED'
|
|
const remedy =
|
|
code === 'UNREACHABLE' || error?.cause?.code === 'ECONNREFUSED'
|
|
? `Is the embedding endpoint at ${settings.embeddingApiBase} running and reachable? Set ` +
|
|
'`embeddingApiBase` through the plugin settings if it moved.'
|
|
: code === 'DIMENSION_MISMATCH'
|
|
? `Set \`embeddingDimensions\` to match "${settings.embeddingModel}".`
|
|
: undefined
|
|
logger.warn('recall search failed: %s', error?.message ?? error)
|
|
return {
|
|
...base,
|
|
ok: false,
|
|
query,
|
|
errorCode: code,
|
|
note: error?.message ?? String(error),
|
|
...(remedy === undefined ? {} : { remedy }),
|
|
}
|
|
}
|
|
},
|
|
}),
|
|
)
|
|
}
|