Not-Expected-Yet/dsh-recall
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

feat: index extra source roots beside the workspace, for the code you write against

Browse Source 
The workspace index cannot answer questions about an installed framework or
SDK, and in plugin development that is where most of the reading goes: 64 of
80 tool calls in one real session went to the harness checkout under
/usr/local/lib/node_modules to look up the API being implemented.

A root is entered whatever it is called, and its dependencies are not: a
path ending in node_modules/... is the normal way to name a package tree,
while a node_modules below the root is somebody else's code. A result
carries the label that says which tree it came from and an absolute
sourcePath, the only form a follow-up read can resolve.

The tool description now also claims the move this replaces - searching
before reading files to find out what a module contains.
This commit is contained in:
NotExpectedYet committed 2026-09-27 07:16:21 +01:00
1 parent bccd112ce5
commit de4d020dcb
8 files changed
+873 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

index.js
+199 -15
View File
@@ -41,7 +41,7 @@
import { readFile, stat } from 'node:fs/promises'
import { createHash } from 'node:crypto'
import { basename, join, resolve } from 'node:path'
import { basename, join, resolve, sep } from 'node:path'
import { defineTool } from '@deepseek-ai/dsh-tools'
// A real dependency rather than a duck-typed `{ parse }`: the projection service
@@ -55,6 +55,7 @@ import { EmbeddingError, createEmbedder } from './lib/embed.js'
import { DEFAULT_FOCUS_MARKERS, discoverScopes, focusKey, resolveFocus } from './lib/focus.js'
import { indexFile, resolveDshHome, workspaceStateDir } from './lib/paths.js'
import { fuseRankings, rankOrder, selectFrom } from './lib/rank.js'
import { listRootFiles, resolveSourceRoots } from './lib/roots.js'
import { RecallIndex } from './lib/store.js'
import { createExpander } from './lib/expand.js'
import { createReranker } from './lib/rerank.js'
@@ -100,6 +101,7 @@ export const SETTING_KEYS = [
'stateDir',
'focus',
'focusMarkers',
'sourceRoots',
'expansionEnabled',
'expansionModel',
'expansionCount',
@@ -209,6 +211,38 @@ export const Config = Schema.object({
focus: Schema.string().default('').volatile(),
/** Filenames that mark a directory as its own scope. Empty means the built-ins. */
focusMarkers: Schema.array(String).default([]).volatile(),
/**
* Extra trees to index beside the workspace, for code that is not in it.
*
* The workspace index cannot answer questions about the code you are writing
* *against* — an installed framework, a platform SDK, a sibling checkout. In
* plugin development that is where most of the reading goes: the harness's own
* packages are the only authoritative copy of the API being implemented, and
* they live outside the workspace.
*
* Each entry is a path, optionally prefixed with `name=`:
*
* - an **absolute** path is taken as given — what a deployment's installed
* tree needs;
* - a **relative** path resolves against the workspace, which is what makes a
* checked-in configuration portable;
* - the **name** defaults to the directory's basename and becomes the first
* segment of every result path from that root (`@deepseek-ai/dsh-agent/…`),
* so a result says which tree it came from. Use `name=` when the basename is
* long or ambiguous.
*
* The root's *contents* are indexed, never its dependencies: `node_modules`,
* build output and VCS directories are skipped below the root while the root
* itself is always entered — which is what lets a root be the scoped-packages
* directory of a package manager. An unusable entry is reported in the result
* rather than being fatal, so one typo cannot cost the other roots.
*
* A result from a root carries an absolute `sourcePath` for the follow-up read,
* because the labelled path is relative to the root rather than the workspace.
* Reading it may still need a scope grant from the caller's own policy; this
* plugin cannot grant one.
*/
sourceRoots: Schema.array(String).default([]).volatile(),
/**
* Ask the local model for alternative phrasings and fuse the rankings.
*
@@ -575,6 +609,7 @@ export function apply(ctx, config) {
'oversized',
'unreadable',
'truncated',
'sourceRoots',
]
for (const key of fields) if (extra[key] !== undefined) out[key] = extra[key]
return out
@@ -655,8 +690,18 @@ export function apply(ctx, config) {
* knows exactly what is stale, so a partial index is described rather than
* guessed at.
*/
async function syncWorkspace(input) {
const { root, prefix = '', index, settings, budgetMs, signal, onProgress } = input
/**
* The candidate file list for one index: the workspace (or focus) tree plus
* every extra root.
*
* Split out from {@link syncWorkspace} because a search reports the absolute
* path of a result that came from an extra root, and the read phase is where
* that mapping is known. Both callers therefore see one list and one set of
* problems.
*/
async function walkIndex(input) {
const { root, prefix = '', settings, signal, sourceRoots = [] } = input
const walk = await listCodeFiles(root, {
...(settings.extensions?.length ? { extensions: settings.extensions } : {}),
skipDirs: DEFAULT_SKIP_DIRS,
@@ -672,8 +717,46 @@ export function apply(ctx, config) {
for (const file of walk.files) file.path = `${prefix}/${file.path}`
}
// Extra roots are walked as their own trees and merged into the same index,
// so one search ranks workspace code and library code together. Each root
// reports its own truncation: a root too large to index must say so rather
// than answer from its first few files.
const files = [...walk.files]
const problems = [...(input.sourceRootProblems ?? [])]
const summary = []
let truncated = walk.truncated
for (const sourceRoot of sourceRoots) {
const walkOfRoot = await listRootFiles({
name: sourceRoot.name,
path: sourceRoot.path,
...(settings.extensions?.length ? { extensions: settings.extensions } : {}),
exclude: settings.exclude ?? [],
maxFiles: Number(settings.maxFiles),
signal,
})
// A root labelled with the focus path itself would index two different
// files under one key — the second write silently replacing the first.
if (prefix !== '' && (prefix === sourceRoot.name || prefix.startsWith(`${sourceRoot.name}/`))) {
problems.push(
`the source root "${sourceRoot.name}" collides with the focus path "${prefix}"; ` +
'give it a different name= or search without that focus',
)
continue
}
files.push(...walkOfRoot.files)
summary.push({ name: sourceRoot.name, path: sourceRoot.path, files: walkOfRoot.files.length })
truncated = truncated || walkOfRoot.truncated
}
return { files, truncated, problems, summary }
}
async function syncWorkspace(input) {
const { index, settings, budgetMs, signal, onProgress } = input
const { files, truncated, problems: rootProblems, summary: rootSummary } = await walkIndex(input)
const known = index.fileHashes()
const present = new Set(walk.files.map((file) => file.path))
const present = new Set(files.map((file) => file.path))
const removed = index.prune([...known.keys()].filter((path) => !present.has(path)))
const maxBytes = Number(settings.maxFileBytes)
@@ -681,7 +764,7 @@ export function apply(ctx, config) {
let oversized = 0
let unreadable = 0
for (const file of walk.files) {
for (const file of files) {
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
let content
try {
@@ -746,8 +829,8 @@ export function apply(ctx, config) {
const stats = index.stats()
return {
...stats,
scanned: walk.files.length,
truncated: walk.truncated,
scanned: files.length,
truncated,
removed,
embedded,
chunksWritten,
@@ -755,6 +838,13 @@ export function apply(ctx, config) {
stale: [...stale],
oversized,
unreadable,
sourceRoots: rootSummary,
sourceRootProblems: rootProblems,
// Not part of the model-facing contract — the schema is closed — but a
// search needs it: a result from an extra root is labelled relative to
// that root, so the absolute path is the only thing `read` can use, and
// only the walk knows it.
fileAbsolutes: new Map(files.map((file) => [file.path, file.absolute])),
}
}
@@ -790,6 +880,13 @@ export function apply(ctx, config) {
excerpt: { type: 'string', required: true },
truncated: { type: 'boolean', required: true },
stale: { type: 'boolean', required: true },
/**
* Absolute path, present only for a result that came from an extra
* index root. The labelled `path` of such a result is relative to
* its root rather than the workspace, so it is not what `read`
* resolves — this is.
*/
sourcePath: { type: 'string' },
},
},
},
@@ -808,8 +905,23 @@ export function apply(ctx, config) {
truncated: { type: 'boolean' },
database: { type: 'string' },
stateDir: { type: 'string' },
/** The extra roots this index holds, and how many files each contributed. */
sourceRoots: {
type: 'array',
items: {
type: 'object',
additionalProperties: false,
properties: {
name: { type: 'string', required: true },
path: { type: 'string', required: true },
files: { type: 'integer', required: true },
},
},
},
},
},
/** Entries of `sourceRoots` that could not be used, one message each. */
sourceRootProblems: { type: 'array', items: { type: 'string' } },
best: { type: 'number' },
endpoint: {
type: 'object',
@@ -880,12 +992,15 @@ export function apply(ctx, config) {
}
if (value.action !== 'search') {
const { files = 0, chunks = 0, pending = 0, scanned = 0, embedded = 0 } = value.index ?? {}
const refused = (value.sourceRootProblems ?? []).length === 0
? ''
: `\n\nRefused source roots: ${value.sourceRootProblems.join('; ')}.`
return [
{
type: 'text',
text:
`Index for ${value.workspace}: ${files} files, ${chunks} chunks. ` +
`Scanned ${scanned}, embedded ${embedded} this call, ${pending} still pending.`,
`Scanned ${scanned}, embedded ${embedded} this call, ${pending} still pending.${refused}`,
},
]
}
@@ -893,11 +1008,15 @@ export function apply(ctx, config) {
const results = value.results ?? []
if (results.length === 0) {
const suffix = (value.index?.pending ?? 0) > 0 ? ` ${value.index.pending} file(s) are still unindexed.` : ''
const refused =
(value.sourceRootProblems ?? []).length === 0
? ''
: ` Refused source roots: ${value.sourceRootProblems.join('; ')}.`
return [
{
type: 'text',
text:
`No match for ${JSON.stringify(value.query)}.${suffix} ` +
`No match for ${JSON.stringify(value.query)}.${suffix}${refused} ` +
'If you know the exact identifier, grep is the better tool for this query.',
},
]
@@ -912,9 +1031,14 @@ export function apply(ctx, config) {
const symbol = result.symbol === undefined ? '' : ` (${result.symbol})`
const stale = result.stale ? ' [stale — file changed since indexing]' : ''
const cut = result.truncated ? '\n… excerpt truncated; read the file for the rest' : ''
// A result from an extra root is labelled relative to that root, so the
// workspace-relative `path` is not something `read` can resolve. The
// absolute path is stated when it is the one that differs — a workspace
// result already has a usable path and does not need the noise.
const source = result.sourcePath === undefined ? '' : `\n at ${result.sourcePath}`
return (
`--- ${at + 1}. ${result.path}:${result.startLine}-${result.endLine}${symbol} ` +
`similarity ${result.similarity}${stale}\n${result.excerpt}${cut}`
`similarity ${result.similarity}${stale}${source}\n${result.excerpt}${cut}`
)
})
// A scope is a window, not a filter on relevance. When it holds nothing
@@ -938,10 +1062,22 @@ export function apply(ctx, config) {
'Search this workspace semantically with a local embedding model and return ranked code excerpts ' +
'with file paths and line ranges — without reading whole files into the conversation. Use it when ' +
'you do not know the exact identifier: "where is X handled", "which module does Y", "how does Z ' +
'recover from failure". Prefer grep when you already know the exact symbol, and read when you need ' +
'the whole file. Because excerpts are returned inline, do not follow up with a read of the same ' +
'range. Actions: "search" (default) answers a query and warms a bounded slice of the index; ' +
'"index" builds or refreshes the whole index, as a background job by default; "status" reports the ' +
'recover from failure". ' +
// The move this exists to replace: reading a module's files one by one to
// find out what is in it. That is a search, and this is cheaper than the
// reads it avoids — including on the very first call, because a search
// warms a bounded slice of the index as it answers.
'Reach for it before reading files to find out what a module contains or ' +
'which of several alike modules does the job: one search returns the relevant lines from wherever ' +
'they are, where reading to find them costs the whole file for each wrong guess and every one of ' +
'those lines is re-sent on every later step. ' +
'Prefer grep when you already know the exact symbol, and read when you need the whole file. ' +
'Because excerpts are returned inline, do not follow up with a read of the same ' +
'range. A result labelled with a leading name (for example "@dsh/…") came from an extra source ' +
'root rather than the workspace; such a result also carries an absolute `sourcePath`, which is the ' +
'one `read` can resolve. Actions: "search" (default) answers a query and warms a bounded slice of ' +
'the index; "index" builds or refreshes the whole index, as a background job by default, which is ' +
'what makes the first search of a session cheap; "status" reports the ' +
'index and endpoint state. The index is incremental and lives outside your repository.',
parameters: {
query: {
@@ -1040,6 +1176,15 @@ export function apply(ctx, config) {
})
const focusPath = focusKey(cwd, focus.root)
// Extra roots are resolved once per call against the live workspace, so
// an entry is judged where it is used rather than at load: a path that
// has not been installed yet is reported on the call that wanted it. A
// bad entry never aborts the call — it is reported beside the result, or
// the caller would lose the workspace index to one typo.
const resolvedRoots = resolveSourceRoots(settings.sourceRoots ?? [], { workspace: cwd })
const sourceRoots = resolvedRoots.roots
const sourceRootProblems = resolvedRoots.problems
let opened
try {
opened = indexFor(settings, cwd, focusPath)
@@ -1094,6 +1239,7 @@ export function apply(ctx, config) {
index: indexView(base.index, stats),
endpoint,
scopes: scopes.map((entry) => ({ path: entry.path, marker: entry.marker })),
...(sourceRootProblems.length === 0 ? {} : { sourceRootProblems }),
note:
`Index ${file} holds ${stats.files} files and ${stats.chunks} chunks, scoped to ` +
`${focusPath === '.' ? 'the whole workspace' : focusPath} (${focus.source}` +
@@ -1108,6 +1254,17 @@ export function apply(ctx, config) {
.slice(0, 12)
.map((entry) => entry.path)
.join(', ')}${scopes.length > 12 ? ` (+${scopes.length - 12} more)` : ''}. `) +
// Extra roots are named even when the index has not been rebuilt
// since they were configured, so their file counts are deliberately
// absent here — `stats` describes the index, not the configuration.
(sourceRoots.length === 0
? ''
: `Extra roots indexed beside the workspace: ${sourceRoots
.map((entry) => `${entry.name} (${entry.path})`)
.join(', ')}. `) +
(sourceRootProblems.length === 0
? ''
: `Source roots refused: ${sourceRootProblems.join('; ')}. `) +
`Embeddings come from "${settings.embeddingModel}" (${settings.embeddingDimensions} dims) at ` +
`${settings.embeddingApiBase} — ` +
(endpoint.ok === true
@@ -1193,6 +1350,8 @@ export function apply(ctx, config) {
settings,
budgetMs: Number.POSITIVE_INFINITY,
signal,
sourceRoots,
sourceRootProblems,
}),
)
@@ -1220,7 +1379,14 @@ export function apply(ctx, config) {
}
const result = await runIndex(exec.signal)
return { ...base, ok: true, index: indexView(base.index, result) }
return {
...base,
ok: true,
index: indexView(base.index, result),
...(result.sourceRootProblems.length === 0
? {}
: { sourceRootProblems: result.sourceRootProblems }),
}
}
// search
@@ -1250,6 +1416,8 @@ export function apply(ctx, config) {
settings,
budgetMs: Math.max(0, Number(settings.syncBudgetMs)),
signal: exec.signal,
sourceRoots,
sourceRootProblems,
}),
)
const matrix = index.matrix()
@@ -1342,6 +1510,18 @@ export function apply(ctx, config) {
const stale = new Set(sync.stale ?? [])
const best = results.length === 0 ? 0 : Math.max(...results.map((result) => result.similarity))
// Where the result actually is, for the cases where the label is not a
// workspace-relative path. A workspace result keeps its plain path: the
// caller resolves it against the workspace and has no use for an
// absolute one. A result from an extra root cannot be resolved that
// way, so it carries the absolute path beside its label.
const absolutes = sync.fileAbsolutes ?? new Map()
const workspacePrefix = resolve(cwd).endsWith(sep) ? resolve(cwd) : `${resolve(cwd)}${sep}`
const sourcePathOf = (result) => {
const absolute = absolutes.get(result.path)
if (absolute === undefined) return undefined
return absolute.startsWith(workspacePrefix) ? undefined : absolute
}
// The alternative to these excerpts: the whole of each file they came
// from. Sizes come from the index, which recorded them at embed time,
@@ -1377,11 +1557,15 @@ export function apply(ctx, config) {
excerpt: result.excerpt,
truncated: result.truncated,
stale: stale.has(result.path),
...(sourcePathOf(result) === undefined ? {} : { sourcePath: sourcePathOf(result) }),
})),
index: indexView(base.index, sync),
best,
weak: results.length === 0 || best < Number(settings.minSimilarity),
measure,
...(sync.sourceRootProblems.length === 0
? {}
: { sourceRootProblems: sync.sourceRootProblems }),
}
} catch (error) {
const code = error instanceof EmbeddingError ? error.code : 'RECALL_FAILED'