feat: index extra source roots beside the workspace, for the code you write against

The workspace index cannot answer questions about an installed framework or
SDK, and in plugin development that is where most of the reading goes: 64 of
80 tool calls in one real session went to the harness checkout under
/usr/local/lib/node_modules to look up the API being implemented.

A root is entered whatever it is called, and its dependencies are not: a
path ending in node_modules/... is the normal way to name a package tree,
while a node_modules below the root is somebody else's code. A result
carries the label that says which tree it came from and an absolute
sourcePath, the only form a follow-up read can resolve.

The tool description now also claims the move this replaces - searching
before reading files to find out what a module contains.

test/helpers.js

@@ -102,8 +102,14 @@ export function makeContext(options = {}) {
 }
 
 /** An `exec` argument for one tool call. */
-export function makeExec(cwd, signal) {
-  return { signal: signal ?? new AbortController().signal, agent: { session: { header: { cwd } } } }
+export function makeExec(cwd, signal, sessionId = 'session-under-test') {
+  // `agent.id` is the SESSION id — the jobs registry resolves a background job's
+  // owner through `agents.get(id)` and refuses anything that is not that string.
+  // A fixture without it cannot catch a call that passes the whole handle.
+  return {
+    signal: signal ?? new AbortController().signal,
+    agent: { id: sessionId, session: { header: { cwd } } },
+  }
 }
 
 /** Write a map of relative path to content into a fresh temporary directory. */

test/plugin.test.js

@@ -1,5 +1,5 @@
 import { strict as assert } from 'node:assert'
-import { readFile, rm, writeFile } from 'node:fs/promises'
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import { test } from 'node:test'
 
@@ -159,6 +159,7 @@ function configFor(stateDir, extra = {}) {
     // the two lists from drifting apart again.
     focus: '',
     focusMarkers: [],
+    sourceRoots: [],
     expansionEnabled: false,
     expansionModel: '',
     expansionCount: 4,
@@ -173,7 +174,7 @@ function configFor(stateDir, extra = {}) {
 }
 
 /** Apply the plugin to a fresh workspace with a stubbed embedding endpoint. */
-async function setup(t, files = FILES, extra = {}, stubOptions = {}) {
+async function setup(t, files = FILES, extra = {}, stubOptions = {}, services = {}) {
   const workspace = await makeWorkspace(files)
   const stateDir = await makeStateDir()
   const stub = makeStubFetch(stubOptions)
@@ -185,7 +186,7 @@ async function setup(t, files = FILES, extra = {}, stubOptions = {}) {
     await rm(stateDir, { recursive: true, force: true })
   })
 
-  const { ctx, tools, dispose } = makeContext()
+  const { ctx, tools, dispose } = makeContext({ services })
   t.after(() => dispose())
   apply(ctx, configFor(stateDir, extra))
   assert.equal(tools.length, 1, 'exactly one tool should be registered')
@@ -814,3 +815,157 @@ test('re-ranking degrades to the fused order when the model has no opinion', asy
   )
   assert.ok(reranked.results.length > 0, 'no opinion must not mean no results')
 })
+
+test('a background index hands the jobs registry a session id, not the agent handle', async (t) => {
+  // The jobs registry resolves `spec.owner` through `agents.get(owner)` and
+  // throws `session "[object Object]" has no live agent` for anything that is not
+  // the session id string. This test is a stand-in for that check: the bug it
+  // catches was invisible in-process and made the DEFAULT `index` call fail on a
+  // real deployment, where a throwing starter leaves the caller no job at all.
+  const started = []
+  const services = {
+    jobs: {
+      start(spec) {
+        started.push(spec)
+        return `recall-index-${started.length}`
+      },
+    },
+  }
+  const { tool, workspace } = await setup(t, FILES, {}, {}, services)
+  const value = await tool.execute({ action: 'index' }, makeExec(workspace, undefined, 'session-42'))
+  assert.deepEqual(schemaErrors(tool.output.schema, value), [])
+  assert.equal(value.jobId, 'recall-index-1')
+  assert.equal(started.length, 1)
+  assert.equal(
+    started[0].owner,
+    'session-42',
+    'owner must be the session id: the registry refuses an Agent handle outright',
+  )
+  assert.equal(typeof started[0].run, 'function')
+  // And the job the registry was handed actually indexes the workspace.
+  const outcome = await started[0].run().done
+  assert.equal(outcome.status, 'completed')
+  assert.match(outcome.output, /Indexed/)
+  const search = await tool.execute({ query: 'session expiry' }, makeExec(workspace))
+  assert.ok(search.results.length > 0, 'the background job must leave a searchable index behind')
+})
+
+/**
+ * A workspace with a library tree beside it, laid out the way a real one is.
+ *
+ * The library is a *scoped packages directory* — its own path runs through
+ * `node_modules` — because that is the shape the workspace walk exists to skip
+ * and the shape this feature has to enter. `root` is the parent of both trees, so
+ * the configured root can be given as a relative path and stays portable.
+ *
+ * @returns {Promise<{root: string, cleanup: () => Promise<void>}>}
+ */
+async function makeRootPair() {
+  const root = await makeWorkspace({})
+  const workspace = join(root, 'My Project')
+  const library = join(root, 'packages', 'node_modules', '@vendor')
+  await mkdir(join(library, 'widget', 'lib'), { recursive: true })
+  await mkdir(join(library, 'widget', 'node_modules', 'dep'), { recursive: true })
+  await mkdir(workspace, { recursive: true })
+  await writeFile(join(workspace, 'app.ts'), 'export const app = 1\n')
+  await writeFile(
+    join(library, 'widget', 'lib', 'index.js'),
+    'export function mountWidget(root, options) { return { root, options } }\n',
+  )
+  await writeFile(
+    join(library, 'widget', 'node_modules', 'dep', 'index.js'),
+    'export const vendored = "should never be indexed"\n',
+  )
+  return { root, workspace, library, cleanup: () => rm(root, { recursive: true, force: true }) }
+}
+
+/** Apply the plugin to a workspace the caller made itself. */
+async function setupAt(t, workspace, extra = {}) {
+  const stateDir = await makeStateDir()
+  const stub = makeStubFetch()
+  const originalFetch = globalThis.fetch
+  globalThis.fetch = stub
+  t.after(async () => {
+    globalThis.fetch = originalFetch
+    await rm(stateDir, { recursive: true, force: true })
+  })
+  const { ctx, tools, dispose } = makeContext()
+  t.after(() => dispose())
+  apply(ctx, configFor(stateDir, extra))
+  return { tool: tools[0], workspace, stub }
+}
+
+test('an extra root is indexed beside the workspace, and its dependencies are not', async (t) => {
+  const { root, workspace, library } = await makeRootPair()
+  t.after(async () => rm(root, { recursive: true, force: true }))
+
+  // Relative to the workspace, and named, because the naming is what makes a
+  // result say which tree it came from.
+  const { tool } = await setupAt(t, workspace, { sourceRoots: ['vendor=../packages/node_modules/@vendor'] })
+
+  const index = await tool.execute({ action: 'index', run_in_background: false }, makeExec(workspace))
+  assert.deepEqual(schemaErrors(tool.output.schema, index), [])
+  assert.deepEqual(index.index.sourceRoots, [{ name: 'vendor', path: library, files: 1 }])
+  assert.equal(index.index.files, 2, 'the workspace file and the library file, and nothing vendored')
+
+  const search = await tool.execute({ query: 'mountWidget root options' }, makeExec(workspace))
+  assert.deepEqual(schemaErrors(tool.output.schema, search), [])
+  const hit = search.results.find((result) => result.path.startsWith('vendor/'))
+  assert.ok(hit, `no library result in ${search.results.map((result) => result.path).join(', ')}`)
+  // The label names the tree; the absolute path is the only thing `read` can use,
+  // because the label is relative to the root rather than the workspace.
+  assert.equal(hit.path, 'vendor/widget/lib/index.js')
+  assert.equal(hit.sourcePath, join(library, 'widget', 'lib', 'index.js'))
+  // A workspace result is unchanged: its own path already resolves.
+  const own = search.results.find((result) => result.path === 'app.ts')
+  if (own !== undefined) assert.equal(own.sourcePath, undefined)
+  // The nested dependency is absent from the index entirely, not merely unranked.
+  assert.equal(
+    search.results.some((result) => result.path.includes('node_modules')),
+    false,
+  )
+})
+
+test('an unusable source root is reported and never costs the workspace index', async (t) => {
+  const { root, workspace } = await makeRootPair()
+  t.after(async () => rm(root, { recursive: true, force: true }))
+  const { tool } = await setupAt(t, workspace, { sourceRoots: ['ghost=../nowhere', 'vendor=../packages/node_modules/@vendor'] })
+
+  const value = await tool.execute({ action: 'index', run_in_background: false }, makeExec(workspace))
+  assert.deepEqual(schemaErrors(tool.output.schema, value), [])
+  // The good root still indexed, and the bad one is named rather than swallowed.
+  assert.deepEqual(value.index.sourceRoots.map((entry) => entry.name), ['vendor'])
+  assert.equal(value.sourceRootProblems.length, 1)
+  assert.match(value.sourceRootProblems[0], /nowhere/)
+  assert.ok(value.index.files >= 2, 'the workspace and the surviving root must both be indexed')
+
+  const status = await tool.execute({ action: 'status' }, makeExec(workspace))
+  assert.deepEqual(schemaErrors(tool.output.schema, status), [])
+  assert.equal(status.sourceRootProblems.length, 1)
+  assert.ok(status.results === undefined, 'status returns no results')
+})
+
+test('a source root whose path is a file, not a directory, is refused', async (t) => {
+  const { root, workspace } = await makeRootPair()
+  t.after(async () => rm(root, { recursive: true, force: true }))
+  const { tool } = await setupAt(t, workspace, { sourceRoots: ['app=./app.ts'] })
+  const value = await tool.execute({ action: 'index', run_in_background: false }, makeExec(workspace))
+  assert.deepEqual(schemaErrors(tool.output.schema, value), [])
+  assert.equal(value.index.sourceRoots.length, 0)
+  assert.match(value.sourceRootProblems[0], /not a directory/)
+})
+
+test('a source root named after the focus path is refused as a collision', async (t) => {
+  // Two different files would be indexed under one key, the second write
+  // replacing the first — so this cannot be left to the walk to get right.
+  const root = await makeWorkspace({ 'vendor/app.ts': 'export const a = 1\n', 'lib/app.ts': 'export const b = 2\n' })
+  t.after(() => rm(root, { recursive: true, force: true }))
+  const { tool } = await setupAt(t, root, { sourceRoots: ['vendor=vendor'] })
+  const value = await tool.execute(
+    { action: 'index', run_in_background: false, focus: 'vendor' },
+    makeExec(root),
+  )
+  assert.deepEqual(schemaErrors(tool.output.schema, value), [])
+  assert.equal(value.index.sourceRoots.length, 0)
+  assert.match(value.sourceRootProblems[0], /collides with the focus path/)
+})

test/roots.test.js

@@ -0,0 +1,138 @@
+import { strict as assert } from 'node:assert'
+import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'node:test'
+
+import {
+  ROOT_SKIP_DIRS,
+  listRootFiles,
+  parseSourceRoot,
+  resolveSourceRoots,
+  rootRelativePath,
+  sanitizeRootName,
+  skipsRootDir,
+} from '../lib/roots.js'
+
+test('a root defaults its name to the directory basename', () => {
+  const parsed = parseSourceRoot('/usr/local/lib/node_modules/@deepseek-ai', '/repo')
+  assert.equal(parsed.name, '@deepseek-ai')
+  assert.equal(parsed.path, '/usr/local/lib/node_modules/@deepseek-ai')
+  assert.equal(parsed.workspaceRelative, false)
+})
+
+test('an explicit name is taken as written', () => {
+  const parsed = parseSourceRoot('dsh=/usr/local/lib/node_modules/@deepseek-ai', '/repo')
+  assert.equal(parsed.name, 'dsh')
+  assert.equal(parsed.path, '/usr/local/lib/node_modules/@deepseek-ai')
+})
+
+test('a relative root resolves against the workspace and says so', () => {
+  const parsed = parseSourceRoot('vendor/lib', '/repo')
+  assert.equal(parsed.path, '/repo/vendor/lib')
+  assert.equal(parsed.name, 'lib')
+  assert.equal(parsed.workspaceRelative, true)
+})
+
+test('a name containing a separator is refused, because it would forge a path', () => {
+  assert.throws(() => parseSourceRoot('a/b=/tmp/x', '/repo'), /path separator/)
+})
+
+test('an entry that names no path or yields no name is refused', () => {
+  assert.throws(() => parseSourceRoot('', '/repo'), /empty entry/)
+  assert.throws(() => parseSourceRoot('name=', '/repo'), /names no path/)
+  // A root that climbs to the filesystem root has no basename to be labelled
+  // with, and `resolve` collapsing the traversal is exactly why the check is on
+  // the resolved path rather than on the text of the entry.
+  assert.throws(() => parseSourceRoot('/../..', '/repo'), /does not yield a usable name/)
+})
+
+test('sanitizeRootName keeps the characters a path segment can hold', () => {
+  assert.equal(sanitizeRootName('@deepseek-ai'), '@deepseek-ai')
+  assert.equal(sanitizeRootName('my root'), 'my-root')
+  assert.equal(sanitizeRootName('..'), '')
+  assert.equal(sanitizeRootName('.'), '')
+  assert.equal(sanitizeRootName('   '), '')
+})
+
+test('rootRelativePath never emits a leading or trailing separator', () => {
+  assert.equal(rootRelativePath('dsh', 'lib/index.js'), 'dsh/lib/index.js')
+  assert.equal(rootRelativePath('dsh', ''), 'dsh')
+})
+
+test('resolveSourceRoots skips a bad entry loudly and keeps the rest', () => {
+  const result = resolveSourceRoots(['/good', 'missing=/nope', '/good'], {
+    workspace: '/repo',
+    isDirectory: (path) => path === '/good',
+  })
+  assert.deepEqual(result.roots, [{ name: 'good', path: '/good' }])
+  // A duplicate name is dropped rather than silently overwriting: two roots with
+  // one label would make every result from the second ambiguous.
+  assert.equal(result.problems.length, 2)
+  assert.match(result.problems[0], /not a directory/)
+  assert.match(result.problems[1], /duplicates the name/)
+})
+
+test('resolveSourceRoots tolerates no configuration at all', () => {
+  assert.deepEqual(resolveSourceRoots([], { workspace: '/repo' }), { roots: [], problems: [] })
+  assert.deepEqual(resolveSourceRoots(undefined, { workspace: '/repo' }), { roots: [], problems: [] })
+})
+
+test('the root itself is entered whatever it is called, and node_modules below it is not', () => {
+  // This asymmetry is the feature: a root is normally named `.../node_modules/...`,
+  // while a `node_modules` inside one of its packages is somebody else's code.
+  assert.equal(skipsRootDir(0, 'node_modules'), false, 'the root must not be skipped')
+  assert.equal(skipsRootDir(0, '.git'), false)
+  assert.equal(skipsRootDir(1, 'node_modules'), true, 'a nested dependency store must be skipped')
+  assert.equal(skipsRootDir(3, 'node_modules'), true)
+  assert.equal(skipsRootDir(1, 'dist'), true)
+  assert.equal(skipsRootDir(1, 'lib'), false)
+  assert.ok(ROOT_SKIP_DIRS.includes('node_modules'), 'the skip list must still name node_modules')
+})
+
+test('listRootFiles indexes a package tree and not its dependencies', async (t) => {
+  const root = await mkdtemp(join(tmpdir(), 'recall-root-'))
+  t.after(() => rm(root, { recursive: true, force: true }))
+  const packages = join(root, 'node_modules', '@scope')
+  await mkdir(join(packages, 'alpha', 'lib'), { recursive: true })
+  await mkdir(join(packages, 'alpha', 'node_modules', 'dep'), { recursive: true })
+  await mkdir(join(packages, 'beta', 'dist'), { recursive: true })
+  await writeFile(join(packages, 'alpha', 'lib', 'index.js'), 'export const a = 1\n')
+  await writeFile(join(packages, 'alpha', 'node_modules', 'dep', 'index.js'), 'module.exports = 0\n')
+  await writeFile(join(packages, 'beta', 'dist', 'bundle.js'), 'var x = 1\n')
+  await writeFile(join(packages, 'beta', 'index.js'), 'export const b = 2\n')
+  await writeFile(join(packages, 'README.md'), '# not code\n')
+
+  const walk = await listRootFiles({ name: 'dsh', path: packages })
+  assert.deepEqual(
+    walk.files.map((file) => file.path),
+    ['dsh/alpha/lib/index.js', 'dsh/beta/index.js'],
+    'the root is entered, a dependency below it is not, and dist is not',
+  )
+  // Every path is labelled but resolvable: `read` needs the absolute one.
+  for (const file of walk.files) assert.ok(file.absolute.startsWith(packages), file.absolute)
+})
+
+test('listRootFiles does not follow a symlink out of the root', async (t) => {
+  const root = await mkdtemp(join(tmpdir(), 'recall-root-'))
+  const outside = await mkdtemp(join(tmpdir(), 'recall-outside-'))
+  t.after(async () => {
+    await rm(root, { recursive: true, force: true })
+    await rm(outside, { recursive: true, force: true })
+  })
+  await mkdir(join(root, 'pkg'), { recursive: true })
+  await writeFile(join(root, 'pkg', 'a.js'), 'export const a = 1\n')
+  await writeFile(join(outside, 'secret.js'), 'const secret = 1\n')
+  await symlink(outside, join(root, 'link'), 'dir')
+
+  const walk = await listRootFiles({ name: 'r', path: root })
+  assert.deepEqual(walk.files.map((file) => file.path), ['r/pkg/a.js'])
+})
+
+test('a file at the root of a root is labelled with the root name alone', async (t) => {
+  const root = await mkdtemp(join(tmpdir(), 'recall-root-'))
+  t.after(() => rm(root, { recursive: true, force: true }))
+  await writeFile(join(root, 'index.js'), 'export const root = 1\n')
+  const walk = await listRootFiles({ name: 'solo', path: root })
+  assert.deepEqual(walk.files.map((file) => file.path), ['solo/index.js'])
+})