dsh-slopo agent
6beb4c0d39
Three test-only spots had baked in the machine this repository was developed
on, and the repository is publicly readable, so they shipped that machine's
shape to anyone who cloned it:
* the end-to-end embedding backend was pinned to a private LAN address;
it is now read from DSH_SLOPO_E2E_API_BASE, defaulting to Ollama's own
loopback address, so the machine running the suite supplies its own;
* the real-slopo test resolved discovery from a hardcoded root home, which
meant it could only ever run for one user — it resolves from the running
user's home now, which is also the fix for everyone else;
* the harness-slot contract tests read the global npm tree by absolute
path; they go through this repository's own node_modules symlink, which
is already relative to the checkout.
No user-facing behaviour changes: the test files are not in the published
package, and the default configuration was already free of machine-specific
values — every address, path and key defaults to empty and is resolved per
user at runtime.
874 lines
37 KiB
JavaScript
874 lines
37 KiB
JavaScript
/**
|
|
* Integration tests: load the real plugin module against a mock Cordis context,
|
|
* then drive the tool through a real shell into the real slopo CLI.
|
|
*
|
|
* The mock is deliberately thin — it implements only the seams the plugin
|
|
* actually touches — so a wrong assumption about a dsh service shows up here as
|
|
* a failing test rather than as a plugin that silently does nothing at runtime.
|
|
*/
|
|
|
|
import assert from 'node:assert/strict'
|
|
import { exec } from 'node:child_process'
|
|
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { test } from 'node:test'
|
|
|
|
import { findSlopo } from '../lib/slopo.js'
|
|
|
|
const testHome = mkdtempSync(join(tmpdir(), 'dsh-slopo-home-'))
|
|
process.env.DSH_HOME = testHome
|
|
|
|
const { Config, apply } = await import('../index.js')
|
|
|
|
const FULL_CONFIG = {
|
|
binary: '',
|
|
autoInstall: true,
|
|
embeddingModel: '',
|
|
embeddingDimensions: 0,
|
|
embeddingApiBase: '',
|
|
embeddingApiKey: '',
|
|
embeddingParamsJson: '{}',
|
|
sourceDir: '',
|
|
sourceDirExclude: [],
|
|
ignoreFile: '',
|
|
stateDir: '',
|
|
embeddingRequestDelay: 0,
|
|
timeoutMs: 600_000,
|
|
useDefaultExcludes: true,
|
|
maxIndexUnits: 5000,
|
|
}
|
|
|
|
/**
|
|
* A shell executor shaped like `ShellExecutor` in dsh 0.1.7-rc.2: `resolve`,
|
|
* then `execute(spec)` returning a process handle whose `result()` is the
|
|
* foreground projection.
|
|
*
|
|
* There is deliberately no `run(spec)`: that single-step method was removed from
|
|
* the seam, and a plugin that still calls it fails with
|
|
* `shell.run is not a function` the first time it needs a command. Keeping the
|
|
* mock honest about the shape is the point — this file is the contract test.
|
|
*/
|
|
const shell = {
|
|
sandboxMode: undefined,
|
|
resolve: (request) => ({
|
|
workdir: process.cwd(),
|
|
timeoutMs: 600_000,
|
|
stdoutMaxBytes: 10_000_000,
|
|
...request,
|
|
}),
|
|
async execute(spec) {
|
|
const result = await new Promise((resolve) => {
|
|
exec(
|
|
spec.command,
|
|
{ cwd: spec.workdir, env: { ...process.env, ...(spec.env ?? {}) }, timeout: spec.timeoutMs, maxBuffer: 64 * 1024 * 1024 },
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
exitCode: error === null ? 0 : typeof error.code === 'number' ? error.code : 1,
|
|
signal: null,
|
|
timedOut: Boolean(error?.killed),
|
|
aborted: false,
|
|
timeoutMs: spec.timeoutMs,
|
|
stdout: { text: String(stdout), truncated: false },
|
|
stderr: { text: String(stderr), truncated: false },
|
|
})
|
|
},
|
|
)
|
|
})
|
|
return { ...result, result: async () => result }
|
|
},
|
|
start() {
|
|
throw new Error('background process start is not used by these tests')
|
|
},
|
|
}
|
|
|
|
/**
|
|
* A stub slopo executable.
|
|
*
|
|
* Discovery itself is `test/slopo.test.js`'s business. Here the tool must never
|
|
* reach `uv tool install`: that is a network round trip, it makes the suite
|
|
* slow and flaky, and a canned runner would answer it with success without
|
|
* installing anything — leaving the plugin to report SLOPO_INSTALL_MISSING.
|
|
* Pointing `binary` at a file that exists takes the "already installed" branch,
|
|
* which is the state every one of these tests means to exercise. A test that
|
|
* wants the missing-executable path passes `binary: ''` and `autoInstall: false`.
|
|
*/
|
|
const stubSlopo = join(mkdtempSync(join(tmpdir(), 'dsh-slopo-bin-')), 'slopo')
|
|
writeFileSync(stubSlopo, '#!/bin/sh\nexit 0\n', { mode: 0o755 })
|
|
|
|
/**
|
|
* Build a context exposing exactly the services the plugin consumes.
|
|
*
|
|
* `jobs` is deliberately absent, which forces the tool down its foreground path
|
|
* so the assertions can see a result rather than a job handle.
|
|
*/
|
|
function harness(overrides = {}, { withSettings = true, withJobs = false, withShell = true, sandboxMode, sandboxPolicyResolver, shellRun } = {}) {
|
|
const config = { ...FULL_CONFIG, binary: stubSlopo, ...overrides }
|
|
// A per-harness shell, so a test can vary sandboxMode and every resolved
|
|
// request is recorded for assertion.
|
|
const scopedShell = {
|
|
...shell,
|
|
sandboxMode,
|
|
resolve(request) {
|
|
state.shellRequests.push(request)
|
|
return { workdir: process.cwd(), timeoutMs: 600_000, stdoutMaxBytes: 10_000_000, ...request }
|
|
},
|
|
// A canned runner lets a test drive the index gate without a real slopo.
|
|
// It replaces only the low-level run, so the `execute().result()` frame
|
|
// stays exactly the shape the real seam has.
|
|
async execute(spec) {
|
|
state.shellExecutions += 1
|
|
const result = shellRun === undefined ? await shell.execute(spec).then((handle) => handle.result()) : await shellRun(spec)
|
|
return { ...result, result: async () => result }
|
|
},
|
|
}
|
|
const state = {
|
|
tools: new Map(),
|
|
updates: [],
|
|
provider: undefined,
|
|
pagePolicy: undefined,
|
|
settingsEffects: 0,
|
|
shell: scopedShell,
|
|
shellRequests: [],
|
|
shellExecutions: 0,
|
|
jobs: undefined,
|
|
}
|
|
|
|
const ctx = {
|
|
fiber: { uid: 'test-fiber' },
|
|
logger: () => ({ info() {}, warn() {}, debug() {}, error() {} }),
|
|
get(name) {
|
|
if (name === 'shell') return withShell ? scopedShell : undefined
|
|
if (name === 'jobs') return withJobs ? state.jobs : undefined
|
|
if (name === 'sandboxPolicy') return sandboxPolicyResolver
|
|
return undefined
|
|
},
|
|
inject(deps, callback) {
|
|
if (!withSettings || !deps.includes('settings')) return
|
|
// The child context `ctx.inject` hands back: its own effect scope plus
|
|
// the settings service. 0.1.7-rc.2 has no `installSection` — an entry's
|
|
// exported Config IS the form — so the only things a plugin does here are
|
|
// claim its page and write through `update`.
|
|
callback({
|
|
effect(effect) {
|
|
state.settingsEffects += 1
|
|
return effect()
|
|
},
|
|
settings: {
|
|
configure(presentation, owner) {
|
|
state.pagePolicy = presentation
|
|
state.pagePolicyOwner = owner
|
|
return () => {
|
|
state.pagePolicy = undefined
|
|
}
|
|
},
|
|
async update(namespace, patch) {
|
|
state.updates.push({ namespace, patch })
|
|
Object.assign(config, patch)
|
|
},
|
|
},
|
|
})
|
|
},
|
|
skills: {
|
|
registerProvider(create) {
|
|
state.provider = create({ signal: new AbortController().signal, invalidate() {} })
|
|
},
|
|
},
|
|
tools: {
|
|
register(definition) {
|
|
state.tools.set(definition.name, definition)
|
|
},
|
|
},
|
|
}
|
|
|
|
apply(ctx, config)
|
|
return { config, state }
|
|
}
|
|
|
|
const execContext = (cwd) => ({ signal: new AbortController().signal, agent: { id: 'session-1', session: { header: { cwd } } } })
|
|
|
|
test('Config is a schemastery schema, which is what cordis requires', () => {
|
|
assert.equal(typeof Config, 'function')
|
|
assert.ok(Config['~standard'] !== undefined, 'expected a Standard Schema')
|
|
})
|
|
|
|
test('apply registers exactly one tool, and it is called slopo', () => {
|
|
const { state } = harness()
|
|
assert.deepEqual([...state.tools.keys()], ['slopo'])
|
|
})
|
|
|
|
test('the ported skills are published user-invocable and NOT model-invocable', async () => {
|
|
const { state } = harness()
|
|
const candidates = await state.provider.list()
|
|
assert.deepEqual(
|
|
candidates.map((c) => c.name),
|
|
['slopo-review', 'slopo-analyze-ignore', 'slopo-analyze-one'],
|
|
)
|
|
for (const candidate of candidates) {
|
|
assert.equal(candidate.invocation.userInvocable, true, candidate.name)
|
|
assert.equal(candidate.invocation.modelInvocable, false, candidate.name)
|
|
assert.equal(candidate.rank, 600)
|
|
assert.equal(candidate.source, 'bundled')
|
|
assert.ok(candidate.description.length > 20, candidate.name)
|
|
assert.ok(candidate.locator !== undefined)
|
|
}
|
|
})
|
|
|
|
test('get() returns the skill body, routed through the locator list() issued', async () => {
|
|
const { state } = harness()
|
|
const [candidate] = await state.provider.list()
|
|
const skill = await state.provider.get(candidate)
|
|
assert.equal(skill.name, 'slopo-review')
|
|
assert.match(skill.content, /# Instructions/)
|
|
assert.match(skill.content, /First-turn output rules/)
|
|
assert.match(skill.content, /slopo.*action: "review"/s)
|
|
})
|
|
|
|
test('every skill tells the agent to call the tool rather than a shell command', async () => {
|
|
const { state } = harness()
|
|
for (const candidate of await state.provider.list()) {
|
|
const skill = await state.provider.get(candidate)
|
|
assert.match(skill.content, /`slopo` tool/, candidate.name)
|
|
// The Claude Code bash-injection directive must not survive the port.
|
|
assert.doesNotMatch(skill.content, /!`/, candidate.name)
|
|
assert.doesNotMatch(skill.content, /allowed-tools/, candidate.name)
|
|
}
|
|
})
|
|
|
|
test('the plugin claims its own settings page instead of a generated one', () => {
|
|
// `auto: false` says "this entry ships a client page" — client/client.js
|
|
// registers one onto the Plugins page. The owner must be the plugin's own
|
|
// fiber, not the settings service's.
|
|
const { state } = harness()
|
|
assert.deepEqual(state.pagePolicy, { auto: false })
|
|
assert.equal(state.pagePolicyOwner?.uid, 'test-fiber', 'the policy belongs to the plugin instance')
|
|
assert.equal(state.settingsEffects, 1, 'the policy is registered as an effect, so a reload drops it')
|
|
})
|
|
|
|
test('the namespace is the Loader entry id the bundle patch declares', async () => {
|
|
const { state } = harness()
|
|
await state.tools.get('slopo').execute({ action: 'configure', config: { embeddingModel: 'x' } }, execContext('/tmp'))
|
|
assert.equal(state.updates[0].namespace, 'dsh-slopo')
|
|
})
|
|
|
|
test('the host half no longer calls the removed installSection API', () => {
|
|
// 0.1.7-rc.2 deleted `ctx.settings.installSection`; a plugin that still calls
|
|
// it loses its whole settings surface, silently, behind a try/catch.
|
|
const source = readFileSync(new URL('../index.js', import.meta.url), 'utf8')
|
|
const code = source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '')
|
|
assert.doesNotMatch(code, /installSection/)
|
|
})
|
|
|
|
test('every Config field is volatile, which is what puts it on the settings page', () => {
|
|
// The framework's form projection keeps only fields under a volatile
|
|
// ancestor, and an entry whose form is empty is dropped from
|
|
// `settings.describe` — so an unmarked field is invisible to every UI.
|
|
// The walk is over the live schema: `Config.toJSON()` is a reference
|
|
// envelope for the wire, not a tree.
|
|
const fields = Object.entries(Config.dict ?? {})
|
|
assert.ok(fields.length > 0, 'the schema must declare fields')
|
|
for (const [key, node] of fields) {
|
|
assert.equal(node.meta?.volatile, true, `${key} must be marked volatile()`)
|
|
}
|
|
})
|
|
|
|
test('the embedding API key is a declared secret, so the settings wire never carries it', () => {
|
|
assert.equal(Config.dict.embeddingApiKey.meta?.role, 'secret')
|
|
// …and no other field claims to be one, because the browser cannot read a
|
|
// secret back and every other field is one the card must display.
|
|
for (const [key, node] of Object.entries(Config.dict)) {
|
|
if (key !== 'embeddingApiKey') assert.notEqual(node.meta?.role, 'secret', `${key} must not be secret`)
|
|
}
|
|
})
|
|
|
|
test('status reports the missing settings instead of pretending to be ready', async () => {
|
|
const { state } = harness()
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'status' }, execContext('/tmp'))
|
|
assert.equal(value.kind, 'foreground')
|
|
assert.equal(value.ok, false)
|
|
assert.deepEqual(value.details.missingSettings, ['embedding_model', 'embedding_dimensions'])
|
|
assert.match(value.message, /Not configured yet/)
|
|
})
|
|
|
|
test('status resolves state under $DSH_HOME, never inside the workspace', async () => {
|
|
const { state } = harness()
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'status' }, execContext('/tmp/some-repo'))
|
|
assert.ok(value.details.stateDir.startsWith(testHome), value.details.stateDir)
|
|
assert.doesNotMatch(value.details.stateDir, /^\/tmp\/some-repo/)
|
|
})
|
|
|
|
test('configure rejects a setting this plugin does not have', async () => {
|
|
const { state } = harness()
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'configure', config: { embeddingModel: 'x', nope: 1 } }, execContext('/tmp'))
|
|
assert.equal(value.ok, false)
|
|
assert.equal(value.errorCode, 'UNKNOWN_SETTING')
|
|
assert.match(value.message, /nope/)
|
|
assert.deepEqual(state.updates, [])
|
|
})
|
|
|
|
test('configure persists through the settings namespace, not a private file', async () => {
|
|
const { state } = harness()
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute(
|
|
{ action: 'configure', config: { embeddingModel: 'ollama/m:latest', embeddingDimensions: 768 } },
|
|
execContext('/tmp'),
|
|
)
|
|
assert.equal(value.ok, true)
|
|
assert.equal(state.updates.length, 1)
|
|
assert.equal(state.updates[0].namespace, 'dsh-slopo')
|
|
assert.deepEqual(state.updates[0].patch, { embeddingModel: 'ollama/m:latest', embeddingDimensions: 768 })
|
|
})
|
|
|
|
test('configure says so plainly when no settings provider is mounted', async () => {
|
|
const { state } = harness({}, { withSettings: false })
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'configure', config: { embeddingModel: 'x' } }, execContext('/tmp'))
|
|
assert.equal(value.ok, false)
|
|
assert.equal(value.errorCode, 'NO_SETTINGS_PROVIDER')
|
|
})
|
|
|
|
test('the plugin still loads and registers its tool without a settings provider', () => {
|
|
// This is the whole reason `settings` goes through ctx.inject rather than the
|
|
// top-level inject: a required dependency would have suppressed apply entirely.
|
|
const { state } = harness({}, { withSettings: false })
|
|
assert.ok(state.tools.has('slopo'))
|
|
assert.ok(state.provider !== undefined)
|
|
})
|
|
|
|
test('review refuses to run before the workspace is configured', async () => {
|
|
const { state } = harness()
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'review' }, execContext('/tmp'))
|
|
assert.equal(value.ok, false)
|
|
assert.equal(value.errorCode, 'NOT_CONFIGURED')
|
|
assert.match(value.remedy, /configure/)
|
|
})
|
|
|
|
test('review writes a config the plugin owns and passes the key only in the environment', async () => {
|
|
// Hermetic on purpose: `binary` points at a stub that exists, so the plugin
|
|
// takes the "already installed" branch instead of running `uv tool install`
|
|
// over the network, and the workspace is a real directory so the shell seam
|
|
// has somewhere to run. The canned runner answers both commands.
|
|
const workspace = mkdtempSync(join(tmpdir(), 'dsh-slopo-ws-'))
|
|
const binary = join(workspace, 'slopo')
|
|
writeFileSync(binary, '#!/bin/sh\nexit 0\n', { mode: 0o755 })
|
|
const ran = []
|
|
const { state } = harness(
|
|
{
|
|
binary,
|
|
embeddingModel: 'ollama/example:latest',
|
|
embeddingDimensions: 768,
|
|
embeddingApiBase: 'http://127.0.0.1:11434',
|
|
embeddingApiKey: 'super-secret',
|
|
},
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 12 code units from 3 files.' }) },
|
|
)
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'review' }, execContext(workspace))
|
|
assert.equal(value.kind, 'foreground')
|
|
assert.ok(existsSync(value.configPath), `expected config at ${value.configPath}`)
|
|
const written = readFileSync(value.configPath, 'utf8')
|
|
assert.match(written, /embedding_model: "ollama\/example:latest"/)
|
|
assert.doesNotMatch(written, /super-secret/)
|
|
// The key rides the environment of the run, not the file.
|
|
const executions = state.shellRequests.filter((request) => /agent-review/.test(request.command))
|
|
assert.equal(executions.length, 1)
|
|
assert.equal(executions[0].env.SLOPO_EMBEDDING_API_KEY, 'super-secret')
|
|
})
|
|
|
|
test('only the warm-up backgrounds by default; the query actions return directly', async () => {
|
|
const started = []
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{ withJobs: true, shellRun: cannedShell(ran, { indexOutput: 'Indexed 1519 code units from 165 files.' }) },
|
|
)
|
|
// ctx.get('jobs') is consulted at call time, not at apply time, so the fake
|
|
// registry can be installed after the plugin has loaded.
|
|
state.jobs = {
|
|
start(spec) {
|
|
started.push(spec)
|
|
return `slopo-${started.length}`
|
|
},
|
|
}
|
|
const tool = state.tools.get('slopo')
|
|
|
|
// A review returns its report. A completion notice is best-effort — a killed
|
|
// job produces none, and an ended turn may only get the message injected — so
|
|
// the query actions must not depend on one.
|
|
const reviewed = await tool.execute({ action: 'review' }, execContext('/tmp'))
|
|
assert.equal(reviewed.kind, 'foreground')
|
|
assert.match(reviewed.report, /cluster 1/)
|
|
assert.equal(started.length, 0, 'a review must not become a job the caller has to collect')
|
|
|
|
// prepare is the long, watchable step, so it does.
|
|
const prepared = await tool.execute({ action: 'prepare' }, execContext('/tmp'))
|
|
assert.equal(prepared.kind, 'background')
|
|
assert.equal(prepared.jobId, 'slopo-1')
|
|
assert.equal(started.length, 1)
|
|
assert.equal(started[0].kind, 'slopo')
|
|
assert.equal(typeof started[0].run, 'function')
|
|
|
|
// And either can be asked for explicitly, under the framework's own name.
|
|
const forced = await tool.execute({ action: 'analyze', run_in_background: true }, execContext('/tmp'))
|
|
assert.equal(forced.kind, 'background')
|
|
assert.equal(started.length, 2)
|
|
})
|
|
|
|
test('a background job is owned by the SESSION ID and settles a JobOutcome', async () => {
|
|
// Two things the 0.1.7-rc.2 job contract requires and the old code got wrong:
|
|
// `owner` is a session id (the registry looks the agent up itself, and a job
|
|
// owned by the wrong object can be neither collected nor cancelled), and the
|
|
// producer's `done` resolves a JobOutcome whose payload field is `result` —
|
|
// `output` is the ring's word, and a job that reports to `output` settles
|
|
// with nothing for job_output to hand back.
|
|
const started = []
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{ withJobs: true, shellRun: cannedShell(ran, { indexOutput: 'Indexed 5 code units from 1 files.' }) },
|
|
)
|
|
state.jobs = {
|
|
start(spec) {
|
|
started.push(spec)
|
|
return 'slopo-1'
|
|
},
|
|
}
|
|
const value = await state.tools.get('slopo').execute({ action: 'prepare' }, execContext('/tmp'))
|
|
assert.equal(value.kind, 'background')
|
|
assert.equal(started[0].owner, 'session-1', 'ownership is fenced by session id')
|
|
|
|
const hooks = started[0].run()
|
|
assert.equal(typeof hooks.cancel, 'function')
|
|
const outcome = await hooks.done
|
|
assert.equal(outcome.status, 'completed')
|
|
assert.equal(outcome.detail, 'exit code: 0')
|
|
assert.equal(typeof outcome.result, 'string', 'the report rides `result`, not `output`')
|
|
})
|
|
|
|
test(
|
|
'review runs the real slopo CLI end to end against a real repository',
|
|
{ skip: process.env.DSH_SLOPO_E2E !== '1' || findSlopo(testHome) === undefined },
|
|
async () => {
|
|
const repo = mkdtempSync(join(tmpdir(), 'dsh-slopo-e2e-'))
|
|
const { execFileSync } = await import('node:child_process')
|
|
const { mkdirSync } = await import('node:fs')
|
|
mkdirSync(join(repo, 'src'), { recursive: true })
|
|
writeFileSync(
|
|
join(repo, 'src', 'orders.py'),
|
|
'def total_for_order(order):\n total = 0\n for line in order["lines"]:\n total += line["price"] * line["qty"]\n return round(total, 2)\n',
|
|
)
|
|
writeFileSync(
|
|
join(repo, 'src', 'invoices.py'),
|
|
'def invoice_total(invoice):\n running = 0\n for item in invoice["items"]:\n running += item["unit_price"] * item["quantity"]\n return round(running, 2)\n',
|
|
)
|
|
execFileSync('git', ['init', '-q'], { cwd: repo })
|
|
execFileSync('git', ['add', '-A'], { cwd: repo })
|
|
execFileSync('git', ['-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'init'], { cwd: repo })
|
|
writeFileSync(
|
|
join(repo, 'src', 'orders.py'),
|
|
'def total_for_order(order):\n total = 0\n for line in order["lines"]:\n total += line["price"] * line["qty"]\n return round(total, 2)\n\n\ndef refund_for_order(order):\n amount = 0\n for line in order["lines"]:\n amount += line["price"] * line["qty"]\n return round(amount, 2)\n',
|
|
)
|
|
|
|
const { state } = harness({
|
|
sourceDir: join(repo, 'src'),
|
|
embeddingModel: 'ollama/unclemusclez/jina-embeddings-v2-base-code:latest',
|
|
embeddingDimensions: 768,
|
|
// Where the real embedding backend lives is a property of the machine
|
|
// running the E2E suite, not of this repository, so it is read from the
|
|
// environment rather than committed. The default is Ollama's own
|
|
// loopback address.
|
|
embeddingApiBase: process.env.DSH_SLOPO_E2E_API_BASE ?? 'http://127.0.0.1:11434',
|
|
})
|
|
const tool = state.tools.get('slopo')
|
|
|
|
const prepared = await tool.execute({ action: 'prepare', run_in_background: false }, execContext(repo))
|
|
assert.equal(prepared.ok, true, JSON.stringify(prepared))
|
|
|
|
const reviewed = await tool.execute({ action: 'review' }, execContext(repo))
|
|
assert.equal(reviewed.ok, true, JSON.stringify(reviewed))
|
|
assert.match(reviewed.report, /cluster 1/)
|
|
assert.match(reviewed.report, /\* /)
|
|
},
|
|
)
|
|
|
|
test('the sandbox policy is resolved for the CALLING SESSION, not the deployment default', async () => {
|
|
// dsh-bash-sandbox stamps `request.sandboxPolicy ?? ctx.sandboxPolicy.resolve()`.
|
|
// Omitting it substitutes the DEPLOYMENT default, which is what made a
|
|
// danger-full-access session fail with SandboxUnavailableError before slopo
|
|
// was ever reached.
|
|
const seen = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{
|
|
sandboxMode: 'workspace-write',
|
|
sandboxPolicyResolver: {
|
|
resolve(arg) {
|
|
seen.push(arg)
|
|
return { mode: 'danger-full-access', workspaceRoot: '/some/workspace' }
|
|
},
|
|
},
|
|
},
|
|
)
|
|
const tool = state.tools.get('slopo')
|
|
await tool.execute({ action: 'review' }, execContext('/tmp'))
|
|
assert.equal(seen.length, 1, 'the policy must be resolved exactly once')
|
|
assert.ok(seen[0].session !== undefined, 'the calling session must travel with the policy')
|
|
// Every command the run issued carries it — not just the first. The count is
|
|
// not the assertion: how many commands a review issues depends on whether the
|
|
// index gate let it through.
|
|
assert.ok(state.shellRequests.length >= 1, 'the run must reach the shell')
|
|
for (const request of state.shellRequests) {
|
|
assert.deepEqual(request.sandboxPolicy, {
|
|
mode: 'danger-full-access',
|
|
workspaceRoot: '/some/workspace',
|
|
})
|
|
}
|
|
})
|
|
|
|
test('a confining executor with no policy service is refused rather than guessed at', async () => {
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{ sandboxMode: 'workspace-write' },
|
|
)
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'review' }, execContext('/tmp'))
|
|
assert.equal(value.ok, false)
|
|
assert.equal(value.errorCode, 'NO_SANDBOX_POLICY')
|
|
assert.deepEqual(state.shellRequests, [], 'nothing may be executed')
|
|
})
|
|
|
|
test('a workspace-write session is told its state lives outside the sandbox', async () => {
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{
|
|
sandboxMode: 'workspace-write',
|
|
sandboxPolicyResolver: { resolve: () => ({ mode: 'workspace-write', workspaceRoot: '/some/workspace' }) },
|
|
},
|
|
)
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'review' }, execContext('/some/workspace'))
|
|
assert.equal(value.ok, false)
|
|
assert.equal(value.errorCode, 'STATE_OUTSIDE_SANDBOX')
|
|
assert.match(value.message, /workspace-write/)
|
|
assert.match(value.remedy, /danger-full-access/)
|
|
assert.match(value.remedy, /stateDir/)
|
|
assert.deepEqual(state.shellRequests, [], 'it must not run where it cannot write')
|
|
})
|
|
|
|
test('a workspace-write session with stateDir inside the workspace proceeds', async () => {
|
|
const workspace = mkdtempSync(join(tmpdir(), 'dsh-slopo-ws-'))
|
|
const { state } = harness(
|
|
{
|
|
embeddingModel: 'ollama/example:latest',
|
|
embeddingDimensions: 768,
|
|
stateDir: join(workspace, '.slopo'),
|
|
},
|
|
{
|
|
sandboxMode: 'workspace-write',
|
|
sandboxPolicyResolver: { resolve: () => ({ mode: 'workspace-write', workspaceRoot: workspace }) },
|
|
},
|
|
)
|
|
const tool = state.tools.get('slopo')
|
|
const value = await tool.execute({ action: 'review' }, execContext(workspace))
|
|
assert.notEqual(value.errorCode, 'STATE_OUTSIDE_SANDBOX')
|
|
assert.ok(state.shellRequests.length > 0, 'it should have attempted the run')
|
|
assert.deepEqual(state.shellRequests[0].sandboxPolicy, { mode: 'workspace-write', workspaceRoot: workspace })
|
|
})
|
|
|
|
test('configure never touches the shell, and status only runs a read-only version check', async () => {
|
|
const { state } = harness({}, { sandboxMode: 'workspace-write' })
|
|
const tool = state.tools.get('slopo')
|
|
await tool.execute({ action: 'configure', config: { embeddingModel: 'x' } }, execContext('/some/workspace'))
|
|
assert.deepEqual(state.shellRequests, [], 'configure must not execute anything')
|
|
await tool.execute({ action: 'status' }, execContext('/some/workspace'))
|
|
for (const request of state.shellRequests) {
|
|
assert.match(request.command, /--version/, 'status may only run a read-only version check')
|
|
}
|
|
})
|
|
|
|
/**
|
|
* Walk a returned value and fail on anything that is not lossless JSON.
|
|
*
|
|
* `assert.deepStrictEqual` against a JSON round trip catches an `undefined`
|
|
* property (Node distinguishes `{a:1,b:undefined}` from `{a:1}`), but the walk
|
|
* names the offending path, which matters when the value is a deep diagnostic.
|
|
*/
|
|
function assertLossless(label, value) {
|
|
assert.deepEqual(value, JSON.parse(JSON.stringify(value)), `${label}: does not survive a JSON round trip`)
|
|
const walk = (node, path) => {
|
|
if (node === null || typeof node !== 'object') return
|
|
if (Array.isArray(node)) {
|
|
node.forEach((item, index) => walk(item, `${path}[${index}]`))
|
|
return
|
|
}
|
|
for (const [key, entry] of Object.entries(node)) {
|
|
assert.notEqual(entry, undefined, `${label}: ${path}.${key} is undefined, which is not lossless JSON`)
|
|
walk(entry, `${path}.${key}`)
|
|
}
|
|
}
|
|
walk(value, label)
|
|
}
|
|
|
|
test('status returns lossless JSON when the endpoint probe SUCCEEDS', async () => {
|
|
// The regression: the probe used to return `error: undefined` on a match, and
|
|
// an explicit undefined property failed the harness's output validation, so a
|
|
// perfectly healthy status call could not be serialised at all.
|
|
const { createServer } = await import('node:http')
|
|
const server = createServer((request, response) => {
|
|
response.writeHead(200, { 'content-type': 'application/json' })
|
|
response.end(JSON.stringify({ data: [{ embedding: new Array(768).fill(0.01) }] }))
|
|
})
|
|
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
|
|
const { port } = server.address()
|
|
try {
|
|
const { state } = harness({
|
|
embeddingModel: 'ollama/example:latest',
|
|
embeddingDimensions: 768,
|
|
embeddingApiBase: `http://127.0.0.1:${port}`,
|
|
})
|
|
const value = await state.tools.get('slopo').execute({ action: 'status' }, execContext('/tmp'))
|
|
assertLossless('status', value)
|
|
assert.equal(value.ok, true)
|
|
assert.equal(value.details.embeddingEndpoint.ok, true)
|
|
assert.equal(value.details.embeddingEndpoint.dimensions, 768)
|
|
assert.equal(Object.hasOwn(value.details.embeddingEndpoint, 'error'), false)
|
|
} finally {
|
|
await new Promise((resolve) => server.close(resolve))
|
|
}
|
|
})
|
|
|
|
test('status returns lossless JSON when the endpoint probe FAILS', async () => {
|
|
const { state } = harness({
|
|
embeddingModel: 'ollama/example:latest',
|
|
embeddingDimensions: 768,
|
|
// Nothing is listening on this port, so the fetch rejects.
|
|
embeddingApiBase: 'http://127.0.0.1:1',
|
|
})
|
|
const value = await state.tools.get('slopo').execute({ action: 'status' }, execContext('/tmp'))
|
|
assertLossless('status (unreachable)', value)
|
|
assert.equal(value.ok, false)
|
|
assert.ok(value.details.embeddingEndpoint.error.length > 0)
|
|
})
|
|
|
|
test('every action that returns without executing anything is lossless JSON', async () => {
|
|
const cases = [
|
|
[{ action: 'status' }, 'status (unconfigured)'],
|
|
[{ action: 'configure', config: { embeddingModel: 'x', nope: 1 } }, 'configure (unknown setting)'],
|
|
[{ action: 'configure', config: { embeddingModel: 'x' } }, 'configure (ok)'],
|
|
[{ action: 'configure', config: 'not-an-object' }, 'configure (bad patch)'],
|
|
[{ action: 'review' }, 'review (unconfigured)'],
|
|
]
|
|
for (const [args, label] of cases) {
|
|
const { state } = harness()
|
|
const value = await state.tools.get('slopo').execute(args, execContext('/tmp'))
|
|
assertLossless(label, value)
|
|
}
|
|
})
|
|
|
|
test('an action refused before it runs is lossless JSON', async () => {
|
|
const configured = {
|
|
embeddingModel: 'ollama/example:latest',
|
|
embeddingDimensions: 768,
|
|
embeddingApiKey: 'secret',
|
|
embeddingApiBase: 'http://127.0.0.1:11434',
|
|
}
|
|
|
|
const noShell = harness(configured, { withShell: false })
|
|
const refused = await noShell.state.tools.get('slopo').execute({ action: 'review' }, execContext('/tmp'))
|
|
assertLossless('review (no shell)', refused)
|
|
assert.equal(refused.errorCode, 'NO_SHELL')
|
|
|
|
const outside = harness(configured, {
|
|
sandboxMode: 'workspace-write',
|
|
sandboxPolicyResolver: { resolve: () => ({ mode: 'workspace-write', workspaceRoot: '/some/workspace' }) },
|
|
})
|
|
const value = await outside.state.tools.get('slopo').execute({ action: 'review' }, execContext('/some/workspace'))
|
|
assertLossless('review (state outside sandbox)', value)
|
|
assert.equal(value.errorCode, 'STATE_OUTSIDE_SANDBOX')
|
|
})
|
|
|
|
/** A shell runner returning canned slopo output, recording every command. */
|
|
function cannedShell(ran, { indexOutput, actionOutput = 'cluster 1:\n * /x/y.py:1-5', exitCode = 0 } = {}) {
|
|
return async (spec) => {
|
|
ran.push(spec.command)
|
|
// The command is shell-quoted, so a bare \sindex\s cannot match `'index'`.
|
|
const isIndex = /\bindex\b/.test(spec.command)
|
|
return {
|
|
exitCode: isIndex || exitCode === 0 ? 0 : exitCode,
|
|
aborted: false,
|
|
timedOut: false,
|
|
timeoutMs: spec.timeoutMs,
|
|
stdout: { text: isIndex ? indexOutput : actionOutput, truncated: false },
|
|
stderr: { text: '', truncated: false },
|
|
}
|
|
}
|
|
}
|
|
|
|
test('a scope over the budget is refused BEFORE anything is embedded', async () => {
|
|
// The regression: the plugin indexed the whole repository — node_modules and
|
|
// all, 113,768 units — and began an embed that would have run for about eight
|
|
// hours. Slopo's own >1000 guard cannot catch this, because it measures the
|
|
// pre-existing backlog and a fresh database has none.
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 113768 code units from 1300 files (0 unchanged, 0 removed).' }) },
|
|
)
|
|
const value = await state.tools.get('slopo').execute({ action: 'review' }, execContext('/tmp'))
|
|
|
|
assert.equal(value.ok, false)
|
|
assert.equal(value.errorCode, 'SCOPE_TOO_LARGE')
|
|
assert.match(value.message, /113,768 code units/)
|
|
assert.match(value.message, /still need embedding/)
|
|
assert.match(value.remedy, /sourceDirExclude/)
|
|
assert.equal(value.details.counted.units, 113_768)
|
|
assertLossless('scope refusal', value)
|
|
|
|
// The point of the gate: the expensive step never started.
|
|
assert.equal(ran.length, 1, `only the index step may run, ran: ${ran.join(' | ')}`)
|
|
assert.match(ran[0], /\bindex\b/)
|
|
assert.doesNotMatch(ran[0], /agent-review/)
|
|
})
|
|
|
|
test('a scope under the budget indexes and then runs the action', async () => {
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 1519 code units from 165 files (0 unchanged, 0 removed).' }) },
|
|
)
|
|
const value = await state.tools.get('slopo').execute({ action: 'review' }, execContext('/tmp'))
|
|
assert.equal(value.ok, true)
|
|
assert.match(value.report, /cluster 1/)
|
|
assert.equal(ran.length, 2)
|
|
assert.match(ran[0], /\bindex\b/)
|
|
assert.match(ran[1], /agent-review/)
|
|
})
|
|
|
|
test('raising the budget lets the same scope through', async () => {
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768, maxIndexUnits: 0 },
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 113768 code units from 1300 files.' }) },
|
|
)
|
|
const value = await state.tools.get('slopo').execute({ action: 'review' }, execContext('/tmp'))
|
|
assert.equal(value.ok, true)
|
|
assert.equal(ran.length, 2)
|
|
})
|
|
|
|
test('the config slopo is handed excludes the defaults, and a user list is added', async () => {
|
|
const ran = []
|
|
const { state } = harness(
|
|
{
|
|
embeddingModel: 'ollama/example:latest',
|
|
embeddingDimensions: 768,
|
|
sourceDirExclude: ['ui/**', 'src/controllers/**'],
|
|
},
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 1519 code units from 165 files.' }) },
|
|
)
|
|
const value = await state.tools.get('slopo').execute({ action: 'review' }, execContext('/tmp'))
|
|
const { readFileSync } = await import('node:fs')
|
|
const written = readFileSync(value.configPath, 'utf8')
|
|
for (const pattern of ['**/node_modules/**', 'ui/**', 'src/controllers/**']) {
|
|
assert.ok(written.includes(pattern), `the config slopo reads must exclude ${pattern}`)
|
|
}
|
|
})
|
|
|
|
test('slopo never sees a directory it should not, even when it is asked twice', async () => {
|
|
// Idempotence matters because indexing now runs on every action.
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768 },
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 1519 code units from 165 files.' }) },
|
|
)
|
|
const tool = state.tools.get('slopo')
|
|
await tool.execute({ action: 'prepare', run_in_background: false }, execContext('/tmp'))
|
|
const afterPrepare = ran.length
|
|
assert.match(ran[0], /\bindex\b/)
|
|
assert.match(ran[1], /\bembed\b/)
|
|
assert.equal(afterPrepare, 2)
|
|
})
|
|
|
|
test('status reports the excludes and the budget, so the levers are visible', async () => {
|
|
const { state } = harness({
|
|
embeddingModel: 'ollama/example:latest',
|
|
embeddingDimensions: 768,
|
|
sourceDirExclude: ['ui/**'],
|
|
})
|
|
const value = await state.tools.get('slopo').execute({ action: 'status' }, execContext('/tmp'))
|
|
assert.ok(value.details.sourceDirExclude.includes('**/node_modules/**'))
|
|
assert.ok(value.details.sourceDirExclude.includes('ui/**'))
|
|
assert.equal(value.details.useDefaultExcludes, true)
|
|
assert.equal(value.details.maxIndexUnits, 5000)
|
|
assertLossless('status', value)
|
|
})
|
|
|
|
test('the gate reads the DATABASE, so a warm index cannot hide a full backlog', async () => {
|
|
// The hole this closes: after a refusal the database exists with the whole
|
|
// scope unembedded, so the next run's `index` reports zero new units and a
|
|
// summary-line gate would wave the hours-long embed straight through.
|
|
const { mkdtempSync } = await import('node:fs')
|
|
const { tmpdir } = await import('node:os')
|
|
const { join } = await import('node:path')
|
|
const { DatabaseSync } = await import('node:sqlite')
|
|
|
|
const stateDir = mkdtempSync(join(tmpdir(), 'dsh-slopo-warm-'))
|
|
const db = new DatabaseSync(join(stateDir, 'slopo.db'))
|
|
for (const table of ['files', 'code_units', 'embeddings']) db.exec(`create table ${table} (id integer primary key)`)
|
|
const fill = (table, count) => {
|
|
const insert = db.prepare(`insert into ${table} (id) values (?)`)
|
|
for (let index = 0; index < count; index += 1) insert.run(index)
|
|
}
|
|
db.exec('begin')
|
|
fill('files', 562)
|
|
fill('code_units', 8607)
|
|
db.exec('commit')
|
|
db.close()
|
|
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768, stateDir },
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 0 code units from 0 files (562 unchanged, 0 removed).' }) },
|
|
)
|
|
const value = await state.tools.get('slopo').execute({ action: 'prepare', run_in_background: false }, execContext('/tmp'))
|
|
|
|
assert.equal(value.errorCode, 'SCOPE_TOO_LARGE', 'the summary line said zero, the database said 8,607')
|
|
assert.equal(value.details.counted.units, 8607)
|
|
assert.equal(value.details.counted.embedded, 0)
|
|
assert.equal(ran.length, 1, `only the index step may run, ran: ${ran.join(' | ')}`)
|
|
assertLossless('warm-database refusal', value)
|
|
})
|
|
|
|
test('a fully embedded index has no backlog and is never refused', async () => {
|
|
const { mkdtempSync } = await import('node:fs')
|
|
const { tmpdir } = await import('node:os')
|
|
const { join } = await import('node:path')
|
|
const { DatabaseSync } = await import('node:sqlite')
|
|
|
|
const stateDir = mkdtempSync(join(tmpdir(), 'dsh-slopo-done-'))
|
|
const db = new DatabaseSync(join(stateDir, 'slopo.db'))
|
|
for (const table of ['files', 'code_units', 'embeddings']) db.exec(`create table ${table} (id integer primary key)`)
|
|
const fill = (table, count) => {
|
|
const insert = db.prepare(`insert into ${table} (id) values (?)`)
|
|
for (let index = 0; index < count; index += 1) insert.run(index)
|
|
}
|
|
db.exec('begin')
|
|
fill('files', 5000)
|
|
fill('code_units', 50000)
|
|
fill('embeddings', 50000)
|
|
db.exec('commit')
|
|
db.close()
|
|
|
|
const ran = []
|
|
const { state } = harness(
|
|
{ embeddingModel: 'ollama/example:latest', embeddingDimensions: 768, stateDir },
|
|
{ shellRun: cannedShell(ran, { indexOutput: 'Indexed 0 code units from 0 files (562 unchanged, 0 removed).' }) },
|
|
)
|
|
const value = await state.tools.get('slopo').execute({ action: 'review' }, execContext('/tmp'))
|
|
assert.equal(value.ok, true, JSON.stringify(value))
|
|
assert.equal(ran.length, 2, 'a done index goes straight to the action')
|
|
})
|