NotExpectedYet
a1c8ecb1e8
Release / release (push) Successful in 6s
`ask` mode promises that an out-of-facet call is put to the user, and that promise needs a channel. A session can be composed without one: the `danger-full-access` permission preset sets `approval: never`, and the approval service's `decide()` then returns "rejected" before consulting any answerer. The call is refused, `facet action=add` is refused with it — widening is itself a question — and the scope can never be widened. The model is told none of that. `serviceAsk` replaces a rejected ask with `the user rejected tool \"…\"`, discarding the reason this guard wrote, so the binding is the only channel left that can state what is true. It now reads the session's effective approval policy and, when there is no channel, says the call was refused rather than asked and names the mechanism that still works: `extraRoots` is configuration applied by the profile, not a grant asked for per conversation. A missing approval service counts as no channel (the tool pipeline denies an ask it cannot route); an unreadable policy counts as a channel, so a transient failure never tells a user to go and edit their configuration.
437 lines
18 KiB
JavaScript
437 lines
18 KiB
JavaScript
/**
|
|
* The guard, driven through the real `apply` against a fake context and a real
|
|
* filesystem.
|
|
*
|
|
* The filesystem is real because the interesting failures are filesystem
|
|
* failures: `/a/b` is not inside `/a/bc`, and a symlink inside the facet that
|
|
* points outside it is the one case where deciding on strings rather than on
|
|
* canonical targets would quietly open the scope. Both are tested against
|
|
* `mkdtemp` and `realpath`, not against a mock's opinion of path semantics.
|
|
*/
|
|
|
|
import assert from 'node:assert/strict'
|
|
import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'
|
|
import { realpath } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
|
import { after, before, test } from 'node:test'
|
|
|
|
import { apply as guard } from '../lib/guard.js'
|
|
import { createFacetPolicy } from '../lib/policy.js'
|
|
import { Config } from '../index.js'
|
|
|
|
/**
|
|
* The symbol cosmokit marks a volatile config reference with. A settings write
|
|
* reaches a running plugin as `target[write](value)` — that is literally what
|
|
* `updateVolatile` does — so a test can commit an edit the way the framework
|
|
* does, without a settings service.
|
|
*/
|
|
const VOLATILE_WRITE = Symbol.for('cosmokit.volatile.write')
|
|
|
|
/** A filesystem provider with `dsh-fs-local`'s two relevant behaviours. */
|
|
const fsLikeLocal = {
|
|
/** Canonical target, walking up past a leaf that does not exist yet. */
|
|
async resolve(path, opts = {}) {
|
|
const displayPath = isAbsolute(path) ? resolve(path) : resolve(opts.cwd ?? process.cwd(), path)
|
|
try {
|
|
return { targetKey: await realpath(displayPath), displayPath }
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT') throw error
|
|
}
|
|
const missing = [basename(displayPath)]
|
|
let ancestor = dirname(displayPath)
|
|
for (;;) {
|
|
try {
|
|
return { targetKey: join(await realpath(ancestor), ...missing), displayPath }
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT') throw error
|
|
}
|
|
const parent = dirname(ancestor)
|
|
if (parent === ancestor) throw new Error(`cannot resolve ${displayPath}`)
|
|
missing.unshift(basename(ancestor))
|
|
ancestor = parent
|
|
}
|
|
},
|
|
contains(parent, child) {
|
|
const path = relative(parent.targetKey, child.targetKey)
|
|
return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path))
|
|
},
|
|
}
|
|
|
|
/**
|
|
* Capture what `apply` registered, so the test can drive it.
|
|
*
|
|
* `approval` stands in for the approval service named by its policy: `'ask'`
|
|
* and `'never'` are the two the schema admits, and leaving it undefined models a
|
|
* deployment that composed no approval service at all — a case the tool pipeline
|
|
* denies outright, so the guard must treat it as no channel.
|
|
*/
|
|
function harness({ mode = 'ask', extraRoots = [], tool = true, config, approval } = {}) {
|
|
const policy = createFacetPolicy(config ?? { mode, extraRoots })
|
|
const captured = { section: undefined, context: undefined, tool: undefined, preExecute: undefined }
|
|
const ctx = {
|
|
facetPolicy: policy,
|
|
fs: fsLikeLocal,
|
|
get: (name) =>
|
|
name === 'approval' && approval !== undefined ? { effectivePolicy: () => approval } : undefined,
|
|
systemPrompt: {
|
|
section: (value) => {
|
|
captured.section = value
|
|
},
|
|
context: (value) => {
|
|
captured.context = value
|
|
},
|
|
},
|
|
tools: {
|
|
register: (value) => {
|
|
captured.tool = value
|
|
},
|
|
},
|
|
on: (event, listener) => {
|
|
if (event === 'tools/pre-execute') captured.preExecute = listener
|
|
},
|
|
}
|
|
guard(ctx, { tool })
|
|
return { policy, ctx, captured }
|
|
}
|
|
|
|
/** Dispatch one call the way the tool pipeline would: our listener wraps `next()`. */
|
|
const dispatch = (captured, { name, args, agent, inner = { kind: 'allow' } }) =>
|
|
captured.preExecute({ name, arguments: args, agent, signal: new AbortController().signal }, async () => inner)
|
|
|
|
/** @type {string} */ let root
|
|
/** @type {string} */ let facet
|
|
const agent = (cwd = facet, id = 'session-1') => ({ session: { id, header: { cwd } } })
|
|
|
|
before(async () => {
|
|
root = await realpath(await mkdtemp(join(tmpdir(), 'dsh-facet-')))
|
|
facet = join(root, 'src', 'controllers', 'room')
|
|
await mkdir(facet, { recursive: true })
|
|
await mkdir(join(root, 'src', 'controllers', 'other'), { recursive: true })
|
|
await mkdir(join(root, 'src', 'shared'), { recursive: true })
|
|
await writeFile(join(facet, 'index.ts'), 'export {}\n')
|
|
await writeFile(join(root, 'src', 'shared', 'types.ts'), 'export type T = 1\n')
|
|
// The case a string comparison would miss.
|
|
await symlink(join(root, 'src', 'shared', 'types.ts'), join(facet, 'link.ts'))
|
|
})
|
|
|
|
after(async () => {
|
|
await rm(root, { recursive: true, force: true })
|
|
})
|
|
|
|
test('a path inside the facet is ordinary work', async () => {
|
|
const { captured } = harness()
|
|
assert.deepEqual(await dispatch(captured, { name: 'read', args: { file_path: 'index.ts' }, agent: agent() }), {
|
|
kind: 'allow',
|
|
})
|
|
assert.deepEqual(
|
|
await dispatch(captured, { name: 'grep', args: { pattern: 'x', path: '.' }, agent: agent() }),
|
|
{ kind: 'allow' },
|
|
)
|
|
})
|
|
|
|
test('a path outside the facet is put to the user', async () => {
|
|
const { captured } = harness()
|
|
const decision = await dispatch(captured, {
|
|
name: 'read',
|
|
args: { file_path: '../../shared/types.ts' },
|
|
agent: agent(),
|
|
})
|
|
assert.equal(decision.kind, 'ask')
|
|
assert.match(decision.reason, /src\/shared\/types\.ts/)
|
|
assert.match(decision.reason, /src\/controllers\/room/)
|
|
})
|
|
|
|
test('the sibling module is outside, which is the entire point', async () => {
|
|
const { captured } = harness()
|
|
const decision = await dispatch(captured, {
|
|
name: 'read',
|
|
args: { file_path: '../other/index.ts' },
|
|
agent: agent(),
|
|
})
|
|
assert.equal(decision.kind, 'ask')
|
|
})
|
|
|
|
test('a symlink inside the facet does not open it', async () => {
|
|
const { captured } = harness()
|
|
const decision = await dispatch(captured, { name: 'read', args: { file_path: 'link.ts' }, agent: agent() })
|
|
assert.equal(decision.kind, 'ask', 'containment must be decided on canonical targets')
|
|
})
|
|
|
|
test('a prefix is not a parent', async () => {
|
|
const { captured } = harness()
|
|
await mkdir(join(root, 'src', 'controllers', 'roommate'), { recursive: true })
|
|
const decision = await dispatch(captured, {
|
|
name: 'read',
|
|
args: { file_path: '../roommate/x.ts' },
|
|
agent: agent(),
|
|
})
|
|
assert.equal(decision.kind, 'ask')
|
|
})
|
|
|
|
test('an omitted path means the session directory, which is in scope', async () => {
|
|
const { captured } = harness()
|
|
assert.deepEqual(await dispatch(captured, { name: 'glob', args: { pattern: '**/*.ts' }, agent: agent() }), {
|
|
kind: 'allow',
|
|
})
|
|
})
|
|
|
|
test('tools the guard cannot judge are left alone, deliberately', async () => {
|
|
const { captured } = harness()
|
|
// A shell command cannot be read reliably from argv, so the facet is enforced
|
|
// on the file tools and `bash` stays a legitimate way out. The prompt says so.
|
|
assert.deepEqual(
|
|
await dispatch(captured, { name: 'bash', args: { command: `cat ${root}/src/shared/types.ts` }, agent: agent() }),
|
|
{ kind: 'allow' },
|
|
)
|
|
assert.deepEqual(await dispatch(captured, { name: 'mcp_files', args: { path: '/etc' }, agent: agent() }), {
|
|
kind: 'allow',
|
|
})
|
|
})
|
|
|
|
test('an agent with no working directory is not scoped', async () => {
|
|
const { captured } = harness()
|
|
assert.deepEqual(await dispatch(captured, { name: 'read', args: { file_path: '/etc/hosts' } }), { kind: 'allow' })
|
|
})
|
|
|
|
test('a standing grant and a session grant both open the scope', async () => {
|
|
const standing = harness({ extraRoots: ['../../shared'] })
|
|
assert.deepEqual(
|
|
await dispatch(standing.captured, { name: 'read', args: { file_path: '../../shared/types.ts' }, agent: agent() }),
|
|
{ kind: 'allow' },
|
|
)
|
|
|
|
const granted = harness()
|
|
granted.policy.grant('session-1', ['../../shared'], facet)
|
|
assert.deepEqual(
|
|
await dispatch(granted.captured, { name: 'read', args: { file_path: '../../shared/types.ts' }, agent: agent() }),
|
|
{ kind: 'allow' },
|
|
)
|
|
})
|
|
|
|
test('deny mode refuses instead of asking, and still allows the way out', async () => {
|
|
const { captured } = harness({ mode: 'deny' })
|
|
const decision = await dispatch(captured, {
|
|
name: 'read',
|
|
args: { file_path: '../../shared/types.ts' },
|
|
agent: agent(),
|
|
})
|
|
assert.equal(decision.kind, 'deny')
|
|
assert.match(decision.reason, /facet action=add/)
|
|
|
|
// Widening is explicit and intentional, so it is asked even here — otherwise
|
|
// the user would have no way to grant anything at all.
|
|
assert.equal(
|
|
(await dispatch(captured, { name: 'facet', args: { action: 'add', paths: ['../../shared'] }, agent: agent() }))
|
|
.kind,
|
|
'ask',
|
|
)
|
|
})
|
|
|
|
test('off mode registers nothing', () => {
|
|
const { captured } = harness({ mode: 'off' })
|
|
assert.equal(captured.preExecute, undefined)
|
|
assert.equal(captured.section, undefined)
|
|
assert.equal(captured.tool, undefined)
|
|
})
|
|
|
|
test('a live mode edit decides the next call, without the guard being remounted', async () => {
|
|
// The loader hands the guard the schema-resolved config, so `mode` arrives as
|
|
// a volatile reference rather than a string. This drives that exact shape.
|
|
const config = Config({ mode: 'ask' })
|
|
const { captured } = harness({ config })
|
|
const outside = { name: 'read', args: { file_path: '../../shared/types.ts' }, agent: agent() }
|
|
|
|
assert.equal((await dispatch(captured, outside)).kind, 'ask')
|
|
|
|
// What a settings write does: commit into the reference in place, and
|
|
// deliberately do not re-apply the plugin. One mount, three answers.
|
|
config.mode[VOLATILE_WRITE]('deny')
|
|
assert.equal((await dispatch(captured, outside)).kind, 'deny')
|
|
|
|
// `off` silences the guard entirely, which is the whole meaning of the mode.
|
|
config.mode[VOLATILE_WRITE]('off')
|
|
assert.equal((await dispatch(captured, outside)).kind, 'allow')
|
|
})
|
|
|
|
test('the guard adds to a decision, never overrides one', async () => {
|
|
const { captured } = harness()
|
|
const denied = await dispatch(captured, {
|
|
name: 'read',
|
|
args: { file_path: '../../shared/types.ts' },
|
|
agent: agent(),
|
|
inner: { kind: 'deny', reason: 'an earlier policy said no' },
|
|
})
|
|
assert.deepEqual(denied, { kind: 'deny', reason: 'an earlier policy said no' })
|
|
})
|
|
|
|
test('the rule is registered with no variables to interpolate', () => {
|
|
const { captured } = harness()
|
|
assert.equal(captured.section.name, 'facet-scope')
|
|
assert.equal(captured.section.interpolate, false)
|
|
assert.match(captured.section.text, /Facet scope/)
|
|
})
|
|
|
|
test('the binding follows the agent, and is empty without one', () => {
|
|
const { captured, policy } = harness()
|
|
assert.equal(captured.context.name, 'facet-binding')
|
|
assert.match(captured.context.text({ agent: agent() }), new RegExp(`facet: ${facet}$`, 'm'))
|
|
|
|
policy.grant('session-1', ['../../shared'], facet)
|
|
assert.match(captured.context.text({ agent: agent() }), /granted outside it: .*src\/shared/)
|
|
|
|
assert.equal(captured.context.text({}), '')
|
|
assert.equal(captured.context.text(undefined), '')
|
|
})
|
|
|
|
test('the binding refuses honestly when the session has no approval channel', () => {
|
|
// `danger-full-access` sets approval: never, so `decide()` returns "rejected"
|
|
// before any answerer is consulted — and `facet action=add`, itself an ask, is
|
|
// refused with it. The reason the guard writes never reaches the model on that
|
|
// path (`serviceAsk` replaces it with `the user rejected tool "…"`), so the
|
|
// binding is the only place the truth can be said.
|
|
const { captured } = harness({ mode: 'ask', approval: 'never' })
|
|
const text = captured.context.text({ agent: agent() })
|
|
|
|
assert.match(text, /approval prompts are off in this session/)
|
|
assert.match(text, /cannot be approved either/)
|
|
assert.match(text, /`extraRoots`/)
|
|
|
|
// The promise `ask` makes cannot be kept here, so it must not be made.
|
|
assert.doesNotMatch(text, /put to the user as a question/)
|
|
})
|
|
|
|
test('an approval channel keeps the promise, and a missing service counts as none', () => {
|
|
const withChannel = harness({ mode: 'ask', approval: 'ask' })
|
|
assert.match(withChannel.captured.context.text({ agent: agent() }), /put to the user as a question/)
|
|
|
|
// No approval service at all: the tool pipeline denies an ask it cannot route,
|
|
// so the binding must not offer one.
|
|
const without = harness({ mode: 'ask' })
|
|
assert.match(without.captured.context.text({ agent: agent() }), /approval prompts are off in this session/)
|
|
})
|
|
|
|
test('deny mode still names the way that works when there is no channel', () => {
|
|
const { captured } = harness({ mode: 'deny', approval: 'never' })
|
|
const text = captured.context.text({ agent: agent() })
|
|
assert.match(text, /refused/)
|
|
assert.match(text, /`extraRoots`/)
|
|
})
|
|
|
|
test('the facet tool reports the scope', async () => {
|
|
const { captured } = harness()
|
|
const value = await captured.tool.execute({ action: 'status' }, { agent: agent() })
|
|
assert.equal(value.ok, true)
|
|
assert.equal(value.root, facet)
|
|
assert.deepEqual(value.granted, [])
|
|
assert.equal(value.mode, 'ask')
|
|
})
|
|
|
|
test('the facet tool grants, narrows and clears', async () => {
|
|
const { captured, policy } = harness()
|
|
const exec = { agent: agent() }
|
|
|
|
await captured.tool.execute({ action: 'add', paths: ['../../shared'] }, exec)
|
|
assert.deepEqual(policy.grantsFor('session-1'), [join(root, 'src', 'shared')])
|
|
|
|
// Granting twice is a no-op, and says so rather than reporting a change.
|
|
const again = await captured.tool.execute({ action: 'add', paths: ['../../shared'] }, exec)
|
|
assert.match(again.note, /Already granted/)
|
|
assert.deepEqual(policy.grantsFor('session-1'), [join(root, 'src', 'shared')])
|
|
|
|
await captured.tool.execute({ action: 'remove', paths: ['../../shared'] }, exec)
|
|
assert.deepEqual(policy.grantsFor('session-1'), [])
|
|
|
|
await captured.tool.execute({ action: 'add', paths: ['../../shared'] }, exec)
|
|
await captured.tool.execute({ action: 'reset' }, exec)
|
|
assert.deepEqual(policy.grantsFor('session-1'), [])
|
|
})
|
|
|
|
test('the facet tool refuses a call it cannot act on', async () => {
|
|
const { captured } = harness()
|
|
const exec = { agent: agent() }
|
|
await assert.rejects(() => captured.tool.execute({ action: 'add' }, exec), /needs `paths`/)
|
|
await assert.rejects(() => captured.tool.execute({ action: 'remove' }, exec), /needs `paths`/)
|
|
await assert.rejects(() => captured.tool.execute({ action: 'explode' }, exec), /unknown action/)
|
|
await assert.rejects(
|
|
() => captured.tool.execute({ action: 'status' }, { agent: { session: { id: 's', header: {} } } }),
|
|
/no working directory/,
|
|
)
|
|
})
|
|
|
|
test('the tool can be left unregistered', () => {
|
|
const { captured } = harness({ tool: false })
|
|
assert.equal(captured.tool, undefined)
|
|
// The guard is the part that matters and it is still there.
|
|
assert.equal(typeof captured.preExecute, 'function')
|
|
})
|
|
|
|
/**
|
|
* Validate a value against the tool's own declared output schema.
|
|
*
|
|
* The host enforces this schema before the model sees a result, and the schema
|
|
* is `additionalProperties: false`, so a field leaking out of `execute` is not a
|
|
* cosmetic extra — it is a failed call. The check reads the declared schema
|
|
* rather than a copy of it, so the two cannot drift apart.
|
|
*/
|
|
function schemaErrors(schema, value, at = '$') {
|
|
const errors = []
|
|
if (schema.oneOf !== undefined) {
|
|
if (!schema.oneOf.some((branch) => schemaErrors(branch, value, at).length === 0)) {
|
|
errors.push(`${at}: matched no oneOf branch`)
|
|
}
|
|
return errors
|
|
}
|
|
if (schema.type === 'object') {
|
|
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
errors.push(`${at}: expected an object`)
|
|
return errors
|
|
}
|
|
const properties = schema.properties ?? {}
|
|
for (const [key, sub] of Object.entries(properties)) {
|
|
if (sub.required === true && value[key] === undefined) errors.push(`${at}.${key}: required but missing`)
|
|
}
|
|
if (schema.additionalProperties === false) {
|
|
for (const key of Object.keys(value)) {
|
|
if (!(key in properties)) errors.push(`${at}.${key}: not declared, but additionalProperties is false`)
|
|
}
|
|
}
|
|
for (const [key, sub] of Object.entries(properties)) {
|
|
if (value[key] !== undefined) errors.push(...schemaErrors(sub, value[key], `${at}.${key}`))
|
|
}
|
|
return errors
|
|
}
|
|
if (schema.type === 'array') {
|
|
if (!Array.isArray(value)) return [`${at}: expected an array`]
|
|
return value.flatMap((item, index) => schemaErrors(schema.items ?? {}, item, `${at}[${index}]`))
|
|
}
|
|
if (schema.type === 'string' && typeof value !== 'string') return [`${at}: expected a string`]
|
|
if (schema.type === 'boolean' && typeof value !== 'boolean') return [`${at}: expected a boolean`]
|
|
return errors
|
|
}
|
|
|
|
test('every facet result satisfies the schema the model is given', async () => {
|
|
const { captured } = harness()
|
|
const schema = captured.tool.output.schema
|
|
const exec = { agent: agent() }
|
|
|
|
const results = [
|
|
await captured.tool.execute({ action: 'status' }, exec),
|
|
await captured.tool.execute({ action: 'add', paths: ['../../shared'] }, exec),
|
|
// The case that produces the optional `note`: granting is idempotent.
|
|
await captured.tool.execute({ action: 'add', paths: ['../../shared'] }, exec),
|
|
await captured.tool.execute({ action: 'remove', paths: ['../../shared'] }, exec),
|
|
await captured.tool.execute({ action: 'reset' }, exec),
|
|
]
|
|
for (const [index, value] of results.entries()) {
|
|
assert.deepEqual(schemaErrors(schema, value), [], `result ${index} does not match the declared schema`)
|
|
}
|
|
|
|
// And the renderer must be a pure function of that value.
|
|
for (const value of results) {
|
|
const blocks = captured.tool.output.render({ action: value.action }, value)
|
|
assert.equal(Array.isArray(blocks), true)
|
|
assert.equal(blocks[0].type, 'text')
|
|
}
|
|
})
|