init
This commit is contained in:
NotExpectedYet committed
commit
0964819cdc
34 files changed
+7912
No files matched your search
@@ -0,0 +1,259 @@
|
||||
/**
|
||||
* The before/after instrument.
|
||||
*
|
||||
* Turns "recall should save tokens" into a number that can be re-run. It reads
|
||||
* two independent stores, because they answer different questions:
|
||||
*
|
||||
* - the cost ledger knows **money and per-request volume**: requests made,
|
||||
* cache-read tokens, and therefore the average context each request carried.
|
||||
* That average is the figure this whole workstream is measured against.
|
||||
* - the session projection cache knows **context shape**: the surface each
|
||||
* session reached, and whether that session used recall at all.
|
||||
*
|
||||
* Run it before the plugin sees real use to capture the baseline, then run it
|
||||
* again after. Sessions that used recall are the "after" group; everything else
|
||||
* is the baseline. Nothing is inferred — when there is no "after" group it says
|
||||
* so instead of printing a table of zeroes.
|
||||
*
|
||||
* node scripts/baseline.mjs [--ledger <path>] [--cache <path>] [--json]
|
||||
*/
|
||||
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const DSH_HOME = process.env.DSH_HOME ?? join(homedir(), '.dsh')
|
||||
|
||||
function argValue(flag, fallback) {
|
||||
const at = process.argv.indexOf(flag)
|
||||
return at === -1 ? fallback : process.argv[at + 1]
|
||||
}
|
||||
|
||||
const LEDGER = argValue('--ledger', join(DSH_HOME, 'storages', 'cost-meter', 'ledger.json'))
|
||||
const CACHE = argValue('--cache', join(DSH_HOME, 'storages', 'session_projcache.json'))
|
||||
const AS_JSON = process.argv.includes('--json')
|
||||
|
||||
/** Format an integer with thousands separators. */
|
||||
const n = (value) => Math.round(value).toLocaleString('en-GB')
|
||||
|
||||
/** A compact human figure. */
|
||||
function short(value) {
|
||||
if (value >= 1e9) return (value / 1e9).toFixed(1) + 'B'
|
||||
if (value >= 1e6) return (value / 1e6).toFixed(1) + 'M'
|
||||
if (value >= 1e3) return Math.round(value / 1e3) + 'k'
|
||||
return String(Math.round(value))
|
||||
}
|
||||
|
||||
async function readJson(path, fallback) {
|
||||
try {
|
||||
return JSON.parse(await readFile(path, 'utf8'))
|
||||
} catch {
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-session request volume and cost, from the ledger.
|
||||
*
|
||||
* The ledger groups by day, and a session that spans midnight appears under both
|
||||
* — so sessions are merged by id rather than listed.
|
||||
*/
|
||||
function sessionsFromLedger(ledger) {
|
||||
const byId = new Map()
|
||||
for (const day of Object.values(ledger?.days ?? {})) {
|
||||
for (const session of day.sessions ?? []) {
|
||||
const id = session.id
|
||||
if (typeof id !== 'string') continue
|
||||
const entry = byId.get(id) ?? { id, calls: 0, cacheRead: 0, input: 0, output: 0, cost: 0 }
|
||||
entry.calls += session.calls ?? 0
|
||||
entry.cacheRead += session.cacheRead ?? 0
|
||||
entry.input += session.input ?? 0
|
||||
entry.output += session.output ?? 0
|
||||
entry.cost += session.cost ?? 0
|
||||
byId.set(id, entry)
|
||||
}
|
||||
}
|
||||
for (const entry of byId.values()) {
|
||||
// What each request carried: the cached prefix plus the new input.
|
||||
entry.contextPerRequest = entry.calls > 0 ? (entry.cacheRead + entry.input) / entry.calls : 0
|
||||
}
|
||||
return [...byId.values()]
|
||||
}
|
||||
|
||||
/** Context shape and recall usage, from the projection cache. */
|
||||
function sessionsFromCache(cache) {
|
||||
const rows = cache?.tables?.sessions ?? {}
|
||||
const out = []
|
||||
for (const [id, entry] of Object.entries(rows)) {
|
||||
const units = entry?.rows ?? {}
|
||||
const pressure = units.contextPressure?.val
|
||||
const stats = units.sessionStats?.val
|
||||
const usage = units.recallUsage?.val
|
||||
out.push({
|
||||
id,
|
||||
cwd: entry?.identity?.cwd ?? '(unknown)',
|
||||
surfaceTokens: pressure?.surfaceTokens ?? 0,
|
||||
contextWindow: pressure?.contextWindow ?? 0,
|
||||
steps: stats?.steps ?? 0,
|
||||
turns: stats?.turns ?? 0,
|
||||
recalls: usage?.searches ?? 0,
|
||||
usedRecall: (usage?.searches ?? 0) > 0,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Mean and median of a numeric list. */
|
||||
function describe(values) {
|
||||
if (values.length === 0) return { count: 0, mean: 0, median: 0, max: 0 }
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length
|
||||
return {
|
||||
count: sorted.length,
|
||||
mean,
|
||||
median: sorted[Math.floor(sorted.length / 2)],
|
||||
max: sorted[sorted.length - 1],
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const ledger = await readJson(LEDGER, undefined)
|
||||
const cache = await readJson(CACHE, undefined)
|
||||
|
||||
if (ledger === undefined && cache === undefined) {
|
||||
console.error(`no data. Looked for:\n ${LEDGER}\n ${CACHE}`)
|
||||
process.exitCode = 1
|
||||
return
|
||||
}
|
||||
|
||||
const money = sessionsFromLedger(ledger)
|
||||
const shape = sessionsFromCache(cache)
|
||||
const shapeById = new Map(shape.map((row) => [row.id, row]))
|
||||
|
||||
const totalCalls = money.reduce((sum, row) => sum + row.calls, 0)
|
||||
const totalCacheRead = money.reduce((sum, row) => sum + row.cacheRead, 0)
|
||||
const totalInput = money.reduce((sum, row) => sum + row.input, 0)
|
||||
const totalCost = money.reduce((sum, row) => sum + row.cost, 0)
|
||||
const averageContext = totalCalls > 0 ? (totalCacheRead + totalInput) / totalCalls : 0
|
||||
|
||||
// Workspaces, so a comparison is not an average over unrelated projects.
|
||||
const byWorkspace = new Map()
|
||||
for (const row of money) {
|
||||
const cwd = shapeById.get(row.id)?.cwd ?? '(not in projection cache)'
|
||||
const entry = byWorkspace.get(cwd) ?? { cwd, sessions: 0, calls: 0, cacheRead: 0, input: 0, cost: 0 }
|
||||
entry.sessions += 1
|
||||
entry.calls += row.calls
|
||||
entry.cacheRead += row.cacheRead
|
||||
entry.input += row.input
|
||||
entry.cost += row.cost
|
||||
byWorkspace.set(cwd, entry)
|
||||
}
|
||||
|
||||
// The comparison the objective names: average context PER REQUEST, which only
|
||||
// the ledger knows, split by whether that session used recall. Peak surface is
|
||||
// reported separately because it answers a different question — what a session
|
||||
// grew to, not what each request carried.
|
||||
const withRecall = money.filter((row) => shapeById.get(row.id)?.usedRecall === true)
|
||||
const without = money.filter((row) => shapeById.get(row.id)?.usedRecall !== true)
|
||||
const pressure = shape.filter((row) => row.surfaceTokens > 0)
|
||||
const window = pressure[0]?.contextWindow ?? 0
|
||||
const trigger = Math.round(window * 0.8)
|
||||
|
||||
const report = {
|
||||
baseline: {
|
||||
sessions: money.length,
|
||||
requests: totalCalls,
|
||||
cacheReadTokens: totalCacheRead,
|
||||
inputTokens: totalInput,
|
||||
costUsd: Number(totalCost.toFixed(2)),
|
||||
// Cost-weighted: every request counts once. This is the figure that maps
|
||||
// to money, and the one the objective names.
|
||||
pooledAverageContextPerRequest: Math.round(averageContext),
|
||||
// Typical: every session counts once. Differs from the pooled figure by
|
||||
// whatever the largest sessions do, which here is a factor of two — so
|
||||
// quoting only one of them would mislead in a different direction
|
||||
// depending on which was chosen.
|
||||
meanSessionContextPerRequest: Math.round(describe(money.map((row) => row.contextPerRequest)).mean),
|
||||
},
|
||||
workspaces: [...byWorkspace.values()]
|
||||
.sort((a, b) => b.calls - a.calls)
|
||||
.map((entry) => ({
|
||||
cwd: entry.cwd,
|
||||
sessions: entry.sessions,
|
||||
requests: entry.calls,
|
||||
averageContextPerRequest: entry.calls > 0 ? Math.round((entry.cacheRead + entry.input) / entry.calls) : 0,
|
||||
costUsd: Number(entry.cost.toFixed(2)),
|
||||
})),
|
||||
contextShape: {
|
||||
sessionsMeasured: pressure.length,
|
||||
contextWindow: window,
|
||||
compactionTrigger: trigger,
|
||||
surface: describe(pressure.map((row) => row.surfaceTokens)),
|
||||
reachedCompaction: pressure.filter((row) => row.surfaceTokens >= trigger && trigger > 0).length,
|
||||
},
|
||||
recall: {
|
||||
sessionsWithRecallUsage: withRecall.length,
|
||||
sessionsWithout: without.length,
|
||||
comparable: withRecall.length > 0 && without.length > 0,
|
||||
after: withRecall.length > 0 ? describe(withRecall.map((row) => row.contextPerRequest)) : null,
|
||||
before: without.length > 0 ? describe(without.map((row) => row.contextPerRequest)) : null,
|
||||
},
|
||||
sources: { ledger: LEDGER, cache: CACHE },
|
||||
}
|
||||
|
||||
if (AS_JSON) {
|
||||
console.log(JSON.stringify(report, null, 2))
|
||||
return
|
||||
}
|
||||
|
||||
const B = report.baseline
|
||||
console.log('BASELINE — every session on this machine')
|
||||
console.log(` sessions ${n(B.sessions)}`)
|
||||
console.log(` requests ${n(B.requests)}`)
|
||||
console.log(` cache-read tokens ${n(B.cacheReadTokens)}`)
|
||||
console.log(` avg context/request (pooled) ${n(B.pooledAverageContextPerRequest)}`)
|
||||
console.log(` avg context/request (typical) ${n(B.meanSessionContextPerRequest)} mean of per-session averages`)
|
||||
console.log(` cost $${B.costUsd.toFixed(2)}`)
|
||||
console.log('')
|
||||
|
||||
console.log('BY WORKSPACE')
|
||||
for (const entry of report.workspaces.slice(0, 6)) {
|
||||
console.log(
|
||||
` ${String(entry.requests).padStart(7)} req ${String(entry.averageContextPerRequest).padStart(8)} ctx/req ` +
|
||||
`$${entry.costUsd.toFixed(2).padStart(8)} ${entry.cwd}`,
|
||||
)
|
||||
}
|
||||
if (report.workspaces.length > 6) console.log(` … and ${report.workspaces.length - 6} more`)
|
||||
console.log('')
|
||||
|
||||
const S = report.contextShape
|
||||
console.log('CONTEXT SHAPE — what sessions reach')
|
||||
console.log(` sessions with a reading ${n(S.sessionsMeasured)} of ${n(shape.length)}`)
|
||||
console.log(` peak surface ${n(S.surface.max)}`)
|
||||
console.log(` median surface ${n(S.surface.median)}`)
|
||||
console.log(` compaction trigger ${n(S.compactionTrigger)}`)
|
||||
console.log(` sessions that reached it ${n(S.reachedCompaction)} of ${n(S.sessionsMeasured)}`)
|
||||
console.log('')
|
||||
|
||||
console.log('BEFORE / AFTER — recall usage')
|
||||
const R = report.recall
|
||||
console.log(` sessions with recall usage ${n(R.sessionsWithRecallUsage)}`)
|
||||
console.log(` sessions without ${n(R.sessionsWithout)}`)
|
||||
console.log(' metric: each session\'s own average context per request (ledger),')
|
||||
console.log(' not peak surface — the two answer different questions')
|
||||
if (!R.comparable) {
|
||||
// The honest output while the plugin has not run. A table here would be a
|
||||
// comparison of nothing against nothing.
|
||||
console.log('')
|
||||
console.log(' Nothing to compare yet: no session has used recall. Run this again')
|
||||
console.log(' after a week of use and the two groups will report side by side.')
|
||||
} else {
|
||||
console.log('')
|
||||
console.log(` before ${n(R.before.mean)} ctx/req over ${n(R.before.count)} sessions (median ${n(R.before.median)})`)
|
||||
console.log(` after ${n(R.after.mean)} ctx/req over ${n(R.after.count)} sessions (median ${n(R.after.median)})`)
|
||||
const delta = R.before.mean > 0 ? ((R.before.mean - R.after.mean) / R.before.mean) * 100 : 0
|
||||
console.log(` change ${delta >= 0 ? '-' : '+'}${Math.abs(delta).toFixed(1)}%`)
|
||||
}
|
||||
}
|
||||
|
||||
await main()
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Real-scale measurement.
|
||||
*
|
||||
* Runs the actual plugin — `apply` plus the registered tool — against a real
|
||||
* workspace and a real embedding endpoint, then reports what the tool costs the
|
||||
* context window versus what reading the files would have cost.
|
||||
*
|
||||
* The comparison is the whole point of the plugin, so it is measured rather than
|
||||
* asserted: the excerpt characters are counted from the tool's own output, and
|
||||
* the "read instead" figure is the character count of the files the results came
|
||||
* from. Token counts are `characters / 4`, which is an approximation and is
|
||||
* labelled as one everywhere it appears.
|
||||
*
|
||||
* node scripts/measure.mjs [workspace] [query] [budgetMs]
|
||||
*/
|
||||
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { apply } from '../index.js'
|
||||
|
||||
const WORKSPACE = process.argv[2] ?? '/root/workspaces/ha-runtime'
|
||||
const QUERY = process.argv[3] ?? 'how does the occupancy state machine decide a room is occupied'
|
||||
const BUDGET_MS = Number(process.argv[4] ?? 90_000)
|
||||
|
||||
const API_BASE = process.env.RECALL_API_BASE ?? 'http://10.50.1.2:11434'
|
||||
const MODEL = process.env.RECALL_MODEL ?? 'unclemusclez/jina-embeddings-v2-base-code:latest'
|
||||
|
||||
const config = {
|
||||
embeddingApiBase: API_BASE,
|
||||
embeddingModel: MODEL,
|
||||
embeddingDimensions: 768,
|
||||
embeddingApiKey: '',
|
||||
embeddingBatchSize: 32,
|
||||
embeddingConcurrency: 4,
|
||||
embeddingTimeoutMs: 120_000,
|
||||
extensions: [],
|
||||
exclude: [],
|
||||
maxFiles: 20_000,
|
||||
maxFileBytes: 1_048_576,
|
||||
chunkLines: 48,
|
||||
maxChunkLines: 80,
|
||||
overlapLines: 8,
|
||||
defaultLimit: 8,
|
||||
maxLimit: 25,
|
||||
perFileLimit: 2,
|
||||
excerptLines: 40,
|
||||
minSimilarity: 0.2,
|
||||
syncBudgetMs: BUDGET_MS,
|
||||
stateDir: '',
|
||||
}
|
||||
|
||||
/** The smallest context `apply` touches, with no settings provider mounted. */
|
||||
function context() {
|
||||
const tools = []
|
||||
const cleanups = []
|
||||
return {
|
||||
tools,
|
||||
ctx: {
|
||||
logger: () => ({ warn() {}, info() {}, error() {} }),
|
||||
effect: (fn) => {
|
||||
const dispose = fn()
|
||||
if (typeof dispose === 'function') cleanups.push(dispose)
|
||||
},
|
||||
get: () => undefined,
|
||||
inject: () => {},
|
||||
tools: { register: (tool) => tools.push(tool) },
|
||||
},
|
||||
dispose: () => cleanups.forEach((fn) => fn()),
|
||||
}
|
||||
}
|
||||
|
||||
const tokens = (text) => Math.round(text.length / 4)
|
||||
|
||||
async function main() {
|
||||
const stateDir = await mkdtemp(join(tmpdir(), 'recall-measure-'))
|
||||
const { ctx, tools, dispose } = context()
|
||||
apply(ctx, { ...config, stateDir })
|
||||
const tool = tools[0]
|
||||
const exec = { signal: new AbortController().signal, agent: { session: { header: { cwd: WORKSPACE } } } }
|
||||
|
||||
console.log(`workspace : ${WORKSPACE}`)
|
||||
console.log(`endpoint : ${API_BASE} model: ${MODEL}`)
|
||||
console.log(`query : ${QUERY}`)
|
||||
console.log('')
|
||||
|
||||
// --- cold start, bounded -------------------------------------------------
|
||||
const coldStart = Date.now()
|
||||
const first = await tool.execute({ query: QUERY }, exec)
|
||||
const coldMs = Date.now() - coldStart
|
||||
console.log('--- FIRST SEARCH (bounded cold start) ---')
|
||||
console.log(`elapsed : ${(coldMs / 1000).toFixed(1)}s (budget ${BUDGET_MS / 1000}s)`)
|
||||
console.log(`scanned files : ${first.index.scanned}`)
|
||||
console.log(`indexed files : ${first.index.files}`)
|
||||
console.log(`chunks : ${first.index.chunks}`)
|
||||
console.log(`embedded now : ${first.index.embedded}`)
|
||||
console.log(`still pending : ${first.index.pending}`)
|
||||
console.log(`skipped (big) : ${first.index.oversized}`)
|
||||
console.log(`results : ${first.results.length} best similarity ${first.best}`)
|
||||
console.log('')
|
||||
|
||||
// --- the token comparison ------------------------------------------------
|
||||
const rendered = tool.output.render({ query: QUERY }, first)[0].text
|
||||
let wholeFileChars = 0
|
||||
const seen = new Set()
|
||||
for (const result of first.results) {
|
||||
if (seen.has(result.path)) continue
|
||||
seen.add(result.path)
|
||||
try {
|
||||
wholeFileChars += (await readFile(join(WORKSPACE, result.path), 'utf8')).length
|
||||
} catch {
|
||||
// A result for a file that has since moved does not change the argument.
|
||||
}
|
||||
}
|
||||
|
||||
console.log('--- WHAT THE MODEL RECEIVED ---')
|
||||
console.log(`recall output : ${rendered.length} chars ~${tokens(rendered)} tokens`)
|
||||
console.log(`whole files : ${wholeFileChars} chars ~${Math.round(wholeFileChars / 4)} tokens across ${seen.size} file(s)`)
|
||||
if (wholeFileChars > 0) {
|
||||
console.log(`ratio : ${(wholeFileChars / rendered.length).toFixed(1)}x smaller`)
|
||||
}
|
||||
console.log('')
|
||||
for (const [at, result] of first.results.entries()) {
|
||||
console.log(`${at + 1}. ${result.path}:${result.startLine}-${result.endLine} sim ${result.similarity} ${result.truncated ? '(truncated)' : ''}`)
|
||||
}
|
||||
console.log('')
|
||||
|
||||
// --- incremental ---------------------------------------------------------
|
||||
const secondStart = Date.now()
|
||||
const second = await tool.execute({ query: 'what happens when the SSE stream reconnects' }, exec)
|
||||
const secondMs = Date.now() - secondStart
|
||||
console.log('--- SECOND SEARCH (warm index, different query) ---')
|
||||
console.log(`elapsed : ${secondMs}ms`)
|
||||
console.log(`embedded : ${second.index.embedded} file(s)`)
|
||||
console.log(`pending : ${second.index.pending}`)
|
||||
console.log(`results : ${second.results.length}`)
|
||||
console.log('')
|
||||
console.log('(token figures are characters/4 — an approximation, not a measurement)')
|
||||
|
||||
dispose()
|
||||
await rm(stateDir, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
await main()
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* What focus scoping does to the index, measured without an embedding endpoint.
|
||||
*
|
||||
* The semantic half of the claim — that a focused search ranks better — needs a
|
||||
* live model. This half does not: index size, per-scope chunk counts, and the
|
||||
* cold-start cost they imply are all arithmetic over the chunker, which is
|
||||
* deterministic and offline.
|
||||
*
|
||||
* The cold-start projection uses the throughput actually measured against the
|
||||
* real endpoint on this repository: 2,257 chunks embedded in 90s, about 25
|
||||
* chunks/s. It is a projection from a real figure, not a guess, and it is
|
||||
* labelled as a projection.
|
||||
*
|
||||
* node scripts/scope-measure.mjs [workspace]
|
||||
*/
|
||||
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { chunkFile } from '../lib/chunk.js'
|
||||
import { listCodeFiles } from '../lib/walk.js'
|
||||
|
||||
const WORKSPACE = process.argv[2] ?? '/root/workspaces/ha-runtime'
|
||||
|
||||
/** Chunks per second, measured on this repository against the live endpoint. */
|
||||
const MEASURED_CHUNKS_PER_SECOND = 2257 / 90
|
||||
|
||||
/** Count files and chunks for one directory, relative to a base for display. */
|
||||
async function measure(root, label) {
|
||||
const { files } = await listCodeFiles(root)
|
||||
let chunks = 0
|
||||
let lines = 0
|
||||
let skipped = 0
|
||||
for (const file of files) {
|
||||
let content
|
||||
try {
|
||||
content = await readFile(file.absolute, 'utf8')
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
lines += content.split('\n').length
|
||||
const split = chunkFile({ path: file.path, content })
|
||||
if (split.chunks.length === 0) skipped += 1
|
||||
chunks += split.chunks.length
|
||||
}
|
||||
return { label, files: files.length, chunks, lines, skipped }
|
||||
}
|
||||
|
||||
/** Directories under src/controllers that hold code directly. */
|
||||
async function controllerRoots(root) {
|
||||
const { files } = await listCodeFiles(root)
|
||||
const seen = new Map()
|
||||
for (const file of files) {
|
||||
if (!file.path.startsWith('src/controllers/')) continue
|
||||
const dir = file.path.slice(0, file.path.lastIndexOf('/'))
|
||||
if (dir === '' || dir.endsWith('__tests__')) continue
|
||||
seen.set(dir, true)
|
||||
}
|
||||
return [...seen.keys()].sort()
|
||||
}
|
||||
|
||||
function seconds(chunks) {
|
||||
return chunks / MEASURED_CHUNKS_PER_SECOND
|
||||
}
|
||||
|
||||
function human(sec) {
|
||||
if (sec < 60) return `${sec.toFixed(0)}s`
|
||||
return `${(sec / 60).toFixed(1)}min`
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`workspace: ${WORKSPACE}`)
|
||||
console.log(`throughput: ${MEASURED_CHUNKS_PER_SECOND.toFixed(1)} chunks/s (measured on this repo)`)
|
||||
console.log('')
|
||||
|
||||
const whole = await measure(WORKSPACE, 'whole workspace')
|
||||
console.log('=== WHOLE WORKSPACE ===')
|
||||
console.log(` files ${whole.files} chunks ${whole.chunks} lines ${whole.lines.toLocaleString()}`)
|
||||
console.log(` cold index (projected): ${human(seconds(whole.chunks))}`)
|
||||
console.log('')
|
||||
|
||||
const roots = await controllerRoots(WORKSPACE)
|
||||
const scopes = []
|
||||
for (const dir of roots) {
|
||||
scopes.push(await measure(join(WORKSPACE, dir), dir))
|
||||
}
|
||||
scopes.sort((left, right) => right.chunks - left.chunks)
|
||||
|
||||
console.log(`=== ${scopes.length} CONTROLLER SCOPES ===`)
|
||||
console.log(` ${'scope'.padEnd(52)} ${'files'.padStart(5)} ${'chunks'.padStart(7)} ${'share'.padStart(7)} ${'cold'.padStart(7)}`)
|
||||
for (const scope of scopes.slice(0, 10)) {
|
||||
const share = ((scope.chunks / whole.chunks) * 100).toFixed(1)
|
||||
console.log(
|
||||
` ${scope.label.replace('src/controllers/', '').padEnd(52)} ${String(scope.files).padStart(5)} ` +
|
||||
`${String(scope.chunks).padStart(7)} ${`${share}%`.padStart(7)} ${human(seconds(scope.chunks)).padStart(7)}`,
|
||||
)
|
||||
}
|
||||
if (scopes.length > 10) console.log(` … and ${scopes.length - 10} more`)
|
||||
console.log('')
|
||||
|
||||
const median = scopes[Math.floor(scopes.length / 2)]
|
||||
const totalScoped = scopes.reduce((sum, scope) => sum + scope.chunks, 0)
|
||||
console.log('=== WHAT SCOPING BUYS ===')
|
||||
console.log(` median scope : ${median.chunks} chunks (${((median.chunks / whole.chunks) * 100).toFixed(1)}% of the workspace)`)
|
||||
console.log(` largest scope : ${scopes[0].chunks} chunks (${((scopes[0].chunks / whole.chunks) * 100).toFixed(1)}%)`)
|
||||
console.log(` candidate pool : ${whole.chunks} -> ${median.chunks} chunks, a ${(whole.chunks / median.chunks).toFixed(1)}x smaller haystack`)
|
||||
console.log(` cold start (projected): ${human(seconds(whole.chunks))} -> ${human(seconds(median.chunks))}`)
|
||||
console.log(` all scopes together : ${totalScoped} chunks, ${((totalScoped / whole.chunks) * 100).toFixed(1)}% of the workspace`)
|
||||
console.log('')
|
||||
console.log('(the candidate-pool figure is the precision argument; whether ranking actually')
|
||||
console.log(' improves needs the live endpoint, which is currently unable to load the model)')
|
||||
}
|
||||
|
||||
await main()
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Live smoke test: the real pipeline against the real embedding endpoint.
|
||||
*
|
||||
* Not part of `npm test`, because it needs a running Ollama and a model that is
|
||||
* already pulled. It exists because the unit tests can only prove the code does
|
||||
* what the test author believed; this proves the retrieval actually returns the
|
||||
* right file for a question about it, which is the only claim that matters.
|
||||
*
|
||||
* node scripts/smoke.mjs [workspace] [query]
|
||||
*/
|
||||
|
||||
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { chunkFile, embedText } from '../lib/chunk.js'
|
||||
import { createEmbedder } from '../lib/embed.js'
|
||||
import { rankChunks } from '../lib/rank.js'
|
||||
import { RecallIndex } from '../lib/store.js'
|
||||
import { listCodeFiles } from '../lib/walk.js'
|
||||
|
||||
const API_BASE = process.env.RECALL_API_BASE ?? 'http://10.50.1.2:11434'
|
||||
const MODEL = process.env.RECALL_MODEL ?? 'unclemusclez/jina-embeddings-v2-base-code:latest'
|
||||
const DIMENSIONS = Number(process.env.RECALL_DIMENSIONS ?? 768)
|
||||
|
||||
const FIXTURE = {
|
||||
'src/services/auth/session.ts': `import { createHash, randomBytes } from 'node:crypto'
|
||||
|
||||
const SESSION_TTL_MS = 1000 * 60 * 60 * 12
|
||||
|
||||
export interface SessionRecord {
|
||||
userId: string
|
||||
issuedAt: number
|
||||
expiresAt: number
|
||||
}
|
||||
|
||||
export function issueSession(userId: string): SessionRecord {
|
||||
const now = Date.now()
|
||||
return { userId, issuedAt: now, expiresAt: now + SESSION_TTL_MS }
|
||||
}
|
||||
|
||||
export function isExpired(record: SessionRecord, now = Date.now()): boolean {
|
||||
return record.expiresAt <= now
|
||||
}
|
||||
|
||||
export function rotateToken(record: SessionRecord): string {
|
||||
if (isExpired(record)) throw new Error('cannot rotate an expired session')
|
||||
return randomBytes(32).toString('hex')
|
||||
}
|
||||
|
||||
export function fingerprint(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex').slice(0, 16)
|
||||
}
|
||||
`,
|
||||
'src/services/weather/forecast.ts': `export interface ForecastDay {
|
||||
date: string
|
||||
minC: number
|
||||
maxC: number
|
||||
rainMm: number
|
||||
}
|
||||
|
||||
export function averageTemperature(days: ForecastDay[]): number {
|
||||
if (days.length === 0) return 0
|
||||
const total = days.reduce((sum, day) => sum + (day.minC + day.maxC) / 2, 0)
|
||||
return total / days.length
|
||||
}
|
||||
|
||||
export function totalRainfall(days: ForecastDay[]): number {
|
||||
return days.reduce((sum, day) => sum + day.rainMm, 0)
|
||||
}
|
||||
`,
|
||||
'src/store/sqlite.ts': `import { DatabaseSync } from 'node:sqlite'
|
||||
|
||||
export function openDatabase(path: string): DatabaseSync {
|
||||
const db = new DatabaseSync(path)
|
||||
db.exec('PRAGMA journal_mode = WAL')
|
||||
return db
|
||||
}
|
||||
|
||||
export function pruneRows(db: DatabaseSync, table: string, olderThan: number): number {
|
||||
const statement = db.prepare(\`DELETE FROM \${table} WHERE created_at < ?\`)
|
||||
const result = statement.run(olderThan)
|
||||
return Number(result.changes)
|
||||
}
|
||||
`,
|
||||
}
|
||||
|
||||
/** Write the fixture into a fresh temporary directory. */
|
||||
async function makeWorkspace() {
|
||||
const root = await mkdtemp(join(tmpdir(), 'recall-smoke-'))
|
||||
for (const [path, content] of Object.entries(FIXTURE)) {
|
||||
const absolute = join(root, path)
|
||||
await mkdir(join(absolute, '..'), { recursive: true })
|
||||
await writeFile(absolute, content, 'utf8')
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const query = process.argv[3] ?? 'how long does a user session stay valid before it expires'
|
||||
const root = process.argv[2] ?? (await makeWorkspace())
|
||||
const indexDir = await mkdtemp(join(tmpdir(), 'recall-index-'))
|
||||
const database = join(indexDir, 'index.sqlite')
|
||||
|
||||
const embedder = createEmbedder({ apiBase: API_BASE, model: MODEL, dimensions: DIMENSIONS })
|
||||
const index = new RecallIndex(database)
|
||||
index.open({ model: MODEL, dimensions: DIMENSIONS, apiBase: API_BASE })
|
||||
|
||||
const started = Date.now()
|
||||
const walk = await listCodeFiles(root)
|
||||
let chunks = 0
|
||||
for (const file of walk.files) {
|
||||
const { readFile } = await import('node:fs/promises')
|
||||
const content = await readFile(file.absolute, 'utf8')
|
||||
const split = chunkFile({ path: file.path, content })
|
||||
if (split.chunks.length === 0) continue
|
||||
const vectors = await embedder.embed(split.chunks.map((chunk) => embedText(file.path, chunk)))
|
||||
index.replaceFile(file.path, 'smoke', Buffer.byteLength(content), split.chunks.map((c, at) => ({ ...c, vec: vectors[at] })))
|
||||
chunks += split.chunks.length
|
||||
}
|
||||
const indexMs = Date.now() - started
|
||||
|
||||
const matrix = index.matrix()
|
||||
const queryVector = await embedder.embedOne(query)
|
||||
const searchStart = Date.now()
|
||||
const ranked = rankChunks({ matrix, queryVector, query, limit: 5 })
|
||||
const searchMs = Date.now() - searchStart
|
||||
|
||||
console.log(`workspace : ${root}`)
|
||||
console.log(`files : ${walk.files.length} chunks: ${chunks} indexed in ${indexMs}ms`)
|
||||
console.log(`query : ${query}`)
|
||||
console.log(`search : ${searchMs}ms batch endpoint: ${embedder.batchEnabled}`)
|
||||
console.log('')
|
||||
for (const [at, result] of ranked.results.entries()) {
|
||||
console.log(`${at + 1}. ${result.path}:${result.startLine}-${result.endLine} similarity ${result.similarity}`)
|
||||
console.log(` ${result.excerpt.split('\n')[0]}`)
|
||||
}
|
||||
console.log('')
|
||||
const top = ranked.results[0]
|
||||
const expected = 'src/services/auth/session.ts'
|
||||
console.log(top?.path === expected ? `PASS top hit is ${expected}` : `FAIL top hit was ${top?.path}, expected ${expected}`)
|
||||
|
||||
index.close()
|
||||
await rm(indexDir, { recursive: true, force: true })
|
||||
if (process.argv[2] === undefined) await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
await main()
|
||||
Reference in new issue
Block a user