/** * What slopo is allowed to look at, and how much of it is too much. * * Two separate failures made a review on this machine index 113,768 code units * and embed for what would have been about eight hours: * * 1. Nothing excluded anything. Slopo does NOT read `.gitignore`, so a * repository with `node_modules` in its ignore file still had all 21,104 * files under `node_modules` parsed — 13,091 of them source files. Vendored * dependencies are not "your code" by any reading, so excluding them is a * correctness default rather than a preference. * * 2. Nothing bounded the work. Slopo's own >1000 guard cannot help on a first * run: it measures the *existing* backlog before indexing, and a fresh * database has no units at all. So the guard that exists precisely to protect * an agent's latency is silent in the one case that most needs it. * * Measured on the repository this was written against: * * no excludes 113,768 units ~7.9 h * default excludes 8,607 units ~36 min * default + a server-only scope 1,519 units ~6 min */ /** * Directories and files that are never the project's own source. * * Globs rather than bare names, because these were verified against slopo 0.8.0 * in this exact setup and a bare name's behaviour is not something to assume. */ import { createRequire } from 'node:module' export const DEFAULT_SOURCE_EXCLUDES = Object.freeze([ '**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/out/**', '**/coverage/**', '**/.venv/**', '**/venv/**', '**/__pycache__/**', '**/target/**', '**/vendor/**', '**/*.min.js', ]) /** * Units per second on the reference backend (a CPU Ollama serving a 768-dim * code model), measured rather than guessed. Used only to turn a unit count * into an estimate in the refusal message — never as a decision input. */ export const REFERENCE_UNITS_PER_SECOND = 4 /** * The excludes slopo should actually be given. * * Defaults and the user's own list are combined rather than chosen between, * because a settings patch REPLACES a whole config value rather than merging * into it: if `sourceDirExclude` were the only knob, setting it to `['ui/**']` * would silently stop excluding `node_modules` and reintroduce the 13x blow-up. * * @param {{ sourceDirExclude?: unknown, useDefaultExcludes?: unknown }} settings * @returns {string[]} */ export function effectiveExcludes(settings) { const defaults = settings.useDefaultExcludes === false ? [] : [...DEFAULT_SOURCE_EXCLUDES] const mine = Array.isArray(settings.sourceDirExclude) ? settings.sourceDirExclude : [] return [...new Set([...defaults, ...mine.filter((entry) => typeof entry === 'string' && entry !== '')])] } /** * Read slopo's own summary line. * * `Indexed 3 code units from 2 files (0 unchanged, 0 removed).` — the count of * code units slopo considered. On a fresh database that is the whole scope, * which is exactly the number a budget should be checked against. * * @param {string} stdout * @returns {{ units: number, files: number } | undefined} */ export function parseIndexOutput(stdout) { // Tolerates a thousands separator, which slopo does not currently emit but // which would silently truncate the count to its first group if it ever did. const match = /Indexed\s+([\d,]+)\s+code units?\s+from\s+([\d,]+)\s+files?/i.exec(String(stdout)) if (match === null) return undefined return { units: Number(match[1].replaceAll(',', '')), files: Number(match[2].replaceAll(',', '')) } } /** A human-scale estimate, so a refusal says minutes rather than a bare count. */ export function estimateDuration(units, unitsPerSecond = REFERENCE_UNITS_PER_SECOND) { const seconds = units / unitsPerSecond if (seconds < 90) return `about ${Math.max(1, Math.round(seconds))} seconds` const minutes = seconds / 60 if (minutes < 90) return `about ${Math.round(minutes)} minutes` return `about ${(minutes / 60).toFixed(1)} hours` } /** * The exact backlog, read from slopo's own database. * * `Indexed N code units` reports only what was NEWLY indexed, so on a warm * database it reads zero while the backlog is still full — and that is exactly * the state a refused run leaves behind, which would let the next attempt slip * straight through the gate. The database has the real numbers, and Node 22.5+ * reads it with no dependency at all. * * Returns undefined on an older runtime, a missing file, or a database another * process is holding — a locked read must not fail the run, so the caller falls * back to the summary line, which is accurate on a first index. * * @param {string} dbFile * @returns {{ units: number, embedded: number, unembedded: number, files: number } | undefined} */ export function readIndexCounts(dbFile) { try { const require = createRequire(import.meta.url) const { DatabaseSync } = require('node:sqlite') const db = new DatabaseSync(dbFile, { readOnly: true }) try { const count = (sql) => Number(db.prepare(sql).get().n) const units = count('select count(*) as n from code_units') const embedded = count('select count(*) as n from embeddings') const files = count('select count(*) as n from files') return { units, embedded, unembedded: Math.max(0, units - embedded), files } } finally { db.close() } } catch { return undefined } } /** * Decide whether the backlog is small enough to embed without asking. * * A refusal is deliberately not a failure: it names the number, the estimate, * and the one lever that changes it, so a large scope becomes a decision the * caller makes once rather than an eight-hour surprise. * * @param {{ units: number, files?: number } | undefined} indexed units STILL TO EMBED * @param {number} budget * @param {string[]} excludes * @returns {{ allowed: true } | { allowed: false, reason: string }} */ export function checkBudget(indexed, budget, excludes) { if (indexed === undefined) return { allowed: true } if (!Number.isFinite(budget) || budget <= 0) return { allowed: true } if (indexed.units <= budget) return { allowed: true } return { allowed: false, reason: `${indexed.units.toLocaleString('en-US')} code units still need embedding` + `${indexed.files === undefined ? '' : ` across ${indexed.files.toLocaleString('en-US')} files`}. ` + `At the reference rate that is ${estimateDuration(indexed.units)}, against a budget of ` + `${budget.toLocaleString('en-US')}.`, } } /** The remedy text for a refusal, shared so it cannot drift from the cause. */ export function budgetRemedy(budget, excludes) { const current = excludes.length === 0 ? 'nothing is currently excluded' : `${excludes.length} patterns are excluded` return ( `Narrow the scope with the \`sourceDirExclude\` setting (${current}), or raise \`maxIndexUnits\` above the ` + `count if you really mean to embed it — set it to 0 to remove the limit. On a repository with a ` + '`node_modules` directory, excluding it is usually most of the difference.' ) }