dsh-slopo/lib/scope.js

/**
 * What slopo is allowed to look at, and how much of it is too much.
 *
 * Two separate failures made a review on this machine index 113,768 code units
 * and embed for what would have been about eight hours:
 *
 * 1. Nothing excluded anything. Slopo does NOT read `.gitignore`, so a
 *    repository with `node_modules` in its ignore file still had all 21,104
 *    files under `node_modules` parsed — 13,091 of them source files. Vendored
 *    dependencies are not "your code" by any reading, so excluding them is a
 *    correctness default rather than a preference.
 *
 * 2. Nothing bounded the work. Slopo's own >1000 guard cannot help on a first
 *    run: it measures the *existing* backlog before indexing, and a fresh
 *    database has no units at all. So the guard that exists precisely to protect
 *    an agent's latency is silent in the one case that most needs it.
 *
 * Measured on the repository this was written against:
 *
 *   no excludes                 113,768 units   ~7.9 h
 *   default excludes              8,607 units   ~36 min
 *   default + a server-only scope 1,519 units   ~6 min
 */

/**
 * Directories and files that are never the project's own source.
 *
 * Globs rather than bare names, because these were verified against slopo 0.8.0
 * in this exact setup and a bare name's behaviour is not something to assume.
 */

import { createRequire } from 'node:module'

export const DEFAULT_SOURCE_EXCLUDES = Object.freeze([
  '**/node_modules/**',
  '**/.git/**',
  '**/dist/**',
  '**/build/**',
  '**/out/**',
  '**/coverage/**',
  '**/.venv/**',
  '**/venv/**',
  '**/__pycache__/**',
  '**/target/**',
  '**/vendor/**',
  '**/*.min.js',
])

/**
 * Units per second on the reference backend (a CPU Ollama serving a 768-dim
 * code model), measured rather than guessed. Used only to turn a unit count
 * into an estimate in the refusal message — never as a decision input.
 */
export const REFERENCE_UNITS_PER_SECOND = 4

/**
 * The excludes slopo should actually be given.
 *
 * Defaults and the user's own list are combined rather than chosen between,
 * because a settings patch REPLACES a whole config value rather than merging
 * into it: if `sourceDirExclude` were the only knob, setting it to `['ui/**']`
 * would silently stop excluding `node_modules` and reintroduce the 13x blow-up.
 *
 * @param {{ sourceDirExclude?: unknown, useDefaultExcludes?: unknown }} settings
 * @returns {string[]}
 */
export function effectiveExcludes(settings) {
  const defaults = settings.useDefaultExcludes === false ? [] : [...DEFAULT_SOURCE_EXCLUDES]
  const mine = Array.isArray(settings.sourceDirExclude) ? settings.sourceDirExclude : []
  return [...new Set([...defaults, ...mine.filter((entry) => typeof entry === 'string' && entry !== '')])]
}

/**
 * Read slopo's own summary line.
 *
 * `Indexed 3 code units from 2 files (0 unchanged, 0 removed).` — the count of
 * code units slopo considered. On a fresh database that is the whole scope,
 * which is exactly the number a budget should be checked against.
 *
 * @param {string} stdout
 * @returns {{ units: number, files: number } | undefined}
 */
export function parseIndexOutput(stdout) {
  // Tolerates a thousands separator, which slopo does not currently emit but
  // which would silently truncate the count to its first group if it ever did.
  const match = /Indexed\s+([\d,]+)\s+code units?\s+from\s+([\d,]+)\s+files?/i.exec(String(stdout))
  if (match === null) return undefined
  return { units: Number(match[1].replaceAll(',', '')), files: Number(match[2].replaceAll(',', '')) }
}

/** A human-scale estimate, so a refusal says minutes rather than a bare count. */
export function estimateDuration(units, unitsPerSecond = REFERENCE_UNITS_PER_SECOND) {
  const seconds = units / unitsPerSecond
  if (seconds < 90) return `about ${Math.max(1, Math.round(seconds))} seconds`
  const minutes = seconds / 60
  if (minutes < 90) return `about ${Math.round(minutes)} minutes`
  return `about ${(minutes / 60).toFixed(1)} hours`
}

/**
 * The exact backlog, read from slopo's own database.
 *
 * `Indexed N code units` reports only what was NEWLY indexed, so on a warm
 * database it reads zero while the backlog is still full — and that is exactly
 * the state a refused run leaves behind, which would let the next attempt slip
 * straight through the gate. The database has the real numbers, and Node 22.5+
 * reads it with no dependency at all.
 *
 * Returns undefined on an older runtime, a missing file, or a database another
 * process is holding — a locked read must not fail the run, so the caller falls
 * back to the summary line, which is accurate on a first index.
 *
 * @param {string} dbFile
 * @returns {{ units: number, embedded: number, unembedded: number, files: number } | undefined}
 */
export function readIndexCounts(dbFile) {
  try {
    const require = createRequire(import.meta.url)
    const { DatabaseSync } = require('node:sqlite')
    const db = new DatabaseSync(dbFile, { readOnly: true })
    try {
      const count = (sql) => Number(db.prepare(sql).get().n)
      const units = count('select count(*) as n from code_units')
      const embedded = count('select count(*) as n from embeddings')
      const files = count('select count(*) as n from files')
      return { units, embedded, unembedded: Math.max(0, units - embedded), files }
    } finally {
      db.close()
    }
  } catch {
    return undefined
  }
}

/**
 * Decide whether the backlog is small enough to embed without asking.
 *
 * A refusal is deliberately not a failure: it names the number, the estimate,
 * and the one lever that changes it, so a large scope becomes a decision the
 * caller makes once rather than an eight-hour surprise.
 *
 * @param {{ units: number, files?: number } | undefined} indexed units STILL TO EMBED
 * @param {number} budget
 * @param {string[]} excludes
 * @returns {{ allowed: true } | { allowed: false, reason: string }}
 */
export function checkBudget(indexed, budget, excludes) {
  if (indexed === undefined) return { allowed: true }
  if (!Number.isFinite(budget) || budget <= 0) return { allowed: true }
  if (indexed.units <= budget) return { allowed: true }
  return {
    allowed: false,
    reason:
      `${indexed.units.toLocaleString('en-US')} code units still need embedding` +
      `${indexed.files === undefined ? '' : ` across ${indexed.files.toLocaleString('en-US')} files`}. ` +
      `At the reference rate that is ${estimateDuration(indexed.units)}, against a budget of ` +
      `${budget.toLocaleString('en-US')}.`,
  }
}

/** The remedy text for a refusal, shared so it cannot drift from the cause. */
export function budgetRemedy(budget, excludes) {
  const current = excludes.length === 0 ? 'nothing is currently excluded' : `${excludes.length} patterns are excluded`
  return (
    `Narrow the scope with the \`sourceDirExclude\` setting (${current}), or raise \`maxIndexUnits\` above the ` +
    `count if you really mean to embed it — set it to 0 to remove the limit. On a repository with a ` +
    '`node_modules` directory, excluding it is usually most of the difference.'
  )
}