dsh-recall/scripts/scope-measure.mjs

/**
 * What focus scoping does to the index, measured without an embedding endpoint.
 *
 * The semantic half of the claim — that a focused search ranks better — needs a
 * live model. This half does not: index size, per-scope chunk counts, and the
 * cold-start cost they imply are all arithmetic over the chunker, which is
 * deterministic and offline.
 *
 * The cold-start projection uses a throughput measured against a live endpoint
 * during development: 2,257 chunks embedded in 90s, about 25 chunks/s. It is a
 * projection from a real figure, not a guess, and it is labelled as one.
 *
 *   node scripts/scope-measure.mjs [workspace]
 */

import { readFile } from 'node:fs/promises'
import { join } from 'node:path'

import { chunkFile } from '../lib/chunk.js'
import { listCodeFiles } from '../lib/walk.js'

// The repository to measure; the working directory by default, so the script
// carries no assumption about where anyone keeps their code.
const WORKSPACE = process.argv[2] ?? process.cwd()

/**
 * Chunks per second, measured against a live endpoint during development
 * (2,257 chunks in 90s). A measured constant, not a guess — and a projection
 * input, which is why the cold-start figures below are labelled as projections.
 */
const MEASURED_CHUNKS_PER_SECOND = 2257 / 90

/** Count files and chunks for one directory, relative to a base for display. */
async function measure(root, label) {
  const { files } = await listCodeFiles(root)
  let chunks = 0
  let lines = 0
  let skipped = 0
  for (const file of files) {
    let content
    try {
      content = await readFile(file.absolute, 'utf8')
    } catch {
      continue
    }
    lines += content.split('\n').length
    const split = chunkFile({ path: file.path, content })
    if (split.chunks.length === 0) skipped += 1
    chunks += split.chunks.length
  }
  return { label, files: files.length, chunks, lines, skipped }
}

/** Directories under src/controllers that hold code directly. */
async function controllerRoots(root) {
  const { files } = await listCodeFiles(root)
  const seen = new Map()
  for (const file of files) {
    if (!file.path.startsWith('src/controllers/')) continue
    const dir = file.path.slice(0, file.path.lastIndexOf('/'))
    if (dir === '' || dir.endsWith('__tests__')) continue
    seen.set(dir, true)
  }
  return [...seen.keys()].sort()
}

function seconds(chunks) {
  return chunks / MEASURED_CHUNKS_PER_SECOND
}

function human(sec) {
  if (sec < 60) return `${sec.toFixed(0)}s`
  return `${(sec / 60).toFixed(1)}min`
}

async function main() {
  console.log(`workspace: ${WORKSPACE}`)
  console.log(`throughput: ${MEASURED_CHUNKS_PER_SECOND.toFixed(1)} chunks/s (measured on this repo)`)
  console.log('')

  const whole = await measure(WORKSPACE, 'whole workspace')
  console.log('=== WHOLE WORKSPACE ===')
  console.log(`  files ${whole.files}  chunks ${whole.chunks}  lines ${whole.lines.toLocaleString()}`)
  console.log(`  cold index (projected): ${human(seconds(whole.chunks))}`)
  console.log('')

  const roots = await controllerRoots(WORKSPACE)
  const scopes = []
  for (const dir of roots) {
    scopes.push(await measure(join(WORKSPACE, dir), dir))
  }
  scopes.sort((left, right) => right.chunks - left.chunks)

  console.log(`=== ${scopes.length} CONTROLLER SCOPES ===`)
  console.log(`  ${'scope'.padEnd(52)} ${'files'.padStart(5)} ${'chunks'.padStart(7)} ${'share'.padStart(7)} ${'cold'.padStart(7)}`)
  for (const scope of scopes.slice(0, 10)) {
    const share = ((scope.chunks / whole.chunks) * 100).toFixed(1)
    console.log(
      `  ${scope.label.replace('src/controllers/', '').padEnd(52)} ${String(scope.files).padStart(5)} ` +
        `${String(scope.chunks).padStart(7)} ${`${share}%`.padStart(7)} ${human(seconds(scope.chunks)).padStart(7)}`,
    )
  }
  if (scopes.length > 10) console.log(`  … and ${scopes.length - 10} more`)
  console.log('')

  // A workspace with no marker files has no scopes to compare, and that is a
  // normal answer rather than a failure — the plugin falls back to the whole
  // workspace there. Say so and stop, instead of dividing by the scope that
  // does not exist.
  if (scopes.length === 0) {
    console.log('=== WHAT SCOPING BUYS ===')
    console.log('  nothing here: no directory in this workspace carries a focus marker,')
    console.log('  so every call indexes the whole workspace. Add a marker file (see')
    console.log('  focusMarkers in the README) to a module you work on to create one.')
    return
  }

  const median = scopes[Math.floor(scopes.length / 2)]
  const totalScoped = scopes.reduce((sum, scope) => sum + scope.chunks, 0)
  console.log('=== WHAT SCOPING BUYS ===')
  console.log(`  median scope          : ${median.chunks} chunks (${((median.chunks / whole.chunks) * 100).toFixed(1)}% of the workspace)`)
  console.log(`  largest scope         : ${scopes[0].chunks} chunks (${((scopes[0].chunks / whole.chunks) * 100).toFixed(1)}%)`)
  console.log(`  candidate pool        : ${whole.chunks} -> ${median.chunks} chunks, a ${(whole.chunks / median.chunks).toFixed(1)}x smaller haystack`)
  console.log(`  cold start (projected): ${human(seconds(whole.chunks))} -> ${human(seconds(median.chunks))}`)
  console.log(`  all scopes together   : ${totalScoped} chunks, ${((totalScoped / whole.chunks) * 100).toFixed(1)}% of the workspace`)
  console.log('')
  console.log('(the candidate-pool figure is the precision argument; whether ranking actually')
  console.log(' improves needs the live endpoint, which is currently unable to load the model)')
}

await main()