dsh-recall/lib/walk.js

/**
 * Workspace file discovery.
 *
 * Two rules shape everything here.
 *
 * **The walk never follows a symlink.** A symlink loop turns a bounded walk into
 * an unbounded one, and a symlink out of the workspace turns "index this
 * project" into "index this machine". Both are worse than missing a file.
 *
 * **The walk is deterministic.** Candidates come back in sorted path order, so
 * a partially-completed cold index covers the same prefix on the next run. A
 * filesystem-ordered walk would re-embed a different subset every time and
 * never converge on a warm index.
 */

import { readdir } from 'node:fs/promises'
import { join, relative, sep } from 'node:path'

/**
 * Directory names skipped at any depth.
 *
 * These are dependency stores and build output: large, machine-generated, and
 * never the answer to a question about this codebase. Indexing them would
 * multiply the embedding cost of a cold start by an order of magnitude and
 * return vendored copies of code the user does not own.
 */
export const DEFAULT_SKIP_DIRS = Object.freeze([
  'node_modules',
  '.git',
  '.hg',
  '.svn',
  'dist',
  'build',
  'out',
  'output',
  'target',
  'coverage',
  '.next',
  '.nuxt',
  '.svelte-kit',
  '.output',
  '.turbo',
  '.cache',
  '.parcel-cache',
  '.venv',
  'venv',
  '__pycache__',
  '.mypy_cache',
  '.pytest_cache',
  '.ruff_cache',
  'vendor',
  'Pods',
  '.idea',
  '.vscode',
  '.gradle',
  '.terraform',
  'tmp',
  '.tmp',
])

/**
 * Extensions indexed by default.
 *
 * Markup and prose are deliberately absent. A question that names a symbol wants
 * the code that defines it, and a README that mentions the symbol is a worse
 * answer that still costs tokens. Deployments that want documentation indexed
 * add the extension through `extensions` in the config.
 */
export const DEFAULT_EXTENSIONS = Object.freeze([
  'ts', 'tsx', 'mts', 'cts',
  'js', 'jsx', 'mjs', 'cjs',
  'py', 'rb', 'go', 'rs', 'java', 'kt', 'kts', 'scala', 'swift',
  'c', 'h', 'cc', 'cpp', 'cxx', 'hpp', 'hh', 'cs',
  'php', 'pl', 'lua', 'sh', 'bash', 'zsh',
  'sql', 'graphql', 'proto', 'vue', 'svelte',
])

/**
 * Translate a glob into an anchored regular expression.
 *
 * Supports `**` (any depth, including none), `*` (within one path segment), `?`
 * (one non-separator character) and a trailing `/` (directory and everything
 * under it). Written out rather than pulled from a dependency because the plugin
 * ships with no build step and no install-time code.
 *
 * @param {string} pattern glob, using `/` as the separator on every platform
 * @returns {RegExp} anchored matcher over a `/`-separated relative path
 */
export function globToRegExp(pattern) {
  const normalized = pattern.trim().replace(/\\/g, '/').replace(/^\.\//, '')
  let source = ''
  for (let at = 0; at < normalized.length; at += 1) {
    const char = normalized[at]
    if (char === '*') {
      const doubled = normalized[at + 1] === '*'
      if (doubled) {
        // `**/` consumes any number of segments, so `**/foo` also matches `foo`.
        if (normalized[at + 2] === '/') {
          source += '(?:[^/]*/)*'
          at += 2
        } else {
          source += '.*'
          at += 1
        }
      } else {
        source += '[^/]*'
      }
    } else if (char === '?') {
      source += '[^/]'
    } else if (char === '/') {
      source += '/'
    } else {
      source += char.replace(/[.+^${}()|[\]\\]/g, '\\$&')
    }
  }
  // A trailing `/` names a directory: match it and everything beneath it.
  if (source.endsWith('/')) source += '.*'
  return new RegExp(`^(?:${source})$`)
}

/** Whether a `/`-separated relative path is matched by any of `patterns`. */
export function matchesAny(relPath, patterns) {
  for (const pattern of patterns) {
    if (pattern === '') continue
    if (globToRegExp(pattern).test(relPath)) return true
  }
  return false
}

/** The extension of a path, lowercased, without the dot. */
export function extensionOf(path) {
  const name = path.slice(path.lastIndexOf('/') + 1)
  const dot = name.lastIndexOf('.')
  return dot <= 0 ? '' : name.slice(dot + 1).toLowerCase()
}

/**
 * List indexable files beneath `root`.
 *
 * @param {string} root absolute directory to walk
 * @param {object} [options]
 * @param {readonly string[]} [options.extensions] allowed extensions, without dots
 * @param {readonly string[]} [options.skipDirs] directory names skipped at any depth
 * @param {readonly string[]} [options.exclude] glob patterns over the relative path
 * @param {(depth: number, name: string, absolute: string) => boolean} [options.isSkipped]
 *   decides directory traversal, replacing the `skipDirs` membership test. This is
 *   the seam an extra index root uses: its top level is a package-manager
 *   `node_modules` tree, which the built-in list exists to skip, so it must opt out
 *   for the root and only the root. Depth 0 is the root's own entries.
 * @param {number} [options.maxFiles] stop after this many candidates
 * @param {number} [options.maxDepth] deepest directory level to enter
 * @param {AbortSignal} [options.signal]
 * @returns {Promise<{files: {path: string, absolute: string}[], truncated: boolean, skippedFiles: number}>}
 */
export async function listCodeFiles(root, options = {}) {
  const {
    extensions = DEFAULT_EXTENSIONS,
    skipDirs = DEFAULT_SKIP_DIRS,
    exclude = [],
    isSkipped,
    maxFiles = 20_000,
    maxDepth = 24,
    signal,
  } = options

  const wanted = new Set(extensions.map((value) => String(value).toLowerCase()))
  const skip = new Set(skipDirs)
  // Materialised once per walk rather than tested with `isSkipped ?? default` at
  // every directory: the hot loop should not carry a branch that cannot change.
  const shouldSkip =
    isSkipped === undefined
      ? (depth, name) => skip.has(name)
      : (depth, name, absolute) => isSkipped(depth, name, absolute)
  const files = []
  let truncated = false
  let skippedFiles = 0

  /** Depth-first, pushing directories in reverse so the pop order is alphabetical. */
  const stack = [{ dir: root, depth: 0 }]
  while (stack.length > 0) {
    if (signal?.aborted) throw signal.reason ?? new Error('aborted')
    const { dir, depth } = stack.pop()
    let entries
    try {
      entries = await readdir(dir, { withFileTypes: true })
    } catch {
      // An unreadable directory is a gap in the index, not a failure of it.
      continue
    }
    for (const entry of entries) {
      // Never traverse a symlink: `dirent.isDirectory()` is false for one, so a
      // link to a directory is skipped rather than followed.
      if (entry.isSymbolicLink()) {
        skippedFiles += 1
        continue
      }
      const absolute = join(dir, entry.name)
      const relPath = relative(root, absolute).split(sep).join('/')
      if (entry.isDirectory()) {
        if (shouldSkip(depth, entry.name, absolute)) continue
        if (depth >= maxDepth) continue
        if (matchesAny(relPath, exclude)) continue
        stack.push({ dir: absolute, depth: depth + 1 })
        continue
      }
      if (!entry.isFile()) continue
      if (!wanted.has(extensionOf(entry.name))) continue
      if (matchesAny(relPath, exclude)) continue
      if (files.length >= maxFiles) {
        truncated = true
        continue
      }
      files.push({ path: relPath, absolute })
    }
  }

  files.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
  return { files, truncated, skippedFiles }
}