init
This commit is contained in:
NotExpectedYet committed
commit
0964819cdc
34 files changed
+7912
No files matched your search
+207
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Workspace file discovery.
|
||||
*
|
||||
* Two rules shape everything here.
|
||||
*
|
||||
* **The walk never follows a symlink.** A symlink loop turns a bounded walk into
|
||||
* an unbounded one, and a symlink out of the workspace turns "index this
|
||||
* project" into "index this machine". Both are worse than missing a file.
|
||||
*
|
||||
* **The walk is deterministic.** Candidates come back in sorted path order, so
|
||||
* a partially-completed cold index covers the same prefix on the next run. A
|
||||
* filesystem-ordered walk would re-embed a different subset every time and
|
||||
* never converge on a warm index.
|
||||
*/
|
||||
|
||||
import { readdir } from 'node:fs/promises'
|
||||
import { join, relative, sep } from 'node:path'
|
||||
|
||||
/**
|
||||
* Directory names skipped at any depth.
|
||||
*
|
||||
* These are dependency stores and build output: large, machine-generated, and
|
||||
* never the answer to a question about this codebase. Indexing them would
|
||||
* multiply the embedding cost of a cold start by an order of magnitude and
|
||||
* return vendored copies of code the user does not own.
|
||||
*/
|
||||
export const DEFAULT_SKIP_DIRS = Object.freeze([
|
||||
'node_modules',
|
||||
'.git',
|
||||
'.hg',
|
||||
'.svn',
|
||||
'dist',
|
||||
'build',
|
||||
'out',
|
||||
'output',
|
||||
'target',
|
||||
'coverage',
|
||||
'.next',
|
||||
'.nuxt',
|
||||
'.svelte-kit',
|
||||
'.output',
|
||||
'.turbo',
|
||||
'.cache',
|
||||
'.parcel-cache',
|
||||
'.venv',
|
||||
'venv',
|
||||
'__pycache__',
|
||||
'.mypy_cache',
|
||||
'.pytest_cache',
|
||||
'.ruff_cache',
|
||||
'vendor',
|
||||
'Pods',
|
||||
'.idea',
|
||||
'.vscode',
|
||||
'.gradle',
|
||||
'.terraform',
|
||||
'tmp',
|
||||
'.tmp',
|
||||
])
|
||||
|
||||
/**
|
||||
* Extensions indexed by default.
|
||||
*
|
||||
* Markup and prose are deliberately absent. A question that names a symbol wants
|
||||
* the code that defines it, and a README that mentions the symbol is a worse
|
||||
* answer that still costs tokens. Deployments that want documentation indexed
|
||||
* add the extension through `extensions` in the config.
|
||||
*/
|
||||
export const DEFAULT_EXTENSIONS = Object.freeze([
|
||||
'ts', 'tsx', 'mts', 'cts',
|
||||
'js', 'jsx', 'mjs', 'cjs',
|
||||
'py', 'rb', 'go', 'rs', 'java', 'kt', 'kts', 'scala', 'swift',
|
||||
'c', 'h', 'cc', 'cpp', 'cxx', 'hpp', 'hh', 'cs',
|
||||
'php', 'pl', 'lua', 'sh', 'bash', 'zsh',
|
||||
'sql', 'graphql', 'proto', 'vue', 'svelte',
|
||||
])
|
||||
|
||||
/**
|
||||
* Translate a glob into an anchored regular expression.
|
||||
*
|
||||
* Supports `**` (any depth, including none), `*` (within one path segment), `?`
|
||||
* (one non-separator character) and a trailing `/` (directory and everything
|
||||
* under it). Written out rather than pulled from a dependency because the plugin
|
||||
* ships with no build step and no install-time code.
|
||||
*
|
||||
* @param {string} pattern glob, using `/` as the separator on every platform
|
||||
* @returns {RegExp} anchored matcher over a `/`-separated relative path
|
||||
*/
|
||||
export function globToRegExp(pattern) {
|
||||
const normalized = pattern.trim().replace(/\\/g, '/').replace(/^\.\//, '')
|
||||
let source = ''
|
||||
for (let at = 0; at < normalized.length; at += 1) {
|
||||
const char = normalized[at]
|
||||
if (char === '*') {
|
||||
const doubled = normalized[at + 1] === '*'
|
||||
if (doubled) {
|
||||
// `**/` consumes any number of segments, so `**/foo` also matches `foo`.
|
||||
if (normalized[at + 2] === '/') {
|
||||
source += '(?:[^/]*/)*'
|
||||
at += 2
|
||||
} else {
|
||||
source += '.*'
|
||||
at += 1
|
||||
}
|
||||
} else {
|
||||
source += '[^/]*'
|
||||
}
|
||||
} else if (char === '?') {
|
||||
source += '[^/]'
|
||||
} else if (char === '/') {
|
||||
source += '/'
|
||||
} else {
|
||||
source += char.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
}
|
||||
// A trailing `/` names a directory: match it and everything beneath it.
|
||||
if (source.endsWith('/')) source += '.*'
|
||||
return new RegExp(`^(?:${source})$`)
|
||||
}
|
||||
|
||||
/** Whether a `/`-separated relative path is matched by any of `patterns`. */
|
||||
export function matchesAny(relPath, patterns) {
|
||||
for (const pattern of patterns) {
|
||||
if (pattern === '') continue
|
||||
if (globToRegExp(pattern).test(relPath)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** The extension of a path, lowercased, without the dot. */
|
||||
export function extensionOf(path) {
|
||||
const name = path.slice(path.lastIndexOf('/') + 1)
|
||||
const dot = name.lastIndexOf('.')
|
||||
return dot <= 0 ? '' : name.slice(dot + 1).toLowerCase()
|
||||
}
|
||||
|
||||
/**
|
||||
* List indexable files beneath `root`.
|
||||
*
|
||||
* @param {string} root absolute directory to walk
|
||||
* @param {object} [options]
|
||||
* @param {readonly string[]} [options.extensions] allowed extensions, without dots
|
||||
* @param {readonly string[]} [options.skipDirs] directory names skipped at any depth
|
||||
* @param {readonly string[]} [options.exclude] glob patterns over the relative path
|
||||
* @param {number} [options.maxFiles] stop after this many candidates
|
||||
* @param {number} [options.maxDepth] deepest directory level to enter
|
||||
* @param {AbortSignal} [options.signal]
|
||||
* @returns {Promise<{files: {path: string, absolute: string}[], truncated: boolean, skippedFiles: number}>}
|
||||
*/
|
||||
export async function listCodeFiles(root, options = {}) {
|
||||
const {
|
||||
extensions = DEFAULT_EXTENSIONS,
|
||||
skipDirs = DEFAULT_SKIP_DIRS,
|
||||
exclude = [],
|
||||
maxFiles = 20_000,
|
||||
maxDepth = 24,
|
||||
signal,
|
||||
} = options
|
||||
|
||||
const wanted = new Set(extensions.map((value) => String(value).toLowerCase()))
|
||||
const skip = new Set(skipDirs)
|
||||
const files = []
|
||||
let truncated = false
|
||||
let skippedFiles = 0
|
||||
|
||||
/** Depth-first, pushing directories in reverse so the pop order is alphabetical. */
|
||||
const stack = [{ dir: root, depth: 0 }]
|
||||
while (stack.length > 0) {
|
||||
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
||||
const { dir, depth } = stack.pop()
|
||||
let entries
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true })
|
||||
} catch {
|
||||
// An unreadable directory is a gap in the index, not a failure of it.
|
||||
continue
|
||||
}
|
||||
for (const entry of entries) {
|
||||
// Never traverse a symlink: `dirent.isDirectory()` is false for one, so a
|
||||
// link to a directory is skipped rather than followed.
|
||||
if (entry.isSymbolicLink()) {
|
||||
skippedFiles += 1
|
||||
continue
|
||||
}
|
||||
const absolute = join(dir, entry.name)
|
||||
const relPath = relative(root, absolute).split(sep).join('/')
|
||||
if (entry.isDirectory()) {
|
||||
if (skip.has(entry.name)) continue
|
||||
if (depth >= maxDepth) continue
|
||||
if (matchesAny(relPath, exclude)) continue
|
||||
stack.push({ dir: absolute, depth: depth + 1 })
|
||||
continue
|
||||
}
|
||||
if (!entry.isFile()) continue
|
||||
if (!wanted.has(extensionOf(entry.name))) continue
|
||||
if (matchesAny(relPath, exclude)) continue
|
||||
if (files.length >= maxFiles) {
|
||||
truncated = true
|
||||
continue
|
||||
}
|
||||
files.push({ path: relPath, absolute })
|
||||
}
|
||||
}
|
||||
|
||||
files.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
|
||||
return { files, truncated, skippedFiles }
|
||||
}
|
||||
Reference in new issue
Block a user