init
This commit is contained in:
NotExpectedYet committed
commit
0964819cdc
34 files changed
+7912
No files matched your search
+273
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* Splitting source into retrievable units.
|
||||
*
|
||||
* This is where retrieval quality is decided, and the rule that matters is that
|
||||
* **a chunk must be a thing a person would name**. Splitting on a fixed line
|
||||
* count produces units that begin mid-function and end mid-conditional; their
|
||||
* embedding sits between two topics and matches neither, so the search returns
|
||||
* plausible-looking noise. Splitting at declarations produces units whose
|
||||
* embedding is about one subject, which is the entire premise of the retrieval.
|
||||
*
|
||||
* The anchor set is deliberately shallow — column-0 declarations only. A real
|
||||
* parser would place boundaries better, but it would also mean shipping a
|
||||
* tree-sitter grammar per language and keeping ten of them current, for a gain
|
||||
* that only shows up on languages with heavy nesting. Since a chunk is a
|
||||
* *search result* and not an edit target, a boundary that is roughly right costs
|
||||
* a few wasted lines and nothing else.
|
||||
*
|
||||
* Fixed-width windows remain as the fallback, because a file whose structure the
|
||||
* anchor set cannot see — a minified bundle, a template, an unusual dialect —
|
||||
* must still be searchable rather than silently absent from the index.
|
||||
*/
|
||||
|
||||
/** Lines a chunk aims for when merging neighbouring declarations. */
|
||||
export const DEFAULT_CHUNK_LINES = 48
|
||||
|
||||
/** Lines a chunk may never exceed; longer ones are split with overlap. */
|
||||
export const DEFAULT_MAX_CHUNK_LINES = 80
|
||||
|
||||
/** Lines shared between the halves of a split chunk, so a boundary is never lost. */
|
||||
export const DEFAULT_OVERLAP_LINES = 8
|
||||
|
||||
/** Chunks taken from one file, so one pathological file cannot dominate the index. */
|
||||
export const DEFAULT_MAX_CHUNKS_PER_FILE = 250
|
||||
|
||||
/** Files whose mean line length exceeds this are treated as generated/minified. */
|
||||
const MINIFIED_MEAN_LINE = 400
|
||||
|
||||
/** Anchor patterns by language family, applied to the raw line. */
|
||||
const FAMILIES = {
|
||||
javascript: /^(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:function|class|const|let|var|interface|type|enum|namespace|declare)\b/,
|
||||
python: /^(?:async\s+)?(?:def|class)\s+[A-Za-z_]/,
|
||||
go: /^(?:func|type|var|const)\b/,
|
||||
rust: /^(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?(?:unsafe\s+)?(?:extern\s+)?(?:fn|struct|enum|impl|trait|mod|const|static|type)\b/,
|
||||
jvm: /^(?:@\w+|public|private|protected|static|final|abstract|sealed|open|class|interface|enum|record|object|fun|val|var|def)\b/,
|
||||
dotnet: /^(?:\[[^\]]*\]|public|private|protected|internal|static|final|abstract|sealed|partial|class|interface|enum|record|struct|namespace)\b/,
|
||||
c: /^(?:class|struct|enum|union|namespace|template|typedef|using|extern)\b|^[A-Za-z_][\w \t*&:<>]*\s+[A-Za-z_]\w*\s*\([^;]*\)\s*(?:const\s*)?$/,
|
||||
ruby: /^(?:def|class|module)\s/,
|
||||
php: /^(?:public|private|protected|static|final|abstract|function|class|interface|trait|namespace)\b/,
|
||||
shell: /^(?:function\s+[A-Za-z_]|[A-Za-z_]\w*\s*\(\)\s*\{)/,
|
||||
sql: /^(?:CREATE|ALTER|DROP|WITH)\b/i,
|
||||
markup: /^<(?:template|script|style|svelte:)/,
|
||||
}
|
||||
|
||||
/** Extension to family. Anything unlisted falls back to the window splitter. */
|
||||
const EXTENSION_FAMILY = {
|
||||
ts: 'javascript', tsx: 'javascript', mts: 'javascript', cts: 'javascript',
|
||||
js: 'javascript', jsx: 'javascript', mjs: 'javascript', cjs: 'javascript',
|
||||
py: 'python', rb: 'ruby', go: 'go', rs: 'rust',
|
||||
java: 'jvm', kt: 'jvm', kts: 'jvm', scala: 'jvm', swift: 'jvm',
|
||||
c: 'c', h: 'c', cc: 'c', cpp: 'c', cxx: 'c', hpp: 'c', hh: 'c',
|
||||
cs: 'dotnet',
|
||||
php: 'php', pl: 'ruby', lua: 'ruby',
|
||||
sh: 'shell', bash: 'shell', zsh: 'shell',
|
||||
sql: 'sql', graphql: 'sql', proto: 'c',
|
||||
vue: 'markup', svelte: 'markup',
|
||||
}
|
||||
|
||||
/** Name of the declaration on an anchor line, when one can be read off it. */
|
||||
const SYMBOL_PATTERNS = [
|
||||
/\b(?:function|class|interface|type|enum|namespace|def|func|fn|struct|trait|impl|mod|const|let|var|record|object|fun|val|module|macro_rules!)\s+([A-Za-z_$][\w$]*)/,
|
||||
/\b([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*(?:const\s*)?(?:\{|:)?\s*$/,
|
||||
]
|
||||
|
||||
/** The language family for a path, or `undefined` when it has none. */
|
||||
export function languageOf(path) {
|
||||
const name = path.slice(path.lastIndexOf('/') + 1).toLowerCase()
|
||||
const dot = name.lastIndexOf('.')
|
||||
if (dot <= 0) return undefined
|
||||
return EXTENSION_FAMILY[name.slice(dot + 1)]
|
||||
}
|
||||
|
||||
/** The declaration regex for a family, or `undefined`. */
|
||||
export function declarationPattern(family) {
|
||||
return family === undefined ? undefined : FAMILIES[family]
|
||||
}
|
||||
|
||||
/** The name declared on a line, when one reads off it. */
|
||||
export function symbolOf(line) {
|
||||
for (const pattern of SYMBOL_PATTERNS) {
|
||||
const match = pattern.exec(line)
|
||||
if (match !== null) return match[1]
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Declaration names for a freshly built segment. */
|
||||
function symbolList(symbol) {
|
||||
return symbol === undefined ? [] : [symbol]
|
||||
}
|
||||
|
||||
/**
|
||||
* The label for a chunk that may have absorbed several declarations.
|
||||
*
|
||||
* A small file merges into a single chunk, and that chunk's label is then the
|
||||
* file's only description in the index — so dropping every name but the first
|
||||
* would describe a whole module as whatever its first declaration happened to
|
||||
* be. The list is capped because beyond a handful of names the label stops
|
||||
* being a description and starts being noise that dilutes the embedding.
|
||||
*/
|
||||
export function symbolLabel(symbols) {
|
||||
const unique = [...new Set(symbols)]
|
||||
if (unique.length === 0) return undefined
|
||||
if (unique.length <= 4) return unique.join(', ')
|
||||
return `${unique.slice(0, 4).join(', ')} +${unique.length - 4}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a file looks machine-generated.
|
||||
*
|
||||
* A minified bundle is one enormous line; embedding windows taken from it
|
||||
* produces chunks of arbitrary identifiers that match every query weakly and
|
||||
* none strongly. The mean line length separates the two cheaply and without
|
||||
* reading the whole file.
|
||||
*/
|
||||
export function isLikelyMinified(content) {
|
||||
const lines = content.split('\n')
|
||||
if (lines.length === 0) return true
|
||||
// Only consider files large enough for the statistic to mean anything.
|
||||
if (lines.length < 4) {
|
||||
return content.length > 20_000
|
||||
}
|
||||
const total = content.length
|
||||
return total / lines.length > MINIFIED_MEAN_LINE
|
||||
}
|
||||
|
||||
/** Trim trailing blank lines from a segment, leaving at least one line. */
|
||||
function trimEnd(lines, start, end) {
|
||||
let last = end
|
||||
while (last > start && lines[last].trim() === '') last -= 1
|
||||
return last
|
||||
}
|
||||
|
||||
/** Split one oversized segment into overlapping windows. */
|
||||
function splitLong(lines, start, end, maxLines, overlap) {
|
||||
const out = []
|
||||
const step = Math.max(1, maxLines - overlap)
|
||||
for (let at = start; at <= end; at += step) {
|
||||
const stop = Math.min(end, at + maxLines - 1)
|
||||
out.push({ startLine: at, endLine: stop, symbol: undefined })
|
||||
if (stop === end) break
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a file into chunks.
|
||||
*
|
||||
* @param {object} input
|
||||
* @param {string} input.path workspace-relative path, used only for language detection
|
||||
* @param {string} input.content file text
|
||||
* @param {number} [input.chunkLines] target chunk size
|
||||
* @param {number} [input.maxChunkLines] hard cap before splitting
|
||||
* @param {number} [input.overlapLines] lines shared when splitting
|
||||
* @param {number} [input.maxChunks] cap on chunks returned for one file
|
||||
* @returns {{chunks: {startLine: number, endLine: number, symbol?: string, text: string}[], skipped?: string}}
|
||||
*/
|
||||
export function chunkFile(input) {
|
||||
const {
|
||||
path,
|
||||
content,
|
||||
chunkLines = DEFAULT_CHUNK_LINES,
|
||||
maxChunkLines = DEFAULT_MAX_CHUNK_LINES,
|
||||
overlapLines = DEFAULT_OVERLAP_LINES,
|
||||
maxChunks = DEFAULT_MAX_CHUNKS_PER_FILE,
|
||||
} = input
|
||||
|
||||
if (isLikelyMinified(content)) return { chunks: [], skipped: 'generated-or-minified' }
|
||||
if (content.trim() === '') return { chunks: [], skipped: 'empty' }
|
||||
|
||||
const lines = content.split('\n')
|
||||
const family = languageOf(path)
|
||||
const pattern = declarationPattern(family)
|
||||
|
||||
/** Anchors: line indices that begin a declaration. */
|
||||
const anchors = []
|
||||
if (pattern !== undefined) {
|
||||
for (let at = 0; at < lines.length; at += 1) {
|
||||
const line = lines[at]
|
||||
if (line === '' || /^\s/.test(line)) continue
|
||||
if (pattern.test(line)) anchors.push(at)
|
||||
}
|
||||
}
|
||||
|
||||
/** Segments: [anchor, next anchor) with trailing blanks trimmed. */
|
||||
const segments = []
|
||||
// One anchor is still structure: a module holding a single top-level function
|
||||
// should carry that function's name rather than lose it to the fallback.
|
||||
if (anchors.length >= 1) {
|
||||
if (anchors[0] > 0) anchors.unshift(0)
|
||||
for (let at = 0; at < anchors.length; at += 1) {
|
||||
const start = anchors[at]
|
||||
const rawEnd = at + 1 < anchors.length ? anchors[at + 1] - 1 : lines.length - 1
|
||||
const end = trimEnd(lines, start, rawEnd)
|
||||
if (end < start) continue
|
||||
segments.push({ startLine: start, endLine: end, symbols: symbolList(symbolOf(lines[start])) })
|
||||
}
|
||||
} else {
|
||||
// No usable structure: one segment, and the splitter below does the work.
|
||||
segments.push({ startLine: 0, endLine: trimEnd(lines, 0, lines.length - 1), symbols: [] })
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge neighbours that fit together.
|
||||
*
|
||||
* A file of one-line getters would otherwise produce one chunk per getter,
|
||||
* each too small to carry meaning. Merging accumulates every declaration name
|
||||
* it swallowed, because a chunk tagged only with its first name would be
|
||||
* embedded under a label describing a fraction of its contents.
|
||||
*/
|
||||
const merged = []
|
||||
let current
|
||||
for (const segment of segments) {
|
||||
if (current !== undefined) {
|
||||
const combined = segment.endLine - current.startLine + 1
|
||||
if (combined <= chunkLines) {
|
||||
current.endLine = segment.endLine
|
||||
current.symbols.push(...segment.symbols)
|
||||
continue
|
||||
}
|
||||
}
|
||||
current = { startLine: segment.startLine, endLine: segment.endLine, symbols: [...segment.symbols] }
|
||||
merged.push(current)
|
||||
}
|
||||
|
||||
const out = []
|
||||
for (const segment of merged) {
|
||||
const label = symbolLabel(segment.symbols)
|
||||
const length = segment.endLine - segment.startLine + 1
|
||||
if (length <= maxChunkLines) {
|
||||
out.push({ startLine: segment.startLine, endLine: segment.endLine, symbol: label })
|
||||
continue
|
||||
}
|
||||
for (const piece of splitLong(lines, segment.startLine, segment.endLine, maxChunkLines, overlapLines)) {
|
||||
out.push({ ...piece, symbol: piece.startLine === segment.startLine ? label : undefined })
|
||||
}
|
||||
}
|
||||
|
||||
const capped = out.length > maxChunks
|
||||
const kept = capped ? out.slice(0, maxChunks) : out
|
||||
return {
|
||||
chunks: kept.map((chunk) => ({
|
||||
// 1-based, so a result can be handed straight to `read` as offset/limit.
|
||||
startLine: chunk.startLine + 1,
|
||||
endLine: chunk.endLine + 1,
|
||||
...(chunk.symbol === undefined ? {} : { symbol: chunk.symbol }),
|
||||
text: lines.slice(chunk.startLine, chunk.endLine + 1).join('\n'),
|
||||
})),
|
||||
...(capped ? { skipped: 'chunk-cap-reached' } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The exact string that gets embedded for a chunk.
|
||||
*
|
||||
* The path is included on purpose. A developer asking "where is auth handled"
|
||||
* means something closer to "under a path called auth" than to any particular
|
||||
* line of code, and the path is the only signal in the chunk that carries that.
|
||||
* The symbol follows for the same reason: it is what the question would name.
|
||||
*/
|
||||
export function embedText(path, chunk) {
|
||||
const header = chunk.symbol === undefined ? path : `${path} :: ${chunk.symbol}`
|
||||
return `${header}\n${chunk.text}`
|
||||
}
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
/**
|
||||
* The embedding client.
|
||||
*
|
||||
* Speaks the Ollama HTTP API, which also covers any OpenAI-compatible gateway
|
||||
* that implements `/api/embed`, and is deliberately the *only* outbound call
|
||||
* this plugin makes. There is no vendor SDK and no configuration surface that
|
||||
* could redirect code anywhere except the endpoint the operator named.
|
||||
*
|
||||
* Two properties the rest of the plugin depends on:
|
||||
*
|
||||
* **Every vector is unit length.** Cosine similarity is then a plain dot
|
||||
* product, which is what makes ranking over tens of thousands of chunks a few
|
||||
* milliseconds of arithmetic rather than a normalisation pass per comparison.
|
||||
*
|
||||
* **A dimension mismatch is fatal and loud.** A vector of the wrong width
|
||||
* cannot be compared with the others, and silently storing one would corrupt
|
||||
* the index in a way that looks like bad ranking rather than a broken config —
|
||||
* the hardest possible failure to diagnose from the outside.
|
||||
*/
|
||||
|
||||
/** Failure carrying a stable code, so the tool can explain rather than dump a stack. */
|
||||
export class EmbeddingError extends Error {
|
||||
/**
|
||||
* @param {string} message human-readable cause
|
||||
* @param {string} code stable identifier
|
||||
*/
|
||||
constructor(message, code) {
|
||||
super(message)
|
||||
this.name = 'EmbeddingError'
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
/** Characters sent per chunk. Comfortably inside an 8k-token code model's window. */
|
||||
const MAX_INPUT_CHARS = 24_000
|
||||
|
||||
/** Attempts per request, including the first. */
|
||||
const MAX_ATTEMPTS = 3
|
||||
|
||||
/** Base backoff between attempts. */
|
||||
const RETRY_BASE_MS = 250
|
||||
|
||||
/** Truncate one input to the character budget. */
|
||||
export function clampInput(text) {
|
||||
return text.length <= MAX_INPUT_CHARS ? text : text.slice(0, MAX_INPUT_CHARS)
|
||||
}
|
||||
|
||||
/** Scale a vector to unit length. A zero vector is returned unchanged. */
|
||||
export function normalize(vector) {
|
||||
let sum = 0
|
||||
for (let at = 0; at < vector.length; at += 1) sum += vector[at] * vector[at]
|
||||
if (sum === 0) return vector
|
||||
const scale = 1 / Math.sqrt(sum)
|
||||
for (let at = 0; at < vector.length; at += 1) vector[at] *= scale
|
||||
return vector
|
||||
}
|
||||
|
||||
/** Dot product, which is cosine similarity for unit vectors. */
|
||||
export function dot(left, right) {
|
||||
let sum = 0
|
||||
const length = Math.min(left.length, right.length)
|
||||
for (let at = 0; at < length; at += 1) sum += left[at] * right[at]
|
||||
return sum
|
||||
}
|
||||
|
||||
/** Sleep that rejects promptly when the caller's signal fires. */
|
||||
function delay(ms, signal) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
resolve()
|
||||
}, ms)
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer)
|
||||
reject(signal.reason ?? new Error('aborted'))
|
||||
}
|
||||
if (signal?.aborted) onAbort()
|
||||
else signal?.addEventListener('abort', onAbort, { once: true })
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an embedder bound to one endpoint.
|
||||
*
|
||||
* @param {object} options
|
||||
* @param {string} options.apiBase base URL, without a trailing slash
|
||||
* @param {string} options.model model id as the endpoint knows it
|
||||
* @param {number} options.dimensions expected vector width
|
||||
* @param {string} [options.apiKey] bearer token, when the endpoint wants one
|
||||
* @param {number} [options.batchSize] inputs per request on the batch endpoint
|
||||
* @param {number} [options.concurrency] requests in flight
|
||||
* @param {number} [options.timeoutMs] per-request timeout
|
||||
* @param {typeof fetch} [options.fetchImpl] injection point for tests
|
||||
*/
|
||||
export function createEmbedder(options) {
|
||||
const {
|
||||
apiBase,
|
||||
model,
|
||||
dimensions,
|
||||
apiKey = '',
|
||||
batchSize = 16,
|
||||
concurrency = 2,
|
||||
timeoutMs = 30_000,
|
||||
maxAttempts = MAX_ATTEMPTS,
|
||||
keepAlive = '',
|
||||
fetchImpl = fetch,
|
||||
} = options
|
||||
|
||||
const base = apiBase.replace(/\/+$/, '')
|
||||
const headers = { 'content-type': 'application/json' }
|
||||
if (apiKey !== '') headers.authorization = `Bearer ${apiKey}`
|
||||
|
||||
/**
|
||||
* Add Ollama's `keep_alive` to a request body, when one is configured.
|
||||
*
|
||||
* Two reasons. An index run issues hundreds of requests, and without a
|
||||
* keep-alive the model can be evicted between them and reloaded from disk
|
||||
* repeatedly. And on a host already holding another model resident, a load
|
||||
* that has to wait for memory is exactly the situation where a bounded,
|
||||
* explicit residency beats Ollama's own default.
|
||||
*/
|
||||
const withKeepAlive = (body) => (keepAlive === '' ? body : { ...body, keep_alive: keepAlive })
|
||||
|
||||
// Discovered on first use: the batch endpoint is preferred, but an older
|
||||
// Ollama build answers it with a 404 and then every request pays that
|
||||
// discovery again. Remembering the answer keeps the fallback cheap.
|
||||
let batchEndpointWorks = true
|
||||
|
||||
/** One HTTP request, with retries on transport faults and 5xx. */
|
||||
async function request(path, body, signal) {
|
||||
let lastError
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
||||
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
||||
const timeout = AbortSignal.timeout(timeoutMs)
|
||||
const combined = signal === undefined ? timeout : AbortSignal.any([signal, timeout])
|
||||
try {
|
||||
const response = await fetchImpl(`${base}${path}`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: combined,
|
||||
})
|
||||
if (response.status === 404 || response.status === 405) {
|
||||
const error = new EmbeddingError(`${path} is not available on ${base}`, 'ENDPOINT_UNSUPPORTED')
|
||||
error.status = response.status
|
||||
throw error
|
||||
}
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => '')
|
||||
const error = new EmbeddingError(
|
||||
`${path} responded ${response.status}${text === '' ? '' : `: ${text.slice(0, 300)}`}`,
|
||||
'HTTP_ERROR',
|
||||
)
|
||||
error.status = response.status
|
||||
throw error
|
||||
}
|
||||
return await response.json()
|
||||
} catch (error) {
|
||||
// A 4xx that is not a retryable status is a configuration fault, not a
|
||||
// transient one, and retrying it just delays the explanation.
|
||||
if (error instanceof EmbeddingError && error.status !== undefined && error.status < 500) throw error
|
||||
if (signal?.aborted) throw signal.reason ?? error
|
||||
lastError = error
|
||||
if (attempt + 1 < MAX_ATTEMPTS) await delay(RETRY_BASE_MS * 2 ** attempt, signal)
|
||||
}
|
||||
}
|
||||
throw new EmbeddingError(
|
||||
`could not reach ${base}: ${lastError?.message ?? lastError}`,
|
||||
'UNREACHABLE',
|
||||
)
|
||||
}
|
||||
|
||||
/** Check a returned vector's width against the configured one. */
|
||||
function check(vector, index) {
|
||||
if (!Array.isArray(vector)) {
|
||||
throw new EmbeddingError(`response item ${index} was not an array`, 'BAD_RESPONSE')
|
||||
}
|
||||
if (dimensions > 0 && vector.length !== dimensions) {
|
||||
throw new EmbeddingError(
|
||||
`model "${model}" returned ${vector.length}-dimensional vectors but ${dimensions} are configured`,
|
||||
'DIMENSION_MISMATCH',
|
||||
)
|
||||
}
|
||||
return normalize(Float32Array.from(vector))
|
||||
}
|
||||
|
||||
/** Embed one batch through whichever endpoint answers. */
|
||||
async function embedBatch(texts, signal) {
|
||||
const inputs = texts.map(clampInput)
|
||||
if (inputs.length === 0) return []
|
||||
if (batchEndpointWorks) {
|
||||
try {
|
||||
const payload = await request('/api/embed', withKeepAlive({ model, input: inputs }), signal)
|
||||
if (!Array.isArray(payload?.embeddings) || payload.embeddings.length !== inputs.length) {
|
||||
throw new EmbeddingError(
|
||||
`/api/embed returned ${payload?.embeddings?.length ?? 0} vectors for ${inputs.length} inputs`,
|
||||
'BAD_RESPONSE',
|
||||
)
|
||||
}
|
||||
return payload.embeddings.map((vector, index) => check(vector, index))
|
||||
} catch (error) {
|
||||
// Only a missing endpoint falls back. A dimension mismatch or a bad
|
||||
// response is a real fault and must not be papered over by a second
|
||||
// endpoint that would fail the same way more slowly.
|
||||
if (!(error instanceof EmbeddingError) || error.code !== 'ENDPOINT_UNSUPPORTED') throw error
|
||||
batchEndpointWorks = false
|
||||
}
|
||||
}
|
||||
const out = []
|
||||
for (const input of inputs) {
|
||||
const payload = await request('/api/embeddings', withKeepAlive({ model, prompt: input }), signal)
|
||||
out.push(check(payload?.embedding, out.length))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
return {
|
||||
model,
|
||||
dimensions,
|
||||
base,
|
||||
/** Whether the batch endpoint has been ruled out. */
|
||||
get batchEnabled() {
|
||||
return batchEndpointWorks
|
||||
},
|
||||
/**
|
||||
* Embed many texts, preserving order.
|
||||
*
|
||||
* @param {string[]} texts inputs
|
||||
* @param {AbortSignal} [signal] caller cancellation
|
||||
* @param {(done: number, total: number) => void} [onProgress]
|
||||
* @returns {Promise<Float32Array[]>} unit vectors in input order
|
||||
*/
|
||||
async embed(texts, signal, onProgress) {
|
||||
const out = new Array(texts.length)
|
||||
const size = Math.max(1, Math.min(batchSize, texts.length))
|
||||
const batches = []
|
||||
for (let at = 0; at < texts.length; at += size) {
|
||||
batches.push({ start: at, items: texts.slice(at, at + size) })
|
||||
}
|
||||
let done = 0
|
||||
let cursor = 0
|
||||
const workers = new Array(Math.max(1, Math.min(concurrency, batches.length))).fill(0).map(async () => {
|
||||
while (cursor < batches.length) {
|
||||
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
||||
const batch = batches[cursor]
|
||||
cursor += 1
|
||||
const vectors = await embedBatch(batch.items, signal)
|
||||
for (let at = 0; at < vectors.length; at += 1) out[batch.start + at] = vectors[at]
|
||||
done += vectors.length
|
||||
onProgress?.(done, texts.length)
|
||||
}
|
||||
})
|
||||
await Promise.all(workers)
|
||||
return out
|
||||
},
|
||||
/**
|
||||
* Embed one query string.
|
||||
*
|
||||
* @param {string} text query
|
||||
* @param {AbortSignal} [signal] caller cancellation
|
||||
* @returns {Promise<Float32Array>} unit vector
|
||||
*/
|
||||
async embedOne(text, signal) {
|
||||
const [vector] = await embedBatch([text], signal)
|
||||
return vector
|
||||
},
|
||||
}
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Query expansion, on the local model.
|
||||
*
|
||||
* A developer's first query is often a description rather than a name — "how
|
||||
* does a room decide it is occupied" — and that phrasing shares few tokens with
|
||||
* the code that answers it. Asking a small local model for a handful of
|
||||
* alternative phrasings, embedding each, and fusing the rankings costs one local
|
||||
* generation and a few local embeddings, and is free in money.
|
||||
*
|
||||
* ## Every failure is a fallback, never an error
|
||||
*
|
||||
* This runs on the critical path of a search, against a model on somebody else's
|
||||
* box. It can be slow, wedged, or absent. None of those may fail the search, so
|
||||
* the contract is narrow: **the returned list always contains the original
|
||||
* query**, and anything that goes wrong simply returns it alone. The caller can
|
||||
* treat expansion as a bonus that is either there or not.
|
||||
*
|
||||
* The internal timeout is deliberately separate from the caller's signal: the
|
||||
* caller cancelling means stop everything, while the model being slow just means
|
||||
* search with what we have.
|
||||
*/
|
||||
|
||||
/** Attempts per request. Expansion is a bonus; one more try is enough. */
|
||||
const MAX_ATTEMPTS = 2
|
||||
|
||||
/** Characters of query allowed through. Longer is a paste, not a question. */
|
||||
const MAX_QUERY_CHARS = 500
|
||||
|
||||
/** A variant this long is a paragraph, not a search phrase. */
|
||||
const MAX_VARIANT_CHARS = 200
|
||||
|
||||
/** Variants shorter than this match everything and rank nothing. */
|
||||
const MIN_VARIANT_CHARS = 3
|
||||
|
||||
/**
|
||||
* Openers that mark a line as the model talking rather than a phrasing.
|
||||
*
|
||||
* A small model asked for a list sometimes answers with a sentence about the
|
||||
* request — "I cannot help with that." A line like that is not a weak variant,
|
||||
* it is not a variant at all: embedding it would search the codebase for the
|
||||
* model's apology and rank whatever matched.
|
||||
*
|
||||
* `i\b` and the refusal check below are the load-bearing ones. A question-shaped
|
||||
* variant ("how does a room decide it is occupied") is entirely legitimate, so
|
||||
* the test is first-person and refusal vocabulary, not sentence punctuation.
|
||||
*/
|
||||
const COMMENTARY = /^(?:i\b|i'm\b|i am\b|sorry\b|unfortunately\b|note\b|explanation\b|here\b|sure\b|of course\b|certainly\b|these\b|those\b|the following\b|as an?\b|phrasings?\b|query\b)/i
|
||||
|
||||
/** Refusal vocabulary, wherever it sits in the line. */
|
||||
const REFUSAL = /\b(?:cannot help|can't help|don't have|do not have|not able to|unable to|no way to)\b/i
|
||||
|
||||
const PROMPT = `You turn a developer's search query into alternative phrasings that would match source code.
|
||||
|
||||
Rules:
|
||||
- One phrasing per line. No numbering, no bullets, no commentary, no blank lines.
|
||||
- Use the vocabulary the code itself would use: identifiers, domain nouns, verb phrases.
|
||||
- Vary the angle: a likely function name, a likely filename fragment, a description of the behaviour.
|
||||
- Do not repeat the original query.
|
||||
|
||||
Query: {{QUERY}}
|
||||
|
||||
Phrasings:`
|
||||
|
||||
/**
|
||||
* Reduce one model line to a usable phrase, or nothing.
|
||||
*
|
||||
* Models are reliably inclined to number a list, wrap it in quotes, or add a
|
||||
* closing remark. None of that is worth an error path, so it is all stripped.
|
||||
*/
|
||||
export function cleanVariant(line) {
|
||||
let text = String(line ?? '')
|
||||
.replace(/^\s*(?:[-*•]|\d+[.)])\s*/, '')
|
||||
.trim()
|
||||
// A trailing aside ("this should find the handler") is common; drop it.
|
||||
text = text.split(/\s+[—–-]{1,2}\s+/)[0].trim()
|
||||
if (text.length >= 2 && ((text.startsWith('"') && text.endsWith('"')) || (text.startsWith("'") && text.endsWith("'")))) {
|
||||
text = text.slice(1, -1).trim()
|
||||
}
|
||||
if (text.length < MIN_VARIANT_CHARS || text.length > MAX_VARIANT_CHARS) return undefined
|
||||
if (COMMENTARY.test(text) || REFUSAL.test(text)) return undefined
|
||||
return text
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a model response into alternative queries, best effort.
|
||||
*
|
||||
* @param {string} body raw model output
|
||||
* @param {string} query the original, excluded from the results
|
||||
* @param {number} count most variants to keep
|
||||
* @returns {string[]} deduplicated variants, in the order the model gave them
|
||||
*/
|
||||
export function parseVariants(body, query, count) {
|
||||
const seen = new Set([String(query).trim().toLowerCase()])
|
||||
const out = []
|
||||
for (const line of String(body ?? '').split('\n')) {
|
||||
if (out.length >= count) break
|
||||
const variant = cleanVariant(line)
|
||||
if (variant === undefined) continue
|
||||
const key = variant.toLowerCase()
|
||||
if (seen.has(key)) continue
|
||||
seen.add(key)
|
||||
out.push(variant)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an expander bound to one local model.
|
||||
*
|
||||
* @param {object} options
|
||||
* @param {string} options.apiBase endpoint, no trailing slash
|
||||
* @param {string} options.model model id
|
||||
* @param {number} [options.timeoutMs] budget for the whole expansion
|
||||
* @param {number} [options.count] most variants to ask for
|
||||
* @param {typeof fetch} [options.fetchImpl] injection point for tests
|
||||
*/
|
||||
export function createExpander(options) {
|
||||
const { apiBase, model, timeoutMs = 8_000, count = 4, fetchImpl = fetch } = options
|
||||
const base = String(apiBase).replace(/\/+$/, '')
|
||||
|
||||
/** One request, bounded by its own deadline. */
|
||||
async function request(query, budget, signal) {
|
||||
// A non-finite budget would make AbortSignal.timeout throw, and the caller's
|
||||
// catch would turn that into a silent fallback — a misconfiguration wearing
|
||||
// the disguise of a slow model. Clamp instead.
|
||||
const deadline = AbortSignal.timeout(Number.isFinite(budget) && budget > 0 ? budget : 8_000)
|
||||
const combined = signal === undefined ? deadline : AbortSignal.any([signal, deadline])
|
||||
const response = await fetchImpl(`${base}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
prompt: PROMPT.replace('{{QUERY}}', query),
|
||||
stream: false,
|
||||
// A handful of short lines; a model allowed to ramble will.
|
||||
options: { num_predict: 120, temperature: 0.4 },
|
||||
}),
|
||||
signal: combined,
|
||||
})
|
||||
if (!response.ok) throw new Error(`${base} responded ${response.status}`)
|
||||
const payload = await response.json()
|
||||
return typeof payload?.response === 'string' ? payload.response : ''
|
||||
}
|
||||
|
||||
return {
|
||||
model,
|
||||
/**
|
||||
* Alternative phrasings for a query.
|
||||
*
|
||||
* @param {string} query the original
|
||||
* @param {AbortSignal} [signal] caller cancellation
|
||||
* @returns {Promise<string[]>} the original first, then whatever else worked
|
||||
*/
|
||||
async expand(query, signal) {
|
||||
const original = String(query ?? '').trim()
|
||||
if (original === '' || original.length > MAX_QUERY_CHARS) return [original]
|
||||
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) {
|
||||
if (signal?.aborted) return [original]
|
||||
try {
|
||||
const body = await request(original, timeoutMs, signal)
|
||||
return [original, ...parseVariants(body, original, count)]
|
||||
} catch {
|
||||
// Slow, wedged, or unreachable all mean the same thing here: search
|
||||
// with the query we were given rather than failing the call.
|
||||
if (signal?.aborted) return [original]
|
||||
}
|
||||
}
|
||||
return [original]
|
||||
},
|
||||
}
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
/**
|
||||
* Focus resolution: which slice of the workspace an agent is confined to.
|
||||
*
|
||||
* ## Why this exists
|
||||
*
|
||||
* A workspace-wide index has to rank across every part of a repository, and in
|
||||
* a codebase built from many near-identical modules that is exactly where
|
||||
* embedding similarity fails. In the repository this was written against there
|
||||
* are eighteen controllers — each a state machine with the same idioms, the same
|
||||
* naming and the same shape — spread across 228k lines. A query about one of
|
||||
* them returns plausible fragments of all eighteen.
|
||||
*
|
||||
* Narrowing the index to the one being worked on removes the confusability
|
||||
* rather than trying to rank through it, and it does three things at once:
|
||||
*
|
||||
* 1. **precision** — there are no near-duplicates to outrank the right answer;
|
||||
* 2. **cold start** — ten files embed in seconds where six hundred took minutes;
|
||||
* 3. **context** — code the agent never sees is code that is never re-sent on
|
||||
* every later step, which is where the money actually is.
|
||||
*
|
||||
* ## Resolution order
|
||||
*
|
||||
* Most explicit wins, and the last fallback is the workspace root — which is the
|
||||
* behaviour that existed before focus did, so adopting this cannot regress a
|
||||
* deployment that never configures it.
|
||||
*
|
||||
* The marker step is the interesting one: a folder that contains its own
|
||||
* instructions *is* a scope. A controller directory holding a `CONTROLLER.md`
|
||||
* becomes the world the agent sees, and the document that describes the module
|
||||
* also defines its boundary. Nothing needed to be invented for that — the
|
||||
* convention already means "this folder is self-describing".
|
||||
*/
|
||||
|
||||
import { existsSync, statSync } from 'node:fs'
|
||||
import { readdir } from 'node:fs/promises'
|
||||
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
||||
|
||||
/**
|
||||
* Files whose presence marks a directory as a scope root.
|
||||
*
|
||||
* Ordered by how specific they are: a `CONTROLLER.md` is a stronger claim about
|
||||
* a directory than the `AGENTS.md` that sits at every repository root.
|
||||
*/
|
||||
export const DEFAULT_FOCUS_MARKERS = Object.freeze(['CONTROLLER.md', 'DEBUG_UI.md', 'AGENTS.md'])
|
||||
|
||||
/** True when `path` is `root` itself or sits underneath it. */
|
||||
export function isInside(root, path) {
|
||||
const prefix = root.endsWith(sep) ? root : root + sep
|
||||
return path === root || path.startsWith(prefix)
|
||||
}
|
||||
|
||||
/** True when `path` exists and is a directory. */
|
||||
function statIsDirectory(path) {
|
||||
try {
|
||||
return statSync(path).isDirectory()
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Every ancestor of `from` up to and including `stop`, nearest first. */
|
||||
export function ancestorsTo(from, stop) {
|
||||
const out = []
|
||||
let current = resolve(from)
|
||||
const limit = resolve(stop)
|
||||
for (;;) {
|
||||
out.push(current)
|
||||
if (current === limit) break
|
||||
const parent = dirname(current)
|
||||
// A path that is not under `stop`, or a filesystem root, ends the walk
|
||||
// rather than looping.
|
||||
if (parent === current || !isInside(limit, parent)) break
|
||||
current = parent
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the focus directory for one session.
|
||||
*
|
||||
* @param {object} input
|
||||
* @param {string} input.workspace absolute workspace root
|
||||
* @param {string} [input.cwd] directory the session is working from
|
||||
* @param {string} [input.explicit] a configured or caller-supplied focus
|
||||
* @param {(path: string) => boolean} [input.isDirectory] injection point for tests
|
||||
* @param {readonly string[]} [input.markers] scope-marker filenames
|
||||
* @param {(path: string) => boolean} [input.exists] injection point for tests
|
||||
* @param {{path: string, marker: string}[]} [input.scopes] scopes found by {@link discoverScopes}
|
||||
* @returns {{root: string, source: 'explicit'|'scope'|'marker'|'workspace', marker?: string, problem?: string}}
|
||||
*/
|
||||
export function resolveFocus(input) {
|
||||
const {
|
||||
workspace,
|
||||
cwd = workspace,
|
||||
explicit = '',
|
||||
markers = DEFAULT_FOCUS_MARKERS,
|
||||
scopes = [],
|
||||
exists = existsSync,
|
||||
isDirectory = statIsDirectory,
|
||||
} = input
|
||||
|
||||
// 1. An explicit focus wins, but a bad one is reported rather than silently
|
||||
// ignored: a typo that quietly fell back to the whole workspace would look
|
||||
// like the feature doing nothing.
|
||||
const requested = String(explicit ?? '').trim()
|
||||
if (requested !== '') {
|
||||
const absolute = isAbsolute(requested) ? resolve(requested) : resolve(workspace, requested)
|
||||
if (!isInside(workspace, absolute)) {
|
||||
return {
|
||||
root: resolve(workspace),
|
||||
source: 'workspace',
|
||||
problem: `focus "${requested}" is outside the workspace`,
|
||||
}
|
||||
}
|
||||
if (!isDirectory(absolute)) {
|
||||
return {
|
||||
root: resolve(workspace),
|
||||
source: 'workspace',
|
||||
problem: `focus "${requested}" is not a directory`,
|
||||
}
|
||||
}
|
||||
return { root: absolute, source: 'explicit' }
|
||||
}
|
||||
|
||||
// 2. Nothing here adopts a discovered scope automatically, and that is
|
||||
// deliberate. A session sitting at the repository root is not thereby
|
||||
// *about* any one controller, and silently confining it to whichever scope
|
||||
// sorted first would hide code the session legitimately needs. Discovery
|
||||
// exists so a caller can be shown the scopes and name one; choosing is an
|
||||
// explicit act, which is what `focus` is for.
|
||||
|
||||
// 3. The nearest ancestor carrying a marker file. `cwd` is normally the
|
||||
// workspace root, at which point the root's own AGENTS.md matches and the
|
||||
// result is the workspace — the same answer the old code gave.
|
||||
const start = isInside(workspace, resolve(cwd)) ? resolve(cwd) : resolve(workspace)
|
||||
for (const directory of ancestorsTo(start, workspace)) {
|
||||
for (const marker of markers) {
|
||||
if (marker !== '' && exists(join(directory, marker))) {
|
||||
return { root: directory, source: 'marker', marker }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. The workspace.
|
||||
return { root: resolve(workspace), source: 'workspace' }
|
||||
}
|
||||
|
||||
/** A stable key for one focus directory, for index filenames. */
|
||||
export function focusKey(workspace, root) {
|
||||
const rel = relative(resolve(workspace), resolve(root)).split(sep).join('/')
|
||||
return rel === '' ? '.' : rel
|
||||
}
|
||||
|
||||
/**
|
||||
* Directories skipped while looking for scopes.
|
||||
*
|
||||
* A marker inside a dependency or a build directory describes somebody else's
|
||||
* project, so discovering it would offer the user a scope they do not own.
|
||||
*/
|
||||
const DISCOVERY_SKIP_DIRS = new Set([
|
||||
'node_modules', '.git', 'dist', 'build', 'out', 'coverage', 'target',
|
||||
'.next', '.nuxt', '.venv', 'venv', '__pycache__', 'vendor', '.cache',
|
||||
])
|
||||
|
||||
/**
|
||||
* Find every directory that declares itself a scope.
|
||||
*
|
||||
* This exists because walking up from the working directory cannot find a
|
||||
* marker in a subdirectory, and in the repository this was written for that is
|
||||
* the whole point: the session sits at the workspace root while the eighteen
|
||||
* controllers each carry their own instructions further down. Inheritance and
|
||||
* discovery are different questions — "what contains me?" versus "what is
|
||||
* available?" — and a scope needs both.
|
||||
*
|
||||
* Results are workspace-relative and sorted, so a list shown to a user or a
|
||||
* model is stable between calls.
|
||||
*
|
||||
* @param {object} input
|
||||
* @param {string} input.workspace absolute workspace root
|
||||
* @param {readonly string[]} [input.markers] marker filenames
|
||||
* @param {number} [input.maxDepth] deepest directory level to scan
|
||||
* @returns {Promise<{path: string, marker: string}[]>} discovered scopes
|
||||
*/
|
||||
export async function discoverScopes(input) {
|
||||
const { workspace, markers = DEFAULT_FOCUS_MARKERS, maxDepth = 8 } = input
|
||||
const root = resolve(workspace)
|
||||
const found = []
|
||||
|
||||
const walk = async (directory, depth) => {
|
||||
if (depth > maxDepth) return
|
||||
let entries
|
||||
try {
|
||||
entries = await readdir(directory, { withFileTypes: true })
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const names = new Set(entries.filter((entry) => entry.isFile()).map((entry) => entry.name))
|
||||
for (const marker of markers) {
|
||||
if (names.has(marker)) {
|
||||
const rel = relative(root, directory).split(sep).join('/')
|
||||
found.push({ path: rel === '' ? '.' : rel, marker })
|
||||
break
|
||||
}
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || entry.isSymbolicLink()) continue
|
||||
if (DISCOVERY_SKIP_DIRS.has(entry.name)) continue
|
||||
await walk(join(directory, entry.name), depth + 1)
|
||||
}
|
||||
}
|
||||
|
||||
await walk(root, 0)
|
||||
found.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
|
||||
return found
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Where this plugin keeps things on disk.
|
||||
*
|
||||
* Everything lives under `$DSH_HOME`, never inside the workspace. That is the
|
||||
* whole point of the plugin's economics: the index holds a vector per code
|
||||
* chunk, which for a 300k-line repository is tens of megabytes — several times
|
||||
* the size of the source it describes. A multi-megabyte binary file appearing
|
||||
* in a repository the user might commit, or that a `git status` might surface
|
||||
* mid-task, is a worse outcome than the tokens it saves.
|
||||
*
|
||||
* The index is also disposable by construction. It is keyed by content hash, so
|
||||
* deleting it costs one re-embed and nothing else — which is what makes it safe
|
||||
* to treat as a cache rather than a document.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto'
|
||||
import { homedir } from 'node:os'
|
||||
import { basename, join, resolve } from 'node:path'
|
||||
|
||||
/** Plugin name. Also the `$DSH_HOME/<name>` directory. */
|
||||
export const PLUGIN_NAME = 'dsh-recall'
|
||||
|
||||
/** The dsh home directory: `$DSH_HOME`, else `~/.dsh`. */
|
||||
export function resolveDshHome(env = process.env) {
|
||||
const configured = typeof env.DSH_HOME === 'string' ? env.DSH_HOME.trim() : ''
|
||||
return configured === '' ? join(homedir(), '.dsh') : resolve(configured)
|
||||
}
|
||||
|
||||
/**
|
||||
* A stable, filesystem-safe key for one workspace.
|
||||
*
|
||||
* The basename is for a human reading the directory listing; the hash is what
|
||||
* actually identifies it, so two checkouts both named `app` cannot collide and
|
||||
* one checkout moved between parents does not silently inherit the other's
|
||||
* index.
|
||||
*/
|
||||
export function workspaceKey(cwd) {
|
||||
const absolute = resolve(cwd)
|
||||
const name =
|
||||
basename(absolute)
|
||||
.replace(/[^A-Za-z0-9._-]+/g, '-')
|
||||
.replace(/^[-.]+|-+$/g, '') || 'workspace'
|
||||
const hash = createHash('sha256').update(absolute).digest('hex').slice(0, 8)
|
||||
return `${name}-${hash}`
|
||||
}
|
||||
|
||||
/** Root of this plugin's own data, independent of any workspace. */
|
||||
export function pluginDataDir(dshHome) {
|
||||
return join(dshHome, PLUGIN_NAME)
|
||||
}
|
||||
|
||||
/** Per-workspace state directory. */
|
||||
export function workspaceStateDir(dshHome, cwd) {
|
||||
return join(pluginDataDir(dshHome), 'workspaces', workspaceKey(cwd))
|
||||
}
|
||||
|
||||
/** The SQLite index for one workspace. */
|
||||
/**
|
||||
* The SQLite index for one workspace, optionally narrowed to one focus.
|
||||
*
|
||||
* The workspace-wide index deliberately keeps its original path. Focus is
|
||||
* additive: a deployment that never sets one must find the index it already
|
||||
* built, not silently re-embed the repository into a new file.
|
||||
*
|
||||
* @param {string} dshHome harness home
|
||||
* @param {string} cwd workspace root
|
||||
* @param {string} [focus] focus key, `.` for the workspace itself
|
||||
*/
|
||||
export function indexFile(dshHome, cwd, focus = '.') {
|
||||
const base = workspaceStateDir(dshHome, cwd)
|
||||
return focus === '.' ? join(base, 'index.sqlite') : join(base, 'focus', focus, 'index.sqlite')
|
||||
}
|
||||
+337
@@ -0,0 +1,337 @@
|
||||
/**
|
||||
* Ranking retrieved chunks.
|
||||
*
|
||||
* **Pure vector search is not good enough for code, and this is the whole reason
|
||||
* the module exists.** An embedding of `getUserById` sits close to every other
|
||||
* lookup function in the repository, so a query naming it returns ten plausible
|
||||
* lookups and buries the actual definition. A keyword match has the opposite
|
||||
* failure: it finds the definition instantly and misses the query that describes
|
||||
* behaviour without naming it.
|
||||
*
|
||||
* The two are combined by **reciprocal rank fusion** rather than by adding
|
||||
* normalised scores. Cosine similarity and a term-frequency score live on
|
||||
* different scales, and the usual fix — min-max normalising each into [0,1] —
|
||||
* is actively harmful here: when every candidate is unrelated, the similarities
|
||||
* are tightly clustered, and stretching that cluster across the full range turns
|
||||
* noise into a confident-looking ranking. Rank fusion is scale-free, so it
|
||||
* cannot make that mistake, and a chunk found by only one of the two methods
|
||||
* still surfaces.
|
||||
*/
|
||||
|
||||
/** RRF damping. 60 is the value from the original TREC work; larger flattens. */
|
||||
const RRF_K = 60
|
||||
|
||||
/** Terms shorter than this carry no signal in code and match everywhere. */
|
||||
const MIN_TERM_LENGTH = 2
|
||||
|
||||
/** Words that appear in almost every source file and therefore rank nothing. */
|
||||
const STOPWORDS = new Set([
|
||||
'the', 'and', 'for', 'with', 'that', 'this', 'from', 'into', 'when', 'then',
|
||||
'const', 'let', 'var', 'function', 'return', 'import', 'export', 'class',
|
||||
'def', 'self', 'new', 'use', 'are', 'was', 'were', 'not', 'but', 'you', 'how',
|
||||
'does', 'what', 'where', 'which', 'who', 'why', 'can', 'should', 'would',
|
||||
])
|
||||
|
||||
/** Lowercased haystacks, keyed by the matrix object they describe. */
|
||||
const haystacks = new WeakMap()
|
||||
|
||||
/**
|
||||
* Split an identifier into its words.
|
||||
*
|
||||
* `getUserById` must match the query terms `get`, `user`, `by` and `id`, because
|
||||
* a caller writing a search phrase naturally writes them separately. Without
|
||||
* this, the single most useful lexical signal in a codebase — a name spelled in
|
||||
* camelCase — is invisible.
|
||||
*
|
||||
* @param {string} identifier
|
||||
* @returns {string[]} lowercased words
|
||||
*/
|
||||
export function splitIdentifier(identifier) {
|
||||
return identifier
|
||||
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
||||
.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
|
||||
.split(/[^A-Za-z0-9]+/)
|
||||
.filter((word) => word !== '')
|
||||
}
|
||||
|
||||
/**
|
||||
* The search terms of a query.
|
||||
*
|
||||
* Both the raw tokens and the sub-words of each identifier are kept, so a query
|
||||
* of `getUserById` matches a chunk that spells it `get_user_by_id` as well as
|
||||
* one that spells it identically.
|
||||
*
|
||||
* @param {string} query
|
||||
* @returns {string[]} de-duplicated terms, longest first
|
||||
*/
|
||||
export function queryTerms(query) {
|
||||
const found = new Set()
|
||||
for (const raw of String(query).split(/[^A-Za-z0-9_$]+/)) {
|
||||
if (raw === '') continue
|
||||
const lower = raw.toLowerCase()
|
||||
if (lower.length >= MIN_TERM_LENGTH && !STOPWORDS.has(lower)) found.add(lower)
|
||||
if (raw.length >= MIN_TERM_LENGTH) {
|
||||
for (const word of splitIdentifier(raw)) {
|
||||
const part = word.toLowerCase()
|
||||
if (part.length >= MIN_TERM_LENGTH && !STOPWORDS.has(part)) found.add(part)
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...found].sort((left, right) => right.length - left.length)
|
||||
}
|
||||
|
||||
/** The lowercased text a chunk is matched against, built once per matrix. */
|
||||
function haystackFor(matrix, row) {
|
||||
let list = haystacks.get(matrix)
|
||||
if (list === undefined) {
|
||||
list = new Array(matrix.rows.length)
|
||||
haystacks.set(matrix, list)
|
||||
}
|
||||
if (list[row.id] === undefined) {
|
||||
const symbol = row.symbol === undefined ? '' : ` ${row.symbol}`
|
||||
list[row.id] = `${row.path}${symbol} ${row.text}`.toLowerCase()
|
||||
}
|
||||
return list[row.id]
|
||||
}
|
||||
|
||||
/**
|
||||
* Score every chunk against the query terms.
|
||||
*
|
||||
* Matches in the path or the symbol are weighted above matches in the body,
|
||||
* because a question about behaviour usually names the thing, and the name is
|
||||
* what the path and symbol carry.
|
||||
*
|
||||
* @param {object} matrix
|
||||
* @param {string[]} terms
|
||||
* @returns {{lexical: Float64Array, matched: Uint8Array}} raw scores and a hit flag
|
||||
*/
|
||||
export function lexicalScores(matrix, terms) {
|
||||
const count = matrix.rows.length
|
||||
const lexical = new Float64Array(count)
|
||||
const matched = new Uint8Array(count)
|
||||
if (terms.length === 0) return { lexical, matched }
|
||||
|
||||
// Document frequency per term, for the inverse-document-frequency weight.
|
||||
const frequency = new Map()
|
||||
for (const term of terms) {
|
||||
let seen = 0
|
||||
for (let at = 0; at < count; at += 1) {
|
||||
if (haystackFor(matrix, matrix.rows[at]).includes(term)) seen += 1
|
||||
}
|
||||
frequency.set(term, seen)
|
||||
}
|
||||
|
||||
for (let at = 0; at < count; at += 1) {
|
||||
const row = matrix.rows[at]
|
||||
const hay = haystackFor(matrix, row)
|
||||
const header = `${row.path} ${row.symbol ?? ''}`.toLowerCase()
|
||||
let score = 0
|
||||
for (const term of terms) {
|
||||
const df = frequency.get(term) ?? 0
|
||||
if (df === 0) continue
|
||||
// Standard IDF, damped so a term present in every file still contributes.
|
||||
const idf = Math.log(1 + count / (1 + df))
|
||||
let occurrences = 0
|
||||
let from = hay.indexOf(term)
|
||||
while (from !== -1) {
|
||||
occurrences += 1
|
||||
from = hay.indexOf(term, from + term.length)
|
||||
}
|
||||
if (occurrences === 0) continue
|
||||
const weight = header.includes(term) ? 3 : 1
|
||||
score += idf * (1 + Math.log(occurrences)) * weight
|
||||
}
|
||||
if (score > 0) {
|
||||
lexical[at] = score
|
||||
matched[at] = 1
|
||||
}
|
||||
}
|
||||
return { lexical, matched }
|
||||
}
|
||||
|
||||
/** Indices ordered by score, descending, ties broken by index for determinism. */
|
||||
function orderBy(score) {
|
||||
const indices = Array.from({ length: score.length }, (_, at) => at)
|
||||
indices.sort((left, right) => score[right] - score[left] || left - right)
|
||||
return indices
|
||||
}
|
||||
|
||||
/**
|
||||
* The excerpt a result shows.
|
||||
*
|
||||
* Head-truncated rather than tail-truncated: a function's signature, its
|
||||
* parameters and its first statements are what identify it, while the tail of a
|
||||
* long function is usually a closing brace and a return.
|
||||
*
|
||||
* @param {string} text chunk text
|
||||
* @param {number} maxLines cap
|
||||
* @returns {{excerpt: string, truncated: boolean}}
|
||||
*/
|
||||
export function excerptOf(text, maxLines) {
|
||||
const lines = text.split('\n')
|
||||
if (lines.length <= maxLines) return { excerpt: text, truncated: false }
|
||||
return { excerpt: lines.slice(0, maxLines).join('\n'), truncated: true }
|
||||
}
|
||||
|
||||
/**
|
||||
* The fused order of every chunk for one query, before selection.
|
||||
*
|
||||
* Split out from ranking so several queries can be ranked against the same rows
|
||||
* and merged: query expansion produces a handful of phrasings, each gets an
|
||||
* order, and the orders are fused. Selecting results is a separate step because
|
||||
* it is the only part that cares about limits and budgets.
|
||||
*
|
||||
* @param {object} input
|
||||
* @param {object} input.matrix vector matrix and rows
|
||||
* @param {Float32Array} input.queryVector unit query vector
|
||||
* @param {string} input.query raw query text
|
||||
* @param {number} [input.vectorWeight] fusion weight for the vector ranking
|
||||
* @param {number} [input.lexicalWeight] fusion weight for the keyword ranking
|
||||
* @returns {{order: number[], similarity: Float64Array, vectorBest: number, lexicalBest: number}}
|
||||
*/
|
||||
export function rankOrder(input) {
|
||||
const { matrix, queryVector, query, vectorWeight = 0.7, lexicalWeight = 0.3 } = input
|
||||
const count = matrix.rows.length
|
||||
if (count === 0) {
|
||||
return { order: [], similarity: new Float64Array(0), vectorBest: 0, lexicalBest: 0 }
|
||||
}
|
||||
|
||||
const dimensions = matrix.dimensions
|
||||
const similarity = new Float64Array(count)
|
||||
for (let at = 0; at < count; at += 1) {
|
||||
let sum = 0
|
||||
const offset = at * dimensions
|
||||
for (let dim = 0; dim < dimensions; dim += 1) sum += matrix.vectors[offset + dim] * queryVector[dim]
|
||||
similarity[at] = sum
|
||||
}
|
||||
|
||||
const terms = queryTerms(query)
|
||||
const { lexical } = lexicalScores(matrix, terms)
|
||||
|
||||
const vectorOrder = orderBy(similarity)
|
||||
const lexicalOrder = orderBy(lexical)
|
||||
const vectorRank = new Int32Array(count).fill(count)
|
||||
const lexicalRank = new Int32Array(count).fill(count)
|
||||
for (let rank = 0; rank < count; rank += 1) vectorRank[vectorOrder[rank]] = rank
|
||||
for (let rank = 0; rank < count; rank += 1) lexicalRank[lexicalOrder[rank]] = rank
|
||||
|
||||
const fused = new Float64Array(count)
|
||||
for (let at = 0; at < count; at += 1) {
|
||||
fused[at] = vectorWeight / (RRF_K + vectorRank[at]) + lexicalWeight / (RRF_K + lexicalRank[at])
|
||||
}
|
||||
|
||||
return {
|
||||
order: orderBy(fused),
|
||||
similarity,
|
||||
vectorBest: Number(similarity[vectorOrder[0]]?.toFixed(4) ?? 0),
|
||||
lexicalBest: Number(lexical[lexicalOrder[0]]?.toFixed(4) ?? 0),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge several rankings of the same rows into one order, by rank.
|
||||
*
|
||||
* Used for query expansion: each alternative phrasing ranks every chunk, and a
|
||||
* chunk that several phrasings agree on should outrank one that a single
|
||||
* phrasing happened to like. Rank fusion again rather than adding scores,
|
||||
* because the rankings come from different query vectors and their similarities
|
||||
* are not on a comparable scale.
|
||||
*
|
||||
* A chunk missing from a ranking is simply absent from that ranking's
|
||||
* contribution rather than penalised, so a phrasing that finds nothing does not
|
||||
* drag anything down.
|
||||
*
|
||||
* @param {readonly (readonly number[])[]} rankings row indices, each in rank order
|
||||
* @returns {number[]} row indices, best first
|
||||
*/
|
||||
export function fuseRankings(rankings) {
|
||||
const present = rankings.filter((ranking) => Array.isArray(ranking) && ranking.length > 0)
|
||||
if (present.length === 0) return []
|
||||
if (present.length === 1) return [...present[0]]
|
||||
|
||||
const scores = new Map()
|
||||
for (const ranking of present) {
|
||||
for (let rank = 0; rank < ranking.length; rank += 1) {
|
||||
const row = ranking[rank]
|
||||
scores.set(row, (scores.get(row) ?? 0) + 1 / (RRF_K + rank))
|
||||
}
|
||||
}
|
||||
// Ties break by row index, so the same input always gives the same output.
|
||||
return [...scores.keys()].sort((left, right) => scores.get(right) - scores.get(left) || left - right)
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a fused order into the results a caller sees.
|
||||
*
|
||||
* @param {number[]} order row indices, best first
|
||||
* @param {object} input
|
||||
* @param {object} input.matrix
|
||||
* @param {Float64Array} input.similarity per-row similarity, for reporting
|
||||
* @param {number} [input.limit]
|
||||
* @param {number} [input.perFile]
|
||||
* @param {number} [input.excerptLines]
|
||||
* @param {number} [input.totalLines]
|
||||
* @returns {object[]} results
|
||||
*/
|
||||
export function selectFrom(order, input) {
|
||||
const { matrix, similarity, limit = 8, perFile = 2, excerptLines = 80, totalLines = 400 } = input
|
||||
const perFileUsed = new Map()
|
||||
const results = []
|
||||
// A shared line budget across the whole result set. The per-excerpt cap alone
|
||||
// does not bound the response — `limit` does — and a chunk is already a
|
||||
// semantic unit, so cutting one in half is how a search turns into a
|
||||
// follow-up read that costs more than the lines it saved. Allocation is by
|
||||
// rank, so the best answer gets the room.
|
||||
let remaining = totalLines
|
||||
for (const at of order) {
|
||||
if (results.length >= limit) break
|
||||
// Always return at least one result, however small the budget.
|
||||
if (remaining <= 0 && results.length > 0) break
|
||||
const row = matrix.rows[at]
|
||||
if (row === undefined) continue
|
||||
const used = perFileUsed.get(row.path) ?? 0
|
||||
if (used >= perFile) continue
|
||||
perFileUsed.set(row.path, used + 1)
|
||||
const allowance = Math.max(1, Math.min(excerptLines, remaining))
|
||||
const { excerpt, truncated } = excerptOf(row.text, allowance)
|
||||
remaining -= excerpt.split('\n').length
|
||||
results.push({
|
||||
path: row.path,
|
||||
startLine: row.startLine,
|
||||
endLine: row.endLine,
|
||||
...(row.symbol === undefined ? {} : { symbol: row.symbol }),
|
||||
similarity: Number(similarity[at].toFixed(4)),
|
||||
excerpt,
|
||||
truncated,
|
||||
})
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank chunks for a query.
|
||||
*
|
||||
* @param {object} input
|
||||
* @param {object} input.matrix vector matrix and rows
|
||||
* @param {Float32Array} input.queryVector unit query vector
|
||||
* @param {string} input.query raw query text
|
||||
* @param {number} [input.limit] results to return
|
||||
* @param {number} [input.perFile] most chunks one file may contribute
|
||||
* @param {number} [input.excerptLines] lines per excerpt
|
||||
* @param {number} [input.totalLines] total line budget
|
||||
* @param {number} [input.vectorWeight] fusion weight for the vector ranking
|
||||
* @param {number} [input.lexicalWeight] fusion weight for the keyword ranking
|
||||
* @returns {{results: object[], vectorBest: number, lexicalBest: number}}
|
||||
*/
|
||||
export function rankChunks(input) {
|
||||
const ranked = rankOrder(input)
|
||||
const results = selectFrom(ranked.order, {
|
||||
matrix: input.matrix,
|
||||
similarity: ranked.similarity,
|
||||
limit: input.limit,
|
||||
perFile: input.perFile,
|
||||
excerptLines: input.excerptLines,
|
||||
totalLines: input.totalLines,
|
||||
})
|
||||
return { results, vectorBest: ranked.vectorBest, lexicalBest: ranked.lexicalBest }
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* Re-ranking candidates on the local model.
|
||||
*
|
||||
* Cosine similarity ranks a chunk against a query in isolation. A small model
|
||||
* reading the query *and* the candidate together does better — it can see that a
|
||||
* chunk is about the right topic but the wrong side of it, which is exactly the
|
||||
* mistake embeddings make.
|
||||
*
|
||||
* ## `undefined` is not zero
|
||||
*
|
||||
* The re-ranker answers with a score per candidate, or with **`undefined`** when
|
||||
* it has no opinion — slow, wedged, unparseable, or scoring a different number
|
||||
* of candidates than it was given. The caller must treat those differently:
|
||||
* replacing a good embedding score with a zero because the model timed out would
|
||||
* turn a working search into an empty one. No opinion means keep the ranking you
|
||||
* had.
|
||||
*
|
||||
* This is why the module returns a bare array rather than an object with a
|
||||
* status: there are exactly two outcomes, and one of them is "carry on".
|
||||
*/
|
||||
|
||||
/** Attempts. Re-ranking is an improvement, not a requirement. */
|
||||
const MAX_ATTEMPTS = 2
|
||||
|
||||
/** Characters sent per candidate. The head of a chunk identifies it. */
|
||||
const MAX_CANDIDATE_CHARS = 700
|
||||
|
||||
/** Candidates scored in one call. Longer prompts get slower and vaguer. */
|
||||
const MAX_CANDIDATES = 12
|
||||
|
||||
const PROMPT = `You score how well each code excerpt answers a developer's search query.
|
||||
|
||||
Give each excerpt a whole number from 0 to 10:
|
||||
- 10: this is the code the query is asking about.
|
||||
- 5: same area, but not what was asked.
|
||||
- 0: unrelated.
|
||||
|
||||
Answer with JSON only, no prose: {"scores": [n, n, ...]} with exactly one score per excerpt, in order.
|
||||
|
||||
Query: {{QUERY}}
|
||||
|
||||
{{CANDIDATES}}`
|
||||
|
||||
/** Pull the first JSON array of numbers out of a model response. */
|
||||
export function parseScores(body) {
|
||||
const text = String(body ?? '')
|
||||
// Prefer the documented shape, then fall back to any bare array: models drift
|
||||
// between the two and both are unambiguous.
|
||||
const wrapped = /\{[^{}]*"scores"\s*:\s*\[([^\]]*)\]/i.exec(text)
|
||||
const bare = wrapped === null ? /\[([^\]]*)\]/.exec(text) : null
|
||||
const inner = wrapped?.[1] ?? bare?.[1]
|
||||
if (inner === undefined) return undefined
|
||||
const numbers = inner
|
||||
.split(',')
|
||||
.map((part) => Number.parseFloat(part.trim()))
|
||||
.filter((value) => Number.isFinite(value))
|
||||
return numbers.length === 0 ? undefined : numbers
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a re-ranker bound to one local model.
|
||||
*
|
||||
* @param {object} options
|
||||
* @param {string} options.apiBase endpoint, no trailing slash
|
||||
* @param {string} options.model model id
|
||||
* @param {number} [options.timeoutMs] budget for the whole call
|
||||
* @param {typeof fetch} [options.fetchImpl] injection point for tests
|
||||
*/
|
||||
export function createReranker(options) {
|
||||
const { apiBase, model, timeoutMs = 15_000, fetchImpl = fetch } = options
|
||||
const base = String(apiBase).replace(/\/+$/, '')
|
||||
|
||||
return {
|
||||
model,
|
||||
/**
|
||||
* Score candidates against a query.
|
||||
*
|
||||
* @param {string} query the search query
|
||||
* @param {readonly string[]} candidates excerpt texts, in rank order
|
||||
* @param {AbortSignal} [signal] caller cancellation
|
||||
* @returns {Promise<number[] | undefined>} one score in 0..1 per candidate, in
|
||||
* order, or `undefined` when the model offered no usable opinion
|
||||
*/
|
||||
async score(query, candidates, signal) {
|
||||
const texts = candidates.slice(0, MAX_CANDIDATES)
|
||||
if (texts.length === 0 || String(query ?? '').trim() === '') return undefined
|
||||
const rendered = texts
|
||||
.map((text, at) => `Excerpt ${at + 1}:\n${String(text).slice(0, MAX_CANDIDATE_CHARS)}`)
|
||||
.join('\n\n')
|
||||
const prompt = PROMPT.replace('{{QUERY}}', String(query).trim()).replace('{{CANDIDATES}}', rendered)
|
||||
|
||||
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) {
|
||||
if (signal?.aborted) return undefined
|
||||
try {
|
||||
// See the expander: a non-finite timeout must not become a silent
|
||||
// "no opinion" that looks exactly like a slow model.
|
||||
const deadline = AbortSignal.timeout(Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 15_000)
|
||||
const combined = signal === undefined ? deadline : AbortSignal.any([signal, deadline])
|
||||
const response = await fetchImpl(`${base}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
prompt,
|
||||
stream: false,
|
||||
options: { num_predict: 80, temperature: 0 },
|
||||
}),
|
||||
signal: combined,
|
||||
})
|
||||
if (!response.ok) throw new Error(`${base} responded ${response.status}`)
|
||||
const payload = await response.json()
|
||||
const scores = parseScores(payload?.response)
|
||||
// A count mismatch is not a partial answer — it is a model that lost
|
||||
// track of the list, and applying it would score the wrong candidates.
|
||||
if (scores === undefined || scores.length !== texts.length) throw new Error('unusable score count')
|
||||
return scores.map((value) => Math.max(0, Math.min(10, value)) / 10)
|
||||
} catch {
|
||||
if (signal?.aborted) return undefined
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
}
|
||||
}
|
||||
+285
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* The SQLite index.
|
||||
*
|
||||
* `node:sqlite` rather than a dependency: the plugin ships with no build step
|
||||
* and no install-time code, so every package it imports is one the user has to
|
||||
* trust and keep current. Node has carried a synchronous SQLite binding since
|
||||
* 22.5, and a synchronous API is genuinely the right shape here — the work is
|
||||
* local, short, and on the critical path of a tool call, so an async wrapper
|
||||
* would only add interleaving to reason about.
|
||||
*
|
||||
* **Why vectors are stored and not merely ranked at write time.** A search must
|
||||
* be able to answer a query it has never seen, so the index has to keep the
|
||||
* vectors themselves. They are written as unit-length `Float32Array` blobs, so
|
||||
* scoring is a dot product and nothing else.
|
||||
*
|
||||
* **Why the model and width are recorded.** Vectors from two different models,
|
||||
* or two widths of the same model, are not comparable, and mixing them produces
|
||||
* plausible-looking but meaningless ranking. A change to either is detected on
|
||||
* open and the index is discarded rather than silently degraded.
|
||||
*/
|
||||
|
||||
import { mkdirSync } from 'node:fs'
|
||||
import { dirname } from 'node:path'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
|
||||
/** Schema version. A bump discards existing indexes. */
|
||||
const SCHEMA_VERSION = '1'
|
||||
|
||||
const SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
path TEXT PRIMARY KEY,
|
||||
hash TEXT NOT NULL,
|
||||
bytes INTEGER NOT NULL,
|
||||
chunks INTEGER NOT NULL,
|
||||
indexed_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chunks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
path TEXT NOT NULL,
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL,
|
||||
symbol TEXT,
|
||||
text TEXT NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS chunks_path_idx ON chunks(path);
|
||||
`
|
||||
|
||||
/** One indexed chunk, metadata only. */
|
||||
/** @typedef {{id: number, path: string, startLine: number, endLine: number, symbol?: string, text: string}} ChunkRow */
|
||||
|
||||
export class RecallIndex {
|
||||
/** @param {string} file absolute path to the index database */
|
||||
constructor(file) {
|
||||
this.file = file
|
||||
/** @type {DatabaseSync | undefined} */
|
||||
this.db = undefined
|
||||
/** Cached scoring matrix, dropped on every write. */
|
||||
this.cached = undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the database, creating it when absent.
|
||||
*
|
||||
* @param {{model: string, dimensions: number, apiBase: string}} identity
|
||||
* the embedding identity this index is only valid under
|
||||
* @returns {{discarded?: string}} what was thrown away, when the identity moved
|
||||
*/
|
||||
open(identity) {
|
||||
mkdirSync(dirname(this.file), { recursive: true })
|
||||
this.db = new DatabaseSync(this.file)
|
||||
// WAL keeps a search from blocking a concurrent sync; the index is a cache
|
||||
// and losing the tail of a write on power loss is acceptable.
|
||||
this.db.exec('PRAGMA journal_mode = WAL')
|
||||
this.db.exec('PRAGMA synchronous = NORMAL')
|
||||
this.db.exec(SCHEMA)
|
||||
return this.recheck(identity)
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-validate the stored embedding identity against a live settings snapshot.
|
||||
*
|
||||
* Called on every tool call, not just on open, because the settings namespace
|
||||
* is editable at runtime: pointing the plugin at a different model must
|
||||
* invalidate the existing vectors rather than silently mix two incompatible
|
||||
* spaces into one ranking.
|
||||
*
|
||||
* @param {{model: string, dimensions: number, apiBase: string}} identity
|
||||
* @returns {{discarded?: string}} what was thrown away, when the identity moved
|
||||
*/
|
||||
recheck(identity) {
|
||||
const previous = {
|
||||
version: this.#meta('schema_version'),
|
||||
model: this.#meta('model'),
|
||||
dimensions: this.#meta('dimensions'),
|
||||
}
|
||||
|
||||
let discarded
|
||||
if (previous.version !== undefined && previous.version !== SCHEMA_VERSION) {
|
||||
discarded = `schema version changed from ${previous.version} to ${SCHEMA_VERSION}`
|
||||
} else if (previous.model !== undefined && previous.model !== identity.model) {
|
||||
discarded = `embedding model changed from "${previous.model}" to "${identity.model}"`
|
||||
} else if (previous.dimensions !== undefined && previous.dimensions !== String(identity.dimensions)) {
|
||||
discarded = `embedding width changed from ${previous.dimensions} to ${identity.dimensions}`
|
||||
}
|
||||
|
||||
if (discarded !== undefined) this.#wipe()
|
||||
|
||||
this.#setMeta('schema_version', SCHEMA_VERSION)
|
||||
this.#setMeta('model', identity.model)
|
||||
this.#setMeta('dimensions', String(identity.dimensions))
|
||||
this.#setMeta('api_base', identity.apiBase)
|
||||
return discarded === undefined ? {} : { discarded }
|
||||
}
|
||||
|
||||
/** Close the database, if open. Safe to call twice. */
|
||||
close() {
|
||||
try {
|
||||
this.db?.close()
|
||||
} catch {
|
||||
// Already closed, or the handle was lost with the process. Nothing to do.
|
||||
}
|
||||
this.db = undefined
|
||||
this.cached = undefined
|
||||
}
|
||||
|
||||
/** Drop every row and reset the identity. */
|
||||
#wipe() {
|
||||
this.db.exec('DELETE FROM chunks; DELETE FROM files; DELETE FROM meta;')
|
||||
this.cached = undefined
|
||||
}
|
||||
|
||||
/** One meta value, or `undefined`. */
|
||||
#meta(key) {
|
||||
const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key)
|
||||
return row === undefined ? undefined : row.value
|
||||
}
|
||||
|
||||
/** Write one meta value. */
|
||||
#setMeta(key, value) {
|
||||
this.db
|
||||
.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value')
|
||||
.run(key, value)
|
||||
}
|
||||
|
||||
/** `path -> content hash` for every indexed file. */
|
||||
fileHashes() {
|
||||
const out = new Map()
|
||||
for (const row of this.db.prepare('SELECT path, hash FROM files').all()) out.set(row.path, row.hash)
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte size of each named file, for the files that are indexed.
|
||||
*
|
||||
* This is what makes an honest "returned less than it would have read" figure
|
||||
* possible: the index already recorded every file's size when it embedded it,
|
||||
* so the alternative to an excerpt can be stated without reading a single
|
||||
* source file at query time.
|
||||
*
|
||||
* @param {readonly string[]} paths workspace-relative paths
|
||||
* @returns {Map<string, number>} only the paths that are indexed
|
||||
*/
|
||||
bytesFor(paths) {
|
||||
const out = new Map()
|
||||
if (paths.length === 0) return out
|
||||
const statement = this.db.prepare('SELECT bytes FROM files WHERE path = ?')
|
||||
for (const path of paths) {
|
||||
const row = statement.get(path)
|
||||
if (row !== undefined) out.set(path, row.bytes)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Counts for the status surface. */
|
||||
stats() {
|
||||
const files = this.db.prepare('SELECT COUNT(*) AS n FROM files').get()?.n ?? 0
|
||||
const chunks = this.db.prepare('SELECT COUNT(*) AS n FROM chunks').get()?.n ?? 0
|
||||
const bytes = this.db.prepare('SELECT COALESCE(SUM(bytes), 0) AS n FROM files').get()?.n ?? 0
|
||||
return { files, chunks, bytes }
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace one file's chunks atomically.
|
||||
*
|
||||
* @param {string} path workspace-relative path
|
||||
* @param {string} hash content hash
|
||||
* @param {number} bytes file size
|
||||
* @param {{startLine: number, endLine: number, symbol?: string, text: string, vec: Float32Array}[]} chunks
|
||||
*/
|
||||
replaceFile(path, hash, bytes, chunks) {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
this.db.prepare('DELETE FROM chunks WHERE path = ?').run(path)
|
||||
const insert = this.db.prepare(
|
||||
'INSERT INTO chunks (path, start_line, end_line, symbol, text, dim, vec) VALUES (?, ?, ?, ?, ?, ?, ?)',
|
||||
)
|
||||
for (const chunk of chunks) {
|
||||
insert.run(
|
||||
path,
|
||||
chunk.startLine,
|
||||
chunk.endLine,
|
||||
chunk.symbol ?? null,
|
||||
chunk.text,
|
||||
chunk.vec.length,
|
||||
new Uint8Array(chunk.vec.buffer, chunk.vec.byteOffset, chunk.vec.byteLength),
|
||||
)
|
||||
}
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO files (path, hash, bytes, chunks, indexed_at) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
hash = excluded.hash, bytes = excluded.bytes,
|
||||
chunks = excluded.chunks, indexed_at = excluded.indexed_at`,
|
||||
)
|
||||
.run(path, hash, bytes, chunks.length, Date.now())
|
||||
this.db.exec('COMMIT')
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
this.cached = undefined
|
||||
}
|
||||
|
||||
/** Forget files that no longer exist on disk. Returns how many were dropped. */
|
||||
prune(toRemove) {
|
||||
if (toRemove.length === 0) return 0
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const dropChunks = this.db.prepare('DELETE FROM chunks WHERE path = ?')
|
||||
const dropFile = this.db.prepare('DELETE FROM files WHERE path = ?')
|
||||
for (const path of toRemove) {
|
||||
dropChunks.run(path)
|
||||
dropFile.run(path)
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
this.cached = undefined
|
||||
return toRemove.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Load every chunk for scoring.
|
||||
*
|
||||
* One contiguous `Float32Array` and a parallel row list, built once and reused
|
||||
* until the next write. At 300k lines of source this is a few tens of
|
||||
* megabytes — the alternative, a blob read per query, would pay SQLite's row
|
||||
* overhead on every search for no benefit, since a search visits every row
|
||||
* anyway.
|
||||
*
|
||||
* @returns {{vectors: Float32Array, rows: ChunkRow[], dimensions: number}}
|
||||
*/
|
||||
matrix() {
|
||||
if (this.cached !== undefined) return this.cached
|
||||
const raw = this.db
|
||||
.prepare('SELECT id, path, start_line, end_line, symbol, text, dim, vec FROM chunks ORDER BY id')
|
||||
.all()
|
||||
const dimensions = raw.length === 0 ? 0 : raw[0].dim
|
||||
const vectors = new Float32Array(raw.length * dimensions)
|
||||
const rows = new Array(raw.length)
|
||||
for (let at = 0; at < raw.length; at += 1) {
|
||||
const row = raw[at]
|
||||
const view = new Float32Array(row.vec.buffer, row.vec.byteOffset, row.vec.byteLength / 4)
|
||||
vectors.set(view, at * dimensions)
|
||||
rows[at] = {
|
||||
id: row.id,
|
||||
path: row.path,
|
||||
startLine: row.start_line,
|
||||
endLine: row.end_line,
|
||||
...(row.symbol === null ? {} : { symbol: row.symbol }),
|
||||
text: row.text,
|
||||
}
|
||||
}
|
||||
this.cached = { vectors, rows, dimensions }
|
||||
return this.cached
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* What one conversation got, and what to do with it.
|
||||
*
|
||||
* Two jobs, both pure:
|
||||
*
|
||||
* 1. **Derive** the small record a tool call leaves in the session log. The
|
||||
* harness persists `output.presentationMeta` verbatim as the `meta` of the
|
||||
* `tool/result` event, so this is the one place a result can state facts
|
||||
* about itself without inventing an event type or a side channel.
|
||||
* 2. **Fold** those records into a running total per conversation. The fold is
|
||||
* the projection unit's transition function, which the framework requires to
|
||||
* be pure and to return the *same reference* for events it does not care
|
||||
* about — an unchanged reference costs nothing downstream, while a fresh
|
||||
* object on every tool call would republish on every step.
|
||||
*
|
||||
* ## Why `sourceBytes` rather than a headline saving
|
||||
*
|
||||
* The honest comparison is between what the caller received and what reading
|
||||
* those same files would have cost. The index already recorded every file's
|
||||
* size when it embedded it, so the second figure costs nothing at query time.
|
||||
*
|
||||
* It is still an **upper bound**, and the numbers are labelled that way
|
||||
* everywhere they surface: a careful agent might have read one of those four
|
||||
* files rather than all four. A figure that flatters the tool is worse than no
|
||||
* figure, because it stops being believed the first time somebody checks.
|
||||
*/
|
||||
|
||||
/** State for a conversation that has never called the tool. */
|
||||
export function emptyUsage() {
|
||||
return {
|
||||
version: 1,
|
||||
searches: 0,
|
||||
indexes: 0,
|
||||
failed: 0,
|
||||
excerpts: 0,
|
||||
returnedBytes: 0,
|
||||
sourceBytes: 0,
|
||||
files: 0,
|
||||
weak: 0,
|
||||
// Bytes of avoided context multiplied by the steps that have re-read them.
|
||||
// Accumulated here rather than derived, because deriving it needs a step
|
||||
// count this state does not own — and the fold sees every step anyway.
|
||||
rereadBytes: 0,
|
||||
lastFocus: '',
|
||||
}
|
||||
}
|
||||
|
||||
/** The context a conversation is carrying that it would not have, in bytes. */
|
||||
function avoidedBytesOf(state) {
|
||||
return Math.max(0, state.sourceBytes - state.returnedBytes)
|
||||
}
|
||||
|
||||
/**
|
||||
* The record one call writes into the session log.
|
||||
*
|
||||
* Deliberately tiny: it is appended for every recall call and replayed on every
|
||||
* projection rebuild, so it carries totals and never content.
|
||||
*
|
||||
* @param {{action?: string, query?: string}} args the validated call arguments
|
||||
* @param {Record<string, any>} value the canonical result
|
||||
* @returns {{recall: Record<string, any>}} the persisted metadata
|
||||
*/
|
||||
export function deriveUsage(args, value) {
|
||||
const action = value?.action ?? args?.action ?? 'search'
|
||||
const record = { action }
|
||||
if (value?.ok === false) record.failed = 1
|
||||
if (typeof value?.focus?.path === 'string') record.focus = value.focus.path
|
||||
if (value?.weak === true) record.weak = 1
|
||||
|
||||
if (action === 'search' && Array.isArray(value?.results)) {
|
||||
record.excerpts = value.results.length
|
||||
record.returnedBytes = value.results.reduce(
|
||||
(sum, result) => sum + Buffer.byteLength(String(result?.excerpt ?? ''), 'utf8'),
|
||||
0,
|
||||
)
|
||||
}
|
||||
if (action === 'index' && typeof value?.index?.embedded === 'number') {
|
||||
record.embedded = value.index.embedded
|
||||
}
|
||||
if (typeof value?.measure?.sourceBytes === 'number') {
|
||||
record.sourceBytes = value.measure.sourceBytes
|
||||
record.files = value.measure.files
|
||||
}
|
||||
return { recall: record }
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold one session event into the running total.
|
||||
*
|
||||
* @param {ReturnType<typeof emptyUsage>} state state covering all prior events
|
||||
* @param {{type?: string, meta?: any}} event the next committed session event
|
||||
* @returns the next state, or **the same reference** when not this unit's event
|
||||
*/
|
||||
export function foldUsage(state, event) {
|
||||
// Every step re-reads whatever the conversation is already carrying, so each
|
||||
// step adds the avoided-so-far to the compounding total. This is what makes
|
||||
// the figure exact rather than an estimate the caller has to supply.
|
||||
if (event?.type === 'step/start') {
|
||||
const carried = avoidedBytesOf(state)
|
||||
if (carried === 0) return state
|
||||
return { ...state, rereadBytes: state.rereadBytes + carried }
|
||||
}
|
||||
|
||||
const record = event?.type === 'tool/result' ? event.meta?.recall : undefined
|
||||
if (record === undefined || record === null || typeof record !== 'object') return state
|
||||
|
||||
const next = {
|
||||
...state,
|
||||
searches: state.searches + (record.action === 'search' ? 1 : 0),
|
||||
indexes: state.indexes + (record.action === 'index' ? 1 : 0),
|
||||
failed: state.failed + (record.failed === 1 ? 1 : 0),
|
||||
excerpts: state.excerpts + (record.excerpts ?? 0),
|
||||
returnedBytes: state.returnedBytes + (record.returnedBytes ?? 0),
|
||||
sourceBytes: state.sourceBytes + (record.sourceBytes ?? 0),
|
||||
files: state.files + (record.files ?? 0),
|
||||
weak: state.weak + (record.weak === 1 ? 1 : 0),
|
||||
lastFocus: typeof record.focus === 'string' ? record.focus : state.lastFocus,
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
/**
|
||||
* The derived figures a surface shows, computed from state.
|
||||
*
|
||||
* Kept out of the state itself so the persisted fold stays a plain running
|
||||
* total: anything derivable is derived on read, which is what lets the formula
|
||||
* change without invalidating every checkpoint in the cache.
|
||||
*
|
||||
* @param {ReturnType<typeof emptyUsage>} state folded totals
|
||||
* @param {{cacheReadPricePerMillion?: number}} [assumptions]
|
||||
* @returns {Record<string, number|string|boolean>}
|
||||
*/
|
||||
export function usageSummary(state, assumptions = {}) {
|
||||
const { cacheReadPricePerMillion = 0.007 } = assumptions
|
||||
// What a read would have cost, less what the excerpt did cost.
|
||||
const avoidedBytes = avoidedBytesOf(state)
|
||||
// bytes/4 is an approximation of tokens; labelled as one wherever it is shown.
|
||||
const avoidedTokens = Math.round(avoidedBytes / 4)
|
||||
const compression = state.sourceBytes > 0 ? state.sourceBytes / Math.max(1, state.returnedBytes) : 0
|
||||
// The compounding figure: context kept out, re-read once per step since.
|
||||
const rereadTokens = Math.round((state.rereadBytes ?? 0) / 4)
|
||||
const avoidedUsd = ((avoidedTokens + rereadTokens) / 1_000_000) * cacheReadPricePerMillion
|
||||
|
||||
return {
|
||||
searches: state.searches,
|
||||
indexes: state.indexes,
|
||||
failed: state.failed,
|
||||
excerpts: state.excerpts,
|
||||
files: state.files,
|
||||
weak: state.weak,
|
||||
returnedBytes: state.returnedBytes,
|
||||
sourceBytes: state.sourceBytes,
|
||||
avoidedBytes,
|
||||
avoidedTokens,
|
||||
compression: Number(compression.toFixed(2)),
|
||||
rereadTokens,
|
||||
avoidedUsd: Number(avoidedUsd.toFixed(4)),
|
||||
lastFocus: state.lastFocus,
|
||||
}
|
||||
}
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Workspace file discovery.
|
||||
*
|
||||
* Two rules shape everything here.
|
||||
*
|
||||
* **The walk never follows a symlink.** A symlink loop turns a bounded walk into
|
||||
* an unbounded one, and a symlink out of the workspace turns "index this
|
||||
* project" into "index this machine". Both are worse than missing a file.
|
||||
*
|
||||
* **The walk is deterministic.** Candidates come back in sorted path order, so
|
||||
* a partially-completed cold index covers the same prefix on the next run. A
|
||||
* filesystem-ordered walk would re-embed a different subset every time and
|
||||
* never converge on a warm index.
|
||||
*/
|
||||
|
||||
import { readdir } from 'node:fs/promises'
|
||||
import { join, relative, sep } from 'node:path'
|
||||
|
||||
/**
|
||||
* Directory names skipped at any depth.
|
||||
*
|
||||
* These are dependency stores and build output: large, machine-generated, and
|
||||
* never the answer to a question about this codebase. Indexing them would
|
||||
* multiply the embedding cost of a cold start by an order of magnitude and
|
||||
* return vendored copies of code the user does not own.
|
||||
*/
|
||||
export const DEFAULT_SKIP_DIRS = Object.freeze([
|
||||
'node_modules',
|
||||
'.git',
|
||||
'.hg',
|
||||
'.svn',
|
||||
'dist',
|
||||
'build',
|
||||
'out',
|
||||
'output',
|
||||
'target',
|
||||
'coverage',
|
||||
'.next',
|
||||
'.nuxt',
|
||||
'.svelte-kit',
|
||||
'.output',
|
||||
'.turbo',
|
||||
'.cache',
|
||||
'.parcel-cache',
|
||||
'.venv',
|
||||
'venv',
|
||||
'__pycache__',
|
||||
'.mypy_cache',
|
||||
'.pytest_cache',
|
||||
'.ruff_cache',
|
||||
'vendor',
|
||||
'Pods',
|
||||
'.idea',
|
||||
'.vscode',
|
||||
'.gradle',
|
||||
'.terraform',
|
||||
'tmp',
|
||||
'.tmp',
|
||||
])
|
||||
|
||||
/**
|
||||
* Extensions indexed by default.
|
||||
*
|
||||
* Markup and prose are deliberately absent. A question that names a symbol wants
|
||||
* the code that defines it, and a README that mentions the symbol is a worse
|
||||
* answer that still costs tokens. Deployments that want documentation indexed
|
||||
* add the extension through `extensions` in the config.
|
||||
*/
|
||||
export const DEFAULT_EXTENSIONS = Object.freeze([
|
||||
'ts', 'tsx', 'mts', 'cts',
|
||||
'js', 'jsx', 'mjs', 'cjs',
|
||||
'py', 'rb', 'go', 'rs', 'java', 'kt', 'kts', 'scala', 'swift',
|
||||
'c', 'h', 'cc', 'cpp', 'cxx', 'hpp', 'hh', 'cs',
|
||||
'php', 'pl', 'lua', 'sh', 'bash', 'zsh',
|
||||
'sql', 'graphql', 'proto', 'vue', 'svelte',
|
||||
])
|
||||
|
||||
/**
|
||||
* Translate a glob into an anchored regular expression.
|
||||
*
|
||||
* Supports `**` (any depth, including none), `*` (within one path segment), `?`
|
||||
* (one non-separator character) and a trailing `/` (directory and everything
|
||||
* under it). Written out rather than pulled from a dependency because the plugin
|
||||
* ships with no build step and no install-time code.
|
||||
*
|
||||
* @param {string} pattern glob, using `/` as the separator on every platform
|
||||
* @returns {RegExp} anchored matcher over a `/`-separated relative path
|
||||
*/
|
||||
export function globToRegExp(pattern) {
|
||||
const normalized = pattern.trim().replace(/\\/g, '/').replace(/^\.\//, '')
|
||||
let source = ''
|
||||
for (let at = 0; at < normalized.length; at += 1) {
|
||||
const char = normalized[at]
|
||||
if (char === '*') {
|
||||
const doubled = normalized[at + 1] === '*'
|
||||
if (doubled) {
|
||||
// `**/` consumes any number of segments, so `**/foo` also matches `foo`.
|
||||
if (normalized[at + 2] === '/') {
|
||||
source += '(?:[^/]*/)*'
|
||||
at += 2
|
||||
} else {
|
||||
source += '.*'
|
||||
at += 1
|
||||
}
|
||||
} else {
|
||||
source += '[^/]*'
|
||||
}
|
||||
} else if (char === '?') {
|
||||
source += '[^/]'
|
||||
} else if (char === '/') {
|
||||
source += '/'
|
||||
} else {
|
||||
source += char.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
}
|
||||
// A trailing `/` names a directory: match it and everything beneath it.
|
||||
if (source.endsWith('/')) source += '.*'
|
||||
return new RegExp(`^(?:${source})$`)
|
||||
}
|
||||
|
||||
/** Whether a `/`-separated relative path is matched by any of `patterns`. */
|
||||
export function matchesAny(relPath, patterns) {
|
||||
for (const pattern of patterns) {
|
||||
if (pattern === '') continue
|
||||
if (globToRegExp(pattern).test(relPath)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** The extension of a path, lowercased, without the dot. */
|
||||
export function extensionOf(path) {
|
||||
const name = path.slice(path.lastIndexOf('/') + 1)
|
||||
const dot = name.lastIndexOf('.')
|
||||
return dot <= 0 ? '' : name.slice(dot + 1).toLowerCase()
|
||||
}
|
||||
|
||||
/**
|
||||
* List indexable files beneath `root`.
|
||||
*
|
||||
* @param {string} root absolute directory to walk
|
||||
* @param {object} [options]
|
||||
* @param {readonly string[]} [options.extensions] allowed extensions, without dots
|
||||
* @param {readonly string[]} [options.skipDirs] directory names skipped at any depth
|
||||
* @param {readonly string[]} [options.exclude] glob patterns over the relative path
|
||||
* @param {number} [options.maxFiles] stop after this many candidates
|
||||
* @param {number} [options.maxDepth] deepest directory level to enter
|
||||
* @param {AbortSignal} [options.signal]
|
||||
* @returns {Promise<{files: {path: string, absolute: string}[], truncated: boolean, skippedFiles: number}>}
|
||||
*/
|
||||
export async function listCodeFiles(root, options = {}) {
|
||||
const {
|
||||
extensions = DEFAULT_EXTENSIONS,
|
||||
skipDirs = DEFAULT_SKIP_DIRS,
|
||||
exclude = [],
|
||||
maxFiles = 20_000,
|
||||
maxDepth = 24,
|
||||
signal,
|
||||
} = options
|
||||
|
||||
const wanted = new Set(extensions.map((value) => String(value).toLowerCase()))
|
||||
const skip = new Set(skipDirs)
|
||||
const files = []
|
||||
let truncated = false
|
||||
let skippedFiles = 0
|
||||
|
||||
/** Depth-first, pushing directories in reverse so the pop order is alphabetical. */
|
||||
const stack = [{ dir: root, depth: 0 }]
|
||||
while (stack.length > 0) {
|
||||
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
||||
const { dir, depth } = stack.pop()
|
||||
let entries
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true })
|
||||
} catch {
|
||||
// An unreadable directory is a gap in the index, not a failure of it.
|
||||
continue
|
||||
}
|
||||
for (const entry of entries) {
|
||||
// Never traverse a symlink: `dirent.isDirectory()` is false for one, so a
|
||||
// link to a directory is skipped rather than followed.
|
||||
if (entry.isSymbolicLink()) {
|
||||
skippedFiles += 1
|
||||
continue
|
||||
}
|
||||
const absolute = join(dir, entry.name)
|
||||
const relPath = relative(root, absolute).split(sep).join('/')
|
||||
if (entry.isDirectory()) {
|
||||
if (skip.has(entry.name)) continue
|
||||
if (depth >= maxDepth) continue
|
||||
if (matchesAny(relPath, exclude)) continue
|
||||
stack.push({ dir: absolute, depth: depth + 1 })
|
||||
continue
|
||||
}
|
||||
if (!entry.isFile()) continue
|
||||
if (!wanted.has(extensionOf(entry.name))) continue
|
||||
if (matchesAny(relPath, exclude)) continue
|
||||
if (files.length >= maxFiles) {
|
||||
truncated = true
|
||||
continue
|
||||
}
|
||||
files.push({ path: relPath, absolute })
|
||||
}
|
||||
}
|
||||
|
||||
files.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
|
||||
return { files, truncated, skippedFiles }
|
||||
}
|
||||
Reference in new issue
Block a user