dsh-recall/scripts/smoke.mjs

/**
 * Live smoke test: the real pipeline against the real embedding endpoint.
 *
 * Not part of `npm test`, because it needs a running Ollama and a model that is
 * already pulled. It exists because the unit tests can only prove the code does
 * what the test author believed; this proves the retrieval actually returns the
 * right file for a question about it, which is the only claim that matters.
 *
 *   node scripts/smoke.mjs [workspace] [query]
 */

import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'

import { chunkFile, embedText } from '../lib/chunk.js'
import { createEmbedder } from '../lib/embed.js'
import { rankChunks } from '../lib/rank.js'
import { RecallIndex } from '../lib/store.js'
import { listCodeFiles } from '../lib/walk.js'

// The endpoint to check against. The default is the Ollama convention, which
// is what the plugin's own schema defaults to; point RECALL_API_BASE at your
// own host when it is somewhere else.
const API_BASE = process.env.RECALL_API_BASE ?? 'http://127.0.0.1:11434'
const MODEL = process.env.RECALL_MODEL ?? 'unclemusclez/jina-embeddings-v2-base-code:latest'
const DIMENSIONS = Number(process.env.RECALL_DIMENSIONS ?? 768)

const FIXTURE = {
  'src/services/auth/session.ts': `import { createHash, randomBytes } from 'node:crypto'

const SESSION_TTL_MS = 1000 * 60 * 60 * 12

export interface SessionRecord {
  userId: string
  issuedAt: number
  expiresAt: number
}

export function issueSession(userId: string): SessionRecord {
  const now = Date.now()
  return { userId, issuedAt: now, expiresAt: now + SESSION_TTL_MS }
}

export function isExpired(record: SessionRecord, now = Date.now()): boolean {
  return record.expiresAt <= now
}

export function rotateToken(record: SessionRecord): string {
  if (isExpired(record)) throw new Error('cannot rotate an expired session')
  return randomBytes(32).toString('hex')
}

export function fingerprint(token: string): string {
  return createHash('sha256').update(token).digest('hex').slice(0, 16)
}
`,
  'src/services/weather/forecast.ts': `export interface ForecastDay {
  date: string
  minC: number
  maxC: number
  rainMm: number
}

export function averageTemperature(days: ForecastDay[]): number {
  if (days.length === 0) return 0
  const total = days.reduce((sum, day) => sum + (day.minC + day.maxC) / 2, 0)
  return total / days.length
}

export function totalRainfall(days: ForecastDay[]): number {
  return days.reduce((sum, day) => sum + day.rainMm, 0)
}
`,
  'src/store/sqlite.ts': `import { DatabaseSync } from 'node:sqlite'

export function openDatabase(path: string): DatabaseSync {
  const db = new DatabaseSync(path)
  db.exec('PRAGMA journal_mode = WAL')
  return db
}

export function pruneRows(db: DatabaseSync, table: string, olderThan: number): number {
  const statement = db.prepare(\`DELETE FROM \${table} WHERE created_at < ?\`)
  const result = statement.run(olderThan)
  return Number(result.changes)
}
`,
}

/** Write the fixture into a fresh temporary directory. */
async function makeWorkspace() {
  const root = await mkdtemp(join(tmpdir(), 'recall-smoke-'))
  for (const [path, content] of Object.entries(FIXTURE)) {
    const absolute = join(root, path)
    await mkdir(join(absolute, '..'), { recursive: true })
    await writeFile(absolute, content, 'utf8')
  }
  return root
}

async function main() {
  const query = process.argv[3] ?? 'how long does a user session stay valid before it expires'
  const root = process.argv[2] ?? (await makeWorkspace())
  const indexDir = await mkdtemp(join(tmpdir(), 'recall-index-'))
  const database = join(indexDir, 'index.sqlite')

  const embedder = createEmbedder({ apiBase: API_BASE, model: MODEL, dimensions: DIMENSIONS })
  const index = new RecallIndex(database)
  index.open({ model: MODEL, dimensions: DIMENSIONS, apiBase: API_BASE })

  const started = Date.now()
  const walk = await listCodeFiles(root)
  let chunks = 0
  for (const file of walk.files) {
    const { readFile } = await import('node:fs/promises')
    const content = await readFile(file.absolute, 'utf8')
    const split = chunkFile({ path: file.path, content })
    if (split.chunks.length === 0) continue
    const vectors = await embedder.embed(split.chunks.map((chunk) => embedText(file.path, chunk)))
    index.replaceFile(file.path, 'smoke', Buffer.byteLength(content), split.chunks.map((c, at) => ({ ...c, vec: vectors[at] })))
    chunks += split.chunks.length
  }
  const indexMs = Date.now() - started

  const matrix = index.matrix()
  const queryVector = await embedder.embedOne(query)
  const searchStart = Date.now()
  const ranked = rankChunks({ matrix, queryVector, query, limit: 5 })
  const searchMs = Date.now() - searchStart

  console.log(`workspace : ${root}`)
  console.log(`files     : ${walk.files.length}  chunks: ${chunks}  indexed in ${indexMs}ms`)
  console.log(`query     : ${query}`)
  console.log(`search    : ${searchMs}ms  batch endpoint: ${embedder.batchEnabled}`)
  console.log('')
  for (const [at, result] of ranked.results.entries()) {
    console.log(`${at + 1}. ${result.path}:${result.startLine}-${result.endLine}  similarity ${result.similarity}`)
    console.log(`   ${result.excerpt.split('\n')[0]}`)
  }
  console.log('')
  const top = ranked.results[0]
  const expected = 'src/services/auth/session.ts'
  console.log(top?.path === expected ? `PASS top hit is ${expected}` : `FAIL top hit was ${top?.path}, expected ${expected}`)

  index.close()
  await rm(indexDir, { recursive: true, force: true })
  if (process.argv[2] === undefined) await rm(root, { recursive: true, force: true })
}

await main()