119 lines
4.7 KiB
JavaScript
119 lines
4.7 KiB
JavaScript
import { strict as assert } from 'node:assert'
|
|
import { test } from 'node:test'
|
|
|
|
import { fuseRankings } from '../lib/rank.js'
|
|
import { createReranker, parseScores } from '../lib/rerank.js'
|
|
|
|
/** A stub endpoint answering with a canned model response. */
|
|
function stubFetch(response, options = {}) {
|
|
const { hang = false, status = 200 } = options
|
|
const calls = []
|
|
const impl = async (url, init) => {
|
|
calls.push({ url, body: JSON.parse(init.body) })
|
|
if (hang) {
|
|
return new Promise((_resolve, reject) => {
|
|
init.signal.addEventListener('abort', () => reject(init.signal.reason), { once: true })
|
|
})
|
|
}
|
|
return { ok: status === 200, status, json: async () => ({ response }) }
|
|
}
|
|
impl.calls = calls
|
|
return impl
|
|
}
|
|
|
|
const MODEL = { apiBase: 'http://stub:11434', model: 'small-model', timeoutMs: 2_000 }
|
|
|
|
test('parseScores accepts the documented shape and a bare array', () => {
|
|
assert.deepEqual(parseScores('{"scores": [10, 4, 0]}'), [10, 4, 0])
|
|
assert.deepEqual(parseScores('Sure: [8, 2, 6]'), [8, 2, 6])
|
|
assert.deepEqual(parseScores('{"scores":[7]}'), [7])
|
|
})
|
|
|
|
test('parseScores returns undefined rather than guessing', () => {
|
|
assert.equal(parseScores('no numbers here'), undefined)
|
|
assert.equal(parseScores('{"scores": []}'), undefined)
|
|
assert.equal(parseScores(''), undefined)
|
|
assert.equal(parseScores(undefined), undefined)
|
|
})
|
|
|
|
test('an unusable response is no opinion, not a zero', async () => {
|
|
// The distinction the whole module rests on: returning zeros here would
|
|
// replace a good embedding ranking with nothing at all.
|
|
for (const fetchImpl of [
|
|
async () => {
|
|
throw new Error('ECONNREFUSED')
|
|
},
|
|
stubFetch('I think the first one is best.'),
|
|
stubFetch('{"scores": [10, 5]}'), // wrong length for 3 candidates
|
|
stubFetch('', { status: 500 }),
|
|
]) {
|
|
assert.equal(await createReranker({ ...MODEL, fetchImpl }).score('q', ['a', 'b', 'c']), undefined)
|
|
}
|
|
})
|
|
|
|
test('a WEDGED endpoint yields no opinion, and does not hang', async () => {
|
|
const started = Date.now()
|
|
const scores = await createReranker({ ...MODEL, fetchImpl: stubFetch('', { hang: true }), timeoutMs: 120 }).score('q', ['a', 'b'])
|
|
assert.equal(scores, undefined)
|
|
assert.ok(Date.now() - started < 3_000, 'the deadline must bound the wait')
|
|
})
|
|
|
|
test('scores come back normalised and clamped, in candidate order', async () => {
|
|
const fetchImpl = stubFetch('{"scores": [10, 0, 5, 42, -3]}')
|
|
const scores = await createReranker({ ...MODEL, fetchImpl }).score('q', ['a', 'b', 'c', 'd', 'e'])
|
|
assert.deepEqual(scores, [1, 0, 0.5, 1, 0], 'out-of-range values are clamped, not trusted')
|
|
})
|
|
|
|
test('a non-finite timeout is clamped rather than read as no opinion', async () => {
|
|
const fetchImpl = stubFetch('{"scores": [9, 1]}')
|
|
const scores = await createReranker({ ...MODEL, fetchImpl, timeoutMs: Number.NaN }).score('q', ['a', 'b'])
|
|
assert.deepEqual(scores, [0.9, 0.1])
|
|
})
|
|
|
|
test('the prompt carries the query and every candidate', async () => {
|
|
const fetchImpl = stubFetch('{"scores": [1, 1]}')
|
|
await createReranker({ ...MODEL, fetchImpl }).score('occupancy detection', ['excerpt one', 'excerpt two'])
|
|
const prompt = fetchImpl.calls[0].body.prompt
|
|
assert.match(prompt, /occupancy detection/)
|
|
assert.match(prompt, /excerpt one/)
|
|
assert.match(prompt, /excerpt two/)
|
|
})
|
|
|
|
test('nothing to score is no opinion, and no request', async () => {
|
|
const fetchImpl = stubFetch('{"scores": []}')
|
|
const reranker = createReranker({ ...MODEL, fetchImpl })
|
|
assert.equal(await reranker.score('q', []), undefined)
|
|
assert.equal(await reranker.score(' ', ['a']), undefined)
|
|
assert.equal(fetchImpl.calls.length, 0)
|
|
})
|
|
|
|
test('caller cancellation yields no opinion without a request', async () => {
|
|
const controller = new AbortController()
|
|
controller.abort()
|
|
const fetchImpl = stubFetch('{"scores": [1]}')
|
|
assert.equal(await createReranker({ ...MODEL, fetchImpl }).score('q', ['a'], controller.signal), undefined)
|
|
assert.equal(fetchImpl.calls.length, 0)
|
|
})
|
|
|
|
test('fusing one ranking keeps its order', () => {
|
|
assert.deepEqual(fuseRankings([[3, 1, 2]]), [3, 1, 2])
|
|
})
|
|
|
|
test('a chunk several queries agree on outranks one that a single query liked', () => {
|
|
// Row 7 is second for both queries; row 1 is first for one and absent from the
|
|
// other. Agreement should win, which is the whole point of expansion.
|
|
const fused = fuseRankings([
|
|
[1, 7, 2],
|
|
[5, 7, 9],
|
|
])
|
|
assert.equal(fused[0], 7)
|
|
})
|
|
|
|
test('fusion is empty for no rankings and stable for ties', () => {
|
|
assert.deepEqual(fuseRankings([]), [])
|
|
assert.deepEqual(fuseRankings([[], []]), [])
|
|
const once = fuseRankings([[2, 1, 3], [1, 2, 3]])
|
|
const twice = fuseRankings([[2, 1, 3], [1, 2, 3]])
|
|
assert.deepEqual(once, twice, 'the same input must give the same order')
|
|
})
|