diff --git a/packages/pipeline/note-core/src/__tests__/note-verifier.test.ts b/packages/pipeline/note-core/src/__tests__/note-verifier.test.ts new file mode 100644 index 0000000..3ff3537 --- /dev/null +++ b/packages/pipeline/note-core/src/__tests__/note-verifier.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { verifyNote } from "../verification/note-verifier.js" + +const sampleTranscript = ` +Doctor: Good morning, what brings you in today? +Patient: I've been having this really bad headache for the past 3 days. +Doctor: Pain severity? +Patient: About 7 or 8 out of 10. +Doctor: Blood pressure is 128/82, temperature 98.4. +` + +const goodNote = "Patient presents with headache for 3 days. Pain severity 7-8/10. Vitals: BP 128/82." +const badNote = "Patient presents with chest pain for 5 days. BP 180/110." + +test("verifyNote validates matching note with non-trivial confidence", async () => { + const result = await verifyNote(goodNote, sampleTranscript) + assert.ok(["verified", "partial"].includes(result.status)) + assert.ok(result.summary.overallConfidence > 0.3) + assert.ok(result.claims.length > 0) +}) + +test("verifyNote lowers confidence for mismatched claims", async () => { + const result = await verifyNote(badNote, sampleTranscript) + assert.ok(result.summary.overallConfidence < 0.3) +}) + +test("verifyNote handles empty note", async () => { + const result = await verifyNote("", sampleTranscript) + assert.equal(result.claims.length, 0) + assert.equal(result.status, "verified") +}) + +test("verifyNote handles empty transcript", async () => { + const result = await verifyNote(goodNote, "") + assert.ok(result.summary.overallConfidence < 0.5) +}) + +test("verifyNote respects factsOnly filter", async () => { + const result = await verifyNote(goodNote, sampleTranscript, { factsOnly: true }) + for (const claim of result.claims) { + assert.equal(claim.kind, "fact") + } +}) diff --git a/packages/pipeline/note-core/src/__tests__/verification-utils.test.ts b/packages/pipeline/note-core/src/__tests__/verification-utils.test.ts new file mode 100644 index 0000000..632cdd9 --- /dev/null +++ b/packages/pipeline/note-core/src/__tests__/verification-utils.test.ts @@ -0,0 +1,39 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { calculateOverlap, classifyClaim, extractNumbers, tokenize } from "../verification/verifier.js" + +test("tokenize extracts tokens and filters stop words", () => { + const tokens = tokenize("Patient reports headache for 3 days") + assert.ok(tokens.includes("headache")) + assert.ok(!tokens.includes("for")) +}) + +test("tokenize handles empty string", () => { + assert.deepEqual(tokenize(""), []) +}) + +test("extractNumbers extracts integer and decimal values", () => { + const numbers = extractNumbers("BP 120/80, temp 98.6") + assert.ok(numbers.includes("120")) + assert.ok(numbers.includes("98.6")) +}) + +test("calculateOverlap returns 1 for identical text", () => { + assert.equal(calculateOverlap("severe headache", "severe headache"), 1) +}) + +test("calculateOverlap returns 0 with no shared tokens", () => { + assert.equal(calculateOverlap("headache pain", "cardiac issues"), 0) +}) + +test("classifyClaim identifies fact claims", () => { + assert.equal(classifyClaim("Patient has hypertension."), "fact") +}) + +test("classifyClaim identifies question claims", () => { + assert.equal(classifyClaim("Does the patient smoke?"), "question") +}) + +test("classifyClaim identifies inference claims", () => { + assert.equal(classifyClaim("I think this might be migraine."), "inference") +}) diff --git a/packages/pipeline/note-core/src/verification/index.ts b/packages/pipeline/note-core/src/verification/index.ts new file mode 100644 index 0000000..1f2e092 --- /dev/null +++ b/packages/pipeline/note-core/src/verification/index.ts @@ -0,0 +1,11 @@ +export { verifyNote } from "./note-verifier" +export { tokenize, extractNumbers, calculateOverlap, classifyClaim } from "./verifier" +export type { + Claim, + ClaimKind, + Evidence, + Verdict, + VerificationOptions, + VerificationResult, + VerificationSummary, +} from "./types" diff --git a/packages/pipeline/note-core/src/verification/note-verifier.ts b/packages/pipeline/note-core/src/verification/note-verifier.ts new file mode 100644 index 0000000..7aed0d4 --- /dev/null +++ b/packages/pipeline/note-core/src/verification/note-verifier.ts @@ -0,0 +1,100 @@ +import type { Claim, Evidence, VerificationOptions, VerificationResult, VerificationSummary } from "./types" +import { classifyClaim, determineVerdict, looksSupported } from "./verifier" + +function extractClaims(text: string): string[] { + return text + .replace(/\n+/g, " ") + .split(/(?<=[.!?])\s+/) + .map((segment) => segment.trim()) + .filter((segment) => segment.length > 10) +} + +function chunkTranscript(transcript: string): { text: string; ref: string }[] { + return transcript + .split("\n") + .filter((line) => line.trim()) + .map((line, index) => ({ text: line.trim(), ref: `line:${index + 1}` })) +} + +function findEvidence( + claim: string, + chunks: { text: string; ref: string }[], + options: VerificationOptions, +): { evidence: Evidence[]; bestScore: number } { + const evidence: Evidence[] = [] + let bestScore = 0 + + for (const chunk of chunks) { + const [, score] = looksSupported(claim, chunk.text, options.minTokenOverlap, options.minNumberCoverage) + if (score > 0.1) { + evidence.push({ ref: chunk.ref, text: chunk.text, score }) + if (score > bestScore) bestScore = score + } + } + + return { + evidence: evidence.sort((left, right) => right.score - left.score).slice(0, 3), + bestScore, + } +} + +function calculateSummary(claims: Claim[]): VerificationSummary { + const facts = claims.filter((claim) => claim.kind === "fact") + const supported = facts.filter((claim) => claim.verdict === "supported").length + const unsupported = facts.filter((claim) => claim.verdict === "unsupported").length + const totalConfidence = facts.reduce((sum, claim) => sum + claim.confidence, 0) + + return { + totalClaims: claims.length, + supportedClaims: supported, + unsupportedClaims: unsupported, + overallConfidence: facts.length > 0 ? Math.round((totalConfidence / facts.length) * 100) / 100 : 1, + } +} + +export async function verifyNote( + noteText: string, + transcript: string, + options: VerificationOptions = {}, +): Promise { + const startTime = performance.now() + const { minTokenOverlap = 0.25, minNumberCoverage = 1, factsOnly = false } = options + + const claimTexts = extractClaims(noteText) + const transcriptChunks = chunkTranscript(transcript) + const claims: Claim[] = [] + + for (let index = 0; index < claimTexts.length; index++) { + const text = claimTexts[index] + const kind = classifyClaim(text) + if (factsOnly && kind !== "fact") continue + + const { evidence, bestScore } = findEvidence(text, transcriptChunks, { minTokenOverlap, minNumberCoverage }) + claims.push({ + id: `claim_${index + 1}`, + text, + kind, + verdict: determineVerdict(bestScore, kind), + confidence: Math.round(bestScore * 100) / 100, + evidence, + }) + } + + const summary = calculateSummary(claims) + const totalResolvedFacts = summary.supportedClaims + summary.unsupportedClaims + let status: "verified" | "partial" | "failed" = "verified" + + if (totalResolvedFacts > 0) { + const supportRate = summary.supportedClaims / totalResolvedFacts + const unsupportedRate = summary.unsupportedClaims / totalResolvedFacts + if (unsupportedRate > 0.3) status = "failed" + else if (supportRate < 0.8 || summary.unsupportedClaims > 0) status = "partial" + } + + return { + status, + summary, + claims, + processingTimeMs: Math.round(performance.now() - startTime), + } +} diff --git a/packages/pipeline/note-core/src/verification/types.ts b/packages/pipeline/note-core/src/verification/types.ts new file mode 100644 index 0000000..ab7ae9b --- /dev/null +++ b/packages/pipeline/note-core/src/verification/types.ts @@ -0,0 +1,37 @@ +export type ClaimKind = "fact" | "inference" | "opinion" | "instruction" | "question" +export type Verdict = "supported" | "uncertain" | "unsupported" + +export interface Claim { + id: string + text: string + kind: ClaimKind + verdict: Verdict + confidence: number + evidence: Evidence[] +} + +export interface Evidence { + ref: string + text: string + score: number +} + +export interface VerificationResult { + status: "verified" | "partial" | "failed" + summary: VerificationSummary + claims: Claim[] + processingTimeMs: number +} + +export interface VerificationSummary { + totalClaims: number + supportedClaims: number + unsupportedClaims: number + overallConfidence: number +} + +export interface VerificationOptions { + minTokenOverlap?: number + minNumberCoverage?: number + factsOnly?: boolean +} diff --git a/packages/pipeline/note-core/src/verification/verifier.ts b/packages/pipeline/note-core/src/verification/verifier.ts new file mode 100644 index 0000000..5baa118 --- /dev/null +++ b/packages/pipeline/note-core/src/verification/verifier.ts @@ -0,0 +1,73 @@ +import type { ClaimKind, Verdict } from "./types" + +const STOP_WORDS = new Set([ + "a", "an", "the", "and", "or", "but", "if", "then", "of", "to", "in", "on", "for", "with", "by", "as", + "is", "are", "was", "were", "be", "been", "it", "this", "that", "at", "from", "not", "can", "do", "does", + "we", "you", "they", "i", "he", "she", "has", "have", "had", "will", "patient", "reports", "denies", +]) + +export function tokenize(text: string): string[] { + const normalized = (text || "").toLowerCase().replace(/[^\w-]+/g, " ").trim() + if (!normalized) return [] + return normalized.split(/\s+/).filter((token) => token.length >= 2 && !STOP_WORDS.has(token)) +} + +export function extractNumbers(text: string): string[] { + return (text || "").match(/(? num.replace(",", ".")) + if (claimNumbers.length === 0) return 1 + + const evidenceNumbers = new Set(extractNumbers(evidence).map((num) => num.replace(",", "."))) + if (evidenceNumbers.size === 0) return 0 + + let hits = 0 + for (const num of claimNumbers) { + if (evidenceNumbers.has(num)) hits++ + } + + return hits / claimNumbers.length +} + +export function looksSupported( + claim: string, + evidence: string, + minOverlap = 0.25, + minNumberCoverage = 1, +): [boolean, number] { + const overlap = calculateOverlap(claim, evidence) + const numberCoverage = calculateNumberCoverage(claim, evidence) + const score = overlap * 0.7 + numberCoverage * 0.3 + return [overlap >= minOverlap && numberCoverage >= minNumberCoverage, score] +} + +export function classifyClaim(text: string): ClaimKind { + const lower = text.toLowerCase().trim() + if (lower.endsWith("?")) return "question" + if (["i think", "i believe", "probably", "likely"].some((phrase) => lower.includes(phrase))) return "inference" + if (["in my opinion", "i feel"].some((phrase) => lower.includes(phrase))) return "opinion" + if (["do ", "please ", "recommend ", "consider "].some((phrase) => lower.startsWith(phrase))) return "instruction" + return "fact" +} + +export function determineVerdict(score: number, kind: ClaimKind): Verdict { + if (kind !== "fact") return "uncertain" + if (score >= 0.5) return "supported" + if (score >= 0.25) return "uncertain" + return "unsupported" +}