feat(pipeline): add standalone clinical note verification library

packages/pipeline/note-core/src/__tests__/note-verifier.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict"
+import test from "node:test"
+import { verifyNote } from "../verification/note-verifier.js"
+
+const sampleTranscript = `
+Doctor: Good morning, what brings you in today?
+Patient: I've been having this really bad headache for the past 3 days.
+Doctor: Pain severity?
+Patient: About 7 or 8 out of 10.
+Doctor: Blood pressure is 128/82, temperature 98.4.
+`
+
+const goodNote = "Patient presents with headache for 3 days. Pain severity 7-8/10. Vitals: BP 128/82."
+const badNote = "Patient presents with chest pain for 5 days. BP 180/110."
+
+test("verifyNote validates matching note with non-trivial confidence", async () => {
+  const result = await verifyNote(goodNote, sampleTranscript)
+  assert.ok(["verified", "partial"].includes(result.status))
+  assert.ok(result.summary.overallConfidence > 0.3)
+  assert.ok(result.claims.length > 0)
+})
+
+test("verifyNote lowers confidence for mismatched claims", async () => {
+  const result = await verifyNote(badNote, sampleTranscript)
+  assert.ok(result.summary.overallConfidence < 0.3)
+})
+
+test("verifyNote handles empty note", async () => {
+  const result = await verifyNote("", sampleTranscript)
+  assert.equal(result.claims.length, 0)
+  assert.equal(result.status, "verified")
+})
+
+test("verifyNote handles empty transcript", async () => {
+  const result = await verifyNote(goodNote, "")
+  assert.ok(result.summary.overallConfidence < 0.5)
+})
+
+test("verifyNote respects factsOnly filter", async () => {
+  const result = await verifyNote(goodNote, sampleTranscript, { factsOnly: true })
+  for (const claim of result.claims) {
+    assert.equal(claim.kind, "fact")
+  }
+})

packages/pipeline/note-core/src/__tests__/verification-utils.test.ts

@@ -0,0 +1,39 @@
+import assert from "node:assert/strict"
+import test from "node:test"
+import { calculateOverlap, classifyClaim, extractNumbers, tokenize } from "../verification/verifier.js"
+
+test("tokenize extracts tokens and filters stop words", () => {
+  const tokens = tokenize("Patient reports headache for 3 days")
+  assert.ok(tokens.includes("headache"))
+  assert.ok(!tokens.includes("for"))
+})
+
+test("tokenize handles empty string", () => {
+  assert.deepEqual(tokenize(""), [])
+})
+
+test("extractNumbers extracts integer and decimal values", () => {
+  const numbers = extractNumbers("BP 120/80, temp 98.6")
+  assert.ok(numbers.includes("120"))
+  assert.ok(numbers.includes("98.6"))
+})
+
+test("calculateOverlap returns 1 for identical text", () => {
+  assert.equal(calculateOverlap("severe headache", "severe headache"), 1)
+})
+
+test("calculateOverlap returns 0 with no shared tokens", () => {
+  assert.equal(calculateOverlap("headache pain", "cardiac issues"), 0)
+})
+
+test("classifyClaim identifies fact claims", () => {
+  assert.equal(classifyClaim("Patient has hypertension."), "fact")
+})
+
+test("classifyClaim identifies question claims", () => {
+  assert.equal(classifyClaim("Does the patient smoke?"), "question")
+})
+
+test("classifyClaim identifies inference claims", () => {
+  assert.equal(classifyClaim("I think this might be migraine."), "inference")
+})

packages/pipeline/note-core/src/verification/index.ts

@@ -0,0 +1,11 @@
+export { verifyNote } from "./note-verifier"
+export { tokenize, extractNumbers, calculateOverlap, classifyClaim } from "./verifier"
+export type {
+  Claim,
+  ClaimKind,
+  Evidence,
+  Verdict,
+  VerificationOptions,
+  VerificationResult,
+  VerificationSummary,
+} from "./types"

packages/pipeline/note-core/src/verification/note-verifier.ts

@@ -0,0 +1,100 @@
+import type { Claim, Evidence, VerificationOptions, VerificationResult, VerificationSummary } from "./types"
+import { classifyClaim, determineVerdict, looksSupported } from "./verifier"
+
+function extractClaims(text: string): string[] {
+  return text
+    .replace(/\n+/g, " ")
+    .split(/(?<=[.!?])\s+/)
+    .map((segment) => segment.trim())
+    .filter((segment) => segment.length > 10)
+}
+
+function chunkTranscript(transcript: string): { text: string; ref: string }[] {
+  return transcript
+    .split("\n")
+    .filter((line) => line.trim())
+    .map((line, index) => ({ text: line.trim(), ref: `line:${index + 1}` }))
+}
+
+function findEvidence(
+  claim: string,
+  chunks: { text: string; ref: string }[],
+  options: VerificationOptions,
+): { evidence: Evidence[]; bestScore: number } {
+  const evidence: Evidence[] = []
+  let bestScore = 0
+
+  for (const chunk of chunks) {
+    const [, score] = looksSupported(claim, chunk.text, options.minTokenOverlap, options.minNumberCoverage)
+    if (score > 0.1) {
+      evidence.push({ ref: chunk.ref, text: chunk.text, score })
+      if (score > bestScore) bestScore = score
+    }
+  }
+
+  return {
+    evidence: evidence.sort((left, right) => right.score - left.score).slice(0, 3),
+    bestScore,
+  }
+}
+
+function calculateSummary(claims: Claim[]): VerificationSummary {
+  const facts = claims.filter((claim) => claim.kind === "fact")
+  const supported = facts.filter((claim) => claim.verdict === "supported").length
+  const unsupported = facts.filter((claim) => claim.verdict === "unsupported").length
+  const totalConfidence = facts.reduce((sum, claim) => sum + claim.confidence, 0)
+
+  return {
+    totalClaims: claims.length,
+    supportedClaims: supported,
+    unsupportedClaims: unsupported,
+    overallConfidence: facts.length > 0 ? Math.round((totalConfidence / facts.length) * 100) / 100 : 1,
+  }
+}
+
+export async function verifyNote(
+  noteText: string,
+  transcript: string,
+  options: VerificationOptions = {},
+): Promise<VerificationResult> {
+  const startTime = performance.now()
+  const { minTokenOverlap = 0.25, minNumberCoverage = 1, factsOnly = false } = options
+
+  const claimTexts = extractClaims(noteText)
+  const transcriptChunks = chunkTranscript(transcript)
+  const claims: Claim[] = []
+
+  for (let index = 0; index < claimTexts.length; index++) {
+    const text = claimTexts[index]
+    const kind = classifyClaim(text)
+    if (factsOnly && kind !== "fact") continue
+
+    const { evidence, bestScore } = findEvidence(text, transcriptChunks, { minTokenOverlap, minNumberCoverage })
+    claims.push({
+      id: `claim_${index + 1}`,
+      text,
+      kind,
+      verdict: determineVerdict(bestScore, kind),
+      confidence: Math.round(bestScore * 100) / 100,
+      evidence,
+    })
+  }
+
+  const summary = calculateSummary(claims)
+  const totalResolvedFacts = summary.supportedClaims + summary.unsupportedClaims
+  let status: "verified" | "partial" | "failed" = "verified"
+
+  if (totalResolvedFacts > 0) {
+    const supportRate = summary.supportedClaims / totalResolvedFacts
+    const unsupportedRate = summary.unsupportedClaims / totalResolvedFacts
+    if (unsupportedRate > 0.3) status = "failed"
+    else if (supportRate < 0.8 || summary.unsupportedClaims > 0) status = "partial"
+  }
+
+  return {
+    status,
+    summary,
+    claims,
+    processingTimeMs: Math.round(performance.now() - startTime),
+  }
+}

packages/pipeline/note-core/src/verification/types.ts

@@ -0,0 +1,37 @@
+export type ClaimKind = "fact" | "inference" | "opinion" | "instruction" | "question"
+export type Verdict = "supported" | "uncertain" | "unsupported"
+
+export interface Claim {
+  id: string
+  text: string
+  kind: ClaimKind
+  verdict: Verdict
+  confidence: number
+  evidence: Evidence[]
+}
+
+export interface Evidence {
+  ref: string
+  text: string
+  score: number
+}
+
+export interface VerificationResult {
+  status: "verified" | "partial" | "failed"
+  summary: VerificationSummary
+  claims: Claim[]
+  processingTimeMs: number
+}
+
+export interface VerificationSummary {
+  totalClaims: number
+  supportedClaims: number
+  unsupportedClaims: number
+  overallConfidence: number
+}
+
+export interface VerificationOptions {
+  minTokenOverlap?: number
+  minNumberCoverage?: number
+  factsOnly?: boolean
+}

packages/pipeline/note-core/src/verification/verifier.ts

@@ -0,0 +1,73 @@
+import type { ClaimKind, Verdict } from "./types"
+
+const STOP_WORDS = new Set([
+  "a", "an", "the", "and", "or", "but", "if", "then", "of", "to", "in", "on", "for", "with", "by", "as",
+  "is", "are", "was", "were", "be", "been", "it", "this", "that", "at", "from", "not", "can", "do", "does",
+  "we", "you", "they", "i", "he", "she", "has", "have", "had", "will", "patient", "reports", "denies",
+])
+
+export function tokenize(text: string): string[] {
+  const normalized = (text || "").toLowerCase().replace(/[^\w-]+/g, " ").trim()
+  if (!normalized) return []
+  return normalized.split(/\s+/).filter((token) => token.length >= 2 && !STOP_WORDS.has(token))
+}
+
+export function extractNumbers(text: string): string[] {
+  return (text || "").match(/(?<![\w])\d+(?:[.,]\d+)?(?![\w])/g) || []
+}
+
+export function calculateOverlap(claim: string, evidence: string): number {
+  const claimTokens = new Set(tokenize(claim))
+  const evidenceTokens = new Set(tokenize(evidence))
+  if (claimTokens.size === 0 || evidenceTokens.size === 0) return 0
+
+  let overlap = 0
+  for (const token of claimTokens) {
+    if (evidenceTokens.has(token)) overlap++
+  }
+
+  return overlap / claimTokens.size
+}
+
+function calculateNumberCoverage(claim: string, evidence: string): number {
+  const claimNumbers = extractNumbers(claim).map((num) => num.replace(",", "."))
+  if (claimNumbers.length === 0) return 1
+
+  const evidenceNumbers = new Set(extractNumbers(evidence).map((num) => num.replace(",", ".")))
+  if (evidenceNumbers.size === 0) return 0
+
+  let hits = 0
+  for (const num of claimNumbers) {
+    if (evidenceNumbers.has(num)) hits++
+  }
+
+  return hits / claimNumbers.length
+}
+
+export function looksSupported(
+  claim: string,
+  evidence: string,
+  minOverlap = 0.25,
+  minNumberCoverage = 1,
+): [boolean, number] {
+  const overlap = calculateOverlap(claim, evidence)
+  const numberCoverage = calculateNumberCoverage(claim, evidence)
+  const score = overlap * 0.7 + numberCoverage * 0.3
+  return [overlap >= minOverlap && numberCoverage >= minNumberCoverage, score]
+}
+
+export function classifyClaim(text: string): ClaimKind {
+  const lower = text.toLowerCase().trim()
+  if (lower.endsWith("?")) return "question"
+  if (["i think", "i believe", "probably", "likely"].some((phrase) => lower.includes(phrase))) return "inference"
+  if (["in my opinion", "i feel"].some((phrase) => lower.includes(phrase))) return "opinion"
+  if (["do ", "please ", "recommend ", "consider "].some((phrase) => lower.startsWith(phrase))) return "instruction"
+  return "fact"
+}
+
+export function determineVerdict(score: number, kind: ClaimKind): Verdict {
+  if (kind !== "fact") return "uncertain"
+  if (score >= 0.5) return "supported"
+  if (score >= 0.25) return "uncertain"
+  return "unsupported"
+}