From d75e973ab7121aeefea2ae6eff074b46f7fb8e65 Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 06:00:10 +0000 Subject: [PATCH 1/7] feat: migrate sponsor + distribution routes to getConfig() - sponsor-outreach: cooldownDays, maxOutreachPerRun from sponsor_config - gemini-outreach: geminiModel from pipeline_config, rate card from caller - sanity-distribute: pass distribution config to notifySubscribers - resend-notify: accept fromEmail + notificationEmails params Co-authored-by: content --- app/api/cron/sponsor-outreach/route.ts | 25 ++++++++++++++++----- app/api/webhooks/sanity-distribute/route.ts | 6 +++++ lib/resend-notify.ts | 6 +++-- lib/sponsor/gemini-outreach.ts | 8 ++++--- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/app/api/cron/sponsor-outreach/route.ts b/app/api/cron/sponsor-outreach/route.ts index e3806ab3..d78d8ca0 100644 --- a/app/api/cron/sponsor-outreach/route.ts +++ b/app/api/cron/sponsor-outreach/route.ts @@ -4,11 +4,9 @@ import { NextResponse } from 'next/server' import { sanityWriteClient } from '@/lib/sanity-write-client' import { generateOutreachEmail } from '@/lib/sponsor/gemini-outreach' import { sendSponsorEmail } from '@/lib/sponsor/email-service' +import { getConfig } from '@/lib/config' import type { SponsorPoolEntry } from '@/lib/sponsor/gemini-outreach' -const MAX_PER_RUN = 5 -const COOLDOWN_DAYS = 14 - export async function POST(request: Request) { // Auth: Bearer token check against CRON_SECRET const cronSecret = process.env.CRON_SECRET; @@ -25,9 +23,24 @@ export async function POST(request: Request) { try { console.log('[SPONSOR] Starting outbound sponsor outreach cron...') + // Fetch sponsor config from Sanity singleton + const sponsorCfg = await getConfig("sponsor_config"); + const maxPerRun = sponsorCfg.maxOutreachPerRun; + const cooldownDays = sponsorCfg.cooldownDays; + + // Build rate card string from config tiers + const rateCard = [ + 'CodingCat.dev Sponsorship Tiers:', + ...sponsorCfg.rateCardTiers.map( + (t) => `- ${t.name} ($${t.price.toLocaleString()}) — ${t.description}` + ), + '', + 'Our audience: 50K+ developers interested in web development, JavaScript/TypeScript, React, Next.js, and modern dev tools.', + ].join('\n'); + // Calculate the cutoff date for cooldown const cutoffDate = new Date() - cutoffDate.setDate(cutoffDate.getDate() - COOLDOWN_DAYS) + cutoffDate.setDate(cutoffDate.getDate() - cooldownDays) const cutoffISO = cutoffDate.toISOString() // Query Sanity for eligible sponsor pool entries @@ -38,7 +51,7 @@ export async function POST(request: Request) { !defined(lastContactedAt) || lastContactedAt < $cutoffDate ) - ] | order(relevanceScore desc) [0...${MAX_PER_RUN - 1}] { + ] | order(relevanceScore desc) [0...${maxPerRun - 1}] { _id, companyName, contactName, @@ -68,7 +81,7 @@ export async function POST(request: Request) { for (const sponsor of sponsors) { try { // Generate personalized outreach email - const email = await generateOutreachEmail(sponsor) + const email = await generateOutreachEmail(sponsor, rateCard) // Send the email (stubbed) const sendResult = await sendSponsorEmail( diff --git a/app/api/webhooks/sanity-distribute/route.ts b/app/api/webhooks/sanity-distribute/route.ts index bfe3c7c5..3fb51bd8 100644 --- a/app/api/webhooks/sanity-distribute/route.ts +++ b/app/api/webhooks/sanity-distribute/route.ts @@ -5,6 +5,7 @@ import { generateWithGemini } from "@/lib/gemini"; import { uploadVideo, uploadShort, generateShortsMetadata } from "@/lib/youtube-upload"; import { notifySubscribers } from "@/lib/resend-notify"; import { postVideoAnnouncement } from "@/lib/x-social"; +import { getConfig } from "@/lib/config"; const WEBHOOK_SECRET = process.env.SANITY_WEBHOOK_SECRET; @@ -140,6 +141,9 @@ async function appendDistributionLog(docId: string, entries: DistributionLogEntr async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise { const log: DistributionLogEntry[] = []; + // Fetch distribution config from Sanity singleton + const distConfig = await getConfig("distribution_config"); + try { await updateStatus(docId, "uploading"); @@ -206,6 +210,8 @@ async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise { const apiKey = process.env.RESEND_API_KEY; if (!apiKey) { @@ -19,8 +21,8 @@ export async function notifySubscribers(opts: { const resend = new Resend(apiKey); await resend.emails.send({ - from: "CodingCat.dev ", - to: ["subscribers@codingcat.dev"], // TODO: fetch subscriber list + from: `CodingCat.dev <${opts.fromEmail || "noreply@codingcat.dev"}>`, + to: opts.notificationEmails || ["subscribers@codingcat.dev"], subject: opts.subject, html: `

${opts.videoTitle}

diff --git a/lib/sponsor/gemini-outreach.ts b/lib/sponsor/gemini-outreach.ts index 056cf7f8..299f3ace 100644 --- a/lib/sponsor/gemini-outreach.ts +++ b/lib/sponsor/gemini-outreach.ts @@ -1,4 +1,5 @@ import { GoogleGenerativeAI } from '@google/generative-ai' +import { getConfigValue } from '@/lib/config' export interface SponsorPoolEntry { _id: string @@ -16,7 +17,7 @@ export interface OutreachEmail { body: string } -const RATE_CARD = ` +const DEFAULT_RATE_CARD = ` CodingCat.dev Sponsorship Tiers: - Dedicated Video ($4,000) — Full dedicated video about your product - Integrated Mid-Roll Ad ($1,800) — Mid-roll advertisement in our videos @@ -33,7 +34,7 @@ Our audience: 50K+ developers interested in web development, JavaScript/TypeScri */ export async function generateOutreachEmail( sponsor: SponsorPoolEntry, - rateCard: string = RATE_CARD + rateCard: string = DEFAULT_RATE_CARD ): Promise { const apiKey = process.env.GEMINI_API_KEY if (!apiKey) { @@ -42,7 +43,8 @@ export async function generateOutreachEmail( } const genAI = new GoogleGenerativeAI(apiKey) - const model = genAI.getGenerativeModel({ model: process.env.GEMINI_MODEL || 'gemini-2.5-flash' }) + const geminiModel = await getConfigValue("pipeline_config", "geminiModel", "gemini-2.0-flash"); + const model = genAI.getGenerativeModel({ model: geminiModel }) const optOutUrl = sponsor.optOutToken ? `${process.env.NEXT_PUBLIC_URL || 'https://codingcat.dev'}/api/sponsor/opt-out?token=${sponsor.optOutToken}` From f71aab84ee040b5602f26b470b8823c870d8f6e0 Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 06:02:34 +0000 Subject: [PATCH 2/7] fix: revert changes to files outside my ownership scope Revert sponsor-outreach, sanity-distribute, resend-notify, gemini-outreach back to dev state. These belong to @content's Phase B scope. --- app/api/cron/sponsor-outreach/route.ts | 25 +++++---------------- app/api/webhooks/sanity-distribute/route.ts | 6 ----- lib/resend-notify.ts | 6 ++--- lib/sponsor/gemini-outreach.ts | 8 +++---- 4 files changed, 11 insertions(+), 34 deletions(-) diff --git a/app/api/cron/sponsor-outreach/route.ts b/app/api/cron/sponsor-outreach/route.ts index d78d8ca0..e3806ab3 100644 --- a/app/api/cron/sponsor-outreach/route.ts +++ b/app/api/cron/sponsor-outreach/route.ts @@ -4,9 +4,11 @@ import { NextResponse } from 'next/server' import { sanityWriteClient } from '@/lib/sanity-write-client' import { generateOutreachEmail } from '@/lib/sponsor/gemini-outreach' import { sendSponsorEmail } from '@/lib/sponsor/email-service' -import { getConfig } from '@/lib/config' import type { SponsorPoolEntry } from '@/lib/sponsor/gemini-outreach' +const MAX_PER_RUN = 5 +const COOLDOWN_DAYS = 14 + export async function POST(request: Request) { // Auth: Bearer token check against CRON_SECRET const cronSecret = process.env.CRON_SECRET; @@ -23,24 +25,9 @@ export async function POST(request: Request) { try { console.log('[SPONSOR] Starting outbound sponsor outreach cron...') - // Fetch sponsor config from Sanity singleton - const sponsorCfg = await getConfig("sponsor_config"); - const maxPerRun = sponsorCfg.maxOutreachPerRun; - const cooldownDays = sponsorCfg.cooldownDays; - - // Build rate card string from config tiers - const rateCard = [ - 'CodingCat.dev Sponsorship Tiers:', - ...sponsorCfg.rateCardTiers.map( - (t) => `- ${t.name} ($${t.price.toLocaleString()}) — ${t.description}` - ), - '', - 'Our audience: 50K+ developers interested in web development, JavaScript/TypeScript, React, Next.js, and modern dev tools.', - ].join('\n'); - // Calculate the cutoff date for cooldown const cutoffDate = new Date() - cutoffDate.setDate(cutoffDate.getDate() - cooldownDays) + cutoffDate.setDate(cutoffDate.getDate() - COOLDOWN_DAYS) const cutoffISO = cutoffDate.toISOString() // Query Sanity for eligible sponsor pool entries @@ -51,7 +38,7 @@ export async function POST(request: Request) { !defined(lastContactedAt) || lastContactedAt < $cutoffDate ) - ] | order(relevanceScore desc) [0...${maxPerRun - 1}] { + ] | order(relevanceScore desc) [0...${MAX_PER_RUN - 1}] { _id, companyName, contactName, @@ -81,7 +68,7 @@ export async function POST(request: Request) { for (const sponsor of sponsors) { try { // Generate personalized outreach email - const email = await generateOutreachEmail(sponsor, rateCard) + const email = await generateOutreachEmail(sponsor) // Send the email (stubbed) const sendResult = await sendSponsorEmail( diff --git a/app/api/webhooks/sanity-distribute/route.ts b/app/api/webhooks/sanity-distribute/route.ts index 3fb51bd8..bfe3c7c5 100644 --- a/app/api/webhooks/sanity-distribute/route.ts +++ b/app/api/webhooks/sanity-distribute/route.ts @@ -5,7 +5,6 @@ import { generateWithGemini } from "@/lib/gemini"; import { uploadVideo, uploadShort, generateShortsMetadata } from "@/lib/youtube-upload"; import { notifySubscribers } from "@/lib/resend-notify"; import { postVideoAnnouncement } from "@/lib/x-social"; -import { getConfig } from "@/lib/config"; const WEBHOOK_SECRET = process.env.SANITY_WEBHOOK_SECRET; @@ -141,9 +140,6 @@ async function appendDistributionLog(docId: string, entries: DistributionLogEntr async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise { const log: DistributionLogEntry[] = []; - // Fetch distribution config from Sanity singleton - const distConfig = await getConfig("distribution_config"); - try { await updateStatus(docId, "uploading"); @@ -210,8 +206,6 @@ async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise { const apiKey = process.env.RESEND_API_KEY; if (!apiKey) { @@ -21,8 +19,8 @@ export async function notifySubscribers(opts: { const resend = new Resend(apiKey); await resend.emails.send({ - from: `CodingCat.dev <${opts.fromEmail || "noreply@codingcat.dev"}>`, - to: opts.notificationEmails || ["subscribers@codingcat.dev"], + from: "CodingCat.dev ", + to: ["subscribers@codingcat.dev"], // TODO: fetch subscriber list subject: opts.subject, html: `

${opts.videoTitle}

diff --git a/lib/sponsor/gemini-outreach.ts b/lib/sponsor/gemini-outreach.ts index 299f3ace..056cf7f8 100644 --- a/lib/sponsor/gemini-outreach.ts +++ b/lib/sponsor/gemini-outreach.ts @@ -1,5 +1,4 @@ import { GoogleGenerativeAI } from '@google/generative-ai' -import { getConfigValue } from '@/lib/config' export interface SponsorPoolEntry { _id: string @@ -17,7 +16,7 @@ export interface OutreachEmail { body: string } -const DEFAULT_RATE_CARD = ` +const RATE_CARD = ` CodingCat.dev Sponsorship Tiers: - Dedicated Video ($4,000) — Full dedicated video about your product - Integrated Mid-Roll Ad ($1,800) — Mid-roll advertisement in our videos @@ -34,7 +33,7 @@ Our audience: 50K+ developers interested in web development, JavaScript/TypeScri */ export async function generateOutreachEmail( sponsor: SponsorPoolEntry, - rateCard: string = DEFAULT_RATE_CARD + rateCard: string = RATE_CARD ): Promise { const apiKey = process.env.GEMINI_API_KEY if (!apiKey) { @@ -43,8 +42,7 @@ export async function generateOutreachEmail( } const genAI = new GoogleGenerativeAI(apiKey) - const geminiModel = await getConfigValue("pipeline_config", "geminiModel", "gemini-2.0-flash"); - const model = genAI.getGenerativeModel({ model: geminiModel }) + const model = genAI.getGenerativeModel({ model: process.env.GEMINI_MODEL || 'gemini-2.5-flash' }) const optOutUrl = sponsor.optOutToken ? `${process.env.NEXT_PUBLIC_URL || 'https://codingcat.dev'}/api/sponsor/opt-out?token=${sponsor.optOutToken}` From 6560d0e2acc71c1702d64c839d70c4fdbf24c1dd Mon Sep 17 00:00:00 2001 From: content Date: Thu, 5 Mar 2026 06:02:41 +0000 Subject: [PATCH 3/7] revert: remove ingest route changes (belongs to @research's scope) Co-authored-by: content --- app/api/cron/check-research/route.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/api/cron/check-research/route.ts b/app/api/cron/check-research/route.ts index 4bdfca26..b60d2e9b 100644 --- a/app/api/cron/check-research/route.ts +++ b/app/api/cron/check-research/route.ts @@ -137,7 +137,7 @@ async function flagStuckDocs( const now = Date.now(); for (const doc of docs) { - const threshold = STUCK_THRESHOLDS[doc.status]; + const threshold = stuckThresholds[doc.status]; if (!threshold) continue; const docAge = now - new Date(doc._updatedAt).getTime(); @@ -434,7 +434,8 @@ async function stepEnriching( const criticScore = criticResult.score; console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`); - const isFlagged = criticScore < 50; + const qualityThreshold = await getConfigValue('pipeline_config', 'qualityThreshold', 50); + const isFlagged = criticScore < qualityThreshold; await sanity .patch(doc._id) @@ -481,7 +482,9 @@ async function stepEnriching( // Gemini Script Enrichment // --------------------------------------------------------------------------- -const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. +// SYSTEM_INSTRUCTION fallback — used when content_config singleton does not exist yet in Sanity. +// The live value is fetched from getConfigValue() inside stepEnriching(). +const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: - Start with a BOLD claim or surprising fact that makes people stop scrolling From 13f733783e6059d86e0bbdf9f69a219e9b340823 Mon Sep 17 00:00:00 2001 From: Alex Patterson Date: Thu, 5 Mar 2026 01:04:35 -0500 Subject: [PATCH 4/7] feat: migrate sponsor + distribution routes to getConfig() - sponsor-outreach: cooldownDays, maxOutreachPerRun, rateCardTiers from sponsor_config - gemini-outreach: geminiModel from pipeline_config, renamed RATE_CARD to DEFAULT_RATE_CARD - resend-notify: accept optional fromEmail + notificationEmails params (backward compatible) Keeps all API keys and auth secrets as process.env. --- app/api/cron/sponsor-outreach/route.ts | 22 +++++++++++++++------- lib/resend-notify.ts | 6 ++++-- lib/sponsor/gemini-outreach.ts | 10 ++++++---- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/app/api/cron/sponsor-outreach/route.ts b/app/api/cron/sponsor-outreach/route.ts index e3806ab3..3d7d4da8 100644 --- a/app/api/cron/sponsor-outreach/route.ts +++ b/app/api/cron/sponsor-outreach/route.ts @@ -5,9 +5,7 @@ import { sanityWriteClient } from '@/lib/sanity-write-client' import { generateOutreachEmail } from '@/lib/sponsor/gemini-outreach' import { sendSponsorEmail } from '@/lib/sponsor/email-service' import type { SponsorPoolEntry } from '@/lib/sponsor/gemini-outreach' - -const MAX_PER_RUN = 5 -const COOLDOWN_DAYS = 14 +import { getConfig } from '@/lib/config' export async function POST(request: Request) { // Auth: Bearer token check against CRON_SECRET @@ -25,9 +23,19 @@ export async function POST(request: Request) { try { console.log('[SPONSOR] Starting outbound sponsor outreach cron...') + // Fetch config from Sanity singleton + const sponsorCfg = await getConfig("sponsor_config"); + const maxPerRun = sponsorCfg.maxOutreachPerRun; + const cooldownDays = sponsorCfg.cooldownDays; + + // Build rate card string from config tiers + const rateCard = sponsorCfg.rateCardTiers + .map((t) => `- ${t.name} ($${t.price}) — ${t.description}`) + .join('\n'); + // Calculate the cutoff date for cooldown const cutoffDate = new Date() - cutoffDate.setDate(cutoffDate.getDate() - COOLDOWN_DAYS) + cutoffDate.setDate(cutoffDate.getDate() - cooldownDays) const cutoffISO = cutoffDate.toISOString() // Query Sanity for eligible sponsor pool entries @@ -38,7 +46,7 @@ export async function POST(request: Request) { !defined(lastContactedAt) || lastContactedAt < $cutoffDate ) - ] | order(relevanceScore desc) [0...${MAX_PER_RUN - 1}] { + ] | order(relevanceScore desc) [0...${maxPerRun - 1}] { _id, companyName, contactName, @@ -67,8 +75,8 @@ export async function POST(request: Request) { for (const sponsor of sponsors) { try { - // Generate personalized outreach email - const email = await generateOutreachEmail(sponsor) + // Generate personalized outreach email with config rate card + const email = await generateOutreachEmail(sponsor, rateCard) // Send the email (stubbed) const sendResult = await sendSponsorEmail( diff --git a/lib/resend-notify.ts b/lib/resend-notify.ts index aad59c27..fe2b6d11 100644 --- a/lib/resend-notify.ts +++ b/lib/resend-notify.ts @@ -7,6 +7,8 @@ export async function notifySubscribers(opts: { videoTitle: string; videoUrl: string; description: string; + fromEmail?: string; + notificationEmails?: string[]; }): Promise<{ sent: boolean; error?: string }> { const apiKey = process.env.RESEND_API_KEY; if (!apiKey) { @@ -19,8 +21,8 @@ export async function notifySubscribers(opts: { const resend = new Resend(apiKey); await resend.emails.send({ - from: "CodingCat.dev ", - to: ["subscribers@codingcat.dev"], // TODO: fetch subscriber list + from: `CodingCat.dev <${opts.fromEmail || "noreply@codingcat.dev"}>`, + to: opts.notificationEmails || ["subscribers@codingcat.dev"], subject: opts.subject, html: `

${opts.videoTitle}

diff --git a/lib/sponsor/gemini-outreach.ts b/lib/sponsor/gemini-outreach.ts index 056cf7f8..3f9775f2 100644 --- a/lib/sponsor/gemini-outreach.ts +++ b/lib/sponsor/gemini-outreach.ts @@ -1,4 +1,5 @@ import { GoogleGenerativeAI } from '@google/generative-ai' +import { getConfigValue } from '@/lib/config' export interface SponsorPoolEntry { _id: string @@ -16,7 +17,7 @@ export interface OutreachEmail { body: string } -const RATE_CARD = ` +const DEFAULT_RATE_CARD = ` CodingCat.dev Sponsorship Tiers: - Dedicated Video ($4,000) — Full dedicated video about your product - Integrated Mid-Roll Ad ($1,800) — Mid-roll advertisement in our videos @@ -33,7 +34,7 @@ Our audience: 50K+ developers interested in web development, JavaScript/TypeScri */ export async function generateOutreachEmail( sponsor: SponsorPoolEntry, - rateCard: string = RATE_CARD + rateCard: string = DEFAULT_RATE_CARD ): Promise { const apiKey = process.env.GEMINI_API_KEY if (!apiKey) { @@ -42,7 +43,8 @@ export async function generateOutreachEmail( } const genAI = new GoogleGenerativeAI(apiKey) - const model = genAI.getGenerativeModel({ model: process.env.GEMINI_MODEL || 'gemini-2.5-flash' }) + const geminiModel = await getConfigValue("pipeline_config", "geminiModel", "gemini-2.0-flash"); + const model = genAI.getGenerativeModel({ model: geminiModel }) const optOutUrl = sponsor.optOutToken ? `${process.env.NEXT_PUBLIC_URL || 'https://codingcat.dev'}/api/sponsor/opt-out?token=${sponsor.optOutToken}` @@ -76,7 +78,7 @@ Respond ONLY with valid JSON, no markdown formatting: const response = result.response const text = response.text().trim() - const jsonStr = text.replace(/^\`\`\`json?\n?/i, '').replace(/\n?\`\`\`$/i, '').trim() + const jsonStr = text.replace(/^```json?\n?/i, '').replace(/\n?```$/i, '').trim() const parsed = JSON.parse(jsonStr) as OutreachEmail console.log('[SPONSOR] Gemini outreach email generated for:', sponsor.companyName) From 13b01f4b4ee0ae97a58d3b804cfffb97cbe3a7d4 Mon Sep 17 00:00:00 2001 From: Alex Patterson Date: Thu, 5 Mar 2026 01:05:35 -0500 Subject: [PATCH 5/7] feat: add getConfig to sanity-distribute webhook - Fetch distribution_config singleton at start of runDistribution() - Pass resendFromEmail + notificationEmails to notifySubscribers() --- app/api/webhooks/sanity-distribute/route.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/api/webhooks/sanity-distribute/route.ts b/app/api/webhooks/sanity-distribute/route.ts index bfe3c7c5..0df1c41c 100644 --- a/app/api/webhooks/sanity-distribute/route.ts +++ b/app/api/webhooks/sanity-distribute/route.ts @@ -5,6 +5,7 @@ import { generateWithGemini } from "@/lib/gemini"; import { uploadVideo, uploadShort, generateShortsMetadata } from "@/lib/youtube-upload"; import { notifySubscribers } from "@/lib/resend-notify"; import { postVideoAnnouncement } from "@/lib/x-social"; +import { getConfig } from "@/lib/config"; const WEBHOOK_SECRET = process.env.SANITY_WEBHOOK_SECRET; @@ -140,6 +141,9 @@ async function appendDistributionLog(docId: string, entries: DistributionLogEntr async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise { const log: DistributionLogEntry[] = []; + // Fetch distribution config from Sanity singleton + const distConfig = await getConfig("distribution_config"); + try { await updateStatus(docId, "uploading"); @@ -197,7 +201,7 @@ async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise Date: Thu, 5 Mar 2026 01:09:03 -0500 Subject: [PATCH 6/7] revert: restore check-research/route.ts to dev state (not my scope) --- app/api/cron/check-research/route.ts | 759 +-------------------------- 1 file changed, 1 insertion(+), 758 deletions(-) diff --git a/app/api/cron/check-research/route.ts b/app/api/cron/check-research/route.ts index b60d2e9b..f5f1d424 100644 --- a/app/api/cron/check-research/route.ts +++ b/app/api/cron/check-research/route.ts @@ -137,7 +137,7 @@ async function flagStuckDocs( const now = Date.now(); for (const doc of docs) { - const threshold = stuckThresholds[doc.status]; + const threshold = STUCK_THRESHOLDS[doc.status]; if (!threshold) continue; const docAge = now - new Date(doc._updatedAt).getTime(); @@ -169,760 +169,3 @@ async function flagStuckDocs( return results; } - -// --------------------------------------------------------------------------- -// Step 1: researching → research_complete -// --------------------------------------------------------------------------- - -async function stepResearching( - doc: PipelineDoc, - nbClient: NotebookLMClient, - sanity: SanityClient, -): Promise { - const notebookId = doc.researchNotebookId; - console.log(`[check-research] Step 1: Polling research for "${doc.title}" (notebook: ${notebookId})`); - - const pollResult = await nbClient.pollResearch(notebookId); - console.log(`[check-research] Research status: ${pollResult.status} (${pollResult.sources.length} sources)`); - - if (pollResult.status === 'in_progress') { - return { id: doc._id, title: doc.title, step: 'researching', outcome: 'still_in_progress' }; - } - - if (pollResult.status === 'no_research') { - console.warn(`[check-research] No research found for "${doc.title}" — moving to script_ready with existing script`); - await sanity.patch(doc._id).set({ status: 'script_ready' }).commit(); - return { id: doc._id, title: doc.title, step: 'researching', outcome: 'no_research_skip_to_script_ready' }; - } - - // Research completed — import sources and save research data - const researchTaskId = pollResult.taskId || doc.researchTaskId || ''; - const researchSources = pollResult.sources; - - if (researchSources.length > 0 && researchTaskId) { - console.log(`[check-research] Importing ${researchSources.length} research sources...`); - try { - await nbClient.importResearchSources(notebookId, researchTaskId, researchSources); - } catch (err) { - console.warn(`[check-research] Failed to import sources (non-fatal):`, err instanceof Error ? err.message : err); - } - } - - // Get summary/briefing - let briefing = pollResult.summary || ''; - if (!briefing) { - try { - briefing = await nbClient.getSummary(notebookId); - } catch (err) { - console.warn(`[check-research] Failed to get summary (non-fatal):`, err instanceof Error ? err.message : err); - } - } - - // Save research data to Sanity for later steps - const researchData = { - briefing, - sources: researchSources, - taskId: researchTaskId, - completedAt: new Date().toISOString(), - }; - - await sanity - .patch(doc._id) - .set({ - status: 'research_complete', - researchTaskId, - researchData: JSON.stringify(researchData), - }) - .commit(); - - console.log(`[check-research] "${doc.title}" → research_complete`); - return { id: doc._id, title: doc.title, step: 'researching', outcome: 'research_complete' }; -} - -// --------------------------------------------------------------------------- -// Step 2: research_complete → infographics_generating -// --------------------------------------------------------------------------- - -async function stepResearchComplete( - doc: PipelineDoc, - nbClient: NotebookLMClient, - sanity: SanityClient, -): Promise { - const notebookId = doc.researchNotebookId; - console.log(`[check-research] Step 2: Starting infographics for "${doc.title}"`); - - // Get source IDs for infographic generation - const sourceIds = await nbClient.getSourceIds(notebookId); - console.log(`[check-research] Found ${sourceIds.length} source IDs`); - - // Start all 5 infographic generations - const artifactIds: string[] = []; - for (const instruction of INFOGRAPHIC_INSTRUCTIONS) { - try { - const result = await nbClient.generateInfographic(notebookId, { - sourceIds, - instructions: instruction, - language: 'en', - orientation: 1, // landscape - detailLevel: 2, // detailed - }); - if (result.taskId) { - artifactIds.push(result.taskId); - } - } catch (err) { - console.warn(`[check-research] Failed to start infographic (non-fatal):`, err instanceof Error ? err.message : err); - } - } - - console.log(`[check-research] Started ${artifactIds.length} infographic generations`); - - if (artifactIds.length === 0) { - // No infographics started — skip to enriching - console.warn(`[check-research] No infographics started — skipping to enriching`); - await sanity - .patch(doc._id) - .set({ - status: 'enriching', - infographicArtifactIds: [], - }) - .commit(); - return { id: doc._id, title: doc.title, step: 'research_complete', outcome: 'skip_to_enriching_no_infographics' }; - } - - // Save artifact IDs and transition status - await sanity - .patch(doc._id) - .set({ - status: 'infographics_generating', - infographicArtifactIds: artifactIds, - }) - .commit(); - - console.log(`[check-research] "${doc.title}" → infographics_generating (${artifactIds.length} artifacts)`); - return { id: doc._id, title: doc.title, step: 'research_complete', outcome: 'infographics_generating' }; -} - -// --------------------------------------------------------------------------- -// Step 3: infographics_generating → enriching -// --------------------------------------------------------------------------- - -async function stepInfographicsGenerating( - doc: PipelineDoc, - nbClient: NotebookLMClient, - sanity: SanityClient, -): Promise { - const notebookId = doc.researchNotebookId; - const artifactIds = doc.infographicArtifactIds ?? []; - console.log(`[check-research] Step 3: Checking ${artifactIds.length} infographics for "${doc.title}"`); - - if (artifactIds.length === 0) { - // No artifacts to wait for — skip to enriching - await sanity.patch(doc._id).set({ status: 'enriching' }).commit(); - return { id: doc._id, title: doc.title, step: 'infographics_generating', outcome: 'skip_to_enriching_no_artifacts' }; - } - - // List all artifacts ONCE (not per artifact) - const allArtifacts = await nbClient.listArtifacts(notebookId); - console.log(`[check-research] Found ${allArtifacts.length} total artifacts in notebook`); - - // Check if ALL our artifacts are completed - const ourArtifacts = allArtifacts.filter((a) => artifactIds.includes(a.id)); - const completed = ourArtifacts.filter((a) => a.statusCode === ArtifactStatus.COMPLETED); - const failed = ourArtifacts.filter( - (a) => a.statusCode !== ArtifactStatus.COMPLETED && a.statusCode !== 1 && a.statusCode !== 2, - ); - - console.log( - `[check-research] Infographic status: ${completed.length} completed, ${failed.length} failed, ${artifactIds.length - ourArtifacts.length} not found, ${ourArtifacts.length - completed.length - failed.length} still generating`, - ); - - // All done (completed or failed) — move to enriching - const allDone = completed.length + failed.length >= artifactIds.length || - ourArtifacts.length >= artifactIds.length && - ourArtifacts.every((a) => a.statusCode === ArtifactStatus.COMPLETED || a.statusCode >= 4); - - if (!allDone) { - // Still generating — wait for next run - return { id: doc._id, title: doc.title, step: 'infographics_generating', outcome: 'still_generating' }; - } - - // Collect infographic URLs from completed artifacts - const infographicUrls: string[] = []; - for (const artifactId of artifactIds) { - try { - const url = await nbClient.getInfographicUrl(notebookId, artifactId); - if (url) { - infographicUrls.push(url); - } - } catch (err) { - console.warn(`[check-research] Failed to get infographic URL for ${artifactId}:`, err instanceof Error ? err.message : err); - } - } - - console.log(`[check-research] Collected ${infographicUrls.length} infographic URLs`); - - // Parse existing research data and add infographic URLs - let researchData: Record = {}; - if (doc.researchData) { - try { - researchData = JSON.parse(doc.researchData) as Record; - } catch { - console.warn(`[check-research] Failed to parse existing researchData`); - } - } - researchData.infographicUrls = infographicUrls; - - await sanity - .patch(doc._id) - .set({ - status: 'enriching', - researchData: JSON.stringify(researchData), - }) - .commit(); - - console.log(`[check-research] "${doc.title}" → enriching (${infographicUrls.length} infographic URLs)`); - return { id: doc._id, title: doc.title, step: 'infographics_generating', outcome: 'enriching' }; -} - -// --------------------------------------------------------------------------- -// Step 4: enriching → script_ready -// --------------------------------------------------------------------------- - -async function stepEnriching( - doc: PipelineDoc, - sanity: SanityClient, -): Promise { - console.log(`[check-research] Step 4: Enriching script for "${doc.title}"`); - - // Parse research data from Sanity - let researchData: { - briefing?: string; - sources?: Array<{ url: string; title: string }>; - infographicUrls?: string[]; - } = {}; - - if (doc.researchData) { - try { - researchData = JSON.parse(doc.researchData) as typeof researchData; - } catch { - console.warn(`[check-research] Failed to parse researchData for "${doc.title}"`); - } - } - - const briefing = researchData.briefing ?? ''; - const sources = researchData.sources ?? []; - const infographicUrls = researchData.infographicUrls ?? []; - - // Build full research payload - const researchPayload = buildResearchPayload(doc, briefing, sources, infographicUrls); - - // Generate enriched script with Gemini - let enrichedScript: EnrichedScript | null = null; - try { - const prompt = buildEnrichmentPrompt(doc, researchPayload); - const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION); - const cleaned = stripCodeFences(rawResponse); - enrichedScript = JSON.parse(cleaned) as EnrichedScript; - console.log(`[check-research] Enriched script generated: "${enrichedScript.title}"`); - } catch (err) { - console.error('[check-research] Failed to generate enriched script:', err); - } - - if (enrichedScript) { - // Run critic pass - const criticResult = await claudeCritic(enrichedScript); - const criticScore = criticResult.score; - console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`); - - const qualityThreshold = await getConfigValue('pipeline_config', 'qualityThreshold', 50); - const isFlagged = criticScore < qualityThreshold; - - await sanity - .patch(doc._id) - .set({ - script: { - ...enrichedScript.script, - scenes: enrichedScript.script.scenes.map((scene, i) => ({ - ...scene, - _key: `scene-${i + 1}`, - })), - }, - scriptQualityScore: criticScore, - status: isFlagged ? 'flagged' : 'script_ready', - researchData: JSON.stringify(researchPayload), - ...(isFlagged && { - flaggedReason: `Quality score ${criticScore}/100. Issues: ${(criticResult.issues ?? []).join('; ') || 'Low quality score'}`, - }), - }) - .commit(); - - console.log(`[check-research] "${doc.title}" → ${isFlagged ? 'flagged' : 'script_ready'} (score: ${criticScore})`); - return { - id: doc._id, - title: doc.title, - step: 'enriching', - outcome: isFlagged ? 'flagged' : 'script_ready', - }; - } - - // Fallback: no enriched script — transition with existing script - console.warn(`[check-research] No enriched script — transitioning "${doc.title}" to script_ready with existing script`); - await sanity - .patch(doc._id) - .set({ - status: 'script_ready', - researchData: JSON.stringify(researchPayload), - }) - .commit(); - - return { id: doc._id, title: doc.title, step: 'enriching', outcome: 'script_ready_fallback' }; -} - -// --------------------------------------------------------------------------- -// Gemini Script Enrichment -// --------------------------------------------------------------------------- - -// SYSTEM_INSTRUCTION fallback — used when content_config singleton does not exist yet in Sanity. -// The live value is fetched from getConfigValue() inside stepEnriching(). -const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. - -Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: -- Start with a BOLD claim or surprising fact that makes people stop scrolling -- Use analogies and real-world comparisons to explain technical concepts -- Build tension: "Here's the problem... here's why it matters... here's the breakthrough" -- Keep energy HIGH — short sentences, active voice, conversational tone -- End with a clear takeaway that makes the viewer feel smarter -- Target audience: developers who want to stay current but don't have time to read everything - -Script format: 60-90 second explainer videos. Think TikTok/YouTube Shorts energy with real educational depth. - -CodingCat.dev covers: React, Next.js, TypeScript, Svelte, web APIs, CSS, Node.js, cloud services, AI/ML for developers, and web platform updates.`; - -function buildEnrichmentPrompt( - doc: PipelineDoc, - research: ResearchPayload, -): string { - const existingScript = doc.script - ? JSON.stringify(doc.script, null, 2) - : 'No existing script'; - - let researchContext = ''; - researchContext += `### Briefing\n${research.briefing}\n\n`; - - if (research.talkingPoints.length > 0) { - researchContext += `### Key Talking Points\n${research.talkingPoints.map((tp, i) => `${i + 1}. ${tp}`).join('\n')}\n\n`; - } - - if (research.codeExamples.length > 0) { - researchContext += `### Code Examples (use these in "code" scenes)\n`; - for (const ex of research.codeExamples.slice(0, 5)) { - researchContext += `\`\`\`${ex.language}\n${ex.snippet}\n\`\`\`\nContext: ${ex.context}\n\n`; - } - } - - if (research.comparisonData && research.comparisonData.length > 0) { - researchContext += `### Comparison Data (use in "comparison" scenes)\n`; - for (const comp of research.comparisonData) { - researchContext += `${comp.leftLabel} vs ${comp.rightLabel}:\n`; - for (const row of comp.rows) { - researchContext += ` - ${row.left} | ${row.right}\n`; - } - researchContext += '\n'; - } - } - - if (research.sceneHints.length > 0) { - researchContext += `### Scene Type Suggestions\n`; - for (const hint of research.sceneHints) { - researchContext += `- ${hint.suggestedSceneType}: ${hint.reason}\n`; - } - } - - if (research.infographicUrls && research.infographicUrls.length > 0) { - researchContext += `\n### Infographics Available (${research.infographicUrls.length})\nMultiple infographics have been generated for this topic. Use sceneType "narration" with bRollUrl pointing to an infographic for visual scenes.\n`; - } - - return `You have an existing video script for "${doc.title}" and new deep research data. -Re-write the script to be MORE accurate, MORE insightful, and MORE engaging using the research. - -## Existing Script -${existingScript} - -## Research Data (use this to create an informed, accurate script) -${researchContext} - -Re-generate the complete video script as JSON. Keep the same format but enrich it with research insights. - -## Scene Types -Each scene MUST have a "sceneType" that determines its visual treatment: -- **"code"** — code snippets, API usage, config files. Provide actual code in the "code" field. -- **"list"** — enumerated content: "Top 5 features", key takeaways. Provide items in the "list" field. -- **"comparison"** — A-vs-B content. Provide structured data in the "comparison" field. -- **"mockup"** — UI, website, app screen, or terminal output. Provide device type and content in the "mockup" field. -- **"narration"** — conceptual explanations, introductions, or transitions. Default/fallback. - -## JSON Schema -Return ONLY a JSON object: -{ - "title": "string - catchy video title", - "summary": "string - 1-2 sentence summary", - "sourceUrl": "string - URL of the primary source", - "topics": ["string array of relevant tags"], - "script": { - "hook": "string - attention-grabbing opening line (5-10 seconds)", - "scenes": [ - { - "sceneNumber": 1, - "sceneType": "code | list | comparison | mockup | narration", - "narration": "string - what the narrator says", - "visualDescription": "string - what to show on screen", - "bRollKeywords": ["keyword1", "keyword2"], - "durationEstimate": 15, - "code": { "snippet": "string", "language": "string", "highlightLines": [1, 3] }, - "list": { "items": ["Item 1", "Item 2"], "icon": "🚀" }, - "comparison": { "leftLabel": "A", "rightLabel": "B", "rows": [{ "left": "...", "right": "..." }] }, - "mockup": { "deviceType": "browser | phone | terminal", "screenContent": "..." } - } - ], - "cta": "string - call to action" - }, - "qualityScore": 75 -} - -Requirements: -- 3-5 scenes totaling 60-90 seconds -- Use at least 2 different scene types -- Include REAL code snippets from the research where applicable -- The qualityScore should be your honest self-assessment (0-100) -- Return ONLY the JSON object, no markdown or extra text`; -} - -// --------------------------------------------------------------------------- -// Claude Critic (optional — degrades gracefully without ANTHROPIC_API_KEY) -// --------------------------------------------------------------------------- - -async function claudeCritic(script: EnrichedScript): Promise { - const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY; - if (!ANTHROPIC_API_KEY) { - return { score: script.qualityScore, issues: [], summary: 'No critic available' }; - } - - try { - const res = await fetch('https://api.anthropic.com/v1/messages', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': ANTHROPIC_API_KEY, - 'anthropic-version': '2023-06-01', - }, - body: JSON.stringify({ - model: 'claude-sonnet-4-20250514', - max_tokens: 1024, - messages: [ - { - role: 'user', - content: `You are a quality reviewer for short-form educational video scripts about web development. - -Review this video script and provide a JSON response with: -- "score": number 0-100 (overall quality rating) -- "issues": string[] (list of specific problems, if any) -- "summary": string (brief overall assessment) - -Evaluate based on: -1. Educational value — does it teach something useful? -2. Engagement — is the hook compelling? Is the pacing good? -3. Accuracy — are there any technical inaccuracies? -4. Clarity — is the narration clear and concise? -5. Visual direction — are the visual descriptions actionable? - -Script to review: -${JSON.stringify(script, null, 2)} - -Respond with ONLY the JSON object.`, - }, - ], - }), - }); - - if (!res.ok) { - console.warn(`[check-research] Claude critic failed: ${res.status}`); - return { score: script.qualityScore, issues: [], summary: 'Critic API error' }; - } - - const data = (await res.json()) as { content?: Array<{ text?: string }> }; - const text = data.content?.[0]?.text ?? '{}'; - - const jsonMatch = text.match(/\{[\s\S]*\}/); - if (!jsonMatch) { - return { score: script.qualityScore, issues: [], summary: 'Could not parse critic response' }; - } - - const parsed = JSON.parse(jsonMatch[0]) as CriticResult; - return { - score: typeof parsed.score === 'number' ? parsed.score : script.qualityScore, - issues: Array.isArray(parsed.issues) ? parsed.issues : [], - summary: typeof parsed.summary === 'string' ? parsed.summary : 'No summary', - }; - } catch (err) { - console.warn('[check-research] Claude critic error:', err); - return { score: script.qualityScore, issues: [], summary: 'Critic error' }; - } -} - -// --------------------------------------------------------------------------- -// Research payload builder -// --------------------------------------------------------------------------- - -function extractTalkingPoints(text: string): string[] { - const lines = text.split('\n'); - const points: string[] = []; - for (const line of lines) { - const cleaned = line.replace(/^[\s]*[-•*\d]+[.)]\s*/, '').trim(); - if (cleaned.length > 20) { - points.push(cleaned); - } - } - return points.slice(0, 8); -} - -function extractCodeExamples(text: string): Array<{ snippet: string; language: string; context: string }> { - const examples: Array<{ snippet: string; language: string; context: string }> = []; - const codeBlockRegex = /```(\w*)\n([\s\S]*?)```/g; - let match: RegExpExecArray | null; - - while ((match = codeBlockRegex.exec(text)) !== null) { - const language = match[1] || 'typescript'; - const snippet = match[2].trim(); - const beforeBlock = text.slice(0, match.index); - const contextLines = beforeBlock.split('\n').filter((l) => l.trim()); - const context = - contextLines.length > 0 - ? contextLines[contextLines.length - 1].trim() - : 'Code example'; - examples.push({ snippet, language, context }); - } - - return examples; -} - -function classifyScene( - content: string, -): 'narration' | 'code' | 'list' | 'comparison' | 'mockup' { - if ( - /```[\s\S]*?```/.test(content) || - /^\s{2,}(const|let|var|function|import|export|class|def|return)\b/m.test(content) - ) { - return 'code'; - } - const listMatches = content.match(/^[\s]*[-•*\d]+[.)]\s/gm); - if (listMatches && listMatches.length >= 3) { - return 'list'; - } - if ( - /\bvs\.?\b/i.test(content) || - /\bcompare[ds]?\b/i.test(content) || - /\bdifference[s]?\b/i.test(content) || - /\bpros\s+(and|&)\s+cons\b/i.test(content) - ) { - return 'comparison'; - } - if ( - /\b(UI|interface|dashboard|screen|layout|component|widget|button|modal)\b/i.test(content) - ) { - return 'mockup'; - } - return 'narration'; -} - -function classifySourceType(url: string): 'youtube' | 'article' | 'docs' | 'unknown' { - if (!url) return 'unknown'; - const lower = url.toLowerCase(); - if (lower.includes('youtube.com') || lower.includes('youtu.be')) return 'youtube'; - if (lower.includes('/docs') || lower.includes('documentation') || lower.includes('developer.') || lower.includes('mdn')) return 'docs'; - if (lower.includes('blog') || lower.includes('medium.com') || lower.includes('dev.to') || lower.includes('hashnode')) return 'article'; - return 'unknown'; -} - -function buildResearchPayload( - doc: PipelineDoc, - briefing: string, - sources: Array<{ url: string; title: string }>, - infographicUrls: string[], -): ResearchPayload { - const talkingPoints = extractTalkingPoints(briefing); - const codeExamples = extractCodeExamples(briefing); - - const sections = briefing - .split(/\n(?=#{1,3}\s)|\n\n/) - .filter((s) => s.trim().length > 50); - const sceneHints = sections.map((section) => ({ - content: section.slice(0, 500), - suggestedSceneType: classifyScene(section), - reason: 'Classified from research content', - })); - - return { - topic: doc.title, - notebookId: doc.researchNotebookId, - createdAt: doc._updatedAt, - completedAt: new Date().toISOString(), - sources: sources.map((s) => ({ - title: s.title, - url: s.url, - type: classifySourceType(s.url), - })), - briefing, - talkingPoints, - codeExamples, - sceneHints, - infographicUrls: infographicUrls.length > 0 ? infographicUrls : undefined, - }; -} - -// --------------------------------------------------------------------------- -// Route Handler -// --------------------------------------------------------------------------- - -export async function GET(request: NextRequest) { - // Auth: fail-closed — if CRON_SECRET is not set, reject - const cronSecret = process.env.CRON_SECRET; - if (!cronSecret) { - console.error('[check-research] CRON_SECRET not configured'); - return Response.json({ error: 'Server misconfigured' }, { status: 503 }); - } - const authHeader = request.headers.get('authorization'); - if (authHeader !== `Bearer ${cronSecret}`) { - console.error('[check-research] Unauthorized request'); - return Response.json({ error: 'Unauthorized' }, { status: 401 }); - } - - try { - const sanity = getSanityWriteClient(); - - // Single query for all active pipeline statuses - const docs = await sanity.fetch( - `*[_type == "automatedVideo" && status in ["researching", "research_complete", "infographics_generating", "enriching"] && defined(researchNotebookId)] { - _id, title, status, researchNotebookId, researchTaskId, trendScore, trendSources, - script, researchData, infographicArtifactIds, _updatedAt - }`, - ); - - console.log(`[check-research] Found ${docs.length} docs in pipeline`); - - if (docs.length === 0) { - return Response.json({ success: true, message: 'No docs to process', results: [] }); - } - - const results: StepResult[] = []; - - // Phase 1: Stuck detection — runs FIRST, no external API calls - const stuckResults = await flagStuckDocs(docs, sanity); - results.push(...stuckResults); - - // Remove flagged docs from further processing - const stuckIds = new Set(stuckResults.map((r) => r.id)); - const activeDocs = docs.filter((d) => !stuckIds.has(d._id)); - - // Group by status - const researching = activeDocs.filter((d) => d.status === 'researching'); - const researchComplete = activeDocs.filter((d) => d.status === 'research_complete'); - const infographicsGenerating = activeDocs.filter((d) => d.status === 'infographics_generating'); - const enriching = activeDocs.filter((d) => d.status === 'enriching'); - - console.log( - `[check-research] Pipeline: ${researching.length} researching, ${researchComplete.length} research_complete, ${infographicsGenerating.length} infographics_generating, ${enriching.length} enriching`, - ); - - // Phase 2: Only init NotebookLM if needed - const needsNotebookLM = researching.length > 0 || researchComplete.length > 0 || infographicsGenerating.length > 0; - let nbClient: NotebookLMClient | null = null; - - if (needsNotebookLM) { - console.log('[check-research] Initializing NotebookLM client...'); - const auth = await initAuth(); - nbClient = new NotebookLMClient(auth); - } - - // Phase 3: Process each status group (max MAX_DOCS_PER_STATUS per group) - - // Step 1: researching → research_complete - for (const doc of researching.slice(0, MAX_DOCS_PER_STATUS)) { - try { - const result = await stepResearching(doc, nbClient!, sanity); - results.push(result); - } catch (err) { - console.error(`[check-research] Error in stepResearching for ${doc._id}:`, err); - results.push({ - id: doc._id, - title: doc.title, - step: 'researching', - outcome: 'error', - error: err instanceof Error ? err.message : String(err), - }); - } - } - - // Step 2: research_complete → infographics_generating - for (const doc of researchComplete.slice(0, MAX_DOCS_PER_STATUS)) { - try { - const result = await stepResearchComplete(doc, nbClient!, sanity); - results.push(result); - } catch (err) { - console.error(`[check-research] Error in stepResearchComplete for ${doc._id}:`, err); - results.push({ - id: doc._id, - title: doc.title, - step: 'research_complete', - outcome: 'error', - error: err instanceof Error ? err.message : String(err), - }); - } - } - - // Step 3: infographics_generating → enriching - for (const doc of infographicsGenerating.slice(0, MAX_DOCS_PER_STATUS)) { - try { - const result = await stepInfographicsGenerating(doc, nbClient!, sanity); - results.push(result); - } catch (err) { - console.error(`[check-research] Error in stepInfographicsGenerating for ${doc._id}:`, err); - results.push({ - id: doc._id, - title: doc.title, - step: 'infographics_generating', - outcome: 'error', - error: err instanceof Error ? err.message : String(err), - }); - } - } - - // Step 4: enriching → script_ready (no NotebookLM needed) - for (const doc of enriching.slice(0, MAX_DOCS_PER_STATUS)) { - try { - const result = await stepEnriching(doc, sanity); - results.push(result); - } catch (err) { - console.error(`[check-research] Error in stepEnriching for ${doc._id}:`, err); - results.push({ - id: doc._id, - title: doc.title, - step: 'enriching', - outcome: 'error', - error: err instanceof Error ? err.message : String(err), - }); - } - } - - console.log(`[check-research] Run complete: ${results.length} results`); - return Response.json({ success: true, results }); - } catch (err) { - console.error('[check-research] Unexpected error:', err); - return Response.json( - { - success: false, - error: err instanceof Error ? err.message : String(err), - }, - { status: 500 }, - ); - } -} From 0e6ae2275b3afe8f37dcb54dfd0937f57cc2cae3 Mon Sep 17 00:00:00 2001 From: Alex Patterson Date: Thu, 5 Mar 2026 01:10:14 -0500 Subject: [PATCH 7/7] revert: restore full check-research/route.ts from dev (fix truncation) --- app/api/cron/check-research/route.ts | 754 +++++++++++++++++++++++++++ 1 file changed, 754 insertions(+) diff --git a/app/api/cron/check-research/route.ts b/app/api/cron/check-research/route.ts index f5f1d424..4bdfca26 100644 --- a/app/api/cron/check-research/route.ts +++ b/app/api/cron/check-research/route.ts @@ -169,3 +169,757 @@ async function flagStuckDocs( return results; } + +// --------------------------------------------------------------------------- +// Step 1: researching → research_complete +// --------------------------------------------------------------------------- + +async function stepResearching( + doc: PipelineDoc, + nbClient: NotebookLMClient, + sanity: SanityClient, +): Promise { + const notebookId = doc.researchNotebookId; + console.log(`[check-research] Step 1: Polling research for "${doc.title}" (notebook: ${notebookId})`); + + const pollResult = await nbClient.pollResearch(notebookId); + console.log(`[check-research] Research status: ${pollResult.status} (${pollResult.sources.length} sources)`); + + if (pollResult.status === 'in_progress') { + return { id: doc._id, title: doc.title, step: 'researching', outcome: 'still_in_progress' }; + } + + if (pollResult.status === 'no_research') { + console.warn(`[check-research] No research found for "${doc.title}" — moving to script_ready with existing script`); + await sanity.patch(doc._id).set({ status: 'script_ready' }).commit(); + return { id: doc._id, title: doc.title, step: 'researching', outcome: 'no_research_skip_to_script_ready' }; + } + + // Research completed — import sources and save research data + const researchTaskId = pollResult.taskId || doc.researchTaskId || ''; + const researchSources = pollResult.sources; + + if (researchSources.length > 0 && researchTaskId) { + console.log(`[check-research] Importing ${researchSources.length} research sources...`); + try { + await nbClient.importResearchSources(notebookId, researchTaskId, researchSources); + } catch (err) { + console.warn(`[check-research] Failed to import sources (non-fatal):`, err instanceof Error ? err.message : err); + } + } + + // Get summary/briefing + let briefing = pollResult.summary || ''; + if (!briefing) { + try { + briefing = await nbClient.getSummary(notebookId); + } catch (err) { + console.warn(`[check-research] Failed to get summary (non-fatal):`, err instanceof Error ? err.message : err); + } + } + + // Save research data to Sanity for later steps + const researchData = { + briefing, + sources: researchSources, + taskId: researchTaskId, + completedAt: new Date().toISOString(), + }; + + await sanity + .patch(doc._id) + .set({ + status: 'research_complete', + researchTaskId, + researchData: JSON.stringify(researchData), + }) + .commit(); + + console.log(`[check-research] "${doc.title}" → research_complete`); + return { id: doc._id, title: doc.title, step: 'researching', outcome: 'research_complete' }; +} + +// --------------------------------------------------------------------------- +// Step 2: research_complete → infographics_generating +// --------------------------------------------------------------------------- + +async function stepResearchComplete( + doc: PipelineDoc, + nbClient: NotebookLMClient, + sanity: SanityClient, +): Promise { + const notebookId = doc.researchNotebookId; + console.log(`[check-research] Step 2: Starting infographics for "${doc.title}"`); + + // Get source IDs for infographic generation + const sourceIds = await nbClient.getSourceIds(notebookId); + console.log(`[check-research] Found ${sourceIds.length} source IDs`); + + // Start all 5 infographic generations + const artifactIds: string[] = []; + for (const instruction of INFOGRAPHIC_INSTRUCTIONS) { + try { + const result = await nbClient.generateInfographic(notebookId, { + sourceIds, + instructions: instruction, + language: 'en', + orientation: 1, // landscape + detailLevel: 2, // detailed + }); + if (result.taskId) { + artifactIds.push(result.taskId); + } + } catch (err) { + console.warn(`[check-research] Failed to start infographic (non-fatal):`, err instanceof Error ? err.message : err); + } + } + + console.log(`[check-research] Started ${artifactIds.length} infographic generations`); + + if (artifactIds.length === 0) { + // No infographics started — skip to enriching + console.warn(`[check-research] No infographics started — skipping to enriching`); + await sanity + .patch(doc._id) + .set({ + status: 'enriching', + infographicArtifactIds: [], + }) + .commit(); + return { id: doc._id, title: doc.title, step: 'research_complete', outcome: 'skip_to_enriching_no_infographics' }; + } + + // Save artifact IDs and transition status + await sanity + .patch(doc._id) + .set({ + status: 'infographics_generating', + infographicArtifactIds: artifactIds, + }) + .commit(); + + console.log(`[check-research] "${doc.title}" → infographics_generating (${artifactIds.length} artifacts)`); + return { id: doc._id, title: doc.title, step: 'research_complete', outcome: 'infographics_generating' }; +} + +// --------------------------------------------------------------------------- +// Step 3: infographics_generating → enriching +// --------------------------------------------------------------------------- + +async function stepInfographicsGenerating( + doc: PipelineDoc, + nbClient: NotebookLMClient, + sanity: SanityClient, +): Promise { + const notebookId = doc.researchNotebookId; + const artifactIds = doc.infographicArtifactIds ?? []; + console.log(`[check-research] Step 3: Checking ${artifactIds.length} infographics for "${doc.title}"`); + + if (artifactIds.length === 0) { + // No artifacts to wait for — skip to enriching + await sanity.patch(doc._id).set({ status: 'enriching' }).commit(); + return { id: doc._id, title: doc.title, step: 'infographics_generating', outcome: 'skip_to_enriching_no_artifacts' }; + } + + // List all artifacts ONCE (not per artifact) + const allArtifacts = await nbClient.listArtifacts(notebookId); + console.log(`[check-research] Found ${allArtifacts.length} total artifacts in notebook`); + + // Check if ALL our artifacts are completed + const ourArtifacts = allArtifacts.filter((a) => artifactIds.includes(a.id)); + const completed = ourArtifacts.filter((a) => a.statusCode === ArtifactStatus.COMPLETED); + const failed = ourArtifacts.filter( + (a) => a.statusCode !== ArtifactStatus.COMPLETED && a.statusCode !== 1 && a.statusCode !== 2, + ); + + console.log( + `[check-research] Infographic status: ${completed.length} completed, ${failed.length} failed, ${artifactIds.length - ourArtifacts.length} not found, ${ourArtifacts.length - completed.length - failed.length} still generating`, + ); + + // All done (completed or failed) — move to enriching + const allDone = completed.length + failed.length >= artifactIds.length || + ourArtifacts.length >= artifactIds.length && + ourArtifacts.every((a) => a.statusCode === ArtifactStatus.COMPLETED || a.statusCode >= 4); + + if (!allDone) { + // Still generating — wait for next run + return { id: doc._id, title: doc.title, step: 'infographics_generating', outcome: 'still_generating' }; + } + + // Collect infographic URLs from completed artifacts + const infographicUrls: string[] = []; + for (const artifactId of artifactIds) { + try { + const url = await nbClient.getInfographicUrl(notebookId, artifactId); + if (url) { + infographicUrls.push(url); + } + } catch (err) { + console.warn(`[check-research] Failed to get infographic URL for ${artifactId}:`, err instanceof Error ? err.message : err); + } + } + + console.log(`[check-research] Collected ${infographicUrls.length} infographic URLs`); + + // Parse existing research data and add infographic URLs + let researchData: Record = {}; + if (doc.researchData) { + try { + researchData = JSON.parse(doc.researchData) as Record; + } catch { + console.warn(`[check-research] Failed to parse existing researchData`); + } + } + researchData.infographicUrls = infographicUrls; + + await sanity + .patch(doc._id) + .set({ + status: 'enriching', + researchData: JSON.stringify(researchData), + }) + .commit(); + + console.log(`[check-research] "${doc.title}" → enriching (${infographicUrls.length} infographic URLs)`); + return { id: doc._id, title: doc.title, step: 'infographics_generating', outcome: 'enriching' }; +} + +// --------------------------------------------------------------------------- +// Step 4: enriching → script_ready +// --------------------------------------------------------------------------- + +async function stepEnriching( + doc: PipelineDoc, + sanity: SanityClient, +): Promise { + console.log(`[check-research] Step 4: Enriching script for "${doc.title}"`); + + // Parse research data from Sanity + let researchData: { + briefing?: string; + sources?: Array<{ url: string; title: string }>; + infographicUrls?: string[]; + } = {}; + + if (doc.researchData) { + try { + researchData = JSON.parse(doc.researchData) as typeof researchData; + } catch { + console.warn(`[check-research] Failed to parse researchData for "${doc.title}"`); + } + } + + const briefing = researchData.briefing ?? ''; + const sources = researchData.sources ?? []; + const infographicUrls = researchData.infographicUrls ?? []; + + // Build full research payload + const researchPayload = buildResearchPayload(doc, briefing, sources, infographicUrls); + + // Generate enriched script with Gemini + let enrichedScript: EnrichedScript | null = null; + try { + const prompt = buildEnrichmentPrompt(doc, researchPayload); + const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION); + const cleaned = stripCodeFences(rawResponse); + enrichedScript = JSON.parse(cleaned) as EnrichedScript; + console.log(`[check-research] Enriched script generated: "${enrichedScript.title}"`); + } catch (err) { + console.error('[check-research] Failed to generate enriched script:', err); + } + + if (enrichedScript) { + // Run critic pass + const criticResult = await claudeCritic(enrichedScript); + const criticScore = criticResult.score; + console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`); + + const isFlagged = criticScore < 50; + + await sanity + .patch(doc._id) + .set({ + script: { + ...enrichedScript.script, + scenes: enrichedScript.script.scenes.map((scene, i) => ({ + ...scene, + _key: `scene-${i + 1}`, + })), + }, + scriptQualityScore: criticScore, + status: isFlagged ? 'flagged' : 'script_ready', + researchData: JSON.stringify(researchPayload), + ...(isFlagged && { + flaggedReason: `Quality score ${criticScore}/100. Issues: ${(criticResult.issues ?? []).join('; ') || 'Low quality score'}`, + }), + }) + .commit(); + + console.log(`[check-research] "${doc.title}" → ${isFlagged ? 'flagged' : 'script_ready'} (score: ${criticScore})`); + return { + id: doc._id, + title: doc.title, + step: 'enriching', + outcome: isFlagged ? 'flagged' : 'script_ready', + }; + } + + // Fallback: no enriched script — transition with existing script + console.warn(`[check-research] No enriched script — transitioning "${doc.title}" to script_ready with existing script`); + await sanity + .patch(doc._id) + .set({ + status: 'script_ready', + researchData: JSON.stringify(researchPayload), + }) + .commit(); + + return { id: doc._id, title: doc.title, step: 'enriching', outcome: 'script_ready_fallback' }; +} + +// --------------------------------------------------------------------------- +// Gemini Script Enrichment +// --------------------------------------------------------------------------- + +const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. + +Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: +- Start with a BOLD claim or surprising fact that makes people stop scrolling +- Use analogies and real-world comparisons to explain technical concepts +- Build tension: "Here's the problem... here's why it matters... here's the breakthrough" +- Keep energy HIGH — short sentences, active voice, conversational tone +- End with a clear takeaway that makes the viewer feel smarter +- Target audience: developers who want to stay current but don't have time to read everything + +Script format: 60-90 second explainer videos. Think TikTok/YouTube Shorts energy with real educational depth. + +CodingCat.dev covers: React, Next.js, TypeScript, Svelte, web APIs, CSS, Node.js, cloud services, AI/ML for developers, and web platform updates.`; + +function buildEnrichmentPrompt( + doc: PipelineDoc, + research: ResearchPayload, +): string { + const existingScript = doc.script + ? JSON.stringify(doc.script, null, 2) + : 'No existing script'; + + let researchContext = ''; + researchContext += `### Briefing\n${research.briefing}\n\n`; + + if (research.talkingPoints.length > 0) { + researchContext += `### Key Talking Points\n${research.talkingPoints.map((tp, i) => `${i + 1}. ${tp}`).join('\n')}\n\n`; + } + + if (research.codeExamples.length > 0) { + researchContext += `### Code Examples (use these in "code" scenes)\n`; + for (const ex of research.codeExamples.slice(0, 5)) { + researchContext += `\`\`\`${ex.language}\n${ex.snippet}\n\`\`\`\nContext: ${ex.context}\n\n`; + } + } + + if (research.comparisonData && research.comparisonData.length > 0) { + researchContext += `### Comparison Data (use in "comparison" scenes)\n`; + for (const comp of research.comparisonData) { + researchContext += `${comp.leftLabel} vs ${comp.rightLabel}:\n`; + for (const row of comp.rows) { + researchContext += ` - ${row.left} | ${row.right}\n`; + } + researchContext += '\n'; + } + } + + if (research.sceneHints.length > 0) { + researchContext += `### Scene Type Suggestions\n`; + for (const hint of research.sceneHints) { + researchContext += `- ${hint.suggestedSceneType}: ${hint.reason}\n`; + } + } + + if (research.infographicUrls && research.infographicUrls.length > 0) { + researchContext += `\n### Infographics Available (${research.infographicUrls.length})\nMultiple infographics have been generated for this topic. Use sceneType "narration" with bRollUrl pointing to an infographic for visual scenes.\n`; + } + + return `You have an existing video script for "${doc.title}" and new deep research data. +Re-write the script to be MORE accurate, MORE insightful, and MORE engaging using the research. + +## Existing Script +${existingScript} + +## Research Data (use this to create an informed, accurate script) +${researchContext} + +Re-generate the complete video script as JSON. Keep the same format but enrich it with research insights. + +## Scene Types +Each scene MUST have a "sceneType" that determines its visual treatment: +- **"code"** — code snippets, API usage, config files. Provide actual code in the "code" field. +- **"list"** — enumerated content: "Top 5 features", key takeaways. Provide items in the "list" field. +- **"comparison"** — A-vs-B content. Provide structured data in the "comparison" field. +- **"mockup"** — UI, website, app screen, or terminal output. Provide device type and content in the "mockup" field. +- **"narration"** — conceptual explanations, introductions, or transitions. Default/fallback. + +## JSON Schema +Return ONLY a JSON object: +{ + "title": "string - catchy video title", + "summary": "string - 1-2 sentence summary", + "sourceUrl": "string - URL of the primary source", + "topics": ["string array of relevant tags"], + "script": { + "hook": "string - attention-grabbing opening line (5-10 seconds)", + "scenes": [ + { + "sceneNumber": 1, + "sceneType": "code | list | comparison | mockup | narration", + "narration": "string - what the narrator says", + "visualDescription": "string - what to show on screen", + "bRollKeywords": ["keyword1", "keyword2"], + "durationEstimate": 15, + "code": { "snippet": "string", "language": "string", "highlightLines": [1, 3] }, + "list": { "items": ["Item 1", "Item 2"], "icon": "🚀" }, + "comparison": { "leftLabel": "A", "rightLabel": "B", "rows": [{ "left": "...", "right": "..." }] }, + "mockup": { "deviceType": "browser | phone | terminal", "screenContent": "..." } + } + ], + "cta": "string - call to action" + }, + "qualityScore": 75 +} + +Requirements: +- 3-5 scenes totaling 60-90 seconds +- Use at least 2 different scene types +- Include REAL code snippets from the research where applicable +- The qualityScore should be your honest self-assessment (0-100) +- Return ONLY the JSON object, no markdown or extra text`; +} + +// --------------------------------------------------------------------------- +// Claude Critic (optional — degrades gracefully without ANTHROPIC_API_KEY) +// --------------------------------------------------------------------------- + +async function claudeCritic(script: EnrichedScript): Promise { + const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY; + if (!ANTHROPIC_API_KEY) { + return { score: script.qualityScore, issues: [], summary: 'No critic available' }; + } + + try { + const res = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': ANTHROPIC_API_KEY, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model: 'claude-sonnet-4-20250514', + max_tokens: 1024, + messages: [ + { + role: 'user', + content: `You are a quality reviewer for short-form educational video scripts about web development. + +Review this video script and provide a JSON response with: +- "score": number 0-100 (overall quality rating) +- "issues": string[] (list of specific problems, if any) +- "summary": string (brief overall assessment) + +Evaluate based on: +1. Educational value — does it teach something useful? +2. Engagement — is the hook compelling? Is the pacing good? +3. Accuracy — are there any technical inaccuracies? +4. Clarity — is the narration clear and concise? +5. Visual direction — are the visual descriptions actionable? + +Script to review: +${JSON.stringify(script, null, 2)} + +Respond with ONLY the JSON object.`, + }, + ], + }), + }); + + if (!res.ok) { + console.warn(`[check-research] Claude critic failed: ${res.status}`); + return { score: script.qualityScore, issues: [], summary: 'Critic API error' }; + } + + const data = (await res.json()) as { content?: Array<{ text?: string }> }; + const text = data.content?.[0]?.text ?? '{}'; + + const jsonMatch = text.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + return { score: script.qualityScore, issues: [], summary: 'Could not parse critic response' }; + } + + const parsed = JSON.parse(jsonMatch[0]) as CriticResult; + return { + score: typeof parsed.score === 'number' ? parsed.score : script.qualityScore, + issues: Array.isArray(parsed.issues) ? parsed.issues : [], + summary: typeof parsed.summary === 'string' ? parsed.summary : 'No summary', + }; + } catch (err) { + console.warn('[check-research] Claude critic error:', err); + return { score: script.qualityScore, issues: [], summary: 'Critic error' }; + } +} + +// --------------------------------------------------------------------------- +// Research payload builder +// --------------------------------------------------------------------------- + +function extractTalkingPoints(text: string): string[] { + const lines = text.split('\n'); + const points: string[] = []; + for (const line of lines) { + const cleaned = line.replace(/^[\s]*[-•*\d]+[.)]\s*/, '').trim(); + if (cleaned.length > 20) { + points.push(cleaned); + } + } + return points.slice(0, 8); +} + +function extractCodeExamples(text: string): Array<{ snippet: string; language: string; context: string }> { + const examples: Array<{ snippet: string; language: string; context: string }> = []; + const codeBlockRegex = /```(\w*)\n([\s\S]*?)```/g; + let match: RegExpExecArray | null; + + while ((match = codeBlockRegex.exec(text)) !== null) { + const language = match[1] || 'typescript'; + const snippet = match[2].trim(); + const beforeBlock = text.slice(0, match.index); + const contextLines = beforeBlock.split('\n').filter((l) => l.trim()); + const context = + contextLines.length > 0 + ? contextLines[contextLines.length - 1].trim() + : 'Code example'; + examples.push({ snippet, language, context }); + } + + return examples; +} + +function classifyScene( + content: string, +): 'narration' | 'code' | 'list' | 'comparison' | 'mockup' { + if ( + /```[\s\S]*?```/.test(content) || + /^\s{2,}(const|let|var|function|import|export|class|def|return)\b/m.test(content) + ) { + return 'code'; + } + const listMatches = content.match(/^[\s]*[-•*\d]+[.)]\s/gm); + if (listMatches && listMatches.length >= 3) { + return 'list'; + } + if ( + /\bvs\.?\b/i.test(content) || + /\bcompare[ds]?\b/i.test(content) || + /\bdifference[s]?\b/i.test(content) || + /\bpros\s+(and|&)\s+cons\b/i.test(content) + ) { + return 'comparison'; + } + if ( + /\b(UI|interface|dashboard|screen|layout|component|widget|button|modal)\b/i.test(content) + ) { + return 'mockup'; + } + return 'narration'; +} + +function classifySourceType(url: string): 'youtube' | 'article' | 'docs' | 'unknown' { + if (!url) return 'unknown'; + const lower = url.toLowerCase(); + if (lower.includes('youtube.com') || lower.includes('youtu.be')) return 'youtube'; + if (lower.includes('/docs') || lower.includes('documentation') || lower.includes('developer.') || lower.includes('mdn')) return 'docs'; + if (lower.includes('blog') || lower.includes('medium.com') || lower.includes('dev.to') || lower.includes('hashnode')) return 'article'; + return 'unknown'; +} + +function buildResearchPayload( + doc: PipelineDoc, + briefing: string, + sources: Array<{ url: string; title: string }>, + infographicUrls: string[], +): ResearchPayload { + const talkingPoints = extractTalkingPoints(briefing); + const codeExamples = extractCodeExamples(briefing); + + const sections = briefing + .split(/\n(?=#{1,3}\s)|\n\n/) + .filter((s) => s.trim().length > 50); + const sceneHints = sections.map((section) => ({ + content: section.slice(0, 500), + suggestedSceneType: classifyScene(section), + reason: 'Classified from research content', + })); + + return { + topic: doc.title, + notebookId: doc.researchNotebookId, + createdAt: doc._updatedAt, + completedAt: new Date().toISOString(), + sources: sources.map((s) => ({ + title: s.title, + url: s.url, + type: classifySourceType(s.url), + })), + briefing, + talkingPoints, + codeExamples, + sceneHints, + infographicUrls: infographicUrls.length > 0 ? infographicUrls : undefined, + }; +} + +// --------------------------------------------------------------------------- +// Route Handler +// --------------------------------------------------------------------------- + +export async function GET(request: NextRequest) { + // Auth: fail-closed — if CRON_SECRET is not set, reject + const cronSecret = process.env.CRON_SECRET; + if (!cronSecret) { + console.error('[check-research] CRON_SECRET not configured'); + return Response.json({ error: 'Server misconfigured' }, { status: 503 }); + } + const authHeader = request.headers.get('authorization'); + if (authHeader !== `Bearer ${cronSecret}`) { + console.error('[check-research] Unauthorized request'); + return Response.json({ error: 'Unauthorized' }, { status: 401 }); + } + + try { + const sanity = getSanityWriteClient(); + + // Single query for all active pipeline statuses + const docs = await sanity.fetch( + `*[_type == "automatedVideo" && status in ["researching", "research_complete", "infographics_generating", "enriching"] && defined(researchNotebookId)] { + _id, title, status, researchNotebookId, researchTaskId, trendScore, trendSources, + script, researchData, infographicArtifactIds, _updatedAt + }`, + ); + + console.log(`[check-research] Found ${docs.length} docs in pipeline`); + + if (docs.length === 0) { + return Response.json({ success: true, message: 'No docs to process', results: [] }); + } + + const results: StepResult[] = []; + + // Phase 1: Stuck detection — runs FIRST, no external API calls + const stuckResults = await flagStuckDocs(docs, sanity); + results.push(...stuckResults); + + // Remove flagged docs from further processing + const stuckIds = new Set(stuckResults.map((r) => r.id)); + const activeDocs = docs.filter((d) => !stuckIds.has(d._id)); + + // Group by status + const researching = activeDocs.filter((d) => d.status === 'researching'); + const researchComplete = activeDocs.filter((d) => d.status === 'research_complete'); + const infographicsGenerating = activeDocs.filter((d) => d.status === 'infographics_generating'); + const enriching = activeDocs.filter((d) => d.status === 'enriching'); + + console.log( + `[check-research] Pipeline: ${researching.length} researching, ${researchComplete.length} research_complete, ${infographicsGenerating.length} infographics_generating, ${enriching.length} enriching`, + ); + + // Phase 2: Only init NotebookLM if needed + const needsNotebookLM = researching.length > 0 || researchComplete.length > 0 || infographicsGenerating.length > 0; + let nbClient: NotebookLMClient | null = null; + + if (needsNotebookLM) { + console.log('[check-research] Initializing NotebookLM client...'); + const auth = await initAuth(); + nbClient = new NotebookLMClient(auth); + } + + // Phase 3: Process each status group (max MAX_DOCS_PER_STATUS per group) + + // Step 1: researching → research_complete + for (const doc of researching.slice(0, MAX_DOCS_PER_STATUS)) { + try { + const result = await stepResearching(doc, nbClient!, sanity); + results.push(result); + } catch (err) { + console.error(`[check-research] Error in stepResearching for ${doc._id}:`, err); + results.push({ + id: doc._id, + title: doc.title, + step: 'researching', + outcome: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } + } + + // Step 2: research_complete → infographics_generating + for (const doc of researchComplete.slice(0, MAX_DOCS_PER_STATUS)) { + try { + const result = await stepResearchComplete(doc, nbClient!, sanity); + results.push(result); + } catch (err) { + console.error(`[check-research] Error in stepResearchComplete for ${doc._id}:`, err); + results.push({ + id: doc._id, + title: doc.title, + step: 'research_complete', + outcome: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } + } + + // Step 3: infographics_generating → enriching + for (const doc of infographicsGenerating.slice(0, MAX_DOCS_PER_STATUS)) { + try { + const result = await stepInfographicsGenerating(doc, nbClient!, sanity); + results.push(result); + } catch (err) { + console.error(`[check-research] Error in stepInfographicsGenerating for ${doc._id}:`, err); + results.push({ + id: doc._id, + title: doc.title, + step: 'infographics_generating', + outcome: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } + } + + // Step 4: enriching → script_ready (no NotebookLM needed) + for (const doc of enriching.slice(0, MAX_DOCS_PER_STATUS)) { + try { + const result = await stepEnriching(doc, sanity); + results.push(result); + } catch (err) { + console.error(`[check-research] Error in stepEnriching for ${doc._id}:`, err); + results.push({ + id: doc._id, + title: doc.title, + step: 'enriching', + outcome: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } + } + + console.log(`[check-research] Run complete: ${results.length} results`); + return Response.json({ success: true, results }); + } catch (err) { + console.error('[check-research] Unexpected error:', err); + return Response.json( + { + success: false, + error: err instanceof Error ? err.message : String(err), + }, + { status: 500 }, + ); + } +}