From d677a013c0b9e9cd9ab37e269868761db8a301bd Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 02:39:42 +0000 Subject: [PATCH 1/7] fix: split remotion.ts to isolate native binary imports from Vercel bundle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move deploySite/deployFunction/getOrCreateBucket into remotion-deploy.ts (one-time CLI setup). remotion.ts now only imports renderMediaOnLambda + getRenderProgress — no @rspack/binding dependency chain in serverless routes. Also remove @remotion/bundler, @remotion/cli, @rspack/core, @rspack/binding from serverExternalPackages in next.config.ts (no longer needed). --- lib/services/remotion-deploy.ts | 173 ++++++++++++++++++++++++++++++++ lib/services/remotion.ts | 149 ++------------------------- next.config.ts | 4 - 3 files changed, 179 insertions(+), 147 deletions(-) create mode 100644 lib/services/remotion-deploy.ts diff --git a/lib/services/remotion-deploy.ts b/lib/services/remotion-deploy.ts new file mode 100644 index 00000000..f9ad5451 --- /dev/null +++ b/lib/services/remotion-deploy.ts @@ -0,0 +1,173 @@ +/** + * Remotion Lambda deployment service — one-time CLI setup only. + * + * This file imports deploySite, deployFunction, and getOrCreateBucket from + * @remotion/lambda, which pull in @rspack/core → @rspack/binding (a native + * binary). These MUST NOT be imported from Vercel serverless routes. + * + * Run this locally or in CI to set up the Lambda infrastructure, then store + * the resulting REMOTION_SERVE_URL and REMOTION_FUNCTION_NAME as env vars. + * + * @module lib/services/remotion-deploy + */ + +import { + deploySite, + deployFunction, + getOrCreateBucket, + type AwsRegion, +} from "@remotion/lambda"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function log(message: string, data?: Record): void { + const ts = new Date().toISOString(); + if (data) { + console.log(`[REMOTION] [${ts}] ${message}`, data); + } else { + console.log(`[REMOTION] [${ts}] ${message}`); + } +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface DeployResult { + functionName: string; + serveUrl: string; + siteName: string; + bucketName: string; + region: string; +} + +// --------------------------------------------------------------------------- +// Deploy helper (one-time setup) +// --------------------------------------------------------------------------- + +/** + * Deploy the Remotion Lambda infrastructure (one-time setup). + * + * This will: + * 1. Create or reuse an S3 bucket for Remotion + * 2. Deploy the Remotion bundle (site) to S3 + * 3. Deploy the Lambda function + * + * After running this, set the returned `serveUrl` as REMOTION_SERVE_URL + * and `functionName` as REMOTION_FUNCTION_NAME in your environment. + * + * @param entryPoint - Path to the Remotion entry file (e.g., "remotion/index.ts") + * @returns Deploy result with function name, serve URL, etc. + */ +export async function deployRemotionLambda( + entryPoint: string = "remotion/index.ts" +): Promise { + const region = (process.env.REMOTION_AWS_REGION || "us-east-1") as AwsRegion; + + if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { + throw new Error( + "[REMOTION] Cannot deploy: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY " + + "must be set in environment variables." + ); + } + + log("Starting Remotion Lambda deployment", { region, entryPoint }); + + // Step 1: Get or create the S3 bucket + log("Step 1/3: Getting or creating S3 bucket..."); + let bucketName: string; + try { + const bucketResult = await getOrCreateBucket({ region }); + bucketName = bucketResult.bucketName; + log(`Bucket ready: ${bucketName}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + throw new Error( + `[REMOTION] Failed to get or create S3 bucket in ${region}: ${message}. ` + + `Ensure your AWS credentials have S3 permissions.` + ); + } + + // Step 2: Deploy the site (bundle) to S3 + log("Step 2/3: Deploying Remotion bundle to S3..."); + let serveUrl: string; + let siteName: string; + try { + const siteResult = await deploySite({ + entryPoint, + bucketName, + region, + siteName: "codingcat-video-pipeline", + options: { + onBundleProgress: (progress: number) => { + if (progress % 25 === 0 || progress === 100) { + log(`Bundle progress: ${progress}%`); + } + }, + onUploadProgress: (upload) => { + if (upload.totalFiles > 0) { + const pct = Math.round( + (upload.filesUploaded / upload.totalFiles) * 100 + ); + if (pct % 25 === 0) { + log( + `Upload progress: ${upload.filesUploaded}/${upload.totalFiles} files (${pct}%)` + ); + } + } + }, + }, + }); + serveUrl = siteResult.serveUrl; + siteName = siteResult.siteName; + log(`Site deployed: ${serveUrl}`, { siteName }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + throw new Error( + `[REMOTION] Failed to deploy site to S3: ${message}. ` + + `Ensure the entry point "${entryPoint}" exists and is a valid Remotion project.` + ); + } + + // Step 3: Deploy the Lambda function + log("Step 3/3: Deploying Lambda function..."); + let functionName: string; + try { + const fnResult = await deployFunction({ + region, + timeoutInSeconds: 240, + memorySizeInMb: 2048, + createCloudWatchLogGroup: true, + cloudWatchLogRetentionPeriodInDays: 14, + diskSizeInMb: 2048, + }); + functionName = fnResult.functionName; + log( + `Lambda function deployed: ${functionName}` + + (fnResult.alreadyExisted ? " (already existed)" : " (newly created)") + ); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + throw new Error( + `[REMOTION] Failed to deploy Lambda function: ${message}. ` + + `Ensure your AWS credentials have Lambda and IAM permissions. ` + + `See: https://www.remotion.dev/docs/lambda/permissions` + ); + } + + log("Deployment complete! Set these environment variables:", { + REMOTION_SERVE_URL: serveUrl, + REMOTION_FUNCTION_NAME: functionName, + REMOTION_AWS_REGION: region, + }); + + return { + functionName, + serveUrl, + siteName, + bucketName, + region, + }; +} diff --git a/lib/services/remotion.ts b/lib/services/remotion.ts index f64f9bee..ca6dc46b 100644 --- a/lib/services/remotion.ts +++ b/lib/services/remotion.ts @@ -1,15 +1,18 @@ /** - * Remotion Lambda rendering service. + * Remotion Lambda rendering service — runtime only. * - * Handles deploying and triggering Remotion Lambda renders for video production. + * Handles triggering and polling Remotion Lambda renders for video production. * Produces both 16:9 (main) and 9:16 (short) video formats. * + * Deploy functions (deploySite, deployFunction, getOrCreateBucket) live in + * `remotion-deploy.ts` to avoid pulling @rspack/binding into Vercel bundles. + * * Requires env vars: * - AWS_ACCESS_KEY_ID * - AWS_SECRET_ACCESS_KEY * - REMOTION_AWS_REGION * - REMOTION_SERVE_URL (generated during Lambda deployment) - * - REMOTION_FUNCTION_NAME (optional, from deployRemotionLambda) + * - REMOTION_FUNCTION_NAME (optional, defaults to DEFAULT_FUNCTION_NAME) * * @module lib/services/remotion */ @@ -17,9 +20,6 @@ import { renderMediaOnLambda, getRenderProgress, - deploySite, - deployFunction, - getOrCreateBucket, type AwsRegion, } from "@remotion/lambda"; @@ -417,140 +417,3 @@ export async function renderBothFormats( return { main, short }; } - -// --------------------------------------------------------------------------- -// Deploy helper (one-time setup) -// --------------------------------------------------------------------------- - -export interface DeployResult { - functionName: string; - serveUrl: string; - siteName: string; - bucketName: string; - region: string; -} - -/** - * Deploy the Remotion Lambda infrastructure (one-time setup). - * - * This will: - * 1. Create or reuse an S3 bucket for Remotion - * 2. Deploy the Remotion bundle (site) to S3 - * 3. Deploy the Lambda function - * - * After running this, set the returned `serveUrl` as REMOTION_SERVE_URL - * and `functionName` as REMOTION_FUNCTION_NAME in your environment. - * - * @param entryPoint - Path to the Remotion entry file (e.g., "remotion/index.ts") - * @returns Deploy result with function name, serve URL, etc. - */ -export async function deployRemotionLambda( - entryPoint: string = "remotion/index.ts" -): Promise { - const region = (process.env.REMOTION_AWS_REGION || "us-east-1") as AwsRegion; - - if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { - throw new Error( - "[REMOTION] Cannot deploy: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY " + - "must be set in environment variables." - ); - } - - log("Starting Remotion Lambda deployment", { region, entryPoint }); - - // Step 1: Get or create the S3 bucket - log("Step 1/3: Getting or creating S3 bucket..."); - let bucketName: string; - try { - const bucketResult = await getOrCreateBucket({ region }); - bucketName = bucketResult.bucketName; - log(`Bucket ready: ${bucketName}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - throw new Error( - `[REMOTION] Failed to get or create S3 bucket in ${region}: ${message}. ` + - `Ensure your AWS credentials have S3 permissions.` - ); - } - - // Step 2: Deploy the site (bundle) to S3 - log("Step 2/3: Deploying Remotion bundle to S3..."); - let serveUrl: string; - let siteName: string; - try { - const siteResult = await deploySite({ - entryPoint, - bucketName, - region, - siteName: "codingcat-video-pipeline", - options: { - onBundleProgress: (progress: number) => { - if (progress % 25 === 0 || progress === 100) { - log(`Bundle progress: ${progress}%`); - } - }, - onUploadProgress: (upload) => { - if (upload.totalFiles > 0) { - const pct = Math.round( - (upload.filesUploaded / upload.totalFiles) * 100 - ); - if (pct % 25 === 0) { - log( - `Upload progress: ${upload.filesUploaded}/${upload.totalFiles} files (${pct}%)` - ); - } - } - }, - }, - }); - serveUrl = siteResult.serveUrl; - siteName = siteResult.siteName; - log(`Site deployed: ${serveUrl}`, { siteName }); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - throw new Error( - `[REMOTION] Failed to deploy site to S3: ${message}. ` + - `Ensure the entry point "${entryPoint}" exists and is a valid Remotion project.` - ); - } - - // Step 3: Deploy the Lambda function - log("Step 3/3: Deploying Lambda function..."); - let functionName: string; - try { - const fnResult = await deployFunction({ - region, - timeoutInSeconds: 240, - memorySizeInMb: 2048, - createCloudWatchLogGroup: true, - cloudWatchLogRetentionPeriodInDays: 14, - diskSizeInMb: 2048, - }); - functionName = fnResult.functionName; - log( - `Lambda function deployed: ${functionName}` + - (fnResult.alreadyExisted ? " (already existed)" : " (newly created)") - ); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - throw new Error( - `[REMOTION] Failed to deploy Lambda function: ${message}. ` + - `Ensure your AWS credentials have Lambda and IAM permissions. ` + - `See: https://www.remotion.dev/docs/lambda/permissions` - ); - } - - log("Deployment complete! Set these environment variables:", { - REMOTION_SERVE_URL: serveUrl, - REMOTION_FUNCTION_NAME: functionName, - REMOTION_AWS_REGION: region, - }); - - return { - functionName, - serveUrl, - siteName, - bucketName, - region, - }; -} diff --git a/next.config.ts b/next.config.ts index 131284c9..b242e004 100644 --- a/next.config.ts +++ b/next.config.ts @@ -14,10 +14,6 @@ const nextConfig: NextConfig = { }, serverExternalPackages: [ "@remotion/lambda", - "@remotion/bundler", - "@remotion/cli", - "@rspack/core", - "@rspack/binding", ], }; From 1d2754f5d01dc00804d18576c71e19f2a49a171e Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 02:53:50 +0000 Subject: [PATCH 2/7] fix: use @remotion/lambda/client to avoid @rspack/binding in serverless MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The main @remotion/lambda entry point re-exports from @remotion/bundler which pulls in @rspack/core → @rspack/binding (native binary). The /client subpath only exports renderMediaOnLambda + getRenderProgress without the bundler dependency chain. This is the official Remotion approach for serverless environments. --- lib/services/remotion.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/remotion.ts b/lib/services/remotion.ts index ca6dc46b..feca1b06 100644 --- a/lib/services/remotion.ts +++ b/lib/services/remotion.ts @@ -21,7 +21,7 @@ import { renderMediaOnLambda, getRenderProgress, type AwsRegion, -} from "@remotion/lambda"; +} from "@remotion/lambda/client"; // --------------------------------------------------------------------------- // Types From 4383df7c0eb217d74b9c30e67a7ce35a01cc0637 Mon Sep 17 00:00:00 2001 From: content Date: Thu, 5 Mar 2026 03:04:21 +0000 Subject: [PATCH 3/7] =?UTF-8?q?fix:=20remove=20Vercel=20cron=20entries=20?= =?UTF-8?q?=E2=80=94=20crons=20run=20via=20Supabase=20pg=5Fcron?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All cron jobs are triggered by Supabase pg_cron + pg_net calling the HTTP endpoints directly. Removes vercel.json cron config to avoid confusion and potential double-triggering. Co-authored-by: content --- vercel.json | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/vercel.json b/vercel.json index f9aa639a..da0952e1 100644 --- a/vercel.json +++ b/vercel.json @@ -1,21 +1,3 @@ { - "$schema": "https://openapi.vercel.sh/vercel.json", - "crons": [ - { - "path": "/api/cron", - "schedule": "0 0 * * *" - }, - { - "path": "/api/cron/ingest", - "schedule": "0 6 * * *" - }, - { - "path": "/api/cron/sponsor-outreach", - "schedule": "0 9 * * 1,4" - }, - { - "path": "/api/cron/check-renders", - "schedule": "0 7 * * *" - } - ] + "$schema": "https://openapi.vercel.sh/vercel.json" } From 775630d6680c05a498736e95eb18b30ab209b718 Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 03:08:25 +0000 Subject: [PATCH 4/7] feat: add pg_cron migration for multi-step pipeline schedules Adds 002_cron_schedules.sql with idempotent schedules for: - ingest-daily (10:00 UTC) - check-research (every 5 min) - check-renders (every 5 min) - sponsor-outreach (Mon/Thu 09:00 UTC) Uses DO blocks for safe unschedule on re-runs. Co-authored-by: research --- supabase/migrations/002_cron_schedules.sql | 104 +++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 supabase/migrations/002_cron_schedules.sql diff --git a/supabase/migrations/002_cron_schedules.sql b/supabase/migrations/002_cron_schedules.sql new file mode 100644 index 00000000..e9ec8971 --- /dev/null +++ b/supabase/migrations/002_cron_schedules.sql @@ -0,0 +1,104 @@ +-- ========================================================================== +-- pg_cron schedules for CodingCat.dev automated video pipeline +-- ========================================================================== +-- +-- Prerequisites: +-- These Supabase config vars must be set before running this migration: +-- ALTER DATABASE postgres SET app.site_url = 'https://your-vercel-url.vercel.app'; +-- ALTER DATABASE postgres SET app.cron_secret = 'your-cron-secret-here'; +-- +-- You can set them in the Supabase dashboard under Database → Extensions, +-- or via SQL in the SQL Editor. +-- +-- Pipeline flow: +-- 1. ingest-daily → discovers trends, creates Sanity doc (status: "researching" or "script_ready") +-- 2. check-research → polls NotebookLM, enriches script, transitions to "script_ready" +-- 3. check-renders → audio gen → Remotion render → upload → publish +-- 4. sponsor-outreach → automated sponsor discovery + outreach emails +-- ========================================================================== + +-- Enable required extensions +CREATE EXTENSION IF NOT EXISTS pg_cron; +CREATE EXTENSION IF NOT EXISTS pg_net; + +-- --------------------------------------------------------------------------- +-- Remove any existing schedules (idempotent re-runs) +-- --------------------------------------------------------------------------- +-- pg_cron's unschedule() throws if the job doesn't exist, so we use DO blocks +DO $$ +BEGIN + PERFORM cron.unschedule('ingest-daily'); +EXCEPTION WHEN OTHERS THEN + NULL; +END $$; + +DO $$ +BEGIN + PERFORM cron.unschedule('check-research'); +EXCEPTION WHEN OTHERS THEN + NULL; +END $$; + +DO $$ +BEGIN + PERFORM cron.unschedule('check-renders'); +EXCEPTION WHEN OTHERS THEN + NULL; +END $$; + +DO $$ +BEGIN + PERFORM cron.unschedule('sponsor-outreach'); +EXCEPTION WHEN OTHERS THEN + NULL; +END $$; + +-- --------------------------------------------------------------------------- +-- Schedule: Ingest — daily at 10:00 UTC +-- --------------------------------------------------------------------------- +SELECT cron.schedule( + 'ingest-daily', + '0 10 * * *', + $$SELECT net.http_get( + url := current_setting('app.site_url') || '/api/cron/ingest', + headers := jsonb_build_object('Authorization', 'Bearer ' || current_setting('app.cron_secret')) + )$$ +); + +-- --------------------------------------------------------------------------- +-- Schedule: Check Research — every 5 minutes +-- Polls NotebookLM for docs in "researching" status +-- --------------------------------------------------------------------------- +SELECT cron.schedule( + 'check-research', + '*/5 * * * *', + $$SELECT net.http_get( + url := current_setting('app.site_url') || '/api/cron/check-research', + headers := jsonb_build_object('Authorization', 'Bearer ' || current_setting('app.cron_secret')) + )$$ +); + +-- --------------------------------------------------------------------------- +-- Schedule: Check Renders — every 5 minutes +-- Polls Remotion render status, uploads completed videos +-- --------------------------------------------------------------------------- +SELECT cron.schedule( + 'check-renders', + '*/5 * * * *', + $$SELECT net.http_get( + url := current_setting('app.site_url') || '/api/cron/check-renders', + headers := jsonb_build_object('Authorization', 'Bearer ' || current_setting('app.cron_secret')) + )$$ +); + +-- --------------------------------------------------------------------------- +-- Schedule: Sponsor Outreach — Mon/Thu at 09:00 UTC +-- --------------------------------------------------------------------------- +SELECT cron.schedule( + 'sponsor-outreach', + '0 9 * * 1,4', + $$SELECT net.http_get( + url := current_setting('app.site_url') || '/api/cron/sponsor-outreach', + headers := jsonb_build_object('Authorization', 'Bearer ' || current_setting('app.cron_secret')) + )$$ +); From 61a25d81c36feddd12fee3cf94e51c1b86e10e7c Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 03:10:23 +0000 Subject: [PATCH 5/7] feat: add check-research cron route for polling NotebookLM New cron route that polls docs in "researching" status: - Queries Sanity for docs with researchNotebookId - Polls NotebookLM research status - On completion: imports sources, generates infographics, gets summary, re-generates enriched Gemini script, runs critic - Updates Sanity doc to "script_ready" with enriched data - Stuck detection: flags docs >30min in "researching" - Follows check-renders patterns (auth, fetchCache, maxDuration) Co-authored-by: research --- app/api/cron/check-research/route.ts | 690 +++++++++++++++++++++++++++ 1 file changed, 690 insertions(+) create mode 100644 app/api/cron/check-research/route.ts diff --git a/app/api/cron/check-research/route.ts b/app/api/cron/check-research/route.ts new file mode 100644 index 00000000..98a9113f --- /dev/null +++ b/app/api/cron/check-research/route.ts @@ -0,0 +1,690 @@ +export const fetchCache = 'force-no-store'; +export const maxDuration = 60; + +import { type NextRequest } from 'next/server'; +import { createClient, type SanityClient } from 'next-sanity'; +import { apiVersion, dataset, projectId } from '@/sanity/lib/api'; +import { NotebookLMClient } from '@/lib/services/notebooklm/client'; +import { initAuth } from '@/lib/services/notebooklm/auth'; +import { sleep } from '@/lib/services/notebooklm/rpc'; +import { generateWithGemini, stripCodeFences } from '@/lib/gemini'; +import type { ResearchPayload } from '@/lib/services/research'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface ResearchingDoc { + _id: string; + title: string; + status: string; + researchNotebookId: string; + researchTaskId?: string; + trendScore?: number; + trendSources?: string; + script?: { + hook: string; + scenes: Array<{ + _key: string; + sceneNumber: number; + sceneType: string; + narration: string; + visualDescription: string; + bRollKeywords: string[]; + durationEstimate: number; + code?: { snippet: string; language: string; highlightLines?: number[] }; + list?: { items: string[]; icon?: string }; + comparison?: { + leftLabel: string; + rightLabel: string; + rows: { left: string; right: string }[]; + }; + mockup?: { deviceType: string; screenContent: string }; + }>; + cta: string; + }; + _createdAt: string; +} + +interface EnrichedScript { + title: string; + summary: string; + sourceUrl: string; + topics: string[]; + script: { + hook: string; + scenes: Array<{ + sceneNumber: number; + sceneType: string; + narration: string; + visualDescription: string; + bRollKeywords: string[]; + durationEstimate: number; + code?: { snippet: string; language: string; highlightLines?: number[] }; + list?: { items: string[]; icon?: string }; + comparison?: { + leftLabel: string; + rightLabel: string; + rows: { left: string; right: string }[]; + }; + mockup?: { deviceType: string; screenContent: string }; + }>; + cta: string; + }; + qualityScore: number; +} + +interface CriticResult { + score: number; + issues: string[]; + summary: string; +} + +interface ProcessResult { + id: string; + title: string; + status: 'script_ready' | 'researching' | 'flagged' | 'error'; + error?: string; +} + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** Docs stuck in "researching" longer than this are flagged (ms) */ +const STUCK_THRESHOLD_MS = 30 * 60 * 1000; // 30 minutes + +/** Max infographic wait time per artifact (ms) */ +const INFOGRAPHIC_WAIT_MS = 120_000; // 2 minutes + +/** Infographic instructions for visual variety */ +const INFOGRAPHIC_INSTRUCTIONS = [ + 'Create a high-level architecture overview diagram', + 'Create a comparison chart of key features and alternatives', + 'Create a step-by-step workflow diagram', + 'Create a timeline of key developments and milestones', + 'Create a pros and cons visual summary', +]; + +// --------------------------------------------------------------------------- +// Sanity Write Client +// --------------------------------------------------------------------------- + +function getSanityWriteClient(): SanityClient { + const token = process.env.SANITY_API_TOKEN || process.env.SANITY_API_WRITE_TOKEN; + if (!token) { + throw new Error('[check-research] Missing SANITY_API_TOKEN environment variable'); + } + return createClient({ projectId, dataset, apiVersion, token, useCdn: false }); +} + +// --------------------------------------------------------------------------- +// Gemini Script Enrichment +// --------------------------------------------------------------------------- + +const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. + +Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: +- Start with a BOLD claim or surprising fact that makes people stop scrolling +- Use analogies and real-world comparisons to explain technical concepts +- Build tension: "Here's the problem... here's why it matters... here's the breakthrough" +- Keep energy HIGH — short sentences, active voice, conversational tone +- End with a clear takeaway that makes the viewer feel smarter +- Target audience: developers who want to stay current but don't have time to read everything + +Script format: 60-90 second explainer videos. Think TikTok/YouTube Shorts energy with real educational depth. + +CodingCat.dev covers: React, Next.js, TypeScript, Svelte, web APIs, CSS, Node.js, cloud services, AI/ML for developers, and web platform updates.`; + +function buildEnrichmentPrompt( + doc: ResearchingDoc, + research: ResearchPayload, +): string { + const existingScript = doc.script + ? JSON.stringify(doc.script, null, 2) + : 'No existing script'; + + let researchContext = ''; + researchContext += `### Briefing\n${research.briefing}\n\n`; + + if (research.talkingPoints.length > 0) { + researchContext += `### Key Talking Points\n${research.talkingPoints.map((tp, i) => `${i + 1}. ${tp}`).join('\n')}\n\n`; + } + + if (research.codeExamples.length > 0) { + researchContext += `### Code Examples (use these in "code" scenes)\n`; + for (const ex of research.codeExamples.slice(0, 5)) { + researchContext += `\`\`\`${ex.language}\n${ex.snippet}\n\`\`\`\nContext: ${ex.context}\n\n`; + } + } + + if (research.comparisonData && research.comparisonData.length > 0) { + researchContext += `### Comparison Data (use in "comparison" scenes)\n`; + for (const comp of research.comparisonData) { + researchContext += `${comp.leftLabel} vs ${comp.rightLabel}:\n`; + for (const row of comp.rows) { + researchContext += ` - ${row.left} | ${row.right}\n`; + } + researchContext += '\n'; + } + } + + if (research.sceneHints.length > 0) { + researchContext += `### Scene Type Suggestions\n`; + for (const hint of research.sceneHints) { + researchContext += `- ${hint.suggestedSceneType}: ${hint.reason}\n`; + } + } + + if (research.infographicUrls && research.infographicUrls.length > 0) { + researchContext += `\n### Infographics Available (${research.infographicUrls.length})\nMultiple infographics have been generated for this topic. Use sceneType "narration" with bRollUrl pointing to an infographic for visual scenes.\n`; + } + + return `You have an existing video script for "${doc.title}" and new deep research data. +Re-write the script to be MORE accurate, MORE insightful, and MORE engaging using the research. + +## Existing Script +${existingScript} + +## Research Data (use this to create an informed, accurate script) +${researchContext} + +Re-generate the complete video script as JSON. Keep the same format but enrich it with research insights. + +## Scene Types +Each scene MUST have a "sceneType" that determines its visual treatment: +- **"code"** — code snippets, API usage, config files. Provide actual code in the "code" field. +- **"list"** — enumerated content: "Top 5 features", key takeaways. Provide items in the "list" field. +- **"comparison"** — A-vs-B content. Provide structured data in the "comparison" field. +- **"mockup"** — UI, website, app screen, or terminal output. Provide device type and content in the "mockup" field. +- **"narration"** — conceptual explanations, introductions, or transitions. Default/fallback. + +## JSON Schema +Return ONLY a JSON object: +{ + "title": "string - catchy video title", + "summary": "string - 1-2 sentence summary", + "sourceUrl": "string - URL of the primary source", + "topics": ["string array of relevant tags"], + "script": { + "hook": "string - attention-grabbing opening line (5-10 seconds)", + "scenes": [ + { + "sceneNumber": 1, + "sceneType": "code | list | comparison | mockup | narration", + "narration": "string - what the narrator says", + "visualDescription": "string - what to show on screen", + "bRollKeywords": ["keyword1", "keyword2"], + "durationEstimate": 15, + "code": { "snippet": "string", "language": "string", "highlightLines": [1, 3] }, + "list": { "items": ["Item 1", "Item 2"], "icon": "🚀" }, + "comparison": { "leftLabel": "A", "rightLabel": "B", "rows": [{ "left": "...", "right": "..." }] }, + "mockup": { "deviceType": "browser | phone | terminal", "screenContent": "..." } + } + ], + "cta": "string - call to action" + }, + "qualityScore": 75 +} + +Requirements: +- 3-5 scenes totaling 60-90 seconds +- Use at least 2 different scene types +- Include REAL code snippets from the research where applicable +- The qualityScore should be your honest self-assessment (0-100) +- Return ONLY the JSON object, no markdown or extra text`; +} + +// --------------------------------------------------------------------------- +// Claude Critic (same as ingest route) +// --------------------------------------------------------------------------- + +async function claudeCritic(script: EnrichedScript): Promise { + const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY; + if (!ANTHROPIC_API_KEY) { + return { score: script.qualityScore, issues: [], summary: 'No critic available' }; + } + + try { + const res = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': ANTHROPIC_API_KEY, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model: 'claude-sonnet-4-20250514', + max_tokens: 1024, + messages: [ + { + role: 'user', + content: `You are a quality reviewer for short-form educational video scripts about web development. + +Review this video script and provide a JSON response with: +- "score": number 0-100 (overall quality rating) +- "issues": string[] (list of specific problems, if any) +- "summary": string (brief overall assessment) + +Evaluate based on: +1. Educational value — does it teach something useful? +2. Engagement — is the hook compelling? Is the pacing good? +3. Accuracy — are there any technical inaccuracies? +4. Clarity — is the narration clear and concise? +5. Visual direction — are the visual descriptions actionable? + +Script to review: +${JSON.stringify(script, null, 2)} + +Respond with ONLY the JSON object.`, + }, + ], + }), + }); + + if (!res.ok) { + console.warn(`[check-research] Claude critic failed: ${res.status}`); + return { score: script.qualityScore, issues: [], summary: 'Critic API error' }; + } + + const data = (await res.json()) as { content?: Array<{ text?: string }> }; + const text = data.content?.[0]?.text ?? '{}'; + + const jsonMatch = text.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + return { score: script.qualityScore, issues: [], summary: 'Could not parse critic response' }; + } + + const parsed = JSON.parse(jsonMatch[0]) as CriticResult; + return { + score: typeof parsed.score === 'number' ? parsed.score : script.qualityScore, + issues: Array.isArray(parsed.issues) ? parsed.issues : [], + summary: typeof parsed.summary === 'string' ? parsed.summary : 'No summary', + }; + } catch (err) { + console.warn('[check-research] Claude critic error:', err); + return { score: script.qualityScore, issues: [], summary: 'Critic error' }; + } +} + +// --------------------------------------------------------------------------- +// Research payload builder (mirrors research.ts logic) +// --------------------------------------------------------------------------- + +function extractTalkingPoints(text: string): string[] { + const lines = text.split('\n'); + const points: string[] = []; + for (const line of lines) { + const cleaned = line.replace(/^[\s]*[-•*\d]+[.)]\s*/, '').trim(); + if (cleaned.length > 20) { + points.push(cleaned); + } + } + return points.slice(0, 8); +} + +function extractCodeExamples(text: string): Array<{ snippet: string; language: string; context: string }> { + const examples: Array<{ snippet: string; language: string; context: string }> = []; + const codeBlockRegex = /```(\w*)\n([\s\S]*?)```/g; + let match: RegExpExecArray | null; + + while ((match = codeBlockRegex.exec(text)) !== null) { + const language = match[1] || 'typescript'; + const snippet = match[2].trim(); + const beforeBlock = text.slice(0, match.index); + const contextLines = beforeBlock.split('\n').filter((l) => l.trim()); + const context = + contextLines.length > 0 + ? contextLines[contextLines.length - 1].trim() + : 'Code example'; + examples.push({ snippet, language, context }); + } + + return examples; +} + +function classifyScene( + content: string, +): 'narration' | 'code' | 'list' | 'comparison' | 'mockup' { + if ( + /```[\s\S]*?```/.test(content) || + /^\s{2,}(const|let|var|function|import|export|class|def|return)\b/m.test(content) + ) { + return 'code'; + } + const listMatches = content.match(/^[\s]*[-•*\d]+[.)]\s/gm); + if (listMatches && listMatches.length >= 3) { + return 'list'; + } + if ( + /\bvs\.?\b/i.test(content) || + /\bcompare[ds]?\b/i.test(content) || + /\bdifference[s]?\b/i.test(content) || + /\bpros\s+(and|&)\s+cons\b/i.test(content) + ) { + return 'comparison'; + } + if ( + /\b(UI|interface|dashboard|screen|layout|component|widget|button|modal)\b/i.test(content) + ) { + return 'mockup'; + } + return 'narration'; +} + +function classifySourceType(url: string): 'youtube' | 'article' | 'docs' | 'unknown' { + if (!url) return 'unknown'; + const lower = url.toLowerCase(); + if (lower.includes('youtube.com') || lower.includes('youtu.be')) return 'youtube'; + if (lower.includes('/docs') || lower.includes('documentation') || lower.includes('developer.') || lower.includes('mdn')) return 'docs'; + if (lower.includes('blog') || lower.includes('medium.com') || lower.includes('dev.to') || lower.includes('hashnode')) return 'article'; + return 'unknown'; +} + +function buildResearchPayload( + doc: ResearchingDoc, + briefing: string, + sources: Array<{ url: string; title: string }>, + infographicUrls: string[], +): ResearchPayload { + const talkingPoints = extractTalkingPoints(briefing); + const codeExamples = extractCodeExamples(briefing); + + const sections = briefing + .split(/\n(?=#{1,3}\s)|\n\n/) + .filter((s) => s.trim().length > 50); + const sceneHints = sections.map((section) => ({ + content: section.slice(0, 500), + suggestedSceneType: classifyScene(section), + reason: `Classified from research content`, + })); + + return { + topic: doc.title, + notebookId: doc.researchNotebookId, + createdAt: doc._createdAt, + completedAt: new Date().toISOString(), + sources: sources.map((s) => ({ + title: s.title, + url: s.url, + type: classifySourceType(s.url), + })), + briefing, + talkingPoints, + codeExamples, + sceneHints, + infographicUrls: infographicUrls.length > 0 ? infographicUrls : undefined, + }; +} + +// --------------------------------------------------------------------------- +// Safe step wrapper +// --------------------------------------------------------------------------- + +async function safeStep( + label: string, + fn: () => Promise, +): Promise { + try { + return await fn(); + } catch (error) { + console.error( + `[check-research] Step "${label}" failed:`, + error instanceof Error ? error.message : error, + ); + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Process a single researching doc +// --------------------------------------------------------------------------- + +async function processResearchingDoc( + doc: ResearchingDoc, + nbClient: NotebookLMClient, + sanityClient: SanityClient, +): Promise { + const notebookId = doc.researchNotebookId; + console.log(`[check-research] Processing doc ${doc._id} ("${doc.title}") — notebook: ${notebookId}`); + + // Step 1: Poll research status + console.log(`[check-research] Polling research status for notebook ${notebookId}...`); + const pollResult = await nbClient.pollResearch(notebookId); + console.log(`[check-research] Research status: ${pollResult.status} (${pollResult.sources.length} sources)`); + + if (pollResult.status === 'in_progress') { + console.log(`[check-research] Research still in progress for "${doc.title}" — will retry next run`); + return { id: doc._id, title: doc.title, status: 'researching' }; + } + + if (pollResult.status === 'no_research') { + console.warn(`[check-research] No research found for "${doc.title}" — marking as script_ready with existing script`); + await sanityClient.patch(doc._id).set({ status: 'script_ready' }).commit(); + return { id: doc._id, title: doc.title, status: 'script_ready' }; + } + + // Research is completed — proceed with enrichment pipeline + console.log(`[check-research] Research completed for "${doc.title}" — starting enrichment pipeline`); + + const researchTaskId = pollResult.taskId || doc.researchTaskId || ''; + const researchSources = pollResult.sources; + + // Step 2: Import research sources + if (researchSources.length > 0 && researchTaskId) { + console.log(`[check-research] Importing ${researchSources.length} research sources...`); + await safeStep('import-sources', () => + nbClient.importResearchSources(notebookId, researchTaskId, researchSources), + ); + } + + // Step 3: Generate infographics + console.log(`[check-research] Generating ${INFOGRAPHIC_INSTRUCTIONS.length} infographics...`); + const infographicTaskIds: string[] = []; + + for (const instruction of INFOGRAPHIC_INSTRUCTIONS) { + const result = await safeStep(`generate-infographic`, () => + nbClient.generateInfographic(notebookId, { + instructions: instruction, + language: 'en', + orientation: 1, // landscape + detailLevel: 2, // detailed + }), + ); + if (result?.taskId) { + infographicTaskIds.push(result.taskId); + } + } + + console.log(`[check-research] Started ${infographicTaskIds.length} infographic generations`); + + // Step 4: Wait for infographics to complete + if (infographicTaskIds.length > 0) { + console.log(`[check-research] Waiting for infographics to complete...`); + const waitPromises = infographicTaskIds.map((taskId) => + safeStep(`wait-infographic-${taskId.substring(0, 8)}`, () => + nbClient.waitForArtifact(notebookId, taskId, { + timeoutMs: INFOGRAPHIC_WAIT_MS, + pollIntervalMs: 10_000, + }), + ), + ); + await Promise.allSettled(waitPromises); + } + + // Step 5: Get notebook summary + console.log(`[check-research] Getting notebook summary...`); + const briefing = (await safeStep('get-summary', () => nbClient.getSummary(notebookId))) ?? ''; + + // Step 6: Collect infographic URLs + console.log(`[check-research] Collecting infographic URLs...`); + const infographicUrls: string[] = []; + for (const taskId of infographicTaskIds) { + const url = await safeStep(`get-infographic-url-${taskId.substring(0, 8)}`, () => + nbClient.getInfographicUrl(notebookId, taskId), + ); + if (url) { + infographicUrls.push(url); + } + } + console.log(`[check-research] Found ${infographicUrls.length} infographic URLs`); + + // Step 7: Build research payload + const researchPayload = buildResearchPayload(doc, briefing, researchSources, infographicUrls); + + // Step 8: Re-generate enriched script with Gemini + console.log(`[check-research] Re-generating enriched script with Gemini...`); + let enrichedScript: EnrichedScript | null = null; + + try { + const prompt = buildEnrichmentPrompt(doc, researchPayload); + const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION); + const cleaned = stripCodeFences(rawResponse); + enrichedScript = JSON.parse(cleaned) as EnrichedScript; + console.log(`[check-research] Enriched script generated: "${enrichedScript.title}"`); + } catch (err) { + console.error('[check-research] Failed to generate enriched script:', err); + // Fall through — we'll use the existing script + } + + // Step 9: Run critic pass + let criticScore = 0; + if (enrichedScript) { + console.log(`[check-research] Running critic pass...`); + const criticResult = await claudeCritic(enrichedScript); + criticScore = criticResult.score; + console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`); + + const isFlagged = criticScore < 50; + + // Step 10: Update Sanity doc with enriched data + console.log(`[check-research] Updating Sanity doc ${doc._id}...`); + await sanityClient + .patch(doc._id) + .set({ + script: { + ...enrichedScript.script, + scenes: enrichedScript.script.scenes.map((scene, i) => ({ + ...scene, + _key: `scene-${i + 1}`, + })), + }, + scriptQualityScore: criticScore, + status: isFlagged ? 'flagged' : 'script_ready', + researchData: JSON.stringify(researchPayload), + ...(isFlagged && { + flaggedReason: `Quality score ${criticScore}/100. Issues: ${(criticResult.issues ?? []).join('; ') || 'Low quality score'}`, + }), + }) + .commit(); + + console.log(`[check-research] Doc ${doc._id} updated to "${isFlagged ? 'flagged' : 'script_ready'}"`); + return { id: doc._id, title: doc.title, status: isFlagged ? 'flagged' : 'script_ready' }; + } + + // Fallback: no enriched script, just transition to script_ready with existing script + console.warn(`[check-research] No enriched script — transitioning ${doc._id} to script_ready with existing script`); + await sanityClient + .patch(doc._id) + .set({ + status: 'script_ready', + researchData: JSON.stringify(researchPayload), + }) + .commit(); + + return { id: doc._id, title: doc.title, status: 'script_ready' }; +} + +// --------------------------------------------------------------------------- +// Route Handler +// --------------------------------------------------------------------------- + +export async function GET(request: NextRequest) { + const authHeader = request.headers.get('authorization'); + if (authHeader !== `Bearer ${process.env.CRON_SECRET}`) { + console.error('[check-research] Unauthorized request: invalid authorization header'); + return Response.json({ error: 'Unauthorized' }, { status: 401 }); + } + + try { + const sanity = getSanityWriteClient(); + + // Query for docs in "researching" status with a notebook ID + const docs = await sanity.fetch( + `*[_type == "automatedVideo" && status == "researching" && defined(researchNotebookId)] { + _id, + title, + status, + researchNotebookId, + researchTaskId, + trendScore, + trendSources, + script, + _createdAt + }`, + ); + + console.log(`[check-research] Found ${docs.length} docs in "researching" status`); + + if (docs.length === 0) { + return Response.json({ success: true, message: 'No docs to process', processed: 0 }); + } + + // Initialize NotebookLM client + console.log('[check-research] Initializing NotebookLM client...'); + const auth = await initAuth(); + const nbClient = new NotebookLMClient(auth); + + const results: ProcessResult[] = []; + const now = Date.now(); + + for (const doc of docs) { + // Stuck detection: flag docs stuck in "researching" for >30 minutes + const docAge = now - new Date(doc._createdAt).getTime(); + if (docAge > STUCK_THRESHOLD_MS) { + console.warn( + `[check-research] Doc ${doc._id} ("${doc.title}") stuck in "researching" for ${Math.round(docAge / 60_000)}min — flagging`, + ); + await sanity + .patch(doc._id) + .set({ + status: 'flagged', + flaggedReason: `Stuck in "researching" for ${Math.round(docAge / 60_000)} minutes (threshold: ${STUCK_THRESHOLD_MS / 60_000}min). Research may have failed.`, + }) + .commit(); + results.push({ id: doc._id, title: doc.title, status: 'flagged', error: 'Stuck in researching' }); + continue; + } + + try { + const result = await processResearchingDoc(doc, nbClient, sanity); + results.push(result); + } catch (err) { + console.error(`[check-research] Error processing doc ${doc._id}:`, err); + results.push({ + id: doc._id, + title: doc.title, + status: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } + } + + console.log('[check-research] Done!', JSON.stringify(results)); + + return Response.json({ + success: true, + processed: results.length, + results, + }); + } catch (err) { + console.error('[check-research] Unexpected error:', err); + return Response.json( + { + success: false, + error: err instanceof Error ? err.message : String(err), + }, + { status: 500 }, + ); + } +} From 7e0f911c5fc8b538bcbf6de60bcb17acca45b6a8 Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 03:11:39 +0000 Subject: [PATCH 6/7] refactor: make ingest fire-and-forget for NotebookLM research MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When ENABLE_NOTEBOOKLM_RESEARCH is set: - Creates notebook + adds sources + starts research (~10s) - Does NOT poll for completion (was blocking 10+ min) - Generates basic script without research data - Creates Sanity doc with status "researching" - Stores researchNotebookId + researchTaskId on doc - check-research cron will poll and enrich later When research is NOT enabled: - Behavior unchanged — straight to "script_ready" Removes conductResearch() import (blocking call). Co-authored-by: research --- app/api/cron/ingest/route.ts | 61 +++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/app/api/cron/ingest/route.ts b/app/api/cron/ingest/route.ts index 297c4932..738c52b0 100644 --- a/app/api/cron/ingest/route.ts +++ b/app/api/cron/ingest/route.ts @@ -5,7 +5,9 @@ import type { NextRequest } from "next/server"; import { generateWithGemini, stripCodeFences } from "@/lib/gemini"; import { writeClient } from "@/lib/sanity-write-client"; import { discoverTrends, type TrendResult } from "@/lib/services/trend-discovery"; -import { conductResearch, type ResearchPayload } from "@/lib/services/research"; +import type { ResearchPayload } from "@/lib/services/research"; +import { NotebookLMClient } from "@/lib/services/notebooklm/client"; +import { initAuth } from "@/lib/services/notebooklm/auth"; // --------------------------------------------------------------------------- // Types @@ -329,8 +331,12 @@ async function createSanityDocuments( criticResult: CriticResult, trends: TrendResult[], research?: ResearchPayload, + researchMeta?: { notebookId: string; taskId: string }, ) { const isFlagged = criticResult.score < 50; + // When research is in-flight, status is "researching" (check-research cron will transition to script_ready) + const isResearching = !!researchMeta?.notebookId; + const status = isFlagged ? "flagged" : isResearching ? "researching" : "script_ready"; const contentIdea = await writeClient.create({ _type: "contentIdea", @@ -360,13 +366,14 @@ async function createSanityDocuments( })), }, scriptQualityScore: criticResult.score, - status: isFlagged ? "flagged" : "script_ready", + status, ...(isFlagged && { flaggedReason: `Quality score ${criticResult.score}/100. Issues: ${criticResult.issues.join("; ") || "Low quality score"}`, }), trendScore: trends[0]?.score, trendSources: trends[0]?.signals.map(s => s.source).join(", "), - researchNotebookId: research?.notebookId, + researchNotebookId: researchMeta?.notebookId ?? research?.notebookId, + ...(researchMeta?.taskId && { researchTaskId: researchMeta.taskId }), }); console.log(`[CRON/ingest] Created automatedVideo: ${automatedVideo._id}`); @@ -374,7 +381,7 @@ async function createSanityDocuments( return { contentIdeaId: contentIdea._id, automatedVideoId: automatedVideo._id, - status: isFlagged ? "flagged" : "script_ready", + status, }; } @@ -409,26 +416,49 @@ export async function GET(request: NextRequest) { trends = FALLBACK_TRENDS; } - // Step 2: Optional deep research on top topic - let research: ResearchPayload | undefined; + // Step 2: Optional deep research on top topic (fire-and-forget) + // When research is enabled, we create a notebook and start research + // but DON'T wait for it — the check-research cron will poll and enrich later + let researchMeta: { notebookId: string; taskId: string } | undefined; if (process.env.ENABLE_NOTEBOOKLM_RESEARCH === "true") { - console.log(`[CRON/ingest] Conducting research on: "${trends[0].topic}"...`); + console.log(`[CRON/ingest] Starting fire-and-forget research on: "${trends[0].topic}"...`); try { - // Extract source URLs from trend signals to seed the notebook + const auth = await initAuth(); + const nbClient = new NotebookLMClient(auth); + + // Create notebook + const notebook = await nbClient.createNotebook(trends[0].topic); + const notebookId = notebook.id; + console.log(`[CRON/ingest] Created notebook: ${notebookId}`); + + // Add source URLs from trend signals const sourceUrls = (trends[0].signals ?? []) .map((s: { url?: string }) => s.url) .filter((u): u is string => !!u && u.startsWith("http")) .slice(0, 5); - research = await conductResearch(trends[0].topic, { sourceUrls }); - console.log(`[CRON/ingest] Research complete: ${research.sources.length} sources, ${research.sceneHints.length} scene hints`); + + for (const url of sourceUrls) { + await nbClient.addSource(notebookId, url).catch((err) => { + console.warn(`[CRON/ingest] Failed to add source ${url}:`, err); + }); + } + console.log(`[CRON/ingest] Added ${sourceUrls.length} source URLs to notebook`); + + // Start deep research (fire-and-forget — don't poll!) + const researchTask = await nbClient.startResearch(notebookId, trends[0].topic, "deep"); + const researchTaskId = researchTask?.taskId ?? ""; + console.log(`[CRON/ingest] Research started — taskId: ${researchTaskId}. check-research cron will poll.`); + + researchMeta = { notebookId, taskId: researchTaskId }; } catch (err) { - console.warn("[CRON/ingest] Research failed, continuing without:", err); + console.warn("[CRON/ingest] Research start failed, continuing without:", err); } } - // Step 3: Generate script with Gemini (enriched with research) + // Step 3: Generate script with Gemini (basic — without research data) + // When research is enabled, check-research will re-generate an enriched script later console.log("[CRON/ingest] Generating script with Gemini..."); - const prompt = buildPrompt(trends, research); + const prompt = buildPrompt(trends); const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION); let script: GeneratedScript; @@ -459,7 +489,7 @@ export async function GET(request: NextRequest) { ); console.log("[CRON/ingest] Creating Sanity documents..."); - const result = await createSanityDocuments(script, criticResult, trends, research); + const result = await createSanityDocuments(script, criticResult, trends, undefined, researchMeta); console.log("[CRON/ingest] Done!", result); @@ -470,7 +500,8 @@ export async function GET(request: NextRequest) { criticScore: criticResult.score, trendCount: trends.length, trendScore: trends[0]?.score, - researchEnabled: !!research, + researchStarted: !!researchMeta, + researchNotebookId: researchMeta?.notebookId, }); } catch (err) { console.error("[CRON/ingest] Unexpected error:", err); From 9faf8fb200db65483f95aa02fe69a72fc42cb4c6 Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 03:15:21 +0000 Subject: [PATCH 7/7] feat: add "researching" status + research fields to automatedVideo schema Adds: - "researching" status option (between draft and script_ready) - researchNotebookId field (hidden, stores NotebookLM notebook UUID) - researchTaskId field (hidden, stores deep research task UUID) - trendScore field (0-100 from trend discovery) - trendSources field (comma-separated signal sources) Co-authored-by: research --- sanity/schemas/documents/automatedVideo.ts | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sanity/schemas/documents/automatedVideo.ts b/sanity/schemas/documents/automatedVideo.ts index 9a3181c0..d920ee9e 100644 --- a/sanity/schemas/documents/automatedVideo.ts +++ b/sanity/schemas/documents/automatedVideo.ts @@ -227,6 +227,7 @@ export default defineType({ options: { list: [ {title: '1 - Draft', value: 'draft'}, + {title: '1.5 - Researching', value: 'researching'}, {title: '2 - Script Ready', value: 'script_ready'}, {title: '3 - Audio Generation', value: 'audio_gen'}, {title: '4 - Rendering', value: 'rendering'}, @@ -318,6 +319,32 @@ export default defineType({ type: 'reference', to: [{type: 'sponsorLead'}], }), + defineField({ + name: 'researchNotebookId', + title: 'NotebookLM Notebook ID', + type: 'string', + description: 'UUID of the NotebookLM notebook used for research', + hidden: true, + }), + defineField({ + name: 'researchTaskId', + title: 'Research Task ID', + type: 'string', + description: 'UUID of the NotebookLM deep research task', + hidden: true, + }), + defineField({ + name: 'trendScore', + title: 'Trend Score', + type: 'number', + description: 'Score from trend discovery (0-100)', + }), + defineField({ + name: 'trendSources', + title: 'Trend Sources', + type: 'string', + description: 'Comma-separated list of trend signal sources', + }), defineField({ name: 'flaggedReason', title: 'Flagged Reason',