From 4066caf9f7d2ada6e1d6a48f7319d5d261825b84 Mon Sep 17 00:00:00 2001 From: Miriad Date: Thu, 5 Mar 2026 06:05:43 +0000 Subject: [PATCH] feat: migrate cron routes to Sanity config singletons Ingest route: ENABLE_NOTEBOOKLM_RESEARCH, quality threshold, system instruction now read from pipeline_config and content_config singletons. Check-research route: stuck thresholds, quality threshold, system instruction now configurable via Sanity. Check-renders route: audited, no tweakable config (Remotion/ElevenLabs config is in service layer). All values use getConfigValue() with existing hardcoded values as fallbacks for graceful degradation. Co-authored-by: research --- app/api/cron/check-renders/route.ts | 13 +++++++---- app/api/cron/check-research/route.ts | 34 ++++++++++++++++++++-------- app/api/cron/ingest/route.ts | 29 ++++++++++++++++++++---- 3 files changed, 57 insertions(+), 19 deletions(-) diff --git a/app/api/cron/check-renders/route.ts b/app/api/cron/check-renders/route.ts index 5a66dd40..58883e6e 100644 --- a/app/api/cron/check-renders/route.ts +++ b/app/api/cron/check-renders/route.ts @@ -1,3 +1,6 @@ +// Config migration: audited — no tweakable config in this route. +// Remotion/ElevenLabs config is in the service layer (owned by @videopipe). +// YouTube SEO prompt is specific to this route, not the shared system instruction. export const fetchCache = 'force-no-store'; export const maxDuration = 60; @@ -253,21 +256,21 @@ async function handleScriptReady(client: SanityClient): Promise<{ claimed: numbe await client.patch(doc._id).set({ status: 'audio_gen' }).commit(); claimedIds.push(doc._id); - // WORK: run video production in background via after() + // WORK: run video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] in background via after() after(async () => { try { - console.log(`[PIPELINE] Starting video production for ${doc._id}`); + console.log(`[PIPELINE] Starting video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] for ${doc._id}`); await processVideoProduction(doc._id); - console.log(`[PIPELINE] ✅ Video production complete for ${doc._id}`); + console.log(`[PIPELINE] ✅ Video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] complete for ${doc._id}`); } catch (error) { const msg = error instanceof Error ? error.message : String(error); - console.error(`[PIPELINE] ❌ Video production failed for ${doc._id}: ${msg}`); + console.error(`[PIPELINE] ❌ Video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] failed for ${doc._id}: ${msg}`); // processVideoProduction already sets flagged on error, but just in case: try { const c = getSanityWriteClient(); await c.patch(doc._id).set({ status: 'flagged', - flaggedReason: `Video production error: ${msg}`, + flaggedReason: `Video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] error: ${msg}`, }).commit(); } catch { /* best-effort */ } } diff --git a/app/api/cron/check-research/route.ts b/app/api/cron/check-research/route.ts index 4bdfca26..205e7848 100644 --- a/app/api/cron/check-research/route.ts +++ b/app/api/cron/check-research/route.ts @@ -8,6 +8,7 @@ import { NotebookLMClient } from '@/lib/services/notebooklm/client'; import { initAuth } from '@/lib/services/notebooklm/auth'; import { ArtifactTypeCode, ArtifactStatus } from '@/lib/services/notebooklm/types'; import { generateWithGemini, stripCodeFences } from '@/lib/gemini'; +import { getConfigValue } from '@/lib/config'; import type { ResearchPayload } from '@/lib/services/research'; // --------------------------------------------------------------------------- @@ -94,12 +95,15 @@ interface StepResult { // Constants // --------------------------------------------------------------------------- -/** Stuck thresholds per status (ms) */ -const STUCK_THRESHOLDS: Record = { - researching: 30 * 60 * 1000, // 30 minutes - infographics_generating: 15 * 60 * 1000, // 15 minutes - enriching: 10 * 60 * 1000, // 10 minutes -}; +/** Build stuck thresholds from config (with fallbacks) */ +async function buildStuckThresholds(): Promise> { + const stuckMinutes = await getConfigValue('pipeline_config', 'stuckTimeoutMinutes', 30); + return { + researching: stuckMinutes * 60 * 1000, + infographics_generating: Math.round(stuckMinutes * 0.5) * 60 * 1000, // half the main timeout + enriching: Math.round(stuckMinutes * 0.33) * 60 * 1000, // third of main timeout + }; +} /** Max docs to process per status per run — keeps total time well under 60s */ const MAX_DOCS_PER_STATUS = 2; @@ -132,12 +136,13 @@ function getSanityWriteClient(): SanityClient { async function flagStuckDocs( docs: PipelineDoc[], sanity: SanityClient, + stuckThresholds: Record, ): Promise { const results: StepResult[] = []; const now = Date.now(); for (const doc of docs) { - const threshold = STUCK_THRESHOLDS[doc.status]; + const threshold = stuckThresholds[doc.status]; if (!threshold) continue; const docAge = now - new Date(doc._updatedAt).getTime(); @@ -419,6 +424,11 @@ async function stepEnriching( // Generate enriched script with Gemini let enrichedScript: EnrichedScript | null = null; try { + const SYSTEM_INSTRUCTION = await getConfigValue( + 'content_config', + 'systemInstruction', + SYSTEM_INSTRUCTION_FALLBACK, + ); const prompt = buildEnrichmentPrompt(doc, researchPayload); const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION); const cleaned = stripCodeFences(rawResponse); @@ -434,7 +444,8 @@ async function stepEnriching( const criticScore = criticResult.score; console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`); - const isFlagged = criticScore < 50; + const qualityThreshold = await getConfigValue('pipeline_config', 'qualityThreshold', 50); + const isFlagged = criticScore < qualityThreshold; await sanity .patch(doc._id) @@ -481,7 +492,9 @@ async function stepEnriching( // Gemini Script Enrichment // --------------------------------------------------------------------------- -const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. +// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity. +// The live value is fetched from getConfigValue() inside stepEnriching(). +const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: - Start with a BOLD claim or surprising fact that makes people stop scrolling @@ -813,7 +826,8 @@ export async function GET(request: NextRequest) { const results: StepResult[] = []; // Phase 1: Stuck detection — runs FIRST, no external API calls - const stuckResults = await flagStuckDocs(docs, sanity); + const stuckThresholds = await buildStuckThresholds(); + const stuckResults = await flagStuckDocs(docs, sanity, stuckThresholds); results.push(...stuckResults); // Remove flagged docs from further processing diff --git a/app/api/cron/ingest/route.ts b/app/api/cron/ingest/route.ts index a6480f30..5a1c4574 100644 --- a/app/api/cron/ingest/route.ts +++ b/app/api/cron/ingest/route.ts @@ -4,6 +4,7 @@ import type { NextRequest } from "next/server"; import { generateWithGemini, stripCodeFences } from "@/lib/gemini"; import { writeClient } from "@/lib/sanity-write-client"; +import { getConfigValue } from "@/lib/config"; import { discoverTrends, type TrendResult } from "@/lib/services/trend-discovery"; import type { ResearchPayload } from "@/lib/services/research"; import { NotebookLMClient } from "@/lib/services/notebooklm/client"; @@ -111,7 +112,9 @@ const FALLBACK_TRENDS: TrendResult[] = [ // Gemini Script Generation // --------------------------------------------------------------------------- -const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. +// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity. +// The live value is fetched from getConfigValue() inside the GET handler. +const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: - Start with a BOLD claim or surprising fact that makes people stop scrolling @@ -330,10 +333,11 @@ async function createSanityDocuments( script: GeneratedScript, criticResult: CriticResult, trends: TrendResult[], + qualityThreshold: number, research?: ResearchPayload, researchMeta?: { notebookId: string; taskId: string }, ) { - const isFlagged = criticResult.score < 50; + const isFlagged = criticResult.score < qualityThreshold; // When research is in-flight, status is "researching" (check-research cron will transition to script_ready) const isResearching = !!researchMeta?.notebookId; const status = isFlagged ? "flagged" : isResearching ? "researching" : "script_ready"; @@ -404,6 +408,23 @@ export async function GET(request: NextRequest) { } try { + // Fetch config values once per invocation (5-min in-memory cache) + const SYSTEM_INSTRUCTION = await getConfigValue( + "content_config", + "systemInstruction", + SYSTEM_INSTRUCTION_FALLBACK, + ); + const enableNotebookLmResearch = await getConfigValue( + "pipeline_config", + "enableNotebookLmResearch", + false, + ); + const qualityThreshold = await getConfigValue( + "pipeline_config", + "qualityThreshold", + 50, + ); + // Step 1: Discover trending topics (replaces fetchTrendingTopics) console.log("[CRON/ingest] Discovering trending topics..."); let trends: TrendResult[]; @@ -425,7 +446,7 @@ export async function GET(request: NextRequest) { // When research is enabled, we create a notebook and start research // but DON'T wait for it — the check-research cron will poll and enrich later let researchMeta: { notebookId: string; taskId: string } | undefined; - if (process.env.ENABLE_NOTEBOOKLM_RESEARCH === "true") { + if (enableNotebookLmResearch) { console.log(`[CRON/ingest] Starting fire-and-forget research on: "${trends[0].topic}"...`); try { const auth = await initAuth(); @@ -494,7 +515,7 @@ export async function GET(request: NextRequest) { ); console.log("[CRON/ingest] Creating Sanity documents..."); - const result = await createSanityDocuments(script, criticResult, trends, undefined, researchMeta); + const result = await createSanityDocuments(script, criticResult, trends, qualityThreshold, undefined, researchMeta); console.log("[CRON/ingest] Done!", result);