Skip to content

feat: migrate cron routes to Sanity config singletons (Phase B) #613

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
codercatdev merged 1 commit into from
Mar 5, 2026
Merged

feat: migrate cron routes to Sanity config singletons (Phase B) #613

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 24 additions & 10 deletions app/api/cron/check-research/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { NotebookLMClient } from '@/lib/services/notebooklm/client';
import { initAuth } from '@/lib/services/notebooklm/auth';
import { ArtifactTypeCode, ArtifactStatus } from '@/lib/services/notebooklm/types';
import { generateWithGemini, stripCodeFences } from '@/lib/gemini';
import { getConfigValue } from '@/lib/config';
import type { ResearchPayload } from '@/lib/services/research';

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -94,12 +95,15 @@ interface StepResult {
// Constants
// ---------------------------------------------------------------------------

/** Stuck thresholds per status (ms) */
const STUCK_THRESHOLDS: Record<string, number> = {
researching: 30 * 60 * 1000, // 30 minutes
infographics_generating: 15 * 60 * 1000, // 15 minutes
enriching: 10 * 60 * 1000, // 10 minutes
};
/** Build stuck thresholds from config (with fallbacks) */
async function buildStuckThresholds(): Promise<Record<string, number>> {
const stuckMinutes = await getConfigValue('pipeline_config', 'stuckTimeoutMinutes', 30);
return {
researching: stuckMinutes * 60 * 1000,
infographics_generating: Math.round(stuckMinutes * 0.5) * 60 * 1000, // half the main timeout
enriching: Math.round(stuckMinutes * 0.33) * 60 * 1000, // third of main timeout
};
}

/** Max docs to process per status per run — keeps total time well under 60s */
const MAX_DOCS_PER_STATUS = 2;
Expand Down Expand Up @@ -132,12 +136,13 @@ function getSanityWriteClient(): SanityClient {
async function flagStuckDocs(
docs: PipelineDoc[],
sanity: SanityClient,
stuckThresholds: Record<string, number>,
): Promise<StepResult[]> {
const results: StepResult[] = [];
const now = Date.now();

for (const doc of docs) {
const threshold = STUCK_THRESHOLDS[doc.status];
const threshold = stuckThresholds[doc.status];
if (!threshold) continue;

const docAge = now - new Date(doc._updatedAt).getTime();
Expand Down Expand Up @@ -419,6 +424,11 @@ async function stepEnriching(
// Generate enriched script with Gemini
let enrichedScript: EnrichedScript | null = null;
try {
const SYSTEM_INSTRUCTION = await getConfigValue(
'content_config',
'systemInstruction',
SYSTEM_INSTRUCTION_FALLBACK,
);
const prompt = buildEnrichmentPrompt(doc, researchPayload);
const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION);
const cleaned = stripCodeFences(rawResponse);
Expand All @@ -434,7 +444,8 @@ async function stepEnriching(
const criticScore = criticResult.score;
console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`);

const isFlagged = criticScore < 50;
const qualityThreshold = await getConfigValue('pipeline_config', 'qualityThreshold', 50);
const isFlagged = criticScore < qualityThreshold;

await sanity
.patch(doc._id)
Expand Down Expand Up @@ -481,7 +492,9 @@ async function stepEnriching(
// Gemini Script Enrichment
// ---------------------------------------------------------------------------

const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.
// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity.
// The live value is fetched from getConfigValue() inside stepEnriching().
const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.

Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles:
- Start with a BOLD claim or surprising fact that makes people stop scrolling
Expand Down Expand Up @@ -813,7 +826,8 @@ export async function GET(request: NextRequest) {
const results: StepResult[] = [];

// Phase 1: Stuck detection — runs FIRST, no external API calls
const stuckResults = await flagStuckDocs(docs, sanity);
const stuckThresholds = await buildStuckThresholds();
const stuckResults = await flagStuckDocs(docs, sanity, stuckThresholds);
results.push(...stuckResults);

// Remove flagged docs from further processing
Expand Down
31 changes: 26 additions & 5 deletions app/api/cron/ingest/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type { NextRequest } from "next/server";

import { generateWithGemini, stripCodeFences } from "@/lib/gemini";
import { writeClient } from "@/lib/sanity-write-client";
import { getConfigValue } from "@/lib/config";
import { discoverTrends, type TrendResult } from "@/lib/services/trend-discovery";
import type { ResearchPayload } from "@/lib/services/research";
import { NotebookLMClient } from "@/lib/services/notebooklm/client";
Expand Down Expand Up @@ -102,7 +103,7 @@ const FALLBACK_TRENDS: TrendResult[] = [
slug: "webassembly-web-apps",
score: 60,
signals: [{ source: "blog", title: "WebAssembly", url: "https://webassembly.org/", score: 60 }],
whyTrending: "WASM adoption growing in [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] apps",
whyTrending: "WASM adoption growing in production apps",
suggestedAngle: "Real-world use cases where WASM outperforms JS",
},
];
Expand All @@ -111,7 +112,9 @@ const FALLBACK_TRENDS: TrendResult[] = [
// Gemini Script Generation
// ---------------------------------------------------------------------------

const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.
// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity.
// The live value is fetched from getConfigValue() inside the GET handler.
const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.

Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles:
- Start with a BOLD claim or surprising fact that makes people stop scrolling
Expand Down Expand Up @@ -330,10 +333,11 @@ async function createSanityDocuments(
script: GeneratedScript,
criticResult: CriticResult,
trends: TrendResult[],
qualityThreshold: number,
research?: ResearchPayload,
researchMeta?: { notebookId: string; taskId: string },
) {
const isFlagged = criticResult.score < 50;
const isFlagged = criticResult.score < qualityThreshold;
// When research is in-flight, status is "researching" (check-research cron will transition to script_ready)
const isResearching = !!researchMeta?.notebookId;
const status = isFlagged ? "flagged" : isResearching ? "researching" : "script_ready";
Expand Down Expand Up @@ -404,6 +408,23 @@ export async function GET(request: NextRequest) {
}

try {
// Fetch config values once per invocation (5-min in-memory cache)
const SYSTEM_INSTRUCTION = await getConfigValue(
"content_config",
"systemInstruction",
SYSTEM_INSTRUCTION_FALLBACK,
);
const enableNotebookLmResearch = await getConfigValue(
"pipeline_config",
"enableNotebookLmResearch",
false,
);
const qualityThreshold = await getConfigValue(
"pipeline_config",
"qualityThreshold",
50,
);

// Step 1: Discover trending topics (replaces fetchTrendingTopics)
console.log("[CRON/ingest] Discovering trending topics...");
let trends: TrendResult[];
Expand All @@ -425,7 +446,7 @@ export async function GET(request: NextRequest) {
// When research is enabled, we create a notebook and start research
// but DON'T wait for it — the check-research cron will poll and enrich later
let researchMeta: { notebookId: string; taskId: string } | undefined;
if (process.env.ENABLE_NOTEBOOKLM_RESEARCH === "true") {
if (enableNotebookLmResearch) {
console.log(`[CRON/ingest] Starting fire-and-forget research on: "${trends[0].topic}"...`);
try {
const auth = await initAuth();
Expand Down Expand Up @@ -494,7 +515,7 @@ export async function GET(request: NextRequest) {
);

console.log("[CRON/ingest] Creating Sanity documents...");
const result = await createSanityDocuments(script, criticResult, trends, undefined, researchMeta);
const result = await createSanityDocuments(script, criticResult, trends, qualityThreshold, undefined, researchMeta);

console.log("[CRON/ingest] Done!", result);

Expand Down
Loading