diff --git a/app/api/cron/check-research/route.ts b/app/api/cron/check-research/route.ts index 4bdfca26..205e7848 100644 --- a/app/api/cron/check-research/route.ts +++ b/app/api/cron/check-research/route.ts @@ -8,6 +8,7 @@ import { NotebookLMClient } from '@/lib/services/notebooklm/client'; import { initAuth } from '@/lib/services/notebooklm/auth'; import { ArtifactTypeCode, ArtifactStatus } from '@/lib/services/notebooklm/types'; import { generateWithGemini, stripCodeFences } from '@/lib/gemini'; +import { getConfigValue } from '@/lib/config'; import type { ResearchPayload } from '@/lib/services/research'; // --------------------------------------------------------------------------- @@ -94,12 +95,15 @@ interface StepResult { // Constants // --------------------------------------------------------------------------- -/** Stuck thresholds per status (ms) */ -const STUCK_THRESHOLDS: Record = { - researching: 30 * 60 * 1000, // 30 minutes - infographics_generating: 15 * 60 * 1000, // 15 minutes - enriching: 10 * 60 * 1000, // 10 minutes -}; +/** Build stuck thresholds from config (with fallbacks) */ +async function buildStuckThresholds(): Promise> { + const stuckMinutes = await getConfigValue('pipeline_config', 'stuckTimeoutMinutes', 30); + return { + researching: stuckMinutes * 60 * 1000, + infographics_generating: Math.round(stuckMinutes * 0.5) * 60 * 1000, // half the main timeout + enriching: Math.round(stuckMinutes * 0.33) * 60 * 1000, // third of main timeout + }; +} /** Max docs to process per status per run — keeps total time well under 60s */ const MAX_DOCS_PER_STATUS = 2; @@ -132,12 +136,13 @@ function getSanityWriteClient(): SanityClient { async function flagStuckDocs( docs: PipelineDoc[], sanity: SanityClient, + stuckThresholds: Record, ): Promise { const results: StepResult[] = []; const now = Date.now(); for (const doc of docs) { - const threshold = STUCK_THRESHOLDS[doc.status]; + const threshold = stuckThresholds[doc.status]; if (!threshold) continue; const docAge = now - new Date(doc._updatedAt).getTime(); @@ -419,6 +424,11 @@ async function stepEnriching( // Generate enriched script with Gemini let enrichedScript: EnrichedScript | null = null; try { + const SYSTEM_INSTRUCTION = await getConfigValue( + 'content_config', + 'systemInstruction', + SYSTEM_INSTRUCTION_FALLBACK, + ); const prompt = buildEnrichmentPrompt(doc, researchPayload); const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION); const cleaned = stripCodeFences(rawResponse); @@ -434,7 +444,8 @@ async function stepEnriching( const criticScore = criticResult.score; console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`); - const isFlagged = criticScore < 50; + const qualityThreshold = await getConfigValue('pipeline_config', 'qualityThreshold', 50); + const isFlagged = criticScore < qualityThreshold; await sanity .patch(doc._id) @@ -481,7 +492,9 @@ async function stepEnriching( // Gemini Script Enrichment // --------------------------------------------------------------------------- -const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. +// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity. +// The live value is fetched from getConfigValue() inside stepEnriching(). +const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: - Start with a BOLD claim or surprising fact that makes people stop scrolling @@ -813,7 +826,8 @@ export async function GET(request: NextRequest) { const results: StepResult[] = []; // Phase 1: Stuck detection — runs FIRST, no external API calls - const stuckResults = await flagStuckDocs(docs, sanity); + const stuckThresholds = await buildStuckThresholds(); + const stuckResults = await flagStuckDocs(docs, sanity, stuckThresholds); results.push(...stuckResults); // Remove flagged docs from further processing diff --git a/app/api/cron/ingest/route.ts b/app/api/cron/ingest/route.ts index a6480f30..fc456f02 100644 --- a/app/api/cron/ingest/route.ts +++ b/app/api/cron/ingest/route.ts @@ -4,6 +4,7 @@ import type { NextRequest } from "next/server"; import { generateWithGemini, stripCodeFences } from "@/lib/gemini"; import { writeClient } from "@/lib/sanity-write-client"; +import { getConfigValue } from "@/lib/config"; import { discoverTrends, type TrendResult } from "@/lib/services/trend-discovery"; import type { ResearchPayload } from "@/lib/services/research"; import { NotebookLMClient } from "@/lib/services/notebooklm/client"; @@ -102,7 +103,7 @@ const FALLBACK_TRENDS: TrendResult[] = [ slug: "webassembly-web-apps", score: 60, signals: [{ source: "blog", title: "WebAssembly", url: "https://webassembly.org/", score: 60 }], - whyTrending: "WASM adoption growing in [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] apps", + whyTrending: "WASM adoption growing in production apps", suggestedAngle: "Real-world use cases where WASM outperforms JS", }, ]; @@ -111,7 +112,9 @@ const FALLBACK_TRENDS: TrendResult[] = [ // Gemini Script Generation // --------------------------------------------------------------------------- -const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. +// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity. +// The live value is fetched from getConfigValue() inside the GET handler. +const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson. Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles: - Start with a BOLD claim or surprising fact that makes people stop scrolling @@ -330,10 +333,11 @@ async function createSanityDocuments( script: GeneratedScript, criticResult: CriticResult, trends: TrendResult[], + qualityThreshold: number, research?: ResearchPayload, researchMeta?: { notebookId: string; taskId: string }, ) { - const isFlagged = criticResult.score < 50; + const isFlagged = criticResult.score < qualityThreshold; // When research is in-flight, status is "researching" (check-research cron will transition to script_ready) const isResearching = !!researchMeta?.notebookId; const status = isFlagged ? "flagged" : isResearching ? "researching" : "script_ready"; @@ -404,6 +408,23 @@ export async function GET(request: NextRequest) { } try { + // Fetch config values once per invocation (5-min in-memory cache) + const SYSTEM_INSTRUCTION = await getConfigValue( + "content_config", + "systemInstruction", + SYSTEM_INSTRUCTION_FALLBACK, + ); + const enableNotebookLmResearch = await getConfigValue( + "pipeline_config", + "enableNotebookLmResearch", + false, + ); + const qualityThreshold = await getConfigValue( + "pipeline_config", + "qualityThreshold", + 50, + ); + // Step 1: Discover trending topics (replaces fetchTrendingTopics) console.log("[CRON/ingest] Discovering trending topics..."); let trends: TrendResult[]; @@ -425,7 +446,7 @@ export async function GET(request: NextRequest) { // When research is enabled, we create a notebook and start research // but DON'T wait for it — the check-research cron will poll and enrich later let researchMeta: { notebookId: string; taskId: string } | undefined; - if (process.env.ENABLE_NOTEBOOKLM_RESEARCH === "true") { + if (enableNotebookLmResearch) { console.log(`[CRON/ingest] Starting fire-and-forget research on: "${trends[0].topic}"...`); try { const auth = await initAuth(); @@ -494,7 +515,7 @@ export async function GET(request: NextRequest) { ); console.log("[CRON/ingest] Creating Sanity documents..."); - const result = await createSanityDocuments(script, criticResult, trends, undefined, researchMeta); + const result = await createSanityDocuments(script, criticResult, trends, qualityThreshold, undefined, researchMeta); console.log("[CRON/ingest] Done!", result); diff --git a/app/api/cron/sponsor-outreach/route.ts b/app/api/cron/sponsor-outreach/route.ts index e3806ab3..3d7d4da8 100644 --- a/app/api/cron/sponsor-outreach/route.ts +++ b/app/api/cron/sponsor-outreach/route.ts @@ -5,9 +5,7 @@ import { sanityWriteClient } from '@/lib/sanity-write-client' import { generateOutreachEmail } from '@/lib/sponsor/gemini-outreach' import { sendSponsorEmail } from '@/lib/sponsor/email-service' import type { SponsorPoolEntry } from '@/lib/sponsor/gemini-outreach' - -const MAX_PER_RUN = 5 -const COOLDOWN_DAYS = 14 +import { getConfig } from '@/lib/config' export async function POST(request: Request) { // Auth: Bearer token check against CRON_SECRET @@ -25,9 +23,19 @@ export async function POST(request: Request) { try { console.log('[SPONSOR] Starting outbound sponsor outreach cron...') + // Fetch config from Sanity singleton + const sponsorCfg = await getConfig("sponsor_config"); + const maxPerRun = sponsorCfg.maxOutreachPerRun; + const cooldownDays = sponsorCfg.cooldownDays; + + // Build rate card string from config tiers + const rateCard = sponsorCfg.rateCardTiers + .map((t) => `- ${t.name} ($${t.price}) — ${t.description}`) + .join('\n'); + // Calculate the cutoff date for cooldown const cutoffDate = new Date() - cutoffDate.setDate(cutoffDate.getDate() - COOLDOWN_DAYS) + cutoffDate.setDate(cutoffDate.getDate() - cooldownDays) const cutoffISO = cutoffDate.toISOString() // Query Sanity for eligible sponsor pool entries @@ -38,7 +46,7 @@ export async function POST(request: Request) { !defined(lastContactedAt) || lastContactedAt < $cutoffDate ) - ] | order(relevanceScore desc) [0...${MAX_PER_RUN - 1}] { + ] | order(relevanceScore desc) [0...${maxPerRun - 1}] { _id, companyName, contactName, @@ -67,8 +75,8 @@ export async function POST(request: Request) { for (const sponsor of sponsors) { try { - // Generate personalized outreach email - const email = await generateOutreachEmail(sponsor) + // Generate personalized outreach email with config rate card + const email = await generateOutreachEmail(sponsor, rateCard) // Send the email (stubbed) const sendResult = await sendSponsorEmail( diff --git a/app/api/webhooks/sanity-distribute/route.ts b/app/api/webhooks/sanity-distribute/route.ts index bfe3c7c5..0df1c41c 100644 --- a/app/api/webhooks/sanity-distribute/route.ts +++ b/app/api/webhooks/sanity-distribute/route.ts @@ -5,6 +5,7 @@ import { generateWithGemini } from "@/lib/gemini"; import { uploadVideo, uploadShort, generateShortsMetadata } from "@/lib/youtube-upload"; import { notifySubscribers } from "@/lib/resend-notify"; import { postVideoAnnouncement } from "@/lib/x-social"; +import { getConfig } from "@/lib/config"; const WEBHOOK_SECRET = process.env.SANITY_WEBHOOK_SECRET; @@ -140,6 +141,9 @@ async function appendDistributionLog(docId: string, entries: DistributionLogEntr async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise { const log: DistributionLogEntry[] = []; + // Fetch distribution config from Sanity singleton + const distConfig = await getConfig("distribution_config"); + try { await updateStatus(docId, "uploading"); @@ -197,7 +201,7 @@ async function runDistribution(docId: string, doc: AutomatedVideoDoc): Promise { const apiKey = process.env.RESEND_API_KEY; if (!apiKey) { @@ -19,8 +21,8 @@ export async function notifySubscribers(opts: { const resend = new Resend(apiKey); await resend.emails.send({ - from: "CodingCat.dev ", - to: ["subscribers@codingcat.dev"], // TODO: fetch subscriber list + from: `CodingCat.dev <${opts.fromEmail || "noreply@codingcat.dev"}>`, + to: opts.notificationEmails || ["subscribers@codingcat.dev"], subject: opts.subject, html: `

${opts.videoTitle}

diff --git a/lib/services/elevenlabs.ts b/lib/services/elevenlabs.ts index faae42f3..c9e7202a 100644 --- a/lib/services/elevenlabs.ts +++ b/lib/services/elevenlabs.ts @@ -12,6 +12,7 @@ import { type WordTimestamp, type SceneAudioResult, } from "@/lib/utils/audio-timestamps"; +import { getConfigValue } from "@/lib/config"; const ELEVENLABS_API_BASE = "https://api.elevenlabs.io/v1"; @@ -62,9 +63,12 @@ interface TTSWithTimestampsResponse { * @returns The resolved {@link ElevenLabsConfig}. * @throws {Error} If required environment variables are missing. */ -function getConfig(): ElevenLabsConfig { +async function getElevenLabsConfig(): Promise { const apiKey = process.env.ELEVENLABS_API_KEY; - const voiceId = process.env.ELEVENLABS_VOICE_ID; + const voiceId = await getConfigValue( + "pipeline_config", "elevenLabsVoiceId", + process.env.ELEVENLABS_VOICE_ID || "pNInz6obpgDQGcFmaJgB" + ); if (!apiKey) { throw new Error( @@ -73,13 +77,6 @@ function getConfig(): ElevenLabsConfig { ); } - if (!voiceId) { - throw new Error( - "Missing ELEVENLABS_VOICE_ID environment variable. " + - "Set it in your .env.local or deployment environment." - ); - } - return { apiKey, voiceId }; } @@ -106,7 +103,7 @@ export async function generateSpeech(text: string): Promise { throw new Error("Cannot generate speech from empty text."); } - const { apiKey, voiceId } = getConfig(); + const { apiKey, voiceId } = await getElevenLabsConfig(); const url = `${ELEVENLABS_API_BASE}/text-to-speech/${voiceId}`; @@ -244,7 +241,7 @@ export async function generateSpeechWithTimestamps( throw new Error("Cannot generate speech from empty text."); } - const { apiKey, voiceId } = getConfig(); + const { apiKey, voiceId } = await getElevenLabsConfig(); const url = `${ELEVENLABS_API_BASE}/text-to-speech/${voiceId}/with-timestamps`; diff --git a/lib/services/gcs.ts b/lib/services/gcs.ts index 64d2f94e..8709cc40 100644 --- a/lib/services/gcs.ts +++ b/lib/services/gcs.ts @@ -14,6 +14,7 @@ */ import * as crypto from "crypto"; +import { getConfigValue } from "@/lib/config"; // --------------------------------------------------------------------------- // Types @@ -63,9 +64,9 @@ const TOKEN_REFRESH_MARGIN_MS = 5 * 60 * 1000; // 5 minutes * The private key may contain literal `\\n` sequences from the env var; * these are converted to real newline characters. */ -export function getGCSConfig(): GCSConfig { - const bucket = process.env.GCS_BUCKET; - const projectId = process.env.GCS_PROJECT_ID; +export async function getGCSConfig(): Promise { + const bucket = await getConfigValue("gcs_config", "bucketName", process.env.GCS_BUCKET); + const projectId = await getConfigValue("gcs_config", "projectId", process.env.GCS_PROJECT_ID); const clientEmail = process.env.GCS_CLIENT_EMAIL; let privateKey = process.env.GCS_PRIVATE_KEY; @@ -205,7 +206,7 @@ async function getAccessToken(): Promise { return cachedToken.accessToken; } - const config = getGCSConfig(); + const config = await getGCSConfig(); const jwt = createServiceAccountJWT(config); cachedToken = await exchangeJWTForToken(jwt); return cachedToken.accessToken; @@ -231,7 +232,7 @@ export async function uploadToGCS( path: string, contentType: string ): Promise { - const config = getGCSConfig(); + const config = await getGCSConfig(); const token = await getAccessToken(); const encodedPath = encodeURIComponent(path); @@ -338,7 +339,7 @@ export async function getSignedUrl( expiresInMinutes = 60 ): Promise { // We still validate config to fail fast if env vars are missing - const config = getGCSConfig(); + const config = await getGCSConfig(); // For public objects, the public URL is sufficient void expiresInMinutes; // acknowledged but unused for public objects diff --git a/lib/services/remotion.ts b/lib/services/remotion.ts index feca1b06..8cb72b54 100644 --- a/lib/services/remotion.ts +++ b/lib/services/remotion.ts @@ -22,12 +22,13 @@ import { getRenderProgress, type AwsRegion, } from "@remotion/lambda/client"; +import { getConfigValue } from "@/lib/config"; // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- -export interface RemotionConfig { +export interface RemotionLambdaConfig { awsAccessKeyId: string; awsSecretAccessKey: string; region: string; @@ -130,11 +131,11 @@ function mapInputProps(input: RenderInput): Record { * Get Remotion Lambda configuration from environment variables. * Throws if any required env var is missing. */ -export function getRemotionConfig(): RemotionConfig { +export async function getRemotionConfig(): Promise { const awsAccessKeyId = process.env.AWS_ACCESS_KEY_ID; const awsSecretAccessKey = process.env.AWS_SECRET_ACCESS_KEY; - const region = process.env.REMOTION_AWS_REGION; - const serveUrl = process.env.REMOTION_SERVE_URL; + const region = await getConfigValue("remotion_config", "awsRegion", process.env.REMOTION_AWS_REGION); + const serveUrl = await getConfigValue("remotion_config", "serveUrl", process.env.REMOTION_SERVE_URL); const missing: string[] = []; if (!awsAccessKeyId) missing.push("AWS_ACCESS_KEY_ID"); @@ -161,8 +162,8 @@ export function getRemotionConfig(): RemotionConfig { /** * Get the Lambda function name from env or use the default. */ -function getFunctionName(): string { - return process.env.REMOTION_FUNCTION_NAME || DEFAULT_FUNCTION_NAME; +async function getFunctionName(): Promise { + return getConfigValue("remotion_config", "functionName", process.env.REMOTION_FUNCTION_NAME || DEFAULT_FUNCTION_NAME); } // --------------------------------------------------------------------------- @@ -202,8 +203,8 @@ async function startRender( composition: string, input: RenderInput ): Promise<{ renderId: string; bucketName: string }> { - const config = getRemotionConfig(); - const functionName = getFunctionName(); + const config = await getRemotionConfig(); + const functionName = await getFunctionName(); const region = config.region as AwsRegion; log(`Starting render for composition "${composition}"`, { @@ -291,8 +292,8 @@ export async function checkRenderProgress( renderId: string, bucketName: string ): Promise { - const config = getRemotionConfig(); - const functionName = getFunctionName(); + const config = await getRemotionConfig(); + const functionName = await getFunctionName(); const region = config.region as AwsRegion; const progress = await getRenderProgress({ diff --git a/lib/sponsor/gemini-outreach.ts b/lib/sponsor/gemini-outreach.ts index 056cf7f8..3f9775f2 100644 --- a/lib/sponsor/gemini-outreach.ts +++ b/lib/sponsor/gemini-outreach.ts @@ -1,4 +1,5 @@ import { GoogleGenerativeAI } from '@google/generative-ai' +import { getConfigValue } from '@/lib/config' export interface SponsorPoolEntry { _id: string @@ -16,7 +17,7 @@ export interface OutreachEmail { body: string } -const RATE_CARD = ` +const DEFAULT_RATE_CARD = ` CodingCat.dev Sponsorship Tiers: - Dedicated Video ($4,000) — Full dedicated video about your product - Integrated Mid-Roll Ad ($1,800) — Mid-roll advertisement in our videos @@ -33,7 +34,7 @@ Our audience: 50K+ developers interested in web development, JavaScript/TypeScri */ export async function generateOutreachEmail( sponsor: SponsorPoolEntry, - rateCard: string = RATE_CARD + rateCard: string = DEFAULT_RATE_CARD ): Promise { const apiKey = process.env.GEMINI_API_KEY if (!apiKey) { @@ -42,7 +43,8 @@ export async function generateOutreachEmail( } const genAI = new GoogleGenerativeAI(apiKey) - const model = genAI.getGenerativeModel({ model: process.env.GEMINI_MODEL || 'gemini-2.5-flash' }) + const geminiModel = await getConfigValue("pipeline_config", "geminiModel", "gemini-2.0-flash"); + const model = genAI.getGenerativeModel({ model: geminiModel }) const optOutUrl = sponsor.optOutToken ? `${process.env.NEXT_PUBLIC_URL || 'https://codingcat.dev'}/api/sponsor/opt-out?token=${sponsor.optOutToken}` @@ -76,7 +78,7 @@ Respond ONLY with valid JSON, no markdown formatting: const response = result.response const text = response.text().trim() - const jsonStr = text.replace(/^\`\`\`json?\n?/i, '').replace(/\n?\`\`\`$/i, '').trim() + const jsonStr = text.replace(/^```json?\n?/i, '').replace(/\n?```$/i, '').trim() const parsed = JSON.parse(jsonStr) as OutreachEmail console.log('[SPONSOR] Gemini outreach email generated for:', sponsor.companyName)