From ccb4f7c0842e9de6524c4459ec25720ab6b81aad Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Tue, 31 Mar 2026 11:53:46 +0000 Subject: [PATCH 1/2] feat(traces): enrich traces list with duration, status, input/output, and age Replace the minimal 3-column traces list (ID, Timestamp, Session) with a rich 7-column table matching the starter toolkit's obs list output. Two-phase CloudWatch query: - Phase 1: Query aws/spans for span-level data (duration, status, counts) - Phase 2: Query runtime logs for input/output message previews Supports Strands, botocore, and generic OTEL message formats. Constraint: Must query two separate log groups (aws/spans + runtime logs) Rejected: Single-query approach | runtime logs lack span duration/status data Rejected: Session-scoped listing | CLI lists all traces per agent, not per session Confidence: high Scope-risk: narrow Directive: Message parser handles 3 formats - add new patterns to extractMessagesFromBody if new frameworks are added Not-tested: Agents using LangChain or GoogleADK instrumentation formats Co-Authored-By: Claude Opus 4.6 --- src/cli/commands/traces/action.ts | 11 +- src/cli/commands/traces/command.tsx | 118 +++++-- src/cli/constants.ts | 10 + src/cli/operations/traces/list-traces.ts | 389 +++++++++++++++++++---- 4 files changed, 449 insertions(+), 79 deletions(-) diff --git a/src/cli/commands/traces/action.ts b/src/cli/commands/traces/action.ts index d761cd4b..cbccab81 100644 --- a/src/cli/commands/traces/action.ts +++ b/src/cli/commands/traces/action.ts @@ -9,7 +9,16 @@ export interface TracesListResult { agentName?: string; targetName?: string; consoleUrl?: string; - traces?: { traceId: string; timestamp: string; sessionId?: string }[]; + traces?: { + traceId: string; + sessionId?: string; + spanCount: number; + errorCount: number; + durationMs: number; + latestEndTimeNano: number; + input?: string; + output?: string; + }[]; error?: string; } diff --git a/src/cli/commands/traces/command.tsx b/src/cli/commands/traces/command.tsx index 0222ce41..79db1b1c 100644 --- a/src/cli/commands/traces/command.tsx +++ b/src/cli/commands/traces/command.tsx @@ -7,14 +7,27 @@ import type { TracesGetOptions, TracesListOptions } from './types'; import type { Command } from '@commander-js/extra-typings'; import { Box, Text, render } from 'ink'; -function formatTimestamp(ts: string): string { - const num = Number(ts); - if (isNaN(num)) return ts; - // Epoch ms → human-readable - return new Date(num) - .toISOString() - .replace('T', ' ') - .replace(/\.\d+Z$/, 'Z'); +function formatDuration(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + return `${(ms / 60_000).toFixed(1)}m`; +} + +function formatAge(nanoTimestamp: number): string { + if (nanoTimestamp === 0) return '-'; + const seconds = Math.floor((Date.now() - nanoTimestamp / 1_000_000) / 1000); + if (seconds < 0) return 'now'; + if (seconds < 60) return `${seconds}s ago`; + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; + return `${Math.floor(seconds / 86400)}d ago`; +} + +function getDurationColor(ms: number): string { + if (ms < 100) return 'green'; + if (ms < 1000) return 'yellow'; + if (ms < 5000) return 'yellowBright'; + return 'red'; } export const registerTraces = (program: Command) => { @@ -53,30 +66,95 @@ export const registerTraces = (program: Command) => { {result.traces && result.traces.length > 0 ? ( <> + {/* Header */} + + + # + + Trace ID - - Timestamp + + Duration + + + Status - - Session ID + + Input + + + Output + + + + Age + + {/* Separator */} + + {'─'.repeat(117)} + + {/* Rows */} {result.traces.map((trace, i) => ( - - - {trace.traceId} - - - {formatTimestamp(trace.timestamp)} + + + + {String(i + 1).padStart(2)} + + + {trace.traceId} + + + {formatDuration(trace.durationMs)} + + + {trace.spanCount} spans + + + {trace.input ?? '-'} + + + {trace.output ?? '-'} + + + {formatAge(trace.latestEndTimeNano)} + - - {trace.sessionId ?? '-'} + {/* Second line: latest marker + status icon */} + + + + + {i === 0 ? (latest) : } + + + + + {trace.errorCount > 0 ? ( + + {'❌'} {trace.errorCount} err + + ) : ( + {'✓'} OK + )} + + + + + + + ))} + + + {'✓'} Found {result.traces.length} traces + ) : ( No traces found in the specified time range. diff --git a/src/cli/constants.ts b/src/cli/constants.ts index 48c3e037..51bd9f21 100644 --- a/src/cli/constants.ts +++ b/src/cli/constants.ts @@ -58,3 +58,13 @@ export const SCHEMA_VERSION = 1; * Default runtime endpoint name used in log group paths and console URLs. */ export const DEFAULT_ENDPOINT_NAME = 'DEFAULT'; + +/** + * CloudWatch log group for OpenTelemetry spans (shared across all agents). + */ +export const SPANS_LOG_GROUP = 'aws/spans'; + +/** + * Maximum character length for input/output message previews in trace list. + */ +export const TRACE_LIST_PREVIEW_LENGTH = 24; diff --git a/src/cli/operations/traces/list-traces.ts b/src/cli/operations/traces/list-traces.ts index 7bff6194..faef2188 100644 --- a/src/cli/operations/traces/list-traces.ts +++ b/src/cli/operations/traces/list-traces.ts @@ -1,12 +1,16 @@ import { getCredentialProvider } from '../../aws'; -import { DEFAULT_ENDPOINT_NAME } from '../../constants'; +import { DEFAULT_ENDPOINT_NAME, SPANS_LOG_GROUP, TRACE_LIST_PREVIEW_LENGTH } from '../../constants'; import { CloudWatchLogsClient, GetQueryResultsCommand, StartQueryCommand } from '@aws-sdk/client-cloudwatch-logs'; export interface TraceEntry { traceId: string; - timestamp: string; sessionId?: string; - spanCount?: string; + spanCount: number; + errorCount: number; + durationMs: number; + latestEndTimeNano: number; + input?: string; + output?: string; } export interface ListTracesOptions { @@ -24,11 +28,289 @@ export interface ListTracesResult { error?: string; } +interface RawSpan { + traceId: string; + spanId: string; + statusCode?: string; + durationMs?: number; + sessionId?: string; + startTimeUnixNano?: number; + endTimeUnixNano?: number; +} + +/** Sanitize a value for use in CloudWatch Insights query strings by removing single quotes. */ +function sanitizeQueryValue(value: string): string { + return value.replace(/'/g, ''); +} + +/** + * Executes a CloudWatch Logs Insights query and polls for results. + * Returns the raw result rows, or throws on failure/timeout. + */ +async function executeQuery( + client: CloudWatchLogsClient, + logGroupName: string, + queryString: string, + startTime: number, + endTime: number +): Promise<{ field?: string; value?: string }[][]> { + const startQuery = await client.send( + new StartQueryCommand({ + logGroupName, + startTime: Math.floor(startTime / 1000), + endTime: Math.floor(endTime / 1000), + queryString, + }) + ); + + if (!startQuery.queryId) { + throw new Error('Failed to start CloudWatch Logs Insights query'); + } + + for (let i = 0; i < 60; i++) { + await new Promise(resolve => setTimeout(resolve, 1000)); + + const queryResults = await client.send(new GetQueryResultsCommand({ queryId: startQuery.queryId })); + const status = queryResults.status ?? 'Unknown'; + + if (status === 'Complete') { + return queryResults.results ?? []; + } + if (status === 'Failed' || status === 'Cancelled') { + throw new Error(`Query ${status.toLowerCase()}`); + } + } + + throw new Error('Query timed out after 60 seconds'); +} + +/** + * Extracts a field value from a CloudWatch Logs Insights result row. + */ +function getField(row: { field?: string; value?: string }[], name: string): string | undefined { + return row.find(f => f.field === name)?.value; +} + +/** + * Parses span rows from CloudWatch Logs Insights results. + */ +function parseSpanRows(rows: { field?: string; value?: string }[][]): RawSpan[] { + return rows.map(row => ({ + traceId: getField(row, 'traceId') ?? 'unknown', + spanId: getField(row, 'spanId') ?? 'unknown', + statusCode: getField(row, 'statusCode'), + durationMs: parseFloat(getField(row, 'durationMs') ?? '0') || undefined, + sessionId: getField(row, 'sessionId'), + startTimeUnixNano: parseInt(getField(row, 'startTimeUnixNano') ?? '0', 10) || undefined, + endTimeUnixNano: parseInt(getField(row, 'endTimeUnixNano') ?? '0', 10) || undefined, + })); +} + +/** + * Groups spans by traceId and computes per-trace aggregates. + */ +function aggregateTraces(spans: RawSpan[], limit: number): Omit[] { + const grouped = new Map(); + for (const span of spans) { + const list = grouped.get(span.traceId); + if (list) { + list.push(span); + } else { + grouped.set(span.traceId, [span]); + } + } + + const traces: Omit[] = []; + + for (const [traceId, traceSpans] of grouped) { + const startTimes = traceSpans.map(s => s.startTimeUnixNano).filter((t): t is number => t != null && t > 0); + const endTimes = traceSpans.map(s => s.endTimeUnixNano).filter((t): t is number => t != null && t > 0); + + let durationMs = 0; + if (startTimes.length > 0 && endTimes.length > 0) { + durationMs = (Math.max(...endTimes) - Math.min(...startTimes)) / 1_000_000; + } else { + durationMs = traceSpans.reduce((sum, s) => sum + (s.durationMs ?? 0), 0); + } + + const errorCount = traceSpans.filter(s => s.statusCode === 'ERROR').length; + const latestEndTimeNano = endTimes.length > 0 ? Math.max(...endTimes) : 0; + const sessionId = traceSpans.find(s => s.sessionId)?.sessionId; + + traces.push({ + traceId, + sessionId, + spanCount: traceSpans.length, + errorCount, + durationMs, + latestEndTimeNano, + }); + } + + // Sort by most recent first + traces.sort((a, b) => b.latestEndTimeNano - a.latestEndTimeNano); + + return traces.slice(0, limit); +} + +/** + * Truncates a string to maxLen characters, appending "..." if truncated. + */ +function truncate(str: string, maxLen: number): string { + if (str.length <= maxLen) return str; + return str.slice(0, maxLen - 3) + '...'; +} + +/** + * Extracts a human-readable text string from a Strands content field. + * Handles nested JSON strings like '[{"text": "hello"}]' and plain strings. + */ +function extractTextFromContent(content: unknown): string | undefined { + if (typeof content === 'string') { + // Try parsing as JSON (Strands wraps content as JSON string e.g. '[{"text": "hello"}]') + try { + const parsed: unknown = JSON.parse(content); + if (Array.isArray(parsed)) { + const texts = parsed.map((item: Record) => item.text).filter(Boolean); + if (texts.length > 0) return texts.join(' '); + } + if (typeof parsed === 'string') return parsed; + } catch { + // Not JSON — use as-is + } + return content; + } + if (content && typeof content === 'object') { + const obj = content as Record; + // Strands output: content.message (string) + if (typeof obj.message === 'string') return obj.message; + // Strands output: content.content (nested string) + if (typeof obj.content === 'string') return extractTextFromContent(obj.content); + // Array of content blocks: [{text: "..."}] + if (Array.isArray(content)) { + const texts = content.map((item: Record) => item.text).filter(Boolean); + if (texts.length > 0) return texts.join(' '); + } + } + return undefined; +} + +/** + * Extracts all user and assistant messages from a parsed runtime log entry. + * Handles multiple instrumentation formats: Strands, botocore, generic OTEL. + */ +function extractMessagesFromBody(parsed: Record): { role: string; content: string }[] { + const results: { role: string; content: string }[] = []; + const body = parsed.body as Record | undefined; + if (!body) return results; + + // Pattern 1: body.message.role + body.message.content (botocore bedrock-runtime) + const message = body.message as Record | undefined; + if (message && typeof message.role === 'string') { + let text: string | undefined; + if (Array.isArray(message.content)) { + text = message.content + .map((c: Record) => c.text) + .filter(Boolean) + .join(' '); + } else if (typeof message.content === 'string') { + text = message.content; + } + if (text) results.push({ role: message.role, content: text }); + } + + // Pattern 2: body.input.messages + body.output.messages (Strands telemetry) + const bodyInput = body.input as Record | undefined; + if (bodyInput?.messages && Array.isArray(bodyInput.messages)) { + for (const msg of bodyInput.messages as Record[]) { + if (typeof msg.role === 'string') { + const text = extractTextFromContent(msg.content); + if (text) results.push({ role: msg.role, content: text }); + } + } + } + + const bodyOutput = body.output as Record | undefined; + if (bodyOutput?.messages && Array.isArray(bodyOutput.messages)) { + for (const msg of bodyOutput.messages as Record[]) { + if (typeof msg.role === 'string') { + const text = extractTextFromContent(msg.content); + if (text) results.push({ role: msg.role, content: text }); + } + } + } + + // Pattern 3: body.role + body.content (simple format) + if (typeof body.role === 'string' && body.content) { + const text = extractTextFromContent(body.content); + if (text) results.push({ role: body.role, content: text }); + } + + return results; +} + +/** + * Parses runtime logs and extracts the last user (input) and assistant (output) messages per trace. + */ +function extractTraceMessages( + logRows: { field?: string; value?: string }[][], + previewLength: number +): Map { + // Collect all messages grouped by traceId + const messagesByTrace = new Map(); + + for (const row of logRows) { + const traceId = getField(row, 'traceId'); + const rawMessage = getField(row, '@message'); + if (!traceId || !rawMessage) continue; + + let parsed: Record; + try { + parsed = JSON.parse(rawMessage) as Record; + } catch { + continue; + } + + const msgs = extractMessagesFromBody(parsed); + if (msgs.length === 0) continue; + + const list = messagesByTrace.get(traceId); + if (list) { + list.push(...msgs); + } else { + messagesByTrace.set(traceId, [...msgs]); + } + } + + // For each trace, find the last user and last assistant message + const result = new Map(); + + for (const [traceId, messages] of messagesByTrace) { + let lastUser: string | undefined; + let lastAssistant: string | undefined; + + for (const msg of messages) { + if (msg.role === 'user') { + lastUser = msg.content; + } else if (msg.role === 'assistant') { + lastAssistant = msg.content; + } + } + + result.set(traceId, { + input: lastUser ? truncate(lastUser, previewLength) : undefined, + output: lastAssistant ? truncate(lastAssistant, previewLength) : undefined, + }); + } + + return result; +} + /** * Lists recent traces for a deployed agent by querying CloudWatch Logs Insights. * - * Log group naming convention: /aws/bedrock-agentcore/runtimes/{runtimeId}-DEFAULT - * Trace data is in the @message JSON body with fields like traceId, spanId, etc. + * Phase 1: Query aws/spans for span-level data (duration, status, counts). + * Phase 2: Query runtime logs for input/output message previews. */ export async function listTraces(options: ListTracesOptions): Promise { const { region, runtimeId, limit = 20 } = options; @@ -38,73 +320,64 @@ export async function listTraces(options: ListTracesOptions): Promise setTimeout(resolve, 1000)); - - const queryResults = await client.send(new GetQueryResultsCommand({ queryId: startQuery.queryId })); - - status = queryResults.status ?? 'Unknown'; - - if (status === 'Complete' || status === 'Failed' || status === 'Cancelled') { - if (status !== 'Complete') { - return { success: false, error: `Query ${status.toLowerCase()}` }; - } - - results = (queryResults.results ?? []).map(row => { - const fields: Record = {}; - for (const field of row) { - if (field.field && field.value) { - fields[field.field] = field.value; - } - } - return { - traceId: fields.traceId ?? 'unknown', - timestamp: fields.lastSeen ?? fields.firstSeen ?? 'unknown', - sessionId: fields.sessionId, - spanCount: fields.spanCount, - }; - }); - break; - } + if (traces.length === 0) { + return { success: true, traces: [] }; } - if (status === 'Running') { - return { success: false, error: 'Query timed out after 60 seconds' }; + // Phase 2: Query runtime logs for input/output messages + const traceIds = traces.map(t => t.traceId); + const runtimeLogGroup = `/aws/bedrock-agentcore/runtimes/${runtimeId}-${DEFAULT_ENDPOINT_NAME}`; + const inClause = traceIds.map(id => `'${sanitizeQueryValue(id)}'`).join(', '); + const runtimeQuery = `fields @timestamp, @message, spanId, traceId, @logStream +| filter traceId in [${inClause}] +| sort @timestamp asc +| limit 10000`; + + let messageMap = new Map(); + + try { + const logRows = await executeQuery(client, runtimeLogGroup, runtimeQuery, startTime, endTime); + messageMap = extractTraceMessages(logRows, TRACE_LIST_PREVIEW_LENGTH); + } catch { + // Runtime logs may not exist — proceed without messages } - return { success: true, traces: results }; + // Merge messages into traces + const enrichedTraces: TraceEntry[] = traces.map(trace => { + const messages = messageMap.get(trace.traceId); + return { + ...trace, + input: messages?.input, + output: messages?.output, + }; + }); + + return { success: true, traces: enrichedTraces }; } catch (error: unknown) { const err = error as Error; if (err.name === 'ResourceNotFoundException') { return { success: false, - error: `Log group '${logGroupName}' not found. The agent may not have been invoked yet, or traces may not be enabled.`, + error: `Log group '${SPANS_LOG_GROUP}' not found. The agent may not have been invoked yet, or traces may not be enabled.`, }; } return { success: false, error: err.message ?? String(error) }; From 9eeaa5e5581d7805ea64254a4954eb12bc08aed5 Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Tue, 31 Mar 2026 15:50:43 +0000 Subject: [PATCH 2/2] docs: add traces list screenshots for PR SVG screenshots captured via TUI harness showing the new 7-column traces list output with color-coded duration, status icons, and input/output message previews. Co-Authored-By: Claude Opus 4.6 --- docs/screenshots/traces-list-light.svg | 45 ++++++++++++++++++++++++++ docs/screenshots/traces-list.svg | 45 ++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 docs/screenshots/traces-list-light.svg create mode 100644 docs/screenshots/traces-list.svg diff --git a/docs/screenshots/traces-list-light.svg b/docs/screenshots/traces-list-light.svg new file mode 100644 index 00000000..c56cde53 --- /dev/null +++ b/docs/screenshots/traces-list-light.svg @@ -0,0 +1,45 @@ + + + + + + + + +@aws-sdk/credential-provider-node - defaultProvider::fromEnv WARNING: + Multiple credential sources detected: + Both AWS_PROFILE and the pair AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY static credentials are set. + This SDK will proceed with the AWS_PROFILE value. + However, a future version may change this behavior to prefer the ENV static credentials. + Please ensure that your environment only sets either the AWS_PROFILE or the + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY pair. +Traces for Agent (target: dev) +# Trace ID Duration Status Input Output Age +───────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + 1 69c1821e0833a477277f7c8434a7c482 2.9s 8 spans what is my name I don't know your nam... 7d ago + (latest) ✓ OK + 2 69c182155d775da239fd98f10671641f 3.0s 8 spans my name is jesse Hello Jesse! Nice to ... 7d ago + ✓ OK + 3 69c180ee40e3f629745b08f409958dea 3.7s 8 spans what is my name I don't know your nam... 7d ago + ✓ OK + 4 69c17fec0125491e652c2a8f6e640dee 3.9s 8 spans my name is jesse Hello Jesse! Nice to ... 7d ago + ✓ OK + 5 69c17fc733a42c8838e995c66679f706 4.5s 8 spans my name is jesse Hello Jesse! Nice to ... 7d ago + ✓ OK + 6 69c17f6f6f6b04121d08a0ff2b0e1a36 4.4s 8 spans my name is jesse Nice to meet you, Jes... 7d ago + ✓ OK +✓ Found 6 traces +Console: https://us-west-2.console.aws.amazon.com/cloudwatch/home?region=us-west-2#/gen-ai-observability/agent-core/agent-alias/deltaAqua_Ag +ent-HkpfaiD6hH/endpoint/DEFAULT/agent/Agent?start=-43200000&resourceId=arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A887863153624%3Aruntime%2F +deltaAqua_Agent-HkpfaiD6hH%2Fruntime-endpoint%2FDEFAULT%3ADEFAULT&serviceName=Agent.DEFAULT&tabId=traces +Note: Traces may take 2-3 minutes to appear in CloudWatch + + + \ No newline at end of file diff --git a/docs/screenshots/traces-list.svg b/docs/screenshots/traces-list.svg new file mode 100644 index 00000000..f476bd16 --- /dev/null +++ b/docs/screenshots/traces-list.svg @@ -0,0 +1,45 @@ + + + + + + + + +@aws-sdk/credential-provider-node - defaultProvider::fromEnv WARNING: + Multiple credential sources detected: + Both AWS_PROFILE and the pair AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY static credentials are set. + This SDK will proceed with the AWS_PROFILE value. + However, a future version may change this behavior to prefer the ENV static credentials. + Please ensure that your environment only sets either the AWS_PROFILE or the + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY pair. +Traces for Agent (target: dev) +# Trace ID Duration Status Input Output Age +───────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + 1 69c1821e0833a477277f7c8434a7c482 2.9s 8 spans what is my name I don't know your nam... 7d ago + (latest) ✓ OK + 2 69c182155d775da239fd98f10671641f 3.0s 8 spans my name is jesse Hello Jesse! Nice to ... 7d ago + ✓ OK + 3 69c180ee40e3f629745b08f409958dea 3.7s 8 spans what is my name I don't know your nam... 7d ago + ✓ OK + 4 69c17fec0125491e652c2a8f6e640dee 3.9s 8 spans my name is jesse Hello Jesse! Nice to ... 7d ago + ✓ OK + 5 69c17fc733a42c8838e995c66679f706 4.5s 8 spans my name is jesse Hello Jesse! Nice to ... 7d ago + ✓ OK + 6 69c17f6f6f6b04121d08a0ff2b0e1a36 4.4s 8 spans my name is jesse Nice to meet you, Jes... 7d ago + ✓ OK +✓ Found 6 traces +Console: https://us-west-2.console.aws.amazon.com/cloudwatch/home?region=us-west-2#/gen-ai-observability/agent-core/agent-alias/deltaAqua_Ag +ent-HkpfaiD6hH/endpoint/DEFAULT/agent/Agent?start=-43200000&resourceId=arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A887863153624%3Aruntime%2F +deltaAqua_Agent-HkpfaiD6hH%2Fruntime-endpoint%2FDEFAULT%3ADEFAULT&serviceName=Agent.DEFAULT&tabId=traces +Note: Traces may take 2-3 minutes to appear in CloudWatch + + + \ No newline at end of file