From 4d0839d3cdc7a18180e4c27a08c6aabd28883806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borna=20=C5=A0unji=C4=87?= Date: Thu, 19 Mar 2026 11:15:13 +0100 Subject: [PATCH 1/4] feat: grafana tests --- tests/grafana/grafana-slo-dashboard.test.ts | 224 ++++++++++++++++++++ tests/grafana/index.test.ts | 40 ++++ tests/grafana/infrastructure/config.ts | 10 + tests/grafana/infrastructure/index.ts | 92 ++++++++ tests/grafana/test-context.ts | 28 +++ 5 files changed, 394 insertions(+) create mode 100644 tests/grafana/grafana-slo-dashboard.test.ts create mode 100644 tests/grafana/index.test.ts create mode 100644 tests/grafana/infrastructure/config.ts create mode 100644 tests/grafana/infrastructure/index.ts create mode 100644 tests/grafana/test-context.ts diff --git a/tests/grafana/grafana-slo-dashboard.test.ts b/tests/grafana/grafana-slo-dashboard.test.ts new file mode 100644 index 00000000..e97db960 --- /dev/null +++ b/tests/grafana/grafana-slo-dashboard.test.ts @@ -0,0 +1,224 @@ +import { it } from 'node:test'; +import * as assert from 'node:assert'; +import { + GetRoleCommand, + GetRolePolicyCommand, + ListRolePoliciesCommand, +} from '@aws-sdk/client-iam'; +import type { Dispatcher } from 'undici'; +import { request } from 'undici'; +import { Unwrap } from '@pulumi/pulumi'; +import { backOff } from '../util'; +import { GrafanaTestContext } from './test-context'; + +const backOffConfig = { numOfAttempts: 15 }; + +export function testGrafanaSloDashboard(ctx: GrafanaTestContext) { + it('should have created the Prometheus data source', async () => { + const grafana = ctx.outputs!.grafanaSloComponent; + const prometheusDataSource = grafana.prometheusDataSource!; + const prometheusDataSourceName = + prometheusDataSource.name as unknown as Unwrap< + typeof prometheusDataSource.name + >; + + await backOff(async () => { + const { body, statusCode } = await grafanaRequest( + ctx, + 'GET', + `/api/datasources/name/${encodeURIComponent(prometheusDataSourceName)}`, + ); + assert.strictEqual(statusCode, 200, 'Expected data source to exist'); + + const data = (await body.json()) as Record; + assert.strictEqual( + data.type, + 'grafana-amazonprometheus-datasource', + 'Expected Amazon Prometheus data source type', + ); + + const workspace = ctx.outputs!.prometheusWorkspace; + const prometheusEndpoint = + workspace.prometheusEndpoint as unknown as Unwrap< + typeof workspace.prometheusEndpoint + >; + assert.ok( + (data.url as string).includes(prometheusEndpoint.replace(/\/$/, '')), + 'Expected data source URL to contain the AMP workspace endpoint', + ); + }, backOffConfig); + }); + + it('should have created the dashboard with expected panels', async () => { + const dashboard = ctx.outputs!.grafanaSloComponent.dashboards[0]; + const dashboardUid = dashboard.uid as unknown as Unwrap< + typeof dashboard.uid + >; + + await backOff(async () => { + const { body, statusCode } = await grafanaRequest( + ctx, + 'GET', + `/api/dashboards/uid/${dashboardUid}`, + ); + assert.strictEqual(statusCode, 200, 'Expected dashboard to exist'); + + const data = (await body.json()) as { + dashboard: { title: string; panels: Array<{ title: string }> }; + }; + assert.strictEqual( + data.dashboard.title, + 'ICB Grafana Test SLO', + 'Expected dashboard title to match', + ); + + const panelTitles = data.dashboard.panels.map(p => p.title).sort(); + const expectedPanels = [ + 'Availability', + 'Availability Burn Rate', + 'Success Rate', + 'Success Rate Burn Rate', + 'HTTP Request Success Rate', + 'Request % below 250ms', + 'Latency Burn Rate', + '99th Percentile Latency', + 'Request percentage below 250ms', + ]; + assert.deepStrictEqual( + panelTitles, + expectedPanels.sort(), + 'Dashboard panels do not match expected panels', + ); + }, backOffConfig); + }); + + it('should display metrics data in the dashboard', async () => { + await requestEndpointWithExpectedStatus(ctx, ctx.config.usersPath, 200); + + const prometheusDataSource = + ctx.outputs!.grafanaSloComponent.prometheusDataSource!; + const prometheusDataSourceName = + prometheusDataSource.name as unknown as Unwrap< + typeof prometheusDataSource.name + >; + const { body: dsBody } = await grafanaRequest( + ctx, + 'GET', + `/api/datasources/name/${encodeURIComponent(prometheusDataSourceName)}`, + ); + const dsData = (await dsBody.json()) as Record; + const dataSourceUid = dsData.uid as string; + + await backOff(async () => { + const { body, statusCode } = await grafanaRequest( + ctx, + 'POST', + '/api/ds/query', + { + queries: [ + { + datasource: { + type: 'grafana-amazonprometheus-datasource', + uid: dataSourceUid, + }, + expr: `{__name__=~"${ctx.config.prometheusNamespace}_.*"}`, + instant: true, + refId: 'A', + }, + ], + from: 'now-5m', + to: 'now', + }, + ); + assert.strictEqual(statusCode, 200, 'Expected query to succeed'); + + const data = (await body.json()) as { + results: Record }>; + }; + const frames = data.results?.A?.frames ?? []; + assert.ok( + frames.length > 0, + `Expected Grafana to return metric frames for namespace '${ctx.config.prometheusNamespace}'`, + ); + }, backOffConfig); + }); + + it('should have created the IAM role with AMP inline policy', async () => { + const iamRole = ctx.outputs!.grafanaSloComponent.grafanaIamRole; + const grafanaAmpRoleArn = iamRole.arn as unknown as Unwrap< + typeof iamRole.arn + >; + const roleName = grafanaAmpRoleArn.split('/').pop()!; + const { Role } = await ctx.clients.iam.send( + new GetRoleCommand({ RoleName: roleName }), + ); + assert.ok(Role, 'Grafana IAM role should exist'); + + const { PolicyNames } = await ctx.clients.iam.send( + new ListRolePoliciesCommand({ RoleName: roleName }), + ); + assert.ok( + PolicyNames && PolicyNames.length > 0, + 'IAM role should have at least one inline policy', + ); + + const { PolicyDocument } = await ctx.clients.iam.send( + new GetRolePolicyCommand({ + RoleName: roleName, + PolicyName: PolicyNames[0], + }), + ); + const policy = JSON.parse(decodeURIComponent(PolicyDocument!)) as { + Statement: Array<{ Action: string[] }>; + }; + const actions = policy.Statement.flatMap(s => s.Action).sort(); + const expectedActions = [ + 'aps:GetSeries', + 'aps:GetLabels', + 'aps:GetMetricMetadata', + 'aps:QueryMetrics', + ].sort(); + assert.deepStrictEqual( + actions, + expectedActions, + 'AMP policy actions do not match expected actions', + ); + }); +} + +async function requestEndpointWithExpectedStatus( + ctx: GrafanaTestContext, + path: string, + expectedStatus: number, +): Promise { + await backOff(async () => { + const webServer = ctx.outputs!.webServer; + const dnsName = webServer.lb.lb.dnsName as unknown as Unwrap< + typeof webServer.lb.lb.dnsName + >; + const endpoint = `http://${dnsName}${path}`; + const response = await request(endpoint); + assert.strictEqual( + response.statusCode, + expectedStatus, + `Endpoint ${endpoint} should return ${expectedStatus}`, + ); + }, backOffConfig); +} + +async function grafanaRequest( + ctx: GrafanaTestContext, + method: Dispatcher.HttpMethod, + path: string, + body?: unknown, +) { + const url = `${ctx.config.grafanaUrl.replace(/\/$/, '')}${path}`; + return request(url, { + method, + headers: { + Authorization: `Bearer ${ctx.config.grafanaAuth}`, + 'Content-Type': 'application/json', + }, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); +} diff --git a/tests/grafana/index.test.ts b/tests/grafana/index.test.ts new file mode 100644 index 00000000..a9ba4ea1 --- /dev/null +++ b/tests/grafana/index.test.ts @@ -0,0 +1,40 @@ +import { before, describe, after } from 'node:test'; +import { InlineProgramArgs, OutputMap } from '@pulumi/pulumi/automation'; +import { IAMClient } from '@aws-sdk/client-iam'; +import * as automation from '../automation'; +import { requireEnv, unwrapOutputs } from '../util'; +import { testGrafanaSloDashboard } from './grafana-slo-dashboard.test'; +import * as infraConfig from './infrastructure/config'; +import { GrafanaTestContext, ProgramOutput } from './test-context'; + +const programArgs: InlineProgramArgs = { + stackName: 'dev', + projectName: 'icb-test-grafana', + program: () => import('./infrastructure'), +}; + +const region = requireEnv('AWS_REGION'); +const ctx: GrafanaTestContext = { + config: { + region, + usersPath: infraConfig.usersPath, + appName: infraConfig.appName, + prometheusNamespace: infraConfig.prometheusNamespace, + grafanaUrl: requireEnv('GRAFANA_URL'), + grafanaAuth: requireEnv('GRAFANA_AUTH'), + }, + clients: { + iam: new IAMClient({ region }), + }, +}; + +describe('Grafana component deployment', () => { + before(async () => { + const outputs: OutputMap = await automation.deploy(programArgs); + ctx.outputs = unwrapOutputs(outputs); + }); + + after(() => automation.destroy(programArgs)); + + describe('SLO dashboard', () => testGrafanaSloDashboard(ctx)); +}); diff --git a/tests/grafana/infrastructure/config.ts b/tests/grafana/infrastructure/config.ts new file mode 100644 index 00000000..5e601b36 --- /dev/null +++ b/tests/grafana/infrastructure/config.ts @@ -0,0 +1,10 @@ +export const appName = 'grafana-test'; + +export const appImage = 'studiondev/observability-sample-app'; +export const appPort = 3000; + +export const usersPath = '/users'; + +export const prometheusNamespace = 'icb_grafana_integration'; + +export const apiFilter = 'http_route=~"/.*"'; diff --git a/tests/grafana/infrastructure/index.ts b/tests/grafana/infrastructure/index.ts new file mode 100644 index 00000000..8ccd438e --- /dev/null +++ b/tests/grafana/infrastructure/index.ts @@ -0,0 +1,92 @@ +import * as aws from '@pulumi/aws'; +import * as pulumi from '@pulumi/pulumi'; +import * as studion from '@studion/infra-code-blocks'; +import { getCommonVpc } from '../../util'; +import { + appImage, + appPort, + appName, + prometheusNamespace, + apiFilter, +} from './config'; + +const stackName = pulumi.getStack(); +const parent = new pulumi.ComponentResource( + 'studion:grafana:TestGroup', + `${appName}-root`, +); +const tags = { + Env: stackName, + Project: appName, +}; + +const vpc = getCommonVpc(); +const cluster = new aws.ecs.Cluster(`${appName}-cluster`, { tags }, { parent }); + +const prometheusWorkspace = new aws.amp.Workspace( + `${appName}-workspace`, + { tags }, + { parent }, +); + +const cloudWatchLogGroup = new aws.cloudwatch.LogGroup( + `${appName}-log-group`, + { + name: `/grafana/test/${appName}-${stackName}`, + tags, + }, + { parent }, +); + +const otelCollector = new studion.openTelemetry.OtelCollectorBuilder( + appName, + stackName, +) + .withDefault({ + prometheusNamespace, + prometheusWorkspace, + region: aws.config.requireRegion(), + logGroup: cloudWatchLogGroup, + logStreamName: `${appName}-stream`, + }) + .build(); + +const ecs = { + cluster, + desiredCount: 1, + size: 'small' as const, + autoscaling: { enabled: false }, +}; + +const webServer = new studion.WebServerBuilder(appName) + .withContainer(appImage, appPort, { + environment: [ + { name: 'OTEL_SERVICE_NAME', value: appName }, + { name: 'OTEL_EXPORTER_OTLP_ENDPOINT', value: 'http://127.0.0.1:4318' }, + { name: 'OTEL_EXPORTER_OTLP_PROTOCOL', value: 'http/json' }, + ], + }) + .withEcsConfig(ecs) + .withVpc(vpc.vpc) + .withOtelCollector(otelCollector) + .build({ parent }); + +const grafanaSloDashboard = + new studion.grafana.dashboard.WebServerSloDashboardBuilder( + `${appName}-slo-dashboard`, + { title: 'ICB Grafana Test SLO' }, + ) + .withAvailability(0.99, '1d', prometheusNamespace) + .withSuccessRate(0.95, '1d', '1h', apiFilter, prometheusNamespace) + .withLatency(0.95, 250, '1d', '1h', apiFilter, prometheusNamespace) + .build(); + +const grafanaSloComponent = new studion.grafana.GrafanaBuilder(`${appName}-slo`) + .withPrometheus({ + endpoint: prometheusWorkspace.prometheusEndpoint, + region: aws.config.requireRegion(), + }) + .addDashboard(grafanaSloDashboard) + .build({ parent }); + +export { webServer, prometheusWorkspace, grafanaSloComponent }; diff --git a/tests/grafana/test-context.ts b/tests/grafana/test-context.ts new file mode 100644 index 00000000..4c0dd13e --- /dev/null +++ b/tests/grafana/test-context.ts @@ -0,0 +1,28 @@ +import * as aws from '@pulumi/aws'; +import * as studion from '@studion/infra-code-blocks'; +import { IAMClient } from '@aws-sdk/client-iam'; +import { AwsContext, ConfigContext, PulumiProgramContext } from '../types'; + +interface Config { + region: string; + usersPath: string; + appName: string; + prometheusNamespace: string; + grafanaUrl: string; + grafanaAuth: string; +} + +interface AwsClients { + iam: IAMClient; +} + +export interface ProgramOutput { + webServer: studion.WebServer; + prometheusWorkspace: aws.amp.Workspace; + grafanaSloComponent: studion.grafana.Grafana; +} + +export interface GrafanaTestContext + extends ConfigContext, + PulumiProgramContext, + AwsContext {} From f8918f309c7b5dc636db13aa95084609c3e29033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borna=20=C5=A0unji=C4=87?= Date: Thu, 19 Mar 2026 11:46:27 +0100 Subject: [PATCH 2/4] test: add custom panel test --- tests/grafana/grafana-slo-dashboard.test.ts | 1 + tests/grafana/infrastructure/index.ts | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/tests/grafana/grafana-slo-dashboard.test.ts b/tests/grafana/grafana-slo-dashboard.test.ts index e97db960..64aea2e2 100644 --- a/tests/grafana/grafana-slo-dashboard.test.ts +++ b/tests/grafana/grafana-slo-dashboard.test.ts @@ -83,6 +83,7 @@ export function testGrafanaSloDashboard(ctx: GrafanaTestContext) { 'Latency Burn Rate', '99th Percentile Latency', 'Request percentage below 250ms', + 'Custom Panel', ]; assert.deepStrictEqual( panelTitles, diff --git a/tests/grafana/infrastructure/index.ts b/tests/grafana/infrastructure/index.ts index 8ccd438e..d15c1ef0 100644 --- a/tests/grafana/infrastructure/index.ts +++ b/tests/grafana/infrastructure/index.ts @@ -79,6 +79,14 @@ const grafanaSloDashboard = .withAvailability(0.99, '1d', prometheusNamespace) .withSuccessRate(0.95, '1d', '1h', apiFilter, prometheusNamespace) .withLatency(0.95, 250, '1d', '1h', apiFilter, prometheusNamespace) + .addPanel(dataSource => ({ + title: 'Custom Panel', + type: 'timeseries', + gridPos: { x: 12, y: 24, w: 12, h: 8 }, + datasource: dataSource.prometheus!, + targets: [{ expr: 'up', legendFormat: 'Up' }], + fieldConfig: { defaults: {} }, + })) .build(); const grafanaSloComponent = new studion.grafana.GrafanaBuilder(`${appName}-slo`) From 6414b6736818f615929c58d51657d0bd64261d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borna=20=C5=A0unji=C4=87?= Date: Mon, 30 Mar 2026 12:49:58 +0200 Subject: [PATCH 3/4] fix: test configuration --- tests/grafana/grafana-slo-dashboard.test.ts | 14 ++++++---- tests/grafana/infrastructure/index.ts | 29 +++++++++------------ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/tests/grafana/grafana-slo-dashboard.test.ts b/tests/grafana/grafana-slo-dashboard.test.ts index 64aea2e2..07178d39 100644 --- a/tests/grafana/grafana-slo-dashboard.test.ts +++ b/tests/grafana/grafana-slo-dashboard.test.ts @@ -1,5 +1,6 @@ import { it } from 'node:test'; import * as assert from 'node:assert'; +import * as studion from '@studion/infra-code-blocks'; import { GetRoleCommand, GetRolePolicyCommand, @@ -16,7 +17,9 @@ const backOffConfig = { numOfAttempts: 15 }; export function testGrafanaSloDashboard(ctx: GrafanaTestContext) { it('should have created the Prometheus data source', async () => { const grafana = ctx.outputs!.grafanaSloComponent; - const prometheusDataSource = grafana.prometheusDataSource!; + const prometheusDataSource = ( + grafana.connections[0] as studion.grafana.AMPConnection + ).dataSource; const prometheusDataSourceName = prometheusDataSource.name as unknown as Unwrap< typeof prometheusDataSource.name @@ -83,7 +86,6 @@ export function testGrafanaSloDashboard(ctx: GrafanaTestContext) { 'Latency Burn Rate', '99th Percentile Latency', 'Request percentage below 250ms', - 'Custom Panel', ]; assert.deepStrictEqual( panelTitles, @@ -96,8 +98,10 @@ export function testGrafanaSloDashboard(ctx: GrafanaTestContext) { it('should display metrics data in the dashboard', async () => { await requestEndpointWithExpectedStatus(ctx, ctx.config.usersPath, 200); - const prometheusDataSource = - ctx.outputs!.grafanaSloComponent.prometheusDataSource!; + const prometheusDataSource = ( + ctx.outputs!.grafanaSloComponent + .connections[0] as studion.grafana.AMPConnection + ).dataSource; const prometheusDataSourceName = prometheusDataSource.name as unknown as Unwrap< typeof prometheusDataSource.name @@ -145,7 +149,7 @@ export function testGrafanaSloDashboard(ctx: GrafanaTestContext) { }); it('should have created the IAM role with AMP inline policy', async () => { - const iamRole = ctx.outputs!.grafanaSloComponent.grafanaIamRole; + const iamRole = ctx.outputs!.grafanaSloComponent.connections[0].role; const grafanaAmpRoleArn = iamRole.arn as unknown as Unwrap< typeof iamRole.arn >; diff --git a/tests/grafana/infrastructure/index.ts b/tests/grafana/infrastructure/index.ts index d15c1ef0..95e5097b 100644 --- a/tests/grafana/infrastructure/index.ts +++ b/tests/grafana/infrastructure/index.ts @@ -72,25 +72,20 @@ const webServer = new studion.WebServerBuilder(appName) .build({ parent }); const grafanaSloDashboard = - new studion.grafana.dashboard.WebServerSloDashboardBuilder( - `${appName}-slo-dashboard`, - { title: 'ICB Grafana Test SLO' }, - ) - .withAvailability(0.99, '1d', prometheusNamespace) - .withSuccessRate(0.95, '1d', '1h', apiFilter, prometheusNamespace) - .withLatency(0.95, 250, '1d', '1h', apiFilter, prometheusNamespace) - .addPanel(dataSource => ({ - title: 'Custom Panel', - type: 'timeseries', - gridPos: { x: 12, y: 24, w: 12, h: 8 }, - datasource: dataSource.prometheus!, - targets: [{ expr: 'up', legendFormat: 'Up' }], - fieldConfig: { defaults: {} }, - })) - .build(); + studion.grafana.dashboard.createWebServerSloDashboard({ + name: `${appName}-slo-dashboard`, + title: 'ICB Grafana Test SLO', + ampNamespace: prometheusNamespace, + filter: apiFilter, + target: 0.99, + window: '1d', + shortWindow: '1h', + targetLatency: 250, + }); const grafanaSloComponent = new studion.grafana.GrafanaBuilder(`${appName}-slo`) - .withPrometheus({ + .addAmp(`${appName}-slo-amp`, { + awsAccountId: '008923505280', endpoint: prometheusWorkspace.prometheusEndpoint, region: aws.config.requireRegion(), }) From a1505bb7d86217a8402addcafdc46cd58a76a5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borna=20=C5=A0unji=C4=87?= Date: Tue, 31 Mar 2026 18:03:01 +0200 Subject: [PATCH 4/4] fix: grafana test infrastructure --- tests/grafana/infrastructure/index.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/grafana/infrastructure/index.ts b/tests/grafana/infrastructure/index.ts index 95e5097b..88bdd938 100644 --- a/tests/grafana/infrastructure/index.ts +++ b/tests/grafana/infrastructure/index.ts @@ -71,25 +71,26 @@ const webServer = new studion.WebServerBuilder(appName) .withOtelCollector(otelCollector) .build({ parent }); -const grafanaSloDashboard = - studion.grafana.dashboard.createWebServerSloDashboard({ +const ampDataSourceName = `${appName}-slo-amp-datasource`; + +const grafanaSloComponent = new studion.grafana.GrafanaBuilder(`${appName}-slo`) + .addAmp(`${appName}-slo-amp`, { + awsAccountId: '008923505280', + endpoint: prometheusWorkspace.prometheusEndpoint, + region: aws.config.requireRegion(), + dataSourceName: ampDataSourceName, + }) + .addSloDashboard({ name: `${appName}-slo-dashboard`, title: 'ICB Grafana Test SLO', ampNamespace: prometheusNamespace, filter: apiFilter, + dataSourceName: ampDataSourceName, target: 0.99, window: '1d', shortWindow: '1h', targetLatency: 250, - }); - -const grafanaSloComponent = new studion.grafana.GrafanaBuilder(`${appName}-slo`) - .addAmp(`${appName}-slo-amp`, { - awsAccountId: '008923505280', - endpoint: prometheusWorkspace.prometheusEndpoint, - region: aws.config.requireRegion(), }) - .addDashboard(grafanaSloDashboard) .build({ parent }); export { webServer, prometheusWorkspace, grafanaSloComponent };