diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..93a7881 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +.gitignore +bin +tmp +.DS_Store diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0cd22c7 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 + +[{Dockerfile,Dockerfile.*}] +indent_size = 4 +tab_width = 4 + +[{Makefile,makefile,GNUmakefile}] +indent_style = tab +indent_size = 4 + +[Makefile.*] +indent_style = tab +indent_size = 4 + +[**/*.{go,mod,sum}] +indent_style = tab +indent_size = unset + +[**/*.py] +indent_size = 4 diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..603f653 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..c8d529f --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,78 @@ +name: ci + +on: + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Setup Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version: '1.26.2' + + - name: Run go vet + run: go vet ./... + + - name: Check formatting + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then + echo "The following files are not formatted:" + echo "$unformatted" + exit 1 + fi + + test-unit: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Setup Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version: '1.26.2' + + - name: Run unit tests + run: make test-unit + + test-integration: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Setup Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version: '1.26.2' + + - name: Run integration tests + run: make test-integration + + build: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Setup Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version: '1.26.2' + + - name: Build + run: make build diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..ef98574 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,29 @@ +name: release + +on: + push: + branches: + - main + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Bump Version + id: tag_version + uses: mathieudutour/github-tag-action@a22cf08638b34d5badda920f9daf6e72c477b07b # v6.2.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + default_bump: minor + custom_release_rules: bug:patch:Fixes,chore:patch:Chores,docs:patch:Documentation,feat:minor:Features,refactor:minor:Refactors,test:patch:Tests,ci:patch:Development,dev:patch:Development + - name: Create Release + uses: ncipollo/release-action@b7eabc95ff50cbeeedec83973935c8f306dfcd0b # v1.20.0 + with: + tag: ${{ steps.tag_version.outputs.new_tag }} + name: ${{ steps.tag_version.outputs.new_tag }} + body: ${{ steps.tag_version.outputs.changelog }} diff --git a/.github/workflows/semantic-check.yaml b/.github/workflows/semantic-check.yaml new file mode 100644 index 0000000..d83a1b6 --- /dev/null +++ b/.github/workflows/semantic-check.yaml @@ -0,0 +1,26 @@ +name: semantic-check +on: + pull_request: + types: + - opened + - edited + - synchronize + +permissions: + contents: read + pull-requests: read + +jobs: + main: + name: Semantic Commit Message Check + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 + name: Check PR for Semantic Commit Message + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + requireScope: false + validateSingleCommit: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..423be19 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +!**/.gitkeep + +tmp/ +dist/ +.DS_Store + +.local/ +.env + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ce301e2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.26.2-alpine AS builder + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server + +FROM alpine:3.21 + +RUN adduser -D -g '' appuser + +USER appuser +WORKDIR /app + +COPY --from=builder /out/server /app/server + +EXPOSE 8080 + +ENTRYPOINT ["/app/server"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c1cb0ea --- /dev/null +++ b/Makefile @@ -0,0 +1,51 @@ +COMPOSE ?= docker compose + +.PHONY: test test-unit test-integration build run docker-build up down logs demo-send demo health report + +test: + go test ./... + +test-unit: + go test ./... + +test-integration: + go test ./... -run Integration -count=1 + +build: + go build ./... + +run: + go run ./cmd/server + +docker-build: + $(COMPOSE) build api + +up: + $(COMPOSE) up --build -d api + +down: + $(COMPOSE) down -v --remove-orphans + +logs: + $(COMPOSE) logs -f api + +demo-send: + $(COMPOSE) run --rm demo-sender + +demo: up demo-send + $(COMPOSE) logs api + +health: + @BASE_PATH="$${BASE_PATH:-/}"; \ + URL_PATH="$${BASE_PATH%/}/v1/manage/healthz"; \ + if [ -z "$${URL_PATH}" ]; then URL_PATH="/v1/manage/healthz"; fi; \ + curl -fsS "http://localhost:8080$${URL_PATH}" + +report: + @BASE_PATH="$${BASE_PATH:-/}"; \ + URL_PATH="$${BASE_PATH%/}/v1/reports"; \ + if [ -z "$${URL_PATH}" ]; then URL_PATH="/v1/reports"; fi; \ + curl -i -X POST \ + -H 'Content-Type: application/reports+json' \ + --data-binary @demo/reports.json \ + "http://localhost:8080$${URL_PATH}" diff --git a/README.md b/README.md index 70e54d0..3fefe32 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,128 @@ # browser-reporting-api Simple, self-hosted Go service for browser Reporting API ingestion. + +## General + +### What + +`browser-reporting-api` is an HTTP service that receives browser reporting +payloads (`application/reports+json` and legacy `application/csp-report` from +`report-uri`), validates each report entry, and streams accepted entries to +stdout as NDJSON (one JSON object per line). + +The payload format and reporting behavior align with the browser Reporting API +documented by +[MDN](https://developer.mozilla.org/en-US/docs/Web/API/Reporting_API) and +[Chrome](https://developer.chrome.com/docs/capabilities/web-apis/reporting-api). + +Endpoints: + +- `POST /v1/reports` (or `{BASE_PATH}/v1/reports`) +- `GET /v1/manage/healthz` +- `GET /v1/manage/readyz` + +### Why + +- Simple to run and reason about +- Safe enough for ingestion (size limits, content-type checks, per-entry + vetting) +- Easy to self-host and observe (accepted reports stream to stdout) +- Collected browser reports (including CSP report traffic) may support + client-side monitoring and script-governance such as those relevant to PCI DSS + payment-page security guidance for Requirements 6.4.3 and 11.6.1 from the + [PCI Security Standards Council](https://blog.pcisecuritystandards.org/new-information-supplement-payment-page-security-and-preventing-e-skimming) + +### How + +Run locally with Go: + +```bash +go run ./cmd/server +``` + +Run a local demo with Docker Compose: + +```bash +make demo +``` + +This starts the API, sends a sample batched report payload, and prints API logs. +To keep watching streamed report lines: + +```bash +make logs +``` + +The demo sender payload is stored at `demo/reports.json`. + +Send a manual sample report: + +```bash +make report +``` + +`make report` sends the same payload file used by the compose demo: +`demo/reports.json`. + +Health check: + +```bash +make health +``` + +Stop containers: + +```bash +make down +``` + +Environment variables: + +- `LISTEN_ADDR` (default `:8080`) +- `BASE_PATH` (default `/`) +- `MAX_BODY_BYTES` (default `1048576`) +- `REPORTS_ALLOWED_ORIGINS` (default `*`) + +Allowed origin examples: + +- `REPORTS_ALLOWED_ORIGINS=*` +- `REPORTS_ALLOWED_ORIGINS=https://app.example.com,https://admin.example.com` +- `REPORTS_ALLOWED_ORIGINS=https://*.example.com,http://localhost:*` + +If `BASE_PATH=/collector`, endpoints become: + +- `POST /collector/v1/reports` +- `GET /collector/v1/manage/healthz` +- `GET /collector/v1/manage/readyz` + +## Development + +Run tests: + +```bash +go test ./... +``` + +Make targets used during development: + +```bash +make test +make run +make up +make demo-send +make demo +make logs +make report +make health +make down +``` + +Implementation notes: + +- Routes are mounted under configurable `BASE_PATH`. +- Reporting ingestion accepts batched arrays and processes entries + independently. +- Invalid entries are rejected while valid entries in the same batch are still + accepted. +- Accepted entries are emitted to stdout in NDJSON format. diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 0000000..981b9c3 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/cruxstack/browser-reporting-api/internal/config" + "github.com/cruxstack/browser-reporting-api/internal/httpx" + "github.com/cruxstack/browser-reporting-api/internal/management" + "github.com/cruxstack/browser-reporting-api/internal/parser" + "github.com/cruxstack/browser-reporting-api/internal/reporting" + "github.com/cruxstack/browser-reporting-api/internal/stream" +) + +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, nil))) + + cfg, err := config.LoadFromEnv() + if err != nil { + slog.Error("load config", "error", err) + os.Exit(1) + } + + writer := stream.NewNDJSONWriter(os.Stdout) + reportsParser := parser.NewJSONBatchParser() + reportsService := reporting.NewService(reportsParser, writer) + originMatcher, err := httpx.NewOriginMatcher(cfg.AllowedOrigins) + if err != nil { + slog.Error("build origin matcher", "error", err) + os.Exit(1) + } + + reportsHandler := reporting.NewHandler(reportsService, cfg.MaxBodyBytes, originMatcher) + managementHandler := management.NewHandler() + handler := httpx.NewRouter(cfg.BasePath, reportsHandler.Routes(), managementHandler.Routes()) + + srv := &http.Server{ + Addr: cfg.ListenAddr, + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + + go func() { + slog.Info("listening", "addr", cfg.ListenAddr, "base_path", cfg.BasePath) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + slog.Error("server error", "error", err) + os.Exit(1) + } + }() + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + <-ctx.Done() + slog.Info("shutting down") + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := srv.Shutdown(shutdownCtx); err != nil { + slog.Error("graceful shutdown failed", "error", err) + if closeErr := srv.Close(); closeErr != nil { + slog.Error("force close failed", "error", closeErr) + } + } +} diff --git a/demo/reports.json b/demo/reports.json new file mode 100644 index 0000000..c1356f1 --- /dev/null +++ b/demo/reports.json @@ -0,0 +1,63 @@ +[ + { + "age": 1, + "type": "csp-violation", + "url": "https://site.example/checkout", + "user_agent": "demo-sender/2.0", + "body": { + "effectiveDirective": "script-src", + "blockedURL": "https://cdn.bad-example.test/tracker.js", + "disposition": "enforce" + } + }, + { + "age": 3, + "type": "deprecation", + "url": "https://site.example/dashboard", + "user_agent": "demo-sender/2.0", + "body": { + "id": "PrefixedStorageInfo", + "message": "Prefixed StorageInfo is deprecated" + } + }, + { + "age": 5, + "type": "coep", + "url": "https://site.example/embed", + "user_agent": "demo-sender/2.0", + "body": { + "blockedURL": "https://thirdparty.example/widget.js", + "destination": "script" + } + }, + { + "age": 2, + "type": "intervention", + "url": "https://site.example/video", + "user_agent": "demo-sender/2.0", + "body": { + "id": "HeavyAdIntervention", + "message": "Ad resource exceeded heavy ad limits" + } + }, + { + "age": 8, + "type": "network-error", + "url": "https://site.example/profile", + "user_agent": "demo-sender/2.0", + "body": { + "status_code": 503, + "phase": "application" + } + }, + { + "age": 13, + "type": "permissions-policy-violation", + "url": "https://site.example/embed/camera", + "user_agent": "demo-sender/2.0", + "body": { + "featureId": "camera", + "sourceFile": "https://site.example/embed/camera" + } + } +] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8176ecd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +services: + api: + build: + context: . + dockerfile: Dockerfile + environment: + LISTEN_ADDR: ${LISTEN_ADDR:-:8080} + BASE_PATH: ${BASE_PATH:-/} + MAX_BODY_BYTES: ${MAX_BODY_BYTES:-1048576} + REPORTS_ALLOWED_ORIGINS: ${REPORTS_ALLOWED_ORIGINS:-*} + ports: + - "8080:8080" + + demo-sender: + image: curlimages/curl:8.12.1 + depends_on: + - api + volumes: + - ./demo/reports.json:/payload/reports.json:ro + environment: + BASE_PATH: ${BASE_PATH:-/} + command: >- + sh -c 'sleep 2; + BASE_PATH="$${BASE_PATH:-/}"; + URL_PATH="$${BASE_PATH%/}/v1/reports"; + if [ -z "$${URL_PATH}" ]; then URL_PATH="/v1/reports"; fi; + curl -sS -i -X POST + -H "Content-Type: application/reports+json" + --data-binary @/payload/reports.json + "http://api:8080$${URL_PATH}"' diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5978707 --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module github.com/cruxstack/browser-reporting-api + +go 1.26.2 + +require ( + github.com/cockroachdb/errors v1.12.0 + github.com/go-chi/chi/v5 v5.2.5 +) + +require ( + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..70fd105 --- /dev/null +++ b/go.sum @@ -0,0 +1,69 @@ +github.com/cockroachdb/errors v1.12.0 h1:d7oCs6vuIMUQRVbi6jWWWEJZahLCfJpnJSVobd1/sUo= +github.com/cockroachdb/errors v1.12.0/go.mod h1:SvzfYNNBshAVbZ8wzNc/UPK3w1vf0dKDUP41ucAIf7g= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= +github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..0da9590 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,139 @@ +package config + +import ( + "log/slog" + "net/url" + "os" + "regexp" + "strconv" + "strings" + + "github.com/cockroachdb/errors" +) + +const ( + defaultListenAddr = ":8080" + defaultBasePath = "/" + defaultMaxBodySize = 1 << 20 + defaultCORSOrigins = "*" +) + +type Config struct { + ListenAddr string + BasePath string + MaxBodyBytes int64 + AllowedOrigins []string +} + +func LoadFromEnv() (Config, error) { + reportsAllowedOrigins := strings.TrimSpace(os.Getenv("REPORTS_ALLOWED_ORIGINS")) + legacyReportsAllowedOrigin := strings.TrimSpace(os.Getenv("REPORTS_ALLOWED_ORIGIN")) + if reportsAllowedOrigins == "" && legacyReportsAllowedOrigin != "" { + slog.Warn("REPORTS_ALLOWED_ORIGIN is deprecated, use REPORTS_ALLOWED_ORIGINS") + } + + originsRaw := reportsAllowedOrigins + if originsRaw == "" { + originsRaw = valueOrDefault("REPORTS_ALLOWED_ORIGIN", defaultCORSOrigins) + } + + cfg := Config{ + ListenAddr: valueOrDefault("LISTEN_ADDR", defaultListenAddr), + BasePath: normalizeBasePath(valueOrDefault("BASE_PATH", defaultBasePath)), + AllowedOrigins: parseList(originsRaw), + } + + if err := validateAllowedOrigins(cfg.AllowedOrigins); err != nil { + return Config{}, err + } + + maxBody := valueOrDefault("MAX_BODY_BYTES", strconv.Itoa(defaultMaxBodySize)) + maxBodyInt, err := strconv.ParseInt(maxBody, 10, 64) + if err != nil || maxBodyInt <= 0 { + return Config{}, errors.Newf("invalid MAX_BODY_BYTES: %q", maxBody) + } + cfg.MaxBodyBytes = maxBodyInt + + return cfg, nil +} + +var wildcardPattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://.*\*.*$`) + +func validateAllowedOrigins(origins []string) error { + if len(origins) == 0 { + return errors.New("REPORTS_ALLOWED_ORIGINS cannot be empty") + } + + for _, origin := range origins { + if origin == "*" { + continue + } + + if strings.Contains(origin, "*") { + if !wildcardPattern.MatchString(origin) { + return errors.Newf("invalid wildcard origin pattern: %q", origin) + } + continue + } + + if !strings.Contains(origin, "://") { + return errors.Newf("invalid origin %q: must include scheme", origin) + } + + parsed, err := url.Parse(origin) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return errors.Newf("invalid origin %q: must be scheme://host[:port]", origin) + } + + if parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.User != nil { + return errors.Newf("invalid origin %q: must be scheme://host[:port]", origin) + } + } + + return nil +} + +func valueOrDefault(name, fallback string) string { + value := strings.TrimSpace(os.Getenv(name)) + if value == "" { + return fallback + } + return value +} + +func parseList(raw string) []string { + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + v := strings.TrimSpace(p) + if v == "" { + continue + } + out = append(out, v) + } + if len(out) == 0 { + return []string{"*"} + } + return out +} + +func normalizeBasePath(basePath string) string { + basePath = strings.TrimSpace(basePath) + if basePath == "" || basePath == "/" { + return "/" + } + + if !strings.HasPrefix(basePath, "/") { + basePath = "/" + basePath + } + + for strings.HasSuffix(basePath, "/") { + basePath = strings.TrimSuffix(basePath, "/") + } + + if basePath == "" { + return "/" + } + + return basePath +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..c43e863 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,24 @@ +package config + +import "testing" + +func TestValidateAllowedOriginsAcceptsValidExactOrigin(t *testing.T) { + err := validateAllowedOrigins([]string{"https://app.example.com"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestValidateAllowedOriginsRejectsOriginWithPath(t *testing.T) { + err := validateAllowedOrigins([]string{"https://app.example.com/path"}) + if err == nil { + t.Fatal("expected error") + } +} + +func TestValidateAllowedOriginsRejectsOriginWithTrailingSlash(t *testing.T) { + err := validateAllowedOrigins([]string{"https://app.example.com/"}) + if err == nil { + t.Fatal("expected error") + } +} diff --git a/internal/domain/ingest.go b/internal/domain/ingest.go new file mode 100644 index 0000000..c3fbfe8 --- /dev/null +++ b/internal/domain/ingest.go @@ -0,0 +1,32 @@ +package domain + +import ( + "context" + "io" +) + +// Parser parses a batch of incoming reports from a reader. +type Parser interface { + ParseBatch(r io.Reader) ([]IncomingReport, error) +} + +// Sink writes an accepted report to an output destination. +type Sink interface { + WriteReport(ctx context.Context, report AcceptedReport) error +} + +// IngestResult summarises the outcome of a single ingest call. +type IngestResult struct { + Received int `json:"received"` + Accepted int `json:"accepted"` + Rejected int `json:"rejected"` +} + +// BadPayloadError wraps errors caused by a malformed or invalid client +// payload. Callers use errors.As to distinguish 4xx from 5xx conditions. +type BadPayloadError struct { + Err error +} + +func (e *BadPayloadError) Error() string { return e.Err.Error() } +func (e *BadPayloadError) Unwrap() error { return e.Err } diff --git a/internal/domain/report.go b/internal/domain/report.go new file mode 100644 index 0000000..34f8b76 --- /dev/null +++ b/internal/domain/report.go @@ -0,0 +1,84 @@ +package domain + +import ( + "bytes" + "encoding/json" + "net/url" + "strings" + "time" + + "github.com/cockroachdb/errors" +) + +const ( + maxTypeLen = 128 + maxURLLen = 4096 + maxUserAgentLen = 1024 +) + +// IncomingReport is the raw, unvalidated report entry received from the browser. +type IncomingReport struct { + Age int64 `json:"age"` + Type string `json:"type"` + URL string `json:"url"` + UserAgent string `json:"user_agent"` + Body json.RawMessage `json:"body"` +} + +// AcceptedReport is a validated, timestamped report ready for output. +type AcceptedReport struct { + ReceivedAt time.Time + Age int64 + Type string + URL string + UserAgent string + Body json.RawMessage +} + +// Vet validates a single IncomingReport and returns an AcceptedReport stamped +// with the provided time. It returns an error if any field fails validation. +func Vet(report IncomingReport, now time.Time) (AcceptedReport, error) { + reportType := strings.TrimSpace(report.Type) + if reportType == "" || len(reportType) > maxTypeLen { + return AcceptedReport{}, errors.New("invalid type") + } + + reportURL := strings.TrimSpace(report.URL) + if reportURL == "" || len(reportURL) > maxURLLen { + return AcceptedReport{}, errors.New("invalid url") + } + + parsedURL, err := url.ParseRequestURI(reportURL) + if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" { + return AcceptedReport{}, errors.New("invalid url") + } + + userAgent := strings.TrimSpace(report.UserAgent) + if len(userAgent) > maxUserAgentLen { + return AcceptedReport{}, errors.New("invalid user_agent") + } + + if report.Age < 0 { + return AcceptedReport{}, errors.New("invalid age") + } + + if len(report.Body) == 0 { + return AcceptedReport{}, errors.New("missing body") + } + + // Validate that body is a JSON object without allocating a full map. + // RawMessage is guaranteed valid JSON by the decoder, so only the type + // (first non-whitespace byte) needs checking. + if trimmed := bytes.TrimSpace(report.Body); len(trimmed) == 0 || trimmed[0] != '{' { + return AcceptedReport{}, errors.New("invalid body: must be a JSON object") + } + + return AcceptedReport{ + ReceivedAt: now.UTC(), + Age: report.Age, + Type: reportType, + URL: reportURL, + UserAgent: userAgent, + Body: report.Body, + }, nil +} diff --git a/internal/domain/report_test.go b/internal/domain/report_test.go new file mode 100644 index 0000000..d24ee35 --- /dev/null +++ b/internal/domain/report_test.go @@ -0,0 +1,50 @@ +package domain + +import ( + "strings" + "testing" + "time" +) + +func TestVetAcceptsValidReport(t *testing.T) { + now := time.Date(2026, 4, 13, 12, 0, 0, 0, time.UTC) + report := IncomingReport{ + Age: 3, + Type: "csp-violation", + URL: "https://site.example/path", + UserAgent: "Mozilla/5.0", + Body: []byte(`{"directive":"script-src"}`), + } + + accepted, err := Vet(report, now) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + if accepted.Type != report.Type { + t.Fatalf("expected type %q, got %q", report.Type, accepted.Type) + } +} + +func TestVetRejectsInvalidURL(t *testing.T) { + _, err := Vet(IncomingReport{ + Type: "csp-violation", + URL: "/relative", + Body: []byte(`{"x":1}`), + }, time.Now()) + if err == nil { + t.Fatal("expected error") + } +} + +func TestVetRejectsOversizedUserAgent(t *testing.T) { + _, err := Vet(IncomingReport{ + Type: "csp-violation", + URL: "https://site.example", + UserAgent: strings.Repeat("a", maxUserAgentLen+1), + Body: []byte(`{"x":1}`), + }, time.Now()) + if err == nil { + t.Fatal("expected error") + } +} diff --git a/internal/httpx/origin_matcher.go b/internal/httpx/origin_matcher.go new file mode 100644 index 0000000..9f0648a --- /dev/null +++ b/internal/httpx/origin_matcher.go @@ -0,0 +1,101 @@ +package httpx + +import ( + "regexp" + "strings" + + "github.com/cockroachdb/errors" +) + +type OriginMatcher struct { + allowAll bool + exact map[string]struct{} + patterns []*regexp.Regexp +} + +func NewOriginMatcher(origins []string) (*OriginMatcher, error) { + m := &OriginMatcher{exact: make(map[string]struct{})} + + for _, raw := range origins { + origin := strings.TrimSpace(raw) + if origin == "" { + continue + } + + if origin == "*" { + m.allowAll = true + continue + } + + if strings.Contains(origin, "*") { + re, err := wildcardPatternToRegex(origin) + if err != nil { + return nil, err + } + m.patterns = append(m.patterns, re) + continue + } + + m.exact[origin] = struct{}{} + } + + if len(m.exact) == 0 && len(m.patterns) == 0 && !m.allowAll { + return nil, errors.New("no valid origins provided") + } + + return m, nil +} + +func (m *OriginMatcher) AllowHeaderValue(origin string) (string, bool) { + if m.allowAll { + return "*", true + } + + origin = strings.TrimSpace(origin) + if origin == "" { + return "", false + } + + if _, ok := m.exact[origin]; ok { + return origin, true + } + + for _, re := range m.patterns { + if re.MatchString(origin) { + return origin, true + } + } + + return "", false +} + +func wildcardPatternToRegex(pattern string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + for i, ch := range pattern { + if ch == '*' { + if isPortWildcard(pattern, i) { + b.WriteString("[0-9]+") + continue + } + + // Match any character except '.' so that a single wildcard + // segment cannot span subdomain or port boundaries, matching + // standard CORS same-origin semantics. + b.WriteString("[^.]*") + continue + } + b.WriteString(regexp.QuoteMeta(string(ch))) + } + b.WriteString("$") + + return regexp.Compile(b.String()) +} + +func isPortWildcard(pattern string, wildcardIndex int) bool { + if wildcardIndex <= 0 { + return false + } + + return pattern[wildcardIndex-1] == ':' +} diff --git a/internal/httpx/origin_matcher_test.go b/internal/httpx/origin_matcher_test.go new file mode 100644 index 0000000..297fb90 --- /dev/null +++ b/internal/httpx/origin_matcher_test.go @@ -0,0 +1,38 @@ +package httpx + +import "testing" + +func TestOriginMatcherAllowAll(t *testing.T) { + m, err := NewOriginMatcher([]string{"*"}) + if err != nil { + t.Fatalf("new matcher: %v", err) + } + + allow, ok := m.AllowHeaderValue("https://any.example") + if !ok || allow != "*" { + t.Fatalf("expected allow all, got allow=%q ok=%t", allow, ok) + } +} + +func TestOriginMatcherWildcardPattern(t *testing.T) { + m, err := NewOriginMatcher([]string{"https://*.example.com", "http://localhost:*"}) + if err != nil { + t.Fatalf("new matcher: %v", err) + } + + if _, ok := m.AllowHeaderValue("https://app.example.com"); !ok { + t.Fatal("expected wildcard host to match") + } + + if _, ok := m.AllowHeaderValue("http://localhost:3000"); !ok { + t.Fatal("expected wildcard port to match") + } + + if _, ok := m.AllowHeaderValue("http://localhost:abc"); ok { + t.Fatal("expected wildcard port to reject non-numeric value") + } + + if _, ok := m.AllowHeaderValue("https://evil.test"); ok { + t.Fatal("expected disallowed origin") + } +} diff --git a/internal/httpx/response.go b/internal/httpx/response.go new file mode 100644 index 0000000..126d105 --- /dev/null +++ b/internal/httpx/response.go @@ -0,0 +1,21 @@ +package httpx + +import ( + "encoding/json" + "log/slog" + "net/http" + "strings" +) + +func WriteJSON(w http.ResponseWriter, status int, payload any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(payload); err != nil { + slog.Error("json encode response", "error", err) + } +} + +func MethodNotAllowed(w http.ResponseWriter, allowed ...string) { + w.Header().Set("Allow", strings.Join(allowed, ", ")) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) +} diff --git a/internal/httpx/router.go b/internal/httpx/router.go new file mode 100644 index 0000000..ceb669f --- /dev/null +++ b/internal/httpx/router.go @@ -0,0 +1,26 @@ +package httpx + +import ( + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" +) + +func NewRouter(basePath string, reports, management chi.Router) http.Handler { + api := chi.NewRouter() + api.Use(middleware.RequestID) + api.Use(middleware.RealIP) + api.Use(middleware.Recoverer) + + api.Mount("/v1/reports", reports) + api.Mount("/v1/manage", management) + + if basePath == "/" { + return api + } + + r := chi.NewRouter() + r.Mount(basePath, api) + return r +} diff --git a/internal/httpx/router_integration_test.go b/internal/httpx/router_integration_test.go new file mode 100644 index 0000000..b6fe080 --- /dev/null +++ b/internal/httpx/router_integration_test.go @@ -0,0 +1,165 @@ +package httpx_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/cruxstack/browser-reporting-api/internal/domain" + "github.com/cruxstack/browser-reporting-api/internal/httpx" + "github.com/cruxstack/browser-reporting-api/internal/management" + "github.com/cruxstack/browser-reporting-api/internal/parser" + "github.com/cruxstack/browser-reporting-api/internal/reporting" +) + +type captureSink struct { + reports []domain.AcceptedReport +} + +func (s *captureSink) WriteReport(_ context.Context, report domain.AcceptedReport) error { + s.reports = append(s.reports, report) + return nil +} + +func TestIntegrationRouterHandlesReportingAndManagement(t *testing.T) { + sink := &captureSink{} + reportsParser := parser.NewJSONBatchParser() + reportsService := reporting.NewService(reportsParser, sink) + origins, err := httpx.NewOriginMatcher([]string{"https://*.example.com"}) + if err != nil { + t.Fatalf("new matcher: %v", err) + } + + reportsHandler := reporting.NewHandler(reportsService, 1024*1024, origins) + managementHandler := management.NewHandler() + router := httpx.NewRouter("/collector", reportsHandler.Routes(), managementHandler.Routes()) + server := httptest.NewServer(router) + defer server.Close() + + payload := `[ + {"age":1,"type":"csp-violation","url":"https://site.example","user_agent":"demo/1.0","body":{"effectiveDirective":"script-src"}}, + {"age":4,"type":"deprecation","url":"https://site.example/page","body":{"id":"PrefixedStorageInfo"}}, + {"age":3,"type":"coep","url":"https://site.example/embed","body":{"blockedURL":"https://cdn.example/script.js"}}, + {"age":7,"type":"intervention","url":"https://site.example/feature","body":{"message":"Feature policy blocked"}}, + {"age":2,"type":"","url":"https://site.example/invalid","body":{"bad":true}} + ]` + + req, err := http.NewRequest(http.MethodPost, server.URL+"/collector/v1/reports", bytes.NewBufferString(payload)) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Origin", "https://app.example.com") + req.Header.Set("Content-Type", "application/reports+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected %d, got %d", http.StatusOK, resp.StatusCode) + } + + var result domain.IngestResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + t.Fatalf("decode response: %v", err) + } + + if result.Received != 5 || result.Accepted != 4 || result.Rejected != 1 { + t.Fatalf("unexpected ingest result: %+v", result) + } + + if len(sink.reports) != 4 { + t.Fatalf("expected 4 accepted reports written to sink, got %d", len(sink.reports)) + } + + healthResp, err := http.Get(server.URL + "/collector/v1/manage/healthz") + if err != nil { + t.Fatalf("health request: %v", err) + } + defer healthResp.Body.Close() + + if healthResp.StatusCode != http.StatusOK { + t.Fatalf("expected health status %d, got %d", http.StatusOK, healthResp.StatusCode) + } + + readyResp, err := http.Get(server.URL + "/collector/v1/manage/readyz") + if err != nil { + t.Fatalf("ready request: %v", err) + } + defer readyResp.Body.Close() + + if readyResp.StatusCode != http.StatusOK { + t.Fatalf("expected ready status %d, got %d", http.StatusOK, readyResp.StatusCode) + } +} + +func TestIntegrationRouterRejectsUnsupportedContentType(t *testing.T) { + sink := &captureSink{} + reportsParser := parser.NewJSONBatchParser() + reportsService := reporting.NewService(reportsParser, sink) + origins, err := httpx.NewOriginMatcher([]string{"*"}) + if err != nil { + t.Fatalf("new matcher: %v", err) + } + + reportsHandler := reporting.NewHandler(reportsService, 1024*1024, origins) + router := httpx.NewRouter("/", reportsHandler.Routes(), management.NewHandler().Routes()) + server := httptest.NewServer(router) + defer server.Close() + + req, err := http.NewRequest(http.MethodPost, server.URL+"/v1/reports", bytes.NewBufferString(`[]`)) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusUnsupportedMediaType { + t.Fatalf("expected %d, got %d", http.StatusUnsupportedMediaType, resp.StatusCode) + } +} + +func TestIntegrationRouterRejectsOversizeBody(t *testing.T) { + sink := &captureSink{} + reportsParser := parser.NewJSONBatchParser() + reportsService := reporting.NewService(reportsParser, sink) + origins, err := httpx.NewOriginMatcher([]string{"*"}) + if err != nil { + t.Fatalf("new matcher: %v", err) + } + + reportsHandler := reporting.NewHandler(reportsService, 32, origins) + router := httpx.NewRouter("/", reportsHandler.Routes(), management.NewHandler().Routes()) + server := httptest.NewServer(router) + defer server.Close() + + payload := `[{"age":1,"type":"csp-violation","url":"https://site.example","body":{"message":"` + strings.Repeat("x", 256) + `"}}]` + req, err := http.NewRequest(http.MethodPost, server.URL+"/v1/reports", bytes.NewBufferString(payload)) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Content-Type", "application/reports+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusRequestEntityTooLarge { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("expected %d, got %d body=%s", http.StatusRequestEntityTooLarge, resp.StatusCode, string(body)) + } +} diff --git a/internal/httpx/router_test.go b/internal/httpx/router_test.go new file mode 100644 index 0000000..3b3cba9 --- /dev/null +++ b/internal/httpx/router_test.go @@ -0,0 +1,31 @@ +package httpx + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi/v5" +) + +func TestRouterUsesBasePath(t *testing.T) { + reports := chi.NewRouter() + reports.Post("/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + manage := chi.NewRouter() + manage.Get("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + h := NewRouter("/collector", reports, manage) + + req := httptest.NewRequest(http.MethodPost, "/collector/v1/reports", nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected %d, got %d", http.StatusOK, rr.Code) + } +} diff --git a/internal/management/handler.go b/internal/management/handler.go new file mode 100644 index 0000000..aff0f66 --- /dev/null +++ b/internal/management/handler.go @@ -0,0 +1,29 @@ +package management + +import ( + "net/http" + + "github.com/cruxstack/browser-reporting-api/internal/httpx" + "github.com/go-chi/chi/v5" +) + +type Handler struct{} + +func NewHandler() *Handler { + return &Handler{} +} + +func (h *Handler) Routes() chi.Router { + r := chi.NewRouter() + r.Get("/healthz", h.handleHealthz) + r.Get("/readyz", h.handleReadyz) + return r +} + +func (h *Handler) handleHealthz(w http.ResponseWriter, _ *http.Request) { + httpx.WriteJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (h *Handler) handleReadyz(w http.ResponseWriter, _ *http.Request) { + httpx.WriteJSON(w, http.StatusOK, map[string]string{"status": "ready"}) +} diff --git a/internal/management/handler_test.go b/internal/management/handler_test.go new file mode 100644 index 0000000..bc90ca9 --- /dev/null +++ b/internal/management/handler_test.go @@ -0,0 +1,51 @@ +package management + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestManagementHealthz(t *testing.T) { + h := NewHandler().Routes() + + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected %d, got %d", http.StatusOK, rr.Code) + } + + if body := rr.Body.String(); body != "{\"status\":\"ok\"}\n" { + t.Fatalf("unexpected body: %q", body) + } +} + +func TestManagementReadyz(t *testing.T) { + h := NewHandler().Routes() + + req := httptest.NewRequest(http.MethodGet, "/readyz", nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected %d, got %d", http.StatusOK, rr.Code) + } + + if body := rr.Body.String(); body != "{\"status\":\"ready\"}\n" { + t.Fatalf("unexpected body: %q", body) + } +} + +func TestManagementMethodNotAllowed(t *testing.T) { + h := NewHandler().Routes() + + req := httptest.NewRequest(http.MethodPost, "/healthz", nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusMethodNotAllowed { + t.Fatalf("expected %d, got %d", http.StatusMethodNotAllowed, rr.Code) + } +} diff --git a/internal/parser/json_parser.go b/internal/parser/json_parser.go new file mode 100644 index 0000000..a99c9db --- /dev/null +++ b/internal/parser/json_parser.go @@ -0,0 +1,95 @@ +package parser + +import ( + "bytes" + "encoding/json" + "io" + + "github.com/cockroachdb/errors" + "github.com/cruxstack/browser-reporting-api/internal/domain" +) + +// JSONBatchParser implements domain.Parser for the application/reports+json +// wire format: a single JSON array of report objects. +type JSONBatchParser struct{} + +func NewJSONBatchParser() *JSONBatchParser { + return &JSONBatchParser{} +} + +func (p *JSONBatchParser) ParseBatch(r io.Reader) ([]domain.IncomingReport, error) { + decoder := json.NewDecoder(r) + + var payload json.RawMessage + if err := decoder.Decode(&payload); err != nil { + return nil, errors.Wrap(err, "decode reports payload") + } + + payload = bytes.TrimSpace(payload) + if len(payload) == 0 { + return nil, errors.New("reports payload must include at least one report") + } + + reports, err := parseReportsPayload(payload) + if err != nil { + return nil, err + } + + if len(reports) == 0 { + return nil, errors.New("reports payload must include at least one report") + } + + // Enforce a single top-level JSON value; trailing junk should be rejected. + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, errors.New("reports payload must contain exactly one JSON value") + } + + return reports, nil +} + +func parseReportsPayload(payload json.RawMessage) ([]domain.IncomingReport, error) { + if payload[0] == '[' { + var reports []domain.IncomingReport + if err := json.Unmarshal(payload, &reports); err != nil { + return nil, errors.Wrap(err, "decode reports payload") + } + return reports, nil + } + + if payload[0] == '{' { + return parseLegacyCSPReportPayload(payload) + } + + return nil, errors.New("reports payload must be a JSON array or object") +} + +func parseLegacyCSPReportPayload(payload json.RawMessage) ([]domain.IncomingReport, error) { + var legacy struct { + CSPReport json.RawMessage `json:"csp-report"` + } + + if err := json.Unmarshal(payload, &legacy); err != nil { + return nil, errors.Wrap(err, "decode reports payload") + } + + trimmed := bytes.TrimSpace(legacy.CSPReport) + if len(trimmed) == 0 { + return nil, errors.New("legacy csp payload must include csp-report") + } + if trimmed[0] != '{' { + return nil, errors.New("legacy csp payload must include csp-report object") + } + + var body struct { + DocumentURI string `json:"document-uri"` + } + if err := json.Unmarshal(trimmed, &body); err != nil { + return nil, errors.Wrap(err, "decode legacy csp-report body") + } + + return []domain.IncomingReport{{ + Type: "csp-violation", + URL: body.DocumentURI, + Body: trimmed, + }}, nil +} diff --git a/internal/parser/json_parser_test.go b/internal/parser/json_parser_test.go new file mode 100644 index 0000000..3b1b612 --- /dev/null +++ b/internal/parser/json_parser_test.go @@ -0,0 +1,57 @@ +package parser + +import ( + "strings" + "testing" +) + +func TestParseBatchAcceptsSingleArrayValue(t *testing.T) { + reports, err := NewJSONBatchParser().ParseBatch(strings.NewReader(`[{"age":1,"type":"csp-violation","url":"https://site.example","body":{"x":1}}]`)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(reports) != 1 { + t.Fatalf("expected 1 report, got %d", len(reports)) + } +} + +func TestParseBatchRejectsEmptyArray(t *testing.T) { + _, err := NewJSONBatchParser().ParseBatch(strings.NewReader(`[]`)) + if err == nil { + t.Fatal("expected error") + } +} + +func TestParseBatchRejectsTrailingTopLevelValue(t *testing.T) { + _, err := NewJSONBatchParser().ParseBatch(strings.NewReader(`[{"age":1,"type":"csp-violation","url":"https://site.example","body":{"x":1}}]{"extra":true}`)) + if err == nil { + t.Fatal("expected error") + } +} + +func TestParseBatchAcceptsLegacyCSPReportURIFormat(t *testing.T) { + reports, err := NewJSONBatchParser().ParseBatch(strings.NewReader(`{"csp-report":{"document-uri":"https://site.example/page","violated-directive":"frame-src"}}`)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(reports) != 1 { + t.Fatalf("expected 1 report, got %d", len(reports)) + } + + if reports[0].Type != "csp-violation" { + t.Fatalf("expected csp-violation type, got %q", reports[0].Type) + } + + if reports[0].URL != "https://site.example/page" { + t.Fatalf("unexpected url %q", reports[0].URL) + } +} + +func TestParseBatchRejectsLegacyCSPReportWithoutBody(t *testing.T) { + _, err := NewJSONBatchParser().ParseBatch(strings.NewReader(`{"type":"csp-violation"}`)) + if err == nil { + t.Fatal("expected error") + } +} diff --git a/internal/reporting/handler.go b/internal/reporting/handler.go new file mode 100644 index 0000000..167ff4b --- /dev/null +++ b/internal/reporting/handler.go @@ -0,0 +1,122 @@ +package reporting + +import ( + "mime" + "net/http" + "strings" + + "github.com/cockroachdb/errors" + "github.com/cruxstack/browser-reporting-api/internal/domain" + "github.com/cruxstack/browser-reporting-api/internal/httpx" + "github.com/go-chi/chi/v5" +) + +// Handler handles HTTP requests for the reporting ingestion endpoint. +type Handler struct { + service *Service + maxBodyBytes int64 + origins *httpx.OriginMatcher +} + +func NewHandler(service *Service, maxBodyBytes int64, origins *httpx.OriginMatcher) *Handler { + return &Handler{service: service, maxBodyBytes: maxBodyBytes, origins: origins} +} + +func (h *Handler) Routes() chi.Router { + r := chi.NewRouter() + r.MethodFunc(http.MethodPost, "/", h.handleIngest) + r.MethodFunc(http.MethodOptions, "/", h.handlePreflight) + return r +} + +func (h *Handler) handlePreflight(w http.ResponseWriter, r *http.Request) { + if !h.setCORSHeaders(w, r) { + httpx.WriteJSON(w, http.StatusForbidden, map[string]string{"error": "origin not allowed"}) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) handleIngest(w http.ResponseWriter, r *http.Request) { + if !h.setCORSHeaders(w, r) { + httpx.WriteJSON(w, http.StatusForbidden, map[string]string{"error": "origin not allowed"}) + return + } + + if err := ensureReportsContentType(r.Header.Get("Content-Type")); err != nil { + httpx.WriteJSON(w, http.StatusUnsupportedMediaType, map[string]string{"error": err.Error()}) + return + } + + r.Body = http.MaxBytesReader(w, r.Body, h.maxBodyBytes) + result, err := h.service.Ingest(r.Context(), r.Body) + if err != nil { + if isPayloadTooLargeErr(err) { + httpx.WriteJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "payload too large"}) + return + } + + if isInvalidPayloadErr(err) { + httpx.WriteJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + + httpx.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to ingest reports"}) + return + } + + httpx.WriteJSON(w, http.StatusOK, result) +} + +func (h *Handler) setCORSHeaders(w http.ResponseWriter, r *http.Request) bool { + origin := strings.TrimSpace(r.Header.Get("Origin")) + if origin == "" { + return true + } + + allowOrigin, ok := h.origins.AllowHeaderValue(origin) + if !ok { + return false + } + + w.Header().Set("Access-Control-Allow-Origin", allowOrigin) + w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Add("Vary", "Origin") + return true +} + +func ensureReportsContentType(value string) error { + if strings.TrimSpace(value) == "" { + return errors.New("content-type must be application/reports+json or application/csp-report") + } + + mediaType, _, err := mime.ParseMediaType(value) + if err != nil { + return errors.New("invalid content-type") + } + + if mediaType != "application/reports+json" && mediaType != "application/csp-report" { + return errors.New("content-type must be application/reports+json or application/csp-report") + } + + return nil +} + +func isInvalidPayloadErr(err error) bool { + if err == nil { + return false + } + + var badPayload *domain.BadPayloadError + return errors.As(err, &badPayload) +} + +func isPayloadTooLargeErr(err error) bool { + if err == nil { + return false + } + + var maxBytesErr *http.MaxBytesError + return errors.As(err, &maxBytesErr) +} diff --git a/internal/reporting/handler_test.go b/internal/reporting/handler_test.go new file mode 100644 index 0000000..83f9d7b --- /dev/null +++ b/internal/reporting/handler_test.go @@ -0,0 +1,126 @@ +package reporting + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/cruxstack/browser-reporting-api/internal/domain" + "github.com/cruxstack/browser-reporting-api/internal/httpx" + "github.com/cruxstack/browser-reporting-api/internal/parser" +) + +type captureSink struct { + reports []domain.AcceptedReport + err error +} + +func (s *captureSink) WriteReport(_ context.Context, report domain.AcceptedReport) error { + if s.err != nil { + return s.err + } + s.reports = append(s.reports, report) + return nil +} + +func newTestHandler(t *testing.T, origins []string) http.Handler { + t.Helper() + + match, err := httpx.NewOriginMatcher(origins) + if err != nil { + t.Fatalf("new matcher: %v", err) + } + + sink := &captureSink{} + service := NewService(parser.NewJSONBatchParser(), sink) + return NewHandler(service, 1024*1024, match).Routes() +} + +func TestIngestAcceptsBatch(t *testing.T) { + h := newTestHandler(t, []string{"*"}) + + body := `[ + {"age":1,"type":"csp-violation","url":"https://site.example","user_agent":"ua","body":{"foo":"bar"}}, + {"age":2,"type":"coep","url":"https://site.example/page","body":{"blockedURL":"https://other.example"}} + ]` + + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/reports+json") + rr := httptest.NewRecorder() + + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected %d, got %d", http.StatusOK, rr.Code) + } + + var response map[string]int + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatalf("decode response: %v", err) + } + + if response["received"] != 2 || response["accepted"] != 2 || response["rejected"] != 0 { + t.Fatalf("unexpected response: %+v", response) + } +} + +func TestIngestRejectsDisallowedOrigin(t *testing.T) { + h := newTestHandler(t, []string{"https://*.example.com"}) + + body := `[{"age":1,"type":"csp-violation","url":"https://site.example","body":{"foo":"bar"}}]` + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBufferString(body)) + req.Header.Set("Origin", "https://attacker.test") + req.Header.Set("Content-Type", "application/reports+json") + rr := httptest.NewRecorder() + + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusForbidden { + t.Fatalf("expected %d, got %d", http.StatusForbidden, rr.Code) + } +} + +func TestIngestAcceptsLegacyCSPReportURIFormat(t *testing.T) { + h := newTestHandler(t, []string{"*"}) + + body := `{"csp-report":{"document-uri":"https://site.example/page","violated-directive":"frame-src"}}` + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/csp-report") + rr := httptest.NewRecorder() + + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected %d, got %d", http.StatusOK, rr.Code) + } + + var response map[string]int + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatalf("decode response: %v", err) + } + + if response["received"] != 1 || response["accepted"] != 1 || response["rejected"] != 0 { + t.Fatalf("unexpected response: %+v", response) + } +} + +func TestPreflightAllowsWildcardPatternOrigin(t *testing.T) { + h := newTestHandler(t, []string{"https://*.example.com"}) + + req := httptest.NewRequest(http.MethodOptions, "/", nil) + req.Header.Set("Origin", "https://app.example.com") + rr := httptest.NewRecorder() + + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusNoContent { + t.Fatalf("expected %d, got %d", http.StatusNoContent, rr.Code) + } + + if got := rr.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" { + t.Fatalf("unexpected allow origin %q", got) + } +} diff --git a/internal/reporting/ingest.go b/internal/reporting/ingest.go new file mode 100644 index 0000000..624dc07 --- /dev/null +++ b/internal/reporting/ingest.go @@ -0,0 +1,67 @@ +package reporting + +import ( + "context" + "io" + "time" + + "github.com/cockroachdb/errors" + "github.com/cruxstack/browser-reporting-api/internal/domain" +) + +// ServiceOption configures a Service. +type ServiceOption func(*Service) + +// WithClock overrides the time source used to stamp accepted reports. +// This is primarily useful for testing. +func WithClock(fn func() time.Time) ServiceOption { + return func(s *Service) { + s.now = fn + } +} + +// Service orchestrates report ingestion: parsing, per-entry validation, +// and writing accepted entries to the configured sink. +type Service struct { + parser domain.Parser + sink domain.Sink + now func() time.Time +} + +func NewService(parser domain.Parser, sink domain.Sink, opts ...ServiceOption) *Service { + s := &Service{ + parser: parser, + sink: sink, + now: time.Now, + } + for _, opt := range opts { + opt(s) + } + return s +} + +func (s *Service) Ingest(ctx context.Context, body io.Reader) (domain.IngestResult, error) { + reports, err := s.parser.ParseBatch(body) + if err != nil { + return domain.IngestResult{}, &domain.BadPayloadError{Err: err} + } + + result := domain.IngestResult{Received: len(reports)} + now := s.now() + + for _, rep := range reports { + accepted, vetErr := domain.Vet(rep, now) + if vetErr != nil { + continue + } + + if err := s.sink.WriteReport(ctx, accepted); err != nil { + return domain.IngestResult{}, errors.Wrap(err, "stream accepted report") + } + + result.Accepted++ + } + + result.Rejected = result.Received - result.Accepted + return result, nil +} diff --git a/internal/stream/stdout.go b/internal/stream/stdout.go new file mode 100644 index 0000000..212ce42 --- /dev/null +++ b/internal/stream/stdout.go @@ -0,0 +1,43 @@ +package stream + +import ( + "context" + "encoding/json" + "io" + "log/slog" + + "github.com/cockroachdb/errors" + + "github.com/cruxstack/browser-reporting-api/internal/domain" +) + +// NDJSONWriter implements domain.Sink, emitting one JSON object per line to +// the provided writer (NDJSON / JSON Lines format). +type NDJSONWriter struct { + logger *slog.Logger +} + +func NewNDJSONWriter(w io.Writer) *NDJSONWriter { + return &NDJSONWriter{ + logger: slog.New(slog.NewJSONHandler(w, nil)), + } +} + +func (w *NDJSONWriter) WriteReport(_ context.Context, report domain.AcceptedReport) error { + var body map[string]any + if err := json.Unmarshal(report.Body, &body); err != nil { + return errors.Wrap(err, "decode report body") + } + + w.logger.Info( + "accepted report", + "received_at", report.ReceivedAt.Format("2006-01-02T15:04:05.000Z07:00"), + "age", report.Age, + "type", report.Type, + "url", report.URL, + "user_agent", report.UserAgent, + "body", body, + ) + + return nil +} diff --git a/internal/stream/stdout_test.go b/internal/stream/stdout_test.go new file mode 100644 index 0000000..dfa8169 --- /dev/null +++ b/internal/stream/stdout_test.go @@ -0,0 +1,98 @@ +package stream + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "github.com/cruxstack/browser-reporting-api/internal/domain" +) + +func TestNDJSONWriterWriteReport(t *testing.T) { + var buf bytes.Buffer + w := NewNDJSONWriter(&buf) + + report := domain.AcceptedReport{ + ReceivedAt: time.Date(2026, time.April, 13, 10, 30, 45, 0, time.UTC), + Age: 2, + Type: "csp-violation", + URL: "https://site.example/path?a=1&b=2", + UserAgent: "demo/1.0", + Body: json.RawMessage(`{"effectiveDirective":"script-src"}`), + } + + if err := w.WriteReport(context.Background(), report); err != nil { + t.Fatalf("write report: %v", err) + } + + line := strings.TrimSpace(buf.String()) + if line == "" { + t.Fatal("expected NDJSON output line") + } + + var got map[string]any + if err := json.Unmarshal([]byte(line), &got); err != nil { + t.Fatalf("decode output: %v", err) + } + + if got["level"] != "INFO" { + t.Fatalf("unexpected level: %v", got["level"]) + } + + if got["msg"] != "accepted report" { + t.Fatalf("unexpected msg: %v", got["msg"]) + } + + if got["received_at"] != "2026-04-13T10:30:45.000Z" { + t.Fatalf("unexpected received_at: %v", got["received_at"]) + } + + if got["type"] != report.Type || got["url"] != report.URL || got["user_agent"] != report.UserAgent { + t.Fatalf("unexpected report output fields: %+v", got) + } + + body, ok := got["body"].(map[string]any) + if !ok { + t.Fatalf("expected body object, got %T", got["body"]) + } + + if body["effectiveDirective"] != "script-src" { + t.Fatalf("unexpected body field: %+v", body) + } +} + +func TestNDJSONWriterConcurrentWrites(t *testing.T) { + var buf bytes.Buffer + w := NewNDJSONWriter(&buf) + + const total = 20 + var wg sync.WaitGroup + wg.Add(total) + + for i := range total { + go func(i int) { + defer wg.Done() + report := domain.AcceptedReport{ + ReceivedAt: time.Date(2026, time.April, 13, 12, 0, i, 0, time.UTC), + Age: int64(i), + Type: "deprecation", + URL: "https://site.example/page", + Body: json.RawMessage(`{"id":"FeatureUse"}`), + } + if err := w.WriteReport(context.Background(), report); err != nil { + t.Errorf("write report %d: %v", i, err) + } + }(i) + } + + wg.Wait() + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + if len(lines) != total { + t.Fatalf("expected %d lines, got %d", total, len(lines)) + } +}