Observability in CI/CD#

# GitLab CI: emit deployment event to Datadog
deploy_prod:
  stage: deploy
  script:
    - ./deploy.sh
    - |
      curl -X POST "https://api.datadoghq.eu/api/v1/events" \
        -H "DD-API-KEY: ${DD_API_KEY}" \
        -d @- <<EOF
      {
        "title": "Production Deployment",
        "text": "Deployed ${CI_PROJECT_NAME} ${CI_COMMIT_TAG}",
        "tags": ["service:${CI_PROJECT_NAME}", "env:prod", "team:${TEAM}"],
        "alert_type": "success"
      }
      EOF

Commit pushed → PR opened → Review → Merge → Build → Test → Deploy (prod)
|←─────────────────── Lead Time ──────────────────────────────────────→|

CFR = (failed deployments / total deployments) × 100

"Failed" = deployment that causes:
  - Rollback
  - Hotfix within 24h
  - Incident created
  - Service degradation alert

MTTR = mean(time_service_restored - time_incident_detected)

# GitHub Actions: output timing to summary
- name: Record pipeline metrics
  if: always()
  run: |
    START=${{ github.event.workflow_run.created_at }}
    END=$(date -u +%Y-%m-%dT%H:%M:%SZ)
    echo "## Pipeline Metrics" >> $GITHUB_STEP_SUMMARY
    echo "- **Duration:** calculated from job timestamps" >> $GITHUB_STEP_SUMMARY
    echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
    echo "- **Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY

notify_failure:
  stage: .post
  script:
    - |
      curl -X POST "$SLACK_WEBHOOK_URL" \
        -H 'Content-Type: application/json' \
        -d "{
          \"text\": \"❌ Pipeline failed on *${CI_PROJECT_NAME}* (${CI_COMMIT_REF_NAME})\",
          \"blocks\": [{
            \"type\": \"section\",
            \"text\": {
              \"type\": \"mrkdwn\",
              \"text\": \"*Pipeline:* <${CI_PIPELINE_URL}|#${CI_PIPELINE_ID}>\n*Branch:* ${CI_COMMIT_REF_NAME}\n*Author:* ${GITLAB_USER_NAME}\"
            }
          }]
        }"
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
      when: on_failure

┌─────────────────────────────────────────────────────┐
│  DORA Metrics (last 30 days)                        │
│  ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌────────┐│
│  │ Deploy   │ │ Lead     │ │ CFR      │ │ MTTR   ││
│  │ Freq: 4/d│ │ Time: 2h │ │ 8%       │ │ 22min  ││
│  └──────────┘ └──────────┘ └──────────┘ └────────┘│
├─────────────────────────────────────────────────────┤
│  Pipeline Health                                     │
│  Success rate: 94% ████████████████░░ (7d)          │
│  Avg duration: 8m 23s (trend: ↓12%)                │
│  Flaky tests:  3 (down from 7 last week)            │
├─────────────────────────────────────────────────────┤
│  Recent Deployments                                  │
│  ✅ v1.4.2 → prod (12 min ago)                     │
│  ✅ v1.4.1 → prod (2 days ago)                     │
│  ❌ v1.4.1-rc1 → pre (rolled back, 3 days ago)     │
└─────────────────────────────────────────────────────┘

post_deploy_verify:
  stage: verify
  script:
    - |
      echo "Waiting for service to be healthy..."
      for i in $(seq 1 30); do
        STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://api.prod.example.com/health)
        if [ "$STATUS" = "200" ]; then
          echo "✅ Health check passed"
          break
        fi
        sleep 10
      done
      if [ "$STATUS" != "200" ]; then
        echo "❌ Health check failed after 5 minutes"
        exit 1
      fi
    - |
      echo "Running smoke tests..."
      curl -sf https://api.prod.example.com/v1/status | jq .version
      curl -sf https://api.prod.example.com/v1/ping
  after_script:
    - |
      if [ "$CI_JOB_STATUS" = "failed" ]; then
        echo "⚠️ Triggering rollback..."
        ./scripts/rollback.sh
      fi

Deploy canary (10% traffic)
         │
         ▼
Monitor for 10 minutes
  ├── Error rate within threshold? ──NO──▶ Rollback
  ├── Latency within threshold?   ──NO──▶ Rollback
  └── All good? ──────────────────────────▶ Promote to 100%

Measure → Identify bottleneck → Experiment → Measure again

Examples:
- Pipeline slow? → Parallelise test stages, add caching
- High CFR? → Add more pre-deploy tests, shrink batch size
- Long lead time? → Reduce review bottlenecks, trunk-based development
- Frequent rollbacks? → Better canary analysis, feature flags