Advanced Topics#

100% ──── handleRequest()
├── 35% ── validateOrder()
│   ├── 20% ── JsonSchema.validate()
│   └── 15% ── RegexMatcher.match()
├── 45% ── processPayment()
│   ├── 40% ── HttpClient.send()    ← 40% of request time waiting on payment API
│   └──  5% ── PaymentMapper.map()
└── 20% ── saveOrder()
    ├── 15% ── JdbcTemplate.update()
    └──  5% ── AuditLogger.log()

import "github.com/grafana/pyroscope-go"

func main() {
    pyroscope.Start(pyroscope.Config{
        ApplicationName: "order-service",
        ServerAddress:   "http://pyroscope:4040",
        ProfileTypes: []pyroscope.ProfileType{
            pyroscope.ProfileCPU,
            pyroscope.ProfileAllocObjects,
            pyroscope.ProfileAllocSpace,
            pyroscope.ProfileInuseObjects,
            pyroscope.ProfileInuseSpace,
            pyroscope.ProfileGoroutines,
            pyroscope.ProfileMutexCount,
            pyroscope.ProfileMutexDuration,
            pyroscope.ProfileBlockCount,
            pyroscope.ProfileBlockDuration,
        },
        Tags: map[string]string{
            "environment": "production",
            "region":      "eu-west-1",
        },
    })
    // ... application code
}

# beyla-config.yaml
open_port: 8080
otel_metrics_export:
  endpoint: http://otel-collector:4318/v1/metrics
otel_traces_export:
  endpoint: http://otel-collector:4318/v1/traces
attributes:
  kubernetes:
    enable: true

# Count system calls by process
bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'

# Histogram of read() sizes
bpftrace -e 'tracepoint:syscalls:sys_exit_read /args->ret > 0/ { @bytes = hist(args->ret); }'

# Track TCP connections by destination port
bpftrace -e 'kprobe:tcp_connect { @[args->sk->__sk_common.skc_dport] = count(); }'

# Latency of DNS lookups
bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:getaddrinfo { @start[tid] = nsecs; }
             uretprobe:/lib/x86_64-linux-gnu/libc.so.6:getaddrinfo { @ns = hist(nsecs - @start[tid]); delete(@start[tid]); }'

# k6 synthetic test
import http from 'k6/http';
import { check, sleep } from 'k6';

export const options = {
  frequency: '60s',
  locations: ['eu-west-1', 'us-east-1', 'ap-southeast-1'],
};

export default function () {
  const res = http.get('https://api.example.com/health');

  check(res, {
    'status is 200': (r) => r.status === 200,
    'response time < 500ms': (r) => r.timings.duration < 500,
    'body contains ok': (r) => r.body.includes('ok'),
  });
}

1. Hypothesis    → "If payment-service fails, checkout degrades gracefully"
2. Observe       → Record baseline SLIs (error rate, latency, throughput)
3. Inject fault  → Kill payment-service pods / inject 500ms latency / drop 50% of packets
4. Observe       → Monitor SLIs during the experiment
5. Analyse       → Compare actual behaviour to hypothesis
6. Improve       → Fix gaps (missing circuit breaker, no fallback, bad timeout)

Normal pattern: Error rate is 0.05% on weekdays, 0.02% on weekends, spikes to 0.3% during deploys
Static threshold: Alert if > 1%    → Misses the 0.3% anomaly during non-deploy hours
Anomaly detection: Alert if > 3σ from expected value → Catches 0.3% if no deploy occurred

Alert: Error rate increased on order-service
Correlated signals:
  - Deploy v2.14.3 occurred 3 minutes before
  - DB connection pool utilisation jumped to 100%
  - payment-service latency increased 10x
  - No infrastructure changes detected

Suggested root cause: Deploy v2.14.3 introduced a regression
affecting database performance. Recommend rollback.

processors:
  # Filter out health check spans
  filter/healthcheck:
    traces:
      span:
        - 'attributes["http.route"] == "/health"'
        - 'attributes["http.route"] == "/ready"'

  # Drop DEBUG logs
  filter/loglevel:
    logs:
      log_record:
        - 'severity_number < 9'  # Drop below INFO

  # Remove noisy attributes
  attributes/cleanup:
    actions:
      - key: http.user_agent
        action: delete
      - key: http.request.header.cookie
        action: delete

  # Tail-based sampling
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors
        type: status_code
        status_code: {status_codes: [ERROR]}
      - name: slow
        type: latency
        latency: {threshold_ms: 2000}
      - name: sample
        type: probabilistic
        probabilistic: {sampling_percentage: 5}

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [filter/healthcheck, tail_sampling, attributes/cleanup, batch]
      exporters: [otlp/tempo]
    logs:
      receivers: [otlp]
      processors: [filter/loglevel, attributes/cleanup, batch]
      exporters: [otlp/loki]