Logging & Monitoring — ELK, Grafana, OpenTelemetry

import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  transport: process.env.NODE_ENV === 'development'
    ? { target: 'pino-pretty' } // Đẹp cho dev
    : undefined,
  redact: {
    paths: ['req.headers.authorization', 'req.body.password', 'db.query'],
    censor: '[REDACTED]',
  },
  serializers: {
    req: pino.stdSerializers.req,
    err: pino.stdSerializers.err,
  },
});

// Express middleware
app.use(pinoHttp({ logger }));

// Usage
app.get('/api/users/:id', async (req, res) => {
  logger.info({ userId: req.params.id }, 'Fetching user');
  try {
    const user = await db.user.findUnique({ where: { id: req.params.id } });
    logger.info({ userId: req.params.id, found: !!user }, 'User fetched');
    res.json(user);
  } catch (err) {
    logger.error({ err, userId: req.params.id }, 'Failed to fetch user');
    res.status(500).json({ error: 'Internal error' });
  }
});

import prometheus from 'prom-client';

// Collect metrics
const register = new prometheus.Registry();
prometheus.collectDefaultMetrics({ register });

// Custom metrics
const httpRequestDuration = new prometheus.Histogram({
  name: 'http_request_duration_seconds',
  help: 'HTTP request duration in seconds',
  labelNames: ['method', 'route', 'status'],
  buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5],
  registers: [register],
});

const activeUsers = new prometheus.Gauge({
  name: 'active_users',
  help: 'Number of active users',
  registers: [register],
});

// Express middleware
app.use((req, res, next) => {
  const end = httpRequestDuration.startTimer();
  res.on('finish', () => {
    end({ method: req.method, route: req.route?.path || 'unknown', status: res.statusCode });
  });
  next();
});

// Metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', register.contentType);
  res.end(await register.metrics());
});

const requestRate = new prometheus.Counter({
  name: 'requests_total',
  help: 'Total requests',
  labelNames: ['service', 'method'],
});

const errorRate = new prometheus.Counter({
  name: 'errors_total',
  help: 'Total errors',
  labelNames: ['service', 'error_type'],
});

const latency = new prometheus.Histogram({
  name: 'request_duration_seconds',
  help: 'Request latency',
  labelNames: ['service'],
  buckets: [0.01, 0.05, 0.1, 0.5, 1],
});

import { trace, context } from '@opentelemetry/api';
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';

// Setup
const provider = new NodeTracerProvider();
provider.addSpanProcessor(new BatchSpanProcessor(new OTLPTraceExporter()));
provider.register();

// Manual tracing
const tracer = trace.getTracer('order-service');

async function createOrder(userId: number, items: Item[]) {
  const span = tracer.startSpan('createOrder', {
    attributes: { userId, itemCount: items.length },
  });

  return await context.with(trace.setSpan(context.active(), span), async () => {
    try {
      const user = await getUser(userId); // span con tự động
      const total = await calculatePrice(items);
      const order = await saveOrder(user, items, total);
      span.setStatus({ code: SpanStatusCode.OK });
      return order;
    } catch (err) {
      span.recordException(err);
      span.setStatus({ code: SpanStatusCode.ERROR });
      throw err;
    } finally {
      span.end();
    }
  });
}

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0
    environment:
      - discovery.type=single-node

  kibana:
    image: docker.elastic.co/kibana/kibana:8.15.0
    ports:
      - "5601:5601"
    depends_on:
      - elasticsearch

  logstash:
    image: docker.elastic.co/logstash/logstash:8.15.0
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf

services:
  prometheus:
    image: prom/prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"

  grafana:
    image: grafana/grafana
    ports:
      - "3000:3000"
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true

  loki:
    image: grafana/loki
    ports:
      - "3100:3100"

  tempo:
    image: grafana/tempo
    ports:
      - "3200:3200"
      - "4317:4317"  # OTLP gRPC

{
  "panels": [
    {
      "title": "Request Rate",
      "type": "graph",
      "targets": [{
        "expr": "rate(http_requests_total[5m])",
        "legendFormat": "{{route}}"
      }]
    },
    {
      "title": "P95 Latency",
      "type": "heatmap",
      "targets": [{
        "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))"
      }]
    },
    {
      "title": "Error Rate",
      "type": "stat",
      "targets": [{
        "expr": "sum(rate(errors_total[5m])) / sum(rate(requests_total[5m])) * 100"
      }]
    }
  ]
}

# prometheus-alerts.yml
groups:
  - name: api-alerts
    rules:
      - alert: HighErrorRate
        expr: rate(errors_total[5m]) > 0.01
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Error rate > 1% for 5 minutes"

      - alert: HighLatency
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 5m
        labels:
          severity: warning

app.get('/health', async (req, res) => {
  const checks = {
    database: { status: 'healthy' },
    redis: { status: 'healthy' },
    externalAPI: { status: 'healthy' },
  };

  try {
    await db.$queryRaw`SELECT 1`;
  } catch {
    checks.database = { status: 'unhealthy', error: 'Cannot connect' };
  }

  const allHealthy = Object.values(checks).every(c => c.status === 'healthy');
  res.status(allHealthy ? 200 : 503).json({
    status: allHealthy ? 'healthy' : 'degraded',
    checks,
    uptime: process.uptime(),
  });
});