Monitoring & Telemetry

node:
  ops:
    port: 9153 # Ops server port (/health, /metrics, /balance, /snapshot)

http://localhost:9153/metrics

metrics:
  interval: "15s"  # Default collection interval
  collectors:
    node:
      enabled: true
      interval: "30s"  # Override interval for specific collector
    genvm:
      enabled: true
      interval: "20s"
    webdriver:
      enabled: true
      interval: "60s"

# Get all available metrics
curl http://localhost:9153/metrics
 
# Check specific metric (example)
curl -s http://localhost:9153/metrics | grep genlayer_node_

# Central monitoring endpoints, from the Foundation team (ask in #bradbury)
CENTRAL_MONITORING_URL=<prometheus-remote-write-url>
CENTRAL_LOKI_URL=<loki-push-url>

# Authentication for central monitoring
# Metrics (Prometheus) credentials
CENTRAL_MONITORING_USERNAME=<metrics-username>
CENTRAL_MONITORING_PASSWORD=<metrics-password>
# Logs (Loki) credentials
CENTRAL_LOKI_USERNAME=<logs-username>
CENTRAL_LOKI_PASSWORD=<logs-password>

# Node identification
NODE_ID=validator-001
VALIDATOR_NAME=MyValidator

# Usually defaults are fine
NODE_METRICS_ENDPOINT=host.docker.internal:9153
LOG_FILE_PATTERN=/var/log/genlayer/node*.log
METRICS_SCRAPE_INTERVAL=15s
# Host directory where the node writes its log files (mounted into Alloy)
NODE_LOGS_PATH=./data/node/logs

  # Grafana Alloy for both logs and metrics forwarding
  # Supports both single node and multi-node configurations
  #
  # Single Node Mode:
  #   Set NODE_ID, VALIDATOR_NAME, NODE_METRICS_ENDPOINT in .env
  #   docker compose --profile monitoring up -d
  #
  # Multi-Node Mode:
  #   Set SCRAPE_TARGETS_JSON in .env
  #   docker compose --profile monitoring up -d
  alloy:
    image: grafana/alloy:v1.20.1
    container_name: genlayer-node-alloy
    # The guard runs before Alloy: with the monitoring profile up but the
    # endpoints or credentials unset, the container stops with a message
    # naming what to set, instead of half-running. Half-running is worse than
    # stopping here in both directions: pushing to a shared endpoint with a
    # shared default credential pollutes it for everyone, and pushing to an
    # unreachable one grows the WAL until the healthcheck below reads the
    # growth as a stall and deletes it. The $$ keeps compose from
    # interpolating; the shell only tests emptiness — no value ever lands on
    # a command line.
    entrypoint: ["/bin/sh", "-c"]
    command:
      - >-
        missing="";
        [ -n "$$CENTRAL_MONITORING_URL" ]      || missing="$$missing CENTRAL_MONITORING_URL";
        [ -n "$$CENTRAL_MONITORING_USERNAME" ] || missing="$$missing CENTRAL_MONITORING_USERNAME";
        [ -n "$$CENTRAL_MONITORING_PASSWORD" ] || missing="$$missing CENTRAL_MONITORING_PASSWORD";
        [ -n "$$CENTRAL_LOKI_URL" ]            || missing="$$missing CENTRAL_LOKI_URL";
        [ -n "$$CENTRAL_LOKI_USERNAME" ]       || missing="$$missing CENTRAL_LOKI_USERNAME";
        [ -n "$$CENTRAL_LOKI_PASSWORD" ]       || missing="$$missing CENTRAL_LOKI_PASSWORD";
        if [ -n "$$missing" ]; then
        echo "ERROR: not set:$$missing" >&2;
        echo "Monitoring pushes to an endpoint YOU configure — set the CENTRAL_* values in .env (see .env.example for a localhost example)." >&2;
        exit 1;
        fi;
        exec /bin/alloy run /etc/alloy/config.river
        --server.http.listen-addr=0.0.0.0:12345
        --storage.path=/var/lib/alloy/data
    volumes:
      - ./alloy-config.river:/etc/alloy/config.river:ro
      - ./alloy-healthcheck.sh:/etc/alloy/healthcheck.sh:ro
      - ${NODE_LOGS_PATH:-./data/node/logs}:/var/log/genlayer:ro
      - alloy_data:/var/lib/alloy
    healthcheck:
      test: ["CMD", "sh", "/etc/alloy/healthcheck.sh"]
      interval: 120s
      timeout: 15s
      retries: 1
      start_period: 120s
    environment:
      # Central monitoring endpoints and credentials: NO defaults, on purpose.
      # An endpoint may be public knowledge; a credential never is, and a
      # default one ships the same key to everyone holding this file. The
      # entrypoint guard above turns "unset" into a clear stop instead of a
      # silent push to the wrong place.
      - CENTRAL_LOKI_URL=${CENTRAL_LOKI_URL:-}
      - CENTRAL_MONITORING_URL=${CENTRAL_MONITORING_URL:-}
 
      # Metrics (Prometheus) authentication
      - CENTRAL_MONITORING_USERNAME=${CENTRAL_MONITORING_USERNAME:-}
      - CENTRAL_MONITORING_PASSWORD=${CENTRAL_MONITORING_PASSWORD:-}
 
      # Logs (Loki) authentication
      - CENTRAL_LOKI_USERNAME=${CENTRAL_LOKI_USERNAME:-}
      - CENTRAL_LOKI_PASSWORD=${CENTRAL_LOKI_PASSWORD:-}
 
      # Single node configuration
      - NODE_ID=${NODE_ID:-validator-001}
      - VALIDATOR_NAME=${VALIDATOR_NAME:-MyValidator}
      - NODE_METRICS_ENDPOINT=${NODE_METRICS_ENDPOINT:-host.docker.internal:9153}
 
      # Multi-node configuration
      # When set, overrides single node config above
      - SCRAPE_TARGETS_JSON=${SCRAPE_TARGETS_JSON:-}
 
      # Scraping configuration
      - METRICS_SCRAPE_INTERVAL=${METRICS_SCRAPE_INTERVAL:-60s}
      - METRICS_SCRAPE_TIMEOUT=${METRICS_SCRAPE_TIMEOUT:-10s}
      - ALLOY_SELF_MONITORING_INTERVAL=${ALLOY_SELF_MONITORING_INTERVAL:-60s}
 
      # Log collection configuration
      - LOG_FILE_PATTERN=${LOG_FILE_PATTERN:-/var/log/genlayer/node*.log}
 
      # Log batching configuration
      - LOKI_BATCH_SIZE=${LOKI_BATCH_SIZE:-1MiB}
      - LOKI_BATCH_WAIT=${LOKI_BATCH_WAIT:-60s}
    ports:
      - "12345:12345"  # Alloy UI for debugging
    restart: unless-stopped
    profiles:
      - monitoring
    extra_hosts:
      - "host.docker.internal:host-gateway"
 
volumes:
  alloy_data:

// Grafana Alloy Configuration for GenLayer Node Telemetry
// Handles both log collection and metrics forwarding

// ==========================================
// Log Collection and Forwarding
// ==========================================

// Discovery component to find log files using local.file_match
// Supports different log file patterns:
// - Single node: "/var/log/genlayer/node.log"
// - Multi-node: "/var/log/genlayer/*/logs/node.log" (each node in subdirectory)
// - Custom pattern via LOG_FILE_PATTERN env var
local.file_match "genlayer_logs" {
  path_targets = [{
    __path__ = coalesce(sys.env("LOG_FILE_PATTERN"), "/var/log/genlayer/node*.log"),
  }]
}

// Relabel to add metadata labels to log entries
discovery.relabel "add_labels" {
  targets = local.file_match.genlayer_logs.targets

  // Add instance label from environment variable
  rule {
    target_label = "instance"
    replacement  = sys.env("NODE_ID")
  }

  // Add validator_name label from environment variable
  rule {
    target_label = "validator_name"
    replacement  = sys.env("VALIDATOR_NAME")
  }

  // Add component label
  rule {
    target_label = "component"
    replacement  = "alloy"
  }

  // Add job label
  rule {
    target_label = "job"
    replacement  = "genlayer-node"
  }
}

// Source component to read log files
loki.source.file "genlayer" {
  targets    = discovery.relabel.add_labels.output
  forward_to = [loki.write.central.receiver]

  // Tail from end to avoid ingesting entire log history on startup
  tail_from_end = true
}

// Write logs to central Loki instance
loki.write "central" {
  endpoint {
    url = sys.env("CENTRAL_LOKI_URL")

    // HTTP Basic Authentication
    basic_auth {
      username = sys.env("CENTRAL_LOKI_USERNAME")
      password = sys.env("CENTRAL_LOKI_PASSWORD")
    }

    // Enable retry with default exponential backoff
    // Note: Alloy's loki.write doesn't have a retry block; retries are handled automatically
    // with exponential backoff by default when the endpoint is unreachable

    // Configurable batch settings for efficient log sending
    batch_size = coalesce(sys.env("LOKI_BATCH_SIZE"), "1MiB")   // Maximum batch size before sending
    batch_wait = coalesce(sys.env("LOKI_BATCH_WAIT"), "60s")    // Maximum wait time before sending partial batch
  }
}

// ==========================================
// Prometheus Metrics Collection and Forwarding
// ==========================================

// Scrape metrics from GenLayer node(s)
// Supports both single node and multi-node configurations
//
// Single Node Mode:
//   Set NODE_METRICS_ENDPOINT, NODE_ID, VALIDATOR_NAME
//
// Multi-Node Mode:
//   Set SCRAPE_TARGETS_JSON with JSON array of target objects
//   Example: [{"__address__":"host.docker.internal:9250","instance":"0x...","validator_name":"node-1"}]
//
// Note: The "network" label is emitted by the node itself (auto-detected from consensus address),
// so it does not need to be configured here.
prometheus.scrape "genlayer_node" {
  // Dynamic targets based on environment variable
  // If SCRAPE_TARGETS_JSON is set, use it (multi-node mode)
  // Otherwise, build single target from individual env vars (single node mode)
  targets = encoding.from_json(coalesce(sys.env("SCRAPE_TARGETS_JSON"), string.format("[{\"__address__\":\"%s\",\"instance\":\"%s\",\"validator_name\":\"%s\"}]", coalesce(sys.env("NODE_METRICS_ENDPOINT"), "host.docker.internal:9153"), coalesce(sys.env("NODE_ID"), "local"), coalesce(sys.env("VALIDATOR_NAME"), "default"))))

  forward_to = [prometheus.relabel.metrics.receiver]

  // Configurable scrape intervals
  scrape_interval = coalesce(sys.env("METRICS_SCRAPE_INTERVAL"), "60s")
  scrape_timeout  = coalesce(sys.env("METRICS_SCRAPE_TIMEOUT"), "10s")
}

// Relabel metrics to filter before forwarding
prometheus.relabel "metrics" {
  forward_to = [prometheus.remote_write.central.receiver]

  // Option 1: Forward all metrics (default)
  // Currently forwarding all metrics from the node.

  // Option 2: Only keep genlayer_node_* metrics to reduce bandwidth (recommended)
  // To enable filtering and reduce bandwidth, uncomment the following rule:
  /*
  rule {
    source_labels = ["__name__"]
    regex        = "genlayer_node_.*"
    action       = "keep"
  }
  */
}

// Remote write configuration for sending metrics to central Prometheus
prometheus.remote_write "central" {
  endpoint {
    url = sys.env("CENTRAL_MONITORING_URL")

    // HTTP Basic Authentication
    basic_auth {
      username = sys.env("CENTRAL_MONITORING_USERNAME")
      password = sys.env("CENTRAL_MONITORING_PASSWORD")
    }

    // Queue configuration for reliability
    queue_config {
      capacity          = 10000
      max_shards        = 5
      max_samples_per_send = 500
      batch_send_deadline = coalesce(sys.env("METRICS_BATCH_SEND_DEADLINE"), "60s")
    }
  }
}

// ==========================================
// Alloy Self-Monitoring
// ==========================================

// Alloy internal exporter for health monitoring
prometheus.exporter.self "alloy" {}

// Expose Alloy's own metrics on the HTTP server
prometheus.scrape "alloy" {
  targets    = prometheus.exporter.self.alloy.targets
  forward_to = []  // Not forwarding Alloy metrics to reduce noise

  // Configurable scrape interval for Alloy's internal health monitoring
  scrape_interval = coalesce(sys.env("ALLOY_SELF_MONITORING_INTERVAL"), "60s")
}

docker compose --profile monitoring up -d

docker logs genlayer-node-alloy | grep "sent batch"

docker logs genlayer-node-alloy | grep "remote_write"

docker inspect --format '{{.State.Health.Status}}' genlayer-node-alloy
docker compose logs --since 30m alloy

http://localhost:12345

curl http://localhost:9153/metrics

docker logs genlayer-node-alloy