OTEL collector — алерт при дропе spans из очереди

#!/usr/bin/env bash
# /etc/cron.d/otel-drops
# */5 * * * * root /opt/otel-drops.sh

# OTEL collector exposes Prometheus metrics on :8888 by default
URL=${OTEL_METRICS:-http://localhost:8888/metrics}
THRESH=${THRESH:-100}                 # spans dropped / 5 min
STATE=/var/lib/otel-drops.state

NOW=$(curl -fsS "$URL" | awk '/^otelcol_exporter_send_failed_spans/ && !/#/ {sum += $2} END {print int(sum)+0}')
PREV=$(cat "$STATE" 2>/dev/null || echo 0)
echo "$NOW" > "$STATE"

DELTA=$((NOW - PREV))

if [ "$DELTA" -gt "$THRESH" ]; then
  curl -fsS "$HEARTBEAT_URL" --data "dropped=$DELTA,window=5m"
  exit 2
fi
echo "OK ($DELTA spans dropped / 5m)"