Blame · Line-by-line history

`gluecron.rules.yml`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

gluecron.rules.ymlBlame133 lines · 1 contributor

Normal view Raw

6d2b36eDictation App	1	# Prometheus alerting rules for gluecron.com.
	2	#
	3	# Drop-in for any Prometheus-compatible system: Prometheus itself,
	4	# Grafana Mimir, VictoriaMetrics, Thanos. Point Alertmanager at the
	5	# rules file or include it via `rule_files:` in prometheus.yml.
	6	#
	7	# Scrape target assumed: /metrics on the gluecron container (Prom format).
	8	# Label `job="gluecron"` assumed — edit if your scrape config uses a
	9	# different job name.
	10	#
	11	# Severity convention:
	12	# critical = page (SMS / phone / Slack-critical channel)
	13	# warning = notify (Slack-warning channel, email)
	14	# info = dashboard only
	15
	16	groups:
	17	- name: gluecron-availability
	18	interval: 30s
	19	rules:
	20	- alert: GluecronDown
	21	expr: up{job="gluecron"} == 0
	22	for: 2m
	23	labels:
	24	severity: critical
	25	service: gluecron
	26	annotations:
	27	summary: "gluecron.com is down"
	28	description: "Prometheus cannot scrape /metrics on {{ $labels.instance }}. Likely the container exited or Caddy can't reach it. Check `docker compose ps` and `docker compose logs gluecron` on 45.76.171.37."
	29	runbook: "https://github.com/ccantynz-alt/Gluecron.com/blob/main/DEPLOY_CHECKLIST.md#rollback"
	30
	31	- alert: GluecronNotReady
	32	expr: probe_success{instance=~".*/readyz"} == 0
	33	for: 5m
	34	labels:
	35	severity: critical
	36	service: gluecron
	37	annotations:
	38	summary: "gluecron /readyz failing for 5m"
	39	description: "App is running but /readyz is returning non-200. Usually means the DB (Neon) is unreachable or the git repos volume is unmounted. Check `docker compose exec gluecron bun run src/scripts/db-check.ts` or equivalent."
	40
	41	- name: gluecron-errors
	42	interval: 30s
	43	rules:
	44	- alert: GluecronHigh5xxRate
	45	expr: \|
	46	(
	47	sum(rate(http_requests_total{job="gluecron",status=~"5.."}[5m]))
	48	/
	49	sum(rate(http_requests_total{job="gluecron"}[5m]))
	50	) > 0.05
	51	for: 10m
	52	labels:
	53	severity: warning
	54	service: gluecron
	55	annotations:
	56	summary: ">5% 5xx rate over 10m"
	57	description: "Current 5xx rate is {{ $value \| humanizePercentage }}. Look at the error-webhook/Sentry sink and recent deploys."
	58
	59	- alert: GluecronSpike5xx
	60	expr: \|
	61	sum(rate(http_requests_total{job="gluecron",status=~"5.."}[1m])) > 1
	62	for: 2m
	63	labels:
	64	severity: critical
	65	service: gluecron
	66	annotations:
	67	summary: "5xx spike (>1/s for 2m)"
	68	description: "Absolute 5xx rate exceeded 1/s. A post-deploy regression or downstream outage."
	69
	70	- alert: GluecronErrorSinkFailing
	71	expr: increase(error_sink_failures_total{job="gluecron"}[15m]) > 0
	72	for: 15m
	73	labels:
	74	severity: warning
	75	service: gluecron
	76	annotations:
	77	summary: "Error webhook / Sentry sink failing"
	78	description: "Errors are being generated but the sink (ERROR_WEBHOOK_URL / SENTRY_DSN) is rejecting them. Check the credentials and the sink's own status page."
	79
	80	- name: gluecron-latency
	81	interval: 30s
	82	rules:
	83	- alert: GluecronHighLatencyP95
	84	expr: \|
	85	histogram_quantile(0.95,
	86	sum by (le) (rate(http_request_duration_seconds_bucket{job="gluecron"}[5m]))
	87	) > 2
	88	for: 15m
	89	labels:
	90	severity: warning
	91	service: gluecron
	92	annotations:
	93	summary: "p95 latency >2s for 15m"
	94	description: "Current p95 is {{ $value }}s. Check DB connection pool, Neon status, and recent slow queries."
	95
	96	- name: gluecron-autopilot
	97	interval: 1m
	98	rules:
	99	- alert: GluecronAutopilotStalled
	100	expr: time() - autopilot_last_run_timestamp_seconds{job="gluecron"} > 900
	101	for: 5m
	102	labels:
	103	severity: warning
	104	service: gluecron
	105	annotations:
	106	summary: "Autopilot ticker stalled >15m"
	107	description: "Autopilot runs every 5m; last tick was {{ $value \| humanizeDuration }} ago. If AUTOPILOT_DISABLED=1 this alert should be muted at the receiver. Otherwise investigate the ticker in `src/lib/autopilot.ts`."
	108
	109	- name: gluecron-resources
	110	interval: 30s
	111	rules:
	112	- alert: GluecronHighMemory
	113	expr: \|
	114	(container_memory_usage_bytes{name=~".gluecron."}
	115	/ container_spec_memory_limit_bytes{name=~".gluecron."}) > 0.9
	116	for: 15m
	117	labels:
	118	severity: warning
	119	service: gluecron
	120	annotations:
	121	summary: "Container memory >90% of limit"
	122	description: "Near the OOM cliff. Either raise the compose memory limit or investigate a leak."
	123
	124	- alert: GluecronDiskFillingFast
	125	expr: \|
	126	predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 24*3600) < 0
	127	for: 30m
	128	labels:
	129	severity: warning
	130	service: gluecron
	131	annotations:
	132	summary: "Disk will fill within 24h at current rate"
	133	description: "Usually the git repos volume growing unchecked. Check `/data/repos` size and retention policy."