# Check Uptime Kuma version
docker exec uptime-kuma node -e "console.log(require('./package.json').version)"

# Test API connectivity
curl -s https://monitor.yourdomain.com/metrics | head -5
# Should return Prometheus-format metrics

# Check the API badge endpoint works (requires a monitor ID)
curl -s https://monitor.yourdomain.com/api/badge/1/status
# Returns: up or down badge JSON

# Install the uptime-kuma-api Python library
pip install uptime-kuma-api

# Basic connection test
python3 << 'EOF'
from uptime_kuma_api import UptimeKumaApi

api = UptimeKumaApi("https://monitor.yourdomain.com")
api.login("admin", "yourpassword")

# List all monitors
monitors = api.get_monitors()
for m in monitors:
    print(f"{m['id']:3} | {m['type']:12} | {m['name']}")

api.disconnect()
EOF

#!/usr/bin/env python3
# register-monitor.py
# Creates an Uptime Kuma monitor for a newly deployed service
# Usage: python3 register-monitor.py --name "Payment API" --url "https://pay.yourdomain.com/health"

import argparse
import os
from uptime_kuma_api import UptimeKumaApi, MonitorType

def register_monitor(name: str, url: str, notification_ids: list[int] = None):
    api = UptimeKumaApi(os.environ["UPTIME_KUMA_URL"])
    api.login(
        os.environ["UPTIME_KUMA_USER"],
        os.environ["UPTIME_KUMA_PASSWORD"]
    )

    try:
        # Check if monitor already exists
        existing = api.get_monitors()
        for m in existing:
            if m.get("url") == url:
                print(f"Monitor already exists: ID={m['id']}, Name={m['name']}")
                return m["id"]

        # Create new monitor
        result = api.add_monitor(
            type=MonitorType.HTTP,
            name=name,
            url=url,
            interval=60,
            retryInterval=30,
            maxretries=3,
            upsideDown=False,
            notificationIDList=notification_ids or [],
            keyword="",
            ignoreTls=False,
            accepted_statuscodes=["200-299"],
            method="GET",
        )

        monitor_id = result["monitorID"]
        print(f"Created monitor: ID={monitor_id}, Name={name}")
        return monitor_id

    finally:
        api.disconnect()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--name", required=True)
    parser.add_argument("--url", required=True)
    parser.add_argument("--notifications", nargs="+", type=int, default=[])
    args = parser.parse_args()

    register_monitor(args.name, args.url, args.notifications)

# .github/workflows/deploy.yml (relevant section)
- name: Register health check monitor
  env:
    UPTIME_KUMA_URL: ${{ secrets.UPTIME_KUMA_URL }}
    UPTIME_KUMA_USER: ${{ secrets.UPTIME_KUMA_USER }}
    UPTIME_KUMA_PASSWORD: ${{ secrets.UPTIME_KUMA_PASSWORD }}
  run: |
    pip install uptime-kuma-api --quiet

    python3 register-monitor.py \
      --name "${{ github.event.repository.name }} (${{ github.ref_name }})" \
      --url "https://${{ env.DEPLOY_DOMAIN }}/health" \
      --notifications ${{ secrets.UPTIME_KUMA_NOTIFICATION_ID }}

    echo "Monitor registered for ${{ env.DEPLOY_DOMAIN }}"

# Similarly, remove the monitor on service teardown:
# python3 << 'EOF'
# from uptime_kuma_api import UptimeKumaApi
# api = UptimeKumaApi(url)
# api.login(user, password)
# monitors = api.get_monitors()
# for m in monitors:
#     if m.get("url") == f"https://{domain}/health":
#         api.delete_monitor(m["id"])
#         print(f"Deleted monitor: {m['id']}")
# api.disconnect()
# EOF

#!/usr/bin/env python3
# export-monitors.py — Export all monitors to a JSON file for version control

import json
import os
from uptime_kuma_api import UptimeKumaApi

api = UptimeKumaApi(os.environ["UPTIME_KUMA_URL"])
api.login(os.environ["UPTIME_KUMA_USER"], os.environ["UPTIME_KUMA_PASSWORD"])

monitors = api.get_monitors()
notifications = api.get_notifications()
status_pages = api.get_status_pages()

# Strip sensitive fields before committing to Git
for m in monitors:
    m.pop("password", None)
    m.pop("authPassword", None)

export = {
    "monitors": monitors,
    "notifications": [
        {k: v for k, v in n.items() if k not in ["config"]}  # Strip notification secrets
        for n in notifications
    ],
    "status_pages": status_pages,
    "exported_at": __import__("datetime").datetime.utcnow().isoformat()
}

with open("monitoring-config.json", "w") as f:
    json.dump(export, f, indent=2)

print(f"Exported {len(monitors)} monitors, {len(notifications)} notifications")
api.disconnect()

# Add to crontab or CI to keep the export current:
# 0 0 * * * cd /opt/monitoring-config && python3 export-monitors.py && \
#   git add monitoring-config.json && \
#   git commit -m "Update monitoring config $(date +%Y-%m-%d)" && \
#   git push

# Deploy Uptime Kuma on secondary monitoring servers (lightweight):
# Server 2: EU West (Frankfurt)
# Server 3: US East (New York)
# Server 4: Asia Pacific (Singapore)

# Each gets a minimal Docker Compose:
mkdir -p /opt/uptime-probe
cat > /opt/uptime-probe/docker-compose.yml << 'EOF'
version: '3.8'

services:
  uptime-kuma:
    image: louislam/uptime-kuma:latest
    container_name: uptime-kuma-probe
    restart: unless-stopped
    ports:
      - "3001:3001"
    volumes:
      - uptime_kuma_probe:/app/data
    environment:
      - TZ=UTC

volumes:
  uptime_kuma_probe:
EOF

docker compose up -d

# Configure the same monitors on each probe via the API:
# Use the same register-monitor.py script from above
# Point it at each probe's URL with separate credentials

UPTIME_KUMA_URL=https://probe-eu.yourdomain.com \
UPTIME_KUMA_USER=admin \
UPTIME_KUMA_PASSWORD=probepassword \
python3 register-monitor.py --name "Main API" --url "https://api.yourdomain.com/health"

#!/usr/bin/env python3
# check-multi-location.py
# Queries multiple Uptime Kuma probes and alerts only on consensus failures
# Run this as a cron job or from a monitoring stack like Grafana

import requests
import re
import json
import os
from datetime import datetime

PROBES = [
    {"name": "EU-West",   "url": "https://probe-eu.yourdomain.com"},
    {"name": "US-East",   "url": "https://probe-us.yourdomain.com"},
    {"name": "AP-South",  "url": "https://probe-ap.yourdomain.com"},
]

ALERT_THRESHOLD = 2  # Alert if this many locations report down
WEBHOOK_URL = os.environ.get("ALERT_WEBHOOK_URL", "")

def get_monitor_status(probe_url: str) -> dict:
    """Fetch monitor statuses from Prometheus metrics endpoint."""
    try:
        resp = requests.get(f"{probe_url}/metrics", timeout=10)
        statuses = {}
        for line in resp.text.split("\n"):
            match = re.match(r'monitor_status\{.*?monitor_name="([^"]+)".*?\} (\d+)', line)
            if match:
                statuses[match.group(1)] = int(match.group(2))
        return statuses
    except Exception as e:
        return {"_error": str(e)}

# Collect status from all probes
results = {}
for probe in PROBES:
    results[probe["name"]] = get_monitor_status(probe["url"])

# Find services that are down in multiple locations
all_monitors = set()
for probe_results in results.values():
    all_monitors.update(k for k in probe_results.keys() if not k.startswith("_"))

for monitor in sorted(all_monitors):
    down_locations = [
        probe_name for probe_name, statuses in results.items()
        if statuses.get(monitor, 1) == 0  # 0 = down
    ]

    if len(down_locations) >= ALERT_THRESHOLD:
        print(f"ALERT: {monitor} is DOWN in {len(down_locations)} locations: {', '.join(down_locations)}")
        if WEBHOOK_URL:
            requests.post(WEBHOOK_URL, json={
                "text": f"🌍 Multi-location outage: *{monitor}* is DOWN in {', '.join(down_locations)}",
                "timestamp": datetime.utcnow().isoformat()
            })
    elif down_locations:
        print(f"WARNING: {monitor} is down in 1 location only ({down_locations[0]}) — may be regional")

#!/usr/bin/env python3
# incident-handler.py
# Flask webhook receiver that creates structured incidents from Uptime Kuma alerts
# Deploy this as a small service on your infrastructure

from flask import Flask, request, jsonify
import requests
import json
import os
from datetime import datetime

app = Flask(__name__)

# Configuration
SLACK_WEBHOOK = os.environ.get("SLACK_WEBHOOK_URL")
GITEA_URL = os.environ.get("GITEA_URL")  # For creating incident issues
GITEA_TOKEN = os.environ.get("GITEA_TOKEN")
GITEA_REPO = os.environ.get("GITEA_REPO", "ops/incidents")
PAGERDUTY_KEY = os.environ.get("PAGERDUTY_INTEGRATION_KEY")

@app.route("/incident", methods=["POST"])
def handle_incident():
    data = request.json
    heartbeat = data.get("heartbeat", {})
    monitor = data.get("monitor", {})

    is_down = heartbeat.get("status") == 0
    monitor_name = monitor.get("name", "Unknown")
    monitor_url = monitor.get("url", "")
    error_msg = heartbeat.get("msg", "")
    duration = heartbeat.get("duration", 0)

    if is_down:
        # 1. Create incident issue in Gitea
        issue_body = f"""## Incident Report

**Monitor:** {monitor_name}
**URL:** {monitor_url}
**Error:** {error_msg}
**Duration:** {duration}s
**Time:** {datetime.utcnow().isoformat()}Z

## Checklist
- [ ] Acknowledge incident
- [ ] Identify root cause
- [ ] Notify stakeholders if > 5 minutes
- [ ] Resolve and document
"""
        if GITEA_URL and GITEA_TOKEN:
            requests.post(
                f"{GITEA_URL}/api/v1/repos/{GITEA_REPO}/issues",
                headers={"Authorization": f"token {GITEA_TOKEN}"},
                json={
                    "title": f"[INCIDENT] {monitor_name} is DOWN",
                    "body": issue_body,
                    "labels": ["incident", "p1"]
                }
            )

        # 2. Send structured Slack alert with context
        if SLACK_WEBHOOK:
            requests.post(SLACK_WEBHOOK, json={
                "blocks": [
                    {"type": "header", "text": {"type": "plain_text", "text": f"🔴 INCIDENT: {monitor_name}"}},
                    {"type": "section", "fields": [
                        {"type": "mrkdwn", "text": f"*URL:*\n{monitor_url}"},
                        {"type": "mrkdwn", "text": f"*Error:*\n{error_msg}"},
                    ]},
                    {"type": "actions", "elements": [
                        {"type": "button", "text": {"type": "plain_text", "text": "View Monitor"},
                         "url": f"{os.environ.get('UPTIME_KUMA_URL')}"},
                    ]}
                ]
            })

    return jsonify({"status": "ok"})

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8090)

#!/bin/bash
# escalate-incident.sh
# Run as a cron job every 5 minutes to check ongoing incidents and escalate

UPTIME_KUMA_URL="https://monitor.yourdomain.com"
SLACK_CRITICAL_CHANNEL="#incidents-critical"
ONCAll_PHONE="+1234567890"

# Query Uptime Kuma metrics for long-running outages
curl -s "${UPTIME_KUMA_URL}/metrics" | grep 'monitor_status' | while read line; do
  STATUS=$(echo "$line" | awk '{print $2}')
  if [ "$STATUS" -eq "0" ]; then
    MONITOR=$(echo "$line" | grep -oP 'monitor_name="\K[^"]+' )
    DURATION=$(echo "$line" | grep -oP 'monitor_response_time{[^}]+} \K\S+' || echo "0")

    # Check how long this monitor has been down from Kuma's history API
    DOWN_SINCE=$(curl -s "${UPTIME_KUMA_URL}/api/badge/${MONITOR_ID}/uptime/24" | jq -r '.uptime')

    # 5+ minute outage → escalate to PagerDuty
    if [ "$DOWNTIME_MINUTES" -ge 5 ]; then
      curl -X POST https://events.pagerduty.com/v2/enqueue \
        -H 'Content-Type: application/json' \
        -d "{
          \"routing_key\": \"${PAGERDUTY_KEY}\",
          \"event_action\": \"trigger\",
          \"payload\": {
            \"summary\": \"${MONITOR} has been DOWN for ${DOWNTIME_MINUTES} minutes\",
            \"severity\": \"critical\",
            \"source\": \"uptime-kuma\"
          }
        }"
    fi
  fi
done

# Add to prometheus/prometheus.yml scrape_configs:
  - job_name: 'uptime-kuma'
    scrape_interval: 30s
    scrape_timeout: 10s
    scheme: https
    static_configs:
      - targets:
          - monitor.yourdomain.com
    metrics_path: '/metrics'
    # If Uptime Kuma is behind basic auth:
    basic_auth:
      username: admin
      password: yourpassword
    # Or use bearer token if configured:
    # bearer_token: your-token

# Reload Prometheus config:
curl -X POST http://localhost:9090/-/reload

# Verify metrics are being scraped:
curl 'http://localhost:9090/api/v1/query?query=monitor_status' | jq '.data.result | length'
# Should return the number of monitors you have

# Useful Uptime Kuma PromQL queries for Grafana dashboards:

# Current status of all monitors (1=up, 0=down)
monitor_status

# Response time in milliseconds per monitor
monitor_response_time

# Count of currently down monitors
count(monitor_status == 0)

# Monitors that have been down (status=0) — alert on this
monitor_status{monitor_name=~".+"} == 0

# 24-hour availability percentage per monitor
# (Note: Kuma doesn't expose this directly — calculate from status changes)
avg_over_time(monitor_status[24h]) * 100

# Response time P95 across all monitors
quantile(0.95, monitor_response_time)

# Monitors slower than 2 seconds
monitor_response_time > 2000

# Alert rule for Grafana (add in Alerting → Alert Rules):
# Alert when any monitor has been down for > 3 minutes:
# count(monitor_status == 0) > 0
# Pending: 3m  (fire only after 3 minutes of consistent down)

#!/bin/bash
# backup-uptime-kuma.sh
# Back up Uptime Kuma configuration and history to S3

set -euo pipefail

DATE=$(date +%Y-%m-%d-%H%M)
BACKUP_DIR="/opt/backups/uptime-kuma"
S3_BUCKET="s3://your-backup-bucket/uptime-kuma"

mkdir -p "$BACKUP_DIR"

# SQLite backup (atomic snapshot — safe to run while Kuma is live)
docker exec uptime-kuma sqlite3 /app/data/kuma.db ".backup '/app/data/kuma-backup.db'"
docker cp uptime-kuma:/app/data/kuma-backup.db "${BACKUP_DIR}/kuma-${DATE}.db"

# Compress
gzip "${BACKUP_DIR}/kuma-${DATE}.db"

# Upload to S3
aws s3 cp "${BACKUP_DIR}/kuma-${DATE}.db.gz" "${S3_BUCKET}/kuma-${DATE}.db.gz"

# Clean up local backups older than 7 days
find "$BACKUP_DIR" -name 'kuma-*.db.gz' -mtime +7 -delete

# Verify backup integrity
sqlite3 "${BACKUP_DIR}/kuma-${DATE}.db.gz" "PRAGMA integrity_check;" 2>/dev/null || \
  echo "Note: Verify integrity on decompressed file"

echo "Backup complete: kuma-${DATE}.db.gz"

# Add to crontab:
# 0 3 * * * /opt/scripts/backup-uptime-kuma.sh >> /var/log/kuma-backup.log 2>&1

#!/bin/bash
# restore-uptime-kuma.sh
# Usage: ./restore-uptime-kuma.sh kuma-2026-04-07-0300.db.gz

BACKUP_FILE="${1:-}"
if [ -z "$BACKUP_FILE" ]; then
  echo "Usage: $0 "
  exit 1
fi

# Download from S3 if file doesn't exist locally
if [ ! -f "$BACKUP_FILE" ]; then
  aws s3 cp "s3://your-backup-bucket/uptime-kuma/${BACKUP_FILE}" "."
fi

# Stop Uptime Kuma
docker compose stop uptime-kuma

# Extract backup
gunzip -c "$BACKUP_FILE" > /tmp/kuma-restore.db

# Verify integrity
sqlite3 /tmp/kuma-restore.db "PRAGMA integrity_check;"

# Copy to data volume
docker cp /tmp/kuma-restore.db uptime-kuma:/app/data/kuma.db
docker exec uptime-kuma chown node:node /app/data/kuma.db

# Restart
docker compose start uptime-kuma

# Verify
sleep 5
curl -I https://monitor.yourdomain.com
echo "Restore complete. Verify monitors at https://monitor.yourdomain.com"

# Pattern: reconnect on each script invocation rather than holding a session
# Use context manager to ensure clean disconnection:

from uptime_kuma_api import UptimeKumaApi
import os

def with_kuma(func):
    """Decorator that handles API connection lifecycle."""
    def wrapper(*args, **kwargs):
        api = UptimeKumaApi(os.environ["UPTIME_KUMA_URL"])
        api.login(
            os.environ["UPTIME_KUMA_USER"],
            os.environ["UPTIME_KUMA_PASSWORD"]
        )
        try:
            return func(api, *args, **kwargs)
        finally:
            api.disconnect()
    return wrapper

@with_kuma
def list_down_monitors(api):
    monitors = api.get_monitors()
    return [m for m in monitors if m.get("active") and not m.get("forceInactive")]

# Test metrics endpoint directly:
curl -v https://monitor.yourdomain.com/metrics

# Common issues:
# 1. Metrics endpoint returns empty if no monitors exist yet
#    → Create at least one monitor first

# 2. Authentication blocking the metrics endpoint
#    → Check if Uptime Kuma requires login for /metrics
#    → In Prometheus: add basic_auth config

# 3. Nginx blocking the /metrics path
#    → Check for location blocks that might return 404
docker exec nginx nginx -T | grep -A5 'location.*metrics'

# 4. Monitor hasn't completed its first check yet
#    → Wait one full interval (60s default) after creating a monitor

# If probes disagree on status, check network path from each:
docker exec uptime-kuma curl -sv --max-time 10 https://api.yourdomain.com/health 2>&1 | \
  grep -E '(Connected to|HTTP|SSL|curl)'

# Check if the disagreement is consistent or intermittent:
# Add a longer observation window before alerting on multi-location failures

# In check-multi-location.py, add temporal consistency:
# Only alert if the same location has been reporting down for 3+ consecutive checks
# Track state in a simple file or Redis:
from pathlib import Path
import json

STATE_FILE = Path("/tmp/monitor-state.json")
state = json.loads(STATE_FILE.read_text()) if STATE_FILE.exists() else {}

# Increment failure count per location
for location in down_locations:
    key = f"{monitor}:{location}"
    state[key] = state.get(key, 0) + 1

# Only alert if consistently down for 3+ checks (~3 minutes at 1min polling)
if all(state.get(f"{monitor}:{loc}", 0) >= 3 for loc in down_locations):
    send_alert(monitor, down_locations)

STATE_FILE.write_text(json.dumps(state))