#!/bin/bash
# =============================================================================
# DNS Maintenance - Graceful Drain & Restore for mTLS Cluster
# =============================================================================
# Entfernt/stellt DNS-Records eines Servers aus/in Cloudflare wieder her.
# Failsafe: Jeder Fehler/Timeout wird per Syslog gemeldet (Wazuh/NewRelic).
#
# Verwendung:
#   dns-maintenance drain     # Server aus DNS entfernen (vor Shutdown)
#   dns-maintenance restore   # Server in DNS wiederherstellen (nach Start)
#   dns-maintenance status    # Aktuelle DNS-Records dieses Servers anzeigen
#   dns-maintenance verify    # Soll/Ist-Abgleich ohne Aenderung
#
# Version: 1.8
# Stand: 2026-04-16
# =============================================================================

set -uo pipefail

# --- Farben ---
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

# --- Konfiguration ---
DRAIN_TIMEOUT=900           # Max. Wartezeit in Sekunden (15 Minuten)
DRAIN_CHECK_INTERVAL=15     # Pruefintervall in Sekunden
DRAIN_QUIET_REQUIRED=300    # Benoetigte Stille auf mTLS-VHost in Sekunden (5 Min)
STATE_FILE="/var/run/dns-maintenance-state.json"
PREDRAIN_MARKER="/run/dns-maintenance-predrained"   # tmpfs: verschwindet beim Reboot
PREDRAIN_MAX_AGE=900                                 # 15 min -- danach ignoriert
PEER_CHECK_INTERVAL=30      # Pruefintervall fuer Peer-Checks (Sekunden)
PEER_FAIL_THRESHOLD=6       # Fehlschlaege bis Vote (6 x 30s = 3 Min)
PEER_QUORUM=3               # Mindestanzahl Votes fuer Drain
PEER_DRAIN_COOLDOWN=600     # 10 Min Cooldown nach Peer-Drain
PEER_DRY_RUN=true           # Dry-Run: nur loggen, nicht drainen
PEER_FAILCOUNT_DIR="/var/run/dns-peer"
PEER_KILL_SWITCH="/etc/dns-maintenance-no-peer-drain"
# Webhook secrets: prefer /etc/cert-pki/alerting.env (0600) — fallback /etc/environment (legacy, 0644)
for _envfile in /etc/cert-pki/alerting.env /etc/environment; do
    [[ -f "$_envfile" && -r "$_envfile" ]] || continue
    while IFS='=' read -r key val; do
        [[ "$key" =~ ^ALERT_WEBHOOK_ ]] && export "$key"="$(echo "$val" | sed 's/^"//;s/"$//')"
    done < "$_envfile"
done
unset _envfile
ALERT_WEBHOOK_SLACK="${ALERT_WEBHOOK_SLACK:-}"
ALERT_WEBHOOK_CANARI="${ALERT_WEBHOOK_CANARI:-}"
LOG_TAG="dns-maintenance"
LOG_FACILITY="local0"
ACCESS_LOG="/var/log/apache2/prd-access.log"

# Per-Zone DNS-TTL (60s = Kompromiss: schnelles Failover + wenig DNS-Overhead).
# Bei Rejection durch CF-Plan-Mindest-TTL wird automatisch auf DNS_TTL_FALLBACK zurueckgegriffen.
declare -A ZONE_TTL
ZONE_TTL[business]=60       # Enterprise (min 1s)
ZONE_TTL[services]=60       # Pro (min 60s)
DNS_TTL_FALLBACK=60         # Safe default

# --- Cloudflare Zone IDs ---
ZONE_ID_BUSINESS="36e79a0dc1c69628fa1e23aae9d5e9b8"
ZONE_ID_SERVICES="ada86c619846d0486e721dd85381ad00"

# --- Server-Mapping ---
declare -A SERVERS_V4
SERVERS_V4[1]="188.245.157.241"  # Cert-Server-1-NBG
SERVERS_V4[2]="116.203.243.240"  # Cert-Server-0-NBG
SERVERS_V4[3]="49.12.76.141"     # Cert-Server-0-FSN
SERVERS_V4[4]="91.98.147.79"     # Cert-Server-1-FSN
SERVERS_V4[5]="46.62.131.226"    # Cert-Server-HEL

declare -A SERVERS_V6
SERVERS_V6[1]="2a01:4f8:c0c:af60::1"   # Cert-Server-1-NBG
SERVERS_V6[2]="2a01:4f8:1c1a:b9f1::1"  # Cert-Server-0-NBG
SERVERS_V6[3]="2a01:4f8:c17:7411::1"   # Cert-Server-0-FSN
SERVERS_V6[4]="2a01:4f8:c012:e96c::1"  # Cert-Server-1-FSN
SERVERS_V6[5]="2a01:4f9:c012:4f8f::1"  # Cert-Server-HEL

declare -A SERVER_NAMES
SERVER_NAMES[1]="Cert-Server-1-NBG"
SERVER_NAMES[2]="Cert-Server-0-NBG"
SERVER_NAMES[3]="Cert-Server-0-FSN"
SERVER_NAMES[4]="Cert-Server-1-FSN"
SERVER_NAMES[5]="Cert-Server-HEL"

# Reverse mapping: hostname -> server number
declare -A HOSTNAME_TO_NUM
HOSTNAME_TO_NUM["Cert-Server-1-NBG"]=1
HOSTNAME_TO_NUM["Cert-Server-0-NBG"]=2
HOSTNAME_TO_NUM["Cert-Server-0-FSN"]=3
HOSTNAME_TO_NUM["Cert-Server-1-FSN"]=4
HOSTNAME_TO_NUM["Cert-Server-HEL"]=5

# VLAN-IP zu Server-Nummer Mapping (fuer SSH-Zugriff bei Peer-Checks)
declare -A VLAN_IPS
VLAN_IPS[1]="10.0.0.2"
VLAN_IPS[2]="10.0.0.3"
VLAN_IPS[3]="10.0.0.4"
VLAN_IPS[4]="10.0.0.5"
VLAN_IPS[5]="10.0.0.6"

# Cloudflare SSH Tunnel URLs (fuer Alerts — Operator kann direkt klicken)
declare -A SSH_URLS
SSH_URLS[1]="https://cert-server-1-nbg-ssh.db-app.dev/"
SSH_URLS[2]="https://cert-server-0-nbg-ssh.db-app.dev/"
SSH_URLS[3]="https://cert-server-0-fsn-ssh.db-app.dev/"
SSH_URLS[4]="https://cert-server-1-fsn-ssh.db-app.dev/"
SSH_URLS[5]="https://cert-server-hel-ssh.db-app.dev/"

# Fixed-IP-Server (Monitoring-only, kein Auto-Drain)
declare -A FIXED_IPS
FIXED_IPS["bund"]="49.12.179.109"
FIXED_IPS["bundeswehr"]="49.12.179.110"

# --- Statische Soll-Liste ---
# Services und Stages die im Round-Robin fuer ALLE 5 Server gelten
SERVICES=(cs cert ccc cs-b cs-bw ccc-b ccc-bw)
STAGES=(dev int tst lup pen pres abn prd)
DOMAINS=(business services)

# Root-Level-Records (ohne Stage) -- existieren nur fuer Server 1 (Cert-Server-1-NBG)
ROOT_RECORDS_SRV1=(
    "ccc.bahn.business"
    "cs.bahn.business"
    "ccc.bahn.services"
    "cert.bahn.services"
    "cs.bahn.services"
)

# Ausnahmen-Logik: Welche Records gehoeren NICHT zum 5er-Cluster?
# Gibt 0 (true) zurueck wenn der Record fuer diesen Server erwartet wird.
is_record_expected() {
    local server_num=$1
    local service=$2
    local stage=$3
    local domain=$4

    # 1. TST Basis-Services (cs, cert, ccc) auf bahn.services: nur Server 1
    #    Hinweis: -b/-bw Varianten auf TST/services haben Round-Robin (alle 5 Server)
    if [[ "$stage" == "tst" && "$domain" == "services" && "$server_num" -ne 1 \
          && ("$service" == "cs" || "$service" == "cert" || "$service" == "ccc") ]]; then
        return 1
    fi

    # 2. cert.* auf bahn.business (ausser dev): CNAME zu pages.dev, keine A-Records
    if [[ "$service" == "cert" && "$domain" == "business" && "$stage" != "dev" ]]; then
        return 1
    fi

    # 3. -b/-bw Varianten auf prd/abn auf bahn.business: Fixed-IP-Server, nicht Cluster
    if [[ ("$service" == "cs-b" || "$service" == "ccc-b" || "$service" == "cs-bw" || "$service" == "ccc-bw") \
          && ("$stage" == "prd" || "$stage" == "abn") \
          && "$domain" == "business" ]]; then
        return 1
    fi

    # 4. -b/-bw Varianten auf dev auf bahn.business: nur Server 1 (Cert-Server-1-NBG)
    if [[ ("$service" == "cs-b" || "$service" == "ccc-b" || "$service" == "cs-bw" || "$service" == "ccc-bw") \
          && "$stage" == "dev" \
          && "$domain" == "business" \
          && "$server_num" -ne 1 ]]; then
        return 1
    fi

    # 5. -b/-bw Varianten auf prd auf bahn.services: nur Server 1 (Cert-Server-1-NBG)
    if [[ ("$service" == "cs-b" || "$service" == "ccc-b" || "$service" == "cs-bw" || "$service" == "ccc-bw") \
          && "$stage" == "prd" \
          && "$domain" == "services" \
          && "$server_num" -ne 1 ]]; then
        return 1
    fi

    return 0
}

# --- Logging ---
log_info() {
    logger -t "$LOG_TAG" -p "${LOG_FACILITY}.info" "$1"
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warning() {
    logger -t "$LOG_TAG" -p "${LOG_FACILITY}.warning" "$1"
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    logger -t "$LOG_TAG" -p "${LOG_FACILITY}.err" "$1"
    echo -e "${RED}[ERROR]${NC} $1"
}

log_critical() {
    logger -t "$LOG_TAG" -p "${LOG_FACILITY}.crit" "$1"
    echo -e "${RED}[CRITICAL]${NC} $1"
}

log_debug() {
    echo -e "${CYAN}[DEBUG]${NC} $1"
}

# --- Hilfsfunktionen ---

load_token() {
    local domain=$1
    if [[ "$domain" == "business" ]]; then
        echo "$CF_TOKEN_BUSINESS"
    else
        echo "$CF_TOKEN_SERVICES"
    fi
}

get_zone_id() {
    local domain=$1
    if [[ "$domain" == "business" ]]; then
        echo "$ZONE_ID_BUSINESS"
    else
        echo "$ZONE_ID_SERVICES"
    fi
}

# Alle Records eines bestimmten Typs fuer eine IP aus einer Zone holen (paginiert!)
get_records_for_ip() {
    local zone_id=$1
    local token=$2
    local record_type=$3  # A oder AAAA
    local ip=$4

    local all_records="[]"
    local page=1

    while true; do
        local result
        result=$(curl -s -X GET \
            "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?type=${record_type}&content=${ip}&per_page=100&page=${page}" \
            -H "Authorization: Bearer ${token}" \
            -H "Content-Type: application/json" 2>/dev/null)

        local count
        count=$(echo "$result" | jq -r '.result | length')

        if [[ "$count" -eq 0 ]]; then
            break
        fi

        all_records=$(echo "$all_records" "$result" | jq -s '.[0] + (.[1].result)')
        page=$((page + 1))

        # Rate limiting
        sleep 0.1
    done

    echo "$all_records"
}

# Einzelnen Record loeschen (Fallback / manueller Gebrauch)
delete_record() {
    local zone_id=$1
    local token=$2
    local record_id=$3

    local result
    result=$(curl -s -X DELETE \
        "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" \
        -H "Authorization: Bearer ${token}" \
        -H "Content-Type: application/json" 2>/dev/null)

    echo "$result" | jq -r '.success'
}

# Batch-Operation via Cloudflare Batch DNS API
# Args: zone_id, token, body (JSON mit deletes/patches/posts/puts)
# Echoes full response JSON
batch_dns_operations() {
    local zone_id=$1
    local token=$2
    local body=$3

    curl -s -X POST \
        "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/batch" \
        -H "Authorization: Bearer ${token}" \
        -H "Content-Type: application/json" \
        --data "$body" 2>/dev/null
}

# Batch-Operation mit TTL-Fallback. Wenn CF die Request wegen zu niedrigem TTL
# ablehnt, wird der Body auf DNS_TTL_FALLBACK umgeschrieben und einmal erneut
# versucht. Args wie batch_dns_operations.
batch_dns_operations_with_retry() {
    local zone_id=$1
    local token=$2
    local body=$3

    local resp
    resp=$(batch_dns_operations "$zone_id" "$token" "$body")

    local success
    success=$(echo "$resp" | jq -r '.success')

    if [[ "$success" == "true" ]]; then
        echo "$resp"
        return 0
    fi

    # Pruefen ob Fehler auf TTL-Limit hinweist (Cloudflare-Fehlermeldungen enthalten
    # "TTL" oder "minimum" wenn das Plan-Mindest-TTL verletzt wird).
    local errs
    errs=$(echo "$resp" | jq -c '.errors // []')
    if echo "$errs" | grep -qiE 'ttl|minimum'; then
        log_warning "Batch rejected due to TTL limit -- retrying with fallback TTL ${DNS_TTL_FALLBACK}. Errors: ${errs}"
        local retry_body
        retry_body=$(echo "$body" | jq --argjson t "$DNS_TTL_FALLBACK" '
            if .posts   then .posts   |= map(.ttl = $t) else . end |
            if .patches then .patches |= map(.ttl = $t) else . end |
            if .puts    then .puts    |= map(.ttl = $t) else . end
        ')
        resp=$(batch_dns_operations "$zone_id" "$token" "$retry_body")
    fi

    echo "$resp"
}

# Record erstellen (Einzel-API, Fallback / manueller Gebrauch). Optional: 6. Arg = domain,
# damit das passende Zone-TTL benutzt wird. Ohne Arg fallback auf DNS_TTL_FALLBACK.
# Bei TTL-Reject einmaliger Retry mit DNS_TTL_FALLBACK.
create_record() {
    local zone_id=$1
    local token=$2
    local record_type=$3
    local hostname=$4
    local content=$5
    local domain=${6:-}

    local ttl="${ZONE_TTL[$domain]:-$DNS_TTL_FALLBACK}"
    local comment_str="managed by dns-maintenance on ${SERVER_NAME} ($(date +'%Y-%m-%d %H:%M'))"

    _create_record_once() {
        local payload
        payload=$(jq -n \
            --arg type "$record_type" \
            --arg name "$hostname" \
            --arg content "$content" \
            --argjson ttl "$1" \
            --arg comment "$comment_str" \
            '{type: $type, name: $name, content: $content, ttl: $ttl, proxied: false, comment: $comment}')
        curl -s -X POST \
            "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
            -H "Authorization: Bearer ${token}" \
            -H "Content-Type: application/json" \
            --data "$payload" 2>/dev/null
    }

    local result
    result=$(_create_record_once "$ttl")
    local success
    success=$(echo "$result" | jq -r '.success')

    if [[ "$success" != "true" ]]; then
        local error_msg
        error_msg=$(echo "$result" | jq -r '.errors[0].message // "Unknown error"')
        if [[ "$error_msg" == *"already exists"* ]]; then
            echo "exists"
            return
        fi
        # TTL-Reject -> einmal mit Fallback retryen
        if echo "$error_msg" | grep -qiE 'ttl|minimum'; then
            log_warning "create_record rejected TTL ${ttl} for ${hostname} -- retrying with ${DNS_TTL_FALLBACK}"
            result=$(_create_record_once "$DNS_TTL_FALLBACK")
            success=$(echo "$result" | jq -r '.success')
            if [[ "$success" == "true" ]]; then
                echo "true"
                return
            fi
            error_msg=$(echo "$result" | jq -r '.errors[0].message // "Unknown error"')
        fi
        echo "false:${error_msg}"
        return
    fi

    echo "true"
}

# Server-Nummer erkennen
detect_server() {
    local hostname
    hostname=$(hostname)

    if [[ -z "${HOSTNAME_TO_NUM[$hostname]+x}" ]]; then
        log_critical "Unknown hostname: $hostname. Not a known cluster server."
        exit 1
    fi

    SERVER_NUM="${HOSTNAME_TO_NUM[$hostname]}"
    SERVER_NAME="${SERVER_NAMES[$SERVER_NUM]}"
    MY_IPV4="${SERVERS_V4[$SERVER_NUM]}"
    MY_IPV6="${SERVERS_V6[$SERVER_NUM]}"

    log_info "Detected server: [$SERVER_NUM] $SERVER_NAME ($MY_IPV4 / $MY_IPV6)"
}

# Token-Validierung
validate_tokens() {
    if [[ -z "${CF_TOKEN_BUSINESS:-}" ]]; then
        log_critical "CF_TOKEN_BUSINESS not set"
        exit 1
    fi
    if [[ -z "${CF_TOKEN_SERVICES:-}" ]]; then
        log_critical "CF_TOKEN_SERVICES not set"
        exit 1
    fi
}

# Erwartete Hostnames fuer diesen Server generieren
get_expected_hostnames() {
    local server_num=$1
    local hostnames=()

    for service in "${SERVICES[@]}"; do
        for stage in "${STAGES[@]}"; do
            for domain in "${DOMAINS[@]}"; do
                if is_record_expected "$server_num" "$service" "$stage" "$domain"; then
                    hostnames+=("${service}.${stage}.bahn.${domain}")
                fi
            done
        done
    done

    # Root-Level-Records fuer Server 1
    if [[ "$server_num" -eq 1 ]]; then
        for root_rec in "${ROOT_RECORDS_SRV1[@]}"; do
            hostnames+=("$root_rec")
        done
    fi

    echo "${hostnames[@]}"
}

# --- Hauptfunktionen ---

do_drain() {
    # Restart-Detection: Nur bei echtem Shutdown drainen, nicht bei Restart.
    # Bei 'systemctl restart dns-maintenance.service' triggert ExecStop sonst ungewollt
    # den produktiven Drain (Incident 2026-04-21: P5-Rollout → 5x paralleler Drain).
    # systemctl is-system-running: 'running'/'degraded' = normaler Betrieb (Restart),
    # 'stopping' = shutdown.target wird angefahren (echter Shutdown/Reboot).
    local sysstate
    sysstate=$(systemctl is-system-running 2>/dev/null || true)
    if [[ "$sysstate" == "running" || "$sysstate" == "degraded" ]]; then
        log_info "=== Restart detected (system-state=$sysstate). Skipping drain (only drains on actual shutdown). ==="
        return 0
    fi

    # Predrain-Skip: wenn Operator bereits via 'drain-and-reboot' gedrained hat,
    # ueberspringt die ExecStop-Ausfuehrung die (dann redundante) zweite Drain-Runde.
    if [[ -f "$PREDRAIN_MARKER" ]]; then
        local marker_age=$(( $(date +%s) - $(stat -c %Y "$PREDRAIN_MARKER") ))
        if [[ "$marker_age" -lt "$PREDRAIN_MAX_AGE" ]]; then
            log_info "=== Pre-drain marker found (${marker_age}s old, < ${PREDRAIN_MAX_AGE}s). Drain already complete. Skipping. ==="
            return 0
        else
            log_warning "Pre-drain marker stale (${marker_age}s old), ignoring and draining fresh"
        fi
    fi

    log_info "=== DRAIN started for $SERVER_NAME ==="
    send_alert "info" "DRAIN gestartet — Server wird aus DNS entfernt (${DRAIN_TIMEOUT}s Timeout, ${DRAIN_QUIET_REQUIRED}s Quiet)"

    local deleted_records="[]"
    local total_deleted=0
    local total_errors=0
    local drain_start
    drain_start=$(date -u +%Y-%m-%dT%H:%M:%SZ)

    for domain in "${DOMAINS[@]}"; do
        local token zone_id
        token=$(load_token "$domain")
        zone_id=$(get_zone_id "$domain")

        log_info "Processing zone: bahn.${domain} (${zone_id})"

        # A-Records fuer unsere IPv4
        local a_records
        a_records=$(get_records_for_ip "$zone_id" "$token" "A" "$MY_IPV4")
        local a_count
        a_count=$(echo "$a_records" | jq 'length')
        log_info "Found ${a_count} A-Records for ${MY_IPV4} in bahn.${domain}"

        # AAAA-Records fuer unsere IPv6
        local aaaa_records
        aaaa_records=$(get_records_for_ip "$zone_id" "$token" "AAAA" "$MY_IPV6")
        local aaaa_count
        aaaa_count=$(echo "$aaaa_records" | jq 'length')
        log_info "Found ${aaaa_count} AAAA-Records for ${MY_IPV6} in bahn.${domain}"

        # Alle Records via Batch-API in EINEM Request loeschen (max. 500 pro Batch,
        # wir haben pro Zone < 100 Records -- passt). Atomar: ganz oder gar nicht.
        local zone_records
        zone_records=$(echo "$a_records" "$aaaa_records" | jq -s '.[0] + .[1]')
        local zone_count
        zone_count=$(echo "$zone_records" | jq 'length')

        if [[ "$zone_count" -eq 0 ]]; then
            log_info "No records to delete in bahn.${domain}"
            continue
        fi

        local batch_body
        batch_body=$(echo "$zone_records" | jq '{deletes: [.[] | {id: .id}]}')

        local batch_result
        batch_result=$(batch_dns_operations_with_retry "$zone_id" "$token" "$batch_body")
        local batch_success
        batch_success=$(echo "$batch_result" | jq -r '.success')

        if [[ "$batch_success" == "true" ]]; then
            log_info "Batch-deleted ${zone_count} records in bahn.${domain}"
            deleted_records=$(echo "$deleted_records" "$zone_records" | jq -s \
                --arg zone "bahn.${domain}" \
                '.[0] + [.[1][] | {zone: $zone, type: .type, name: .name, id: .id, content: .content}]')
            total_deleted=$((total_deleted + zone_count))
        else
            local errors_msg
            errors_msg=$(echo "$batch_result" | jq -c '.errors // []')
            log_error "Batch delete failed for bahn.${domain}: ${errors_msg}"
            total_errors=$((total_errors + zone_count))
        fi
    done

    log_info "Deletion phase complete: ${total_deleted} deleted, ${total_errors} errors"

    if [[ "$total_errors" -gt 0 ]]; then
        log_warning "There were ${total_errors} deletion errors"
    fi

    # State-File schreiben
    jq -n \
        --arg server "$SERVER_NAME" \
        --arg ipv4 "$MY_IPV4" \
        --arg ipv6 "$MY_IPV6" \
        --arg drain_started "$drain_start" \
        --arg drain_completed "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
        --argjson records_deleted "$deleted_records" \
        --argjson total_deleted "$total_deleted" \
        '{server: $server, ipv4: $ipv4, ipv6: $ipv6, drain_started: $drain_started, drain_completed: $drain_completed, records_deleted: $records_deleted, total_deleted: $total_deleted}' \
        > "$STATE_FILE"

    log_info "State file written: ${STATE_FILE} (${total_deleted} records)"

    # --- Verifikationsschleife ---
    log_info "Starting verification (timeout: ${DRAIN_TIMEOUT}s, interval: ${DRAIN_CHECK_INTERVAL}s, quiet required: ${DRAIN_QUIET_REQUIRED}s on mTLS vhost)"

    local elapsed=0
    local quiet_seconds=0
    local sample_hostnames=("cs.prd.bahn.business" "cs.prd.bahn.services" "cert.prd.bahn.services" "ccc.prd.bahn.business")

    # Cluster-IPs zusammenbauen (fuer Filterung der Peer-Health-Checks)
    # IPv4, IPv6 UND VLAN-IPs muessen raus, sonst blockiert Peer-Traffic die Quiet-Period
    local cluster_ip_pattern=""
    for num in "${!SERVERS_V4[@]}"; do
        [[ -n "$cluster_ip_pattern" ]] && cluster_ip_pattern+="|"
        cluster_ip_pattern+="${SERVERS_V4[$num]}"
    done
    for num in "${!SERVERS_V6[@]}"; do
        cluster_ip_pattern+="|${SERVERS_V6[$num]}"
    done
    for num in "${!VLAN_IPS[@]}"; do
        cluster_ip_pattern+="|${VLAN_IPS[$num]}"
    done
    # Localhost ebenfalls ausschliessen (Apache-Status, lokales Monitoring)
    cluster_ip_pattern+="|127\\.0\\.0\\.1|::1"
    log_debug "Cluster-IP filter pattern: ${cluster_ip_pattern}"

    while [[ $elapsed -lt $DRAIN_TIMEOUT ]]; do
        sleep "$DRAIN_CHECK_INTERVAL"
        elapsed=$((elapsed + DRAIN_CHECK_INTERVAL))

        # Check 1: DNS - eigene IP darf nicht mehr aufgeloest werden
        local dns_clean=true
        for hostname in "${sample_hostnames[@]}"; do
            local resolved
            resolved=$(dig +short "$hostname" 2>/dev/null)
            if echo "$resolved" | grep -q "$MY_IPV4"; then
                dns_clean=false
                log_warning "DNS still resolving ${hostname} -> ${MY_IPV4} after ${elapsed}s"
                break
            fi
        done

        # Check 2: Aktive HTTPS-Connections
        local active_conns
        active_conns=$(ss -tnp sport = :443 2>/dev/null | grep -c ESTAB || true)

        # Check 3: Apache Access-Log (neue Eintraege, Peer-Health-Checks ausgefiltert)
        # Peer-Checks von Cluster-IPs zaehlen NICHT als Traffic, da sie auch waehrend
        # des Drains weiterlaufen und sonst die Quiet-Period verhindern wuerden.
        local recent_requests=0
        if [[ -f "$ACCESS_LOG" ]]; then
            local cutoff_epoch
            cutoff_epoch=$(date -d "${DRAIN_CHECK_INTERVAL} seconds ago" +%s 2>/dev/null)
            # Letzte 200 Zeilen lesen, Cluster-IPs rausfiltern, nur Zeilen im Zeitfenster zaehlen
            # mtls_vhost LogFormat: "%v:%p %{Host}i %h ..." — Client-IP ist Feld 3
            recent_requests=$(tail -n 200 "$ACCESS_LOG" 2>/dev/null \
                | grep -vE "^\S+ \S+ (${cluster_ip_pattern}) " \
                | grep -oP '\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}' \
                | while IFS= read -r ts; do
                    ts_epoch=$(date -d "$(echo "$ts" | sed 's|/| |g; s/:/ /')" +%s 2>/dev/null || echo 0)
                    [[ "$ts_epoch" -ge "$cutoff_epoch" ]] && echo 1
                done | wc -l)
            recent_requests=${recent_requests:-0}
        fi

        # Stille-Counter auf mTLS-VHost: erst wenn DRAIN_QUIET_REQUIRED Sekunden
        # in Folge KEINE Requests mehr ankamen, gilt der Drain als abgeschlossen.
        if [[ "$recent_requests" -eq 0 ]]; then
            quiet_seconds=$((quiet_seconds + DRAIN_CHECK_INTERVAL))
        else
            if [[ "$quiet_seconds" -gt 0 ]]; then
                log_warning "mTLS request seen at ${elapsed}s - resetting quiet timer (was ${quiet_seconds}s)"
            fi
            quiet_seconds=0
        fi

        log_debug "Check at ${elapsed}s: dns_clean=${dns_clean}, active_conns=${active_conns}, recent_requests=${recent_requests}, quiet=${quiet_seconds}/${DRAIN_QUIET_REQUIRED}s"

        if [[ "$dns_clean" == "true" && "$active_conns" -le 1 && "$quiet_seconds" -ge "$DRAIN_QUIET_REQUIRED" ]]; then
            log_info "=== DRAIN complete after ${elapsed}s (${quiet_seconds}s silent on mTLS vhost). No traffic for ${SERVER_NAME}. ==="
            send_alert "info" "DRAIN abgeschlossen nach ${elapsed}s (${quiet_seconds}s silent). ${total_deleted} Records entfernt."
            return 0
        fi
    done

    # Timeout erreicht
    log_critical "DRAIN TIMEOUT after ${DRAIN_TIMEOUT}s for ${SERVER_NAME}. Traffic may still arrive! (conns=${active_conns}, dns_clean=${dns_clean}, quiet=${quiet_seconds}/${DRAIN_QUIET_REQUIRED}s)"
    send_alert "critical" "DRAIN TIMEOUT nach ${DRAIN_TIMEOUT}s! Traffic kommt moeglicherweise noch an. (conns=${active_conns}, dns_clean=${dns_clean}, quiet=${quiet_seconds}/${DRAIN_QUIET_REQUIRED}s)"
    return 1
}

# Interaktiver Pre-Reboot Drain: drained synchronously, dann systemctl reboot.
# Ziel: Der Operator sieht Drain-Fortschritt im eigenen Shell, die SSH-Session
# bleibt aktiv bis zum *echten* Reboot-Zeitpunkt. ExecStop erkennt den Marker
# und ueberspringt die zweite Drain-Runde.
do_drain_and_reboot() {
    log_info "=== INTERACTIVE PRE-REBOOT DRAIN for $SERVER_NAME ==="
    log_info "Drain laeuft im aktuellen Shell -- Ctrl-C bricht AB (kein Reboot)"
    log_info "HINWEIS: SSH-Session trennt sich erst am Ende, wenn 'systemctl reboot' getriggert wird"
    echo ""

    if do_drain; then
        touch "$PREDRAIN_MARKER"
        log_info "Pre-drain marker set: $PREDRAIN_MARKER"
        log_info "=== PRE-DRAIN COMPLETE. Initiating reboot in 3s... ==="
        send_alert "info" "REBOOT wird ausgefuehrt — Drain erfolgreich, Server startet in 3s neu"
        sleep 3
        exec systemctl reboot
    else
        log_critical "Pre-drain FAILED (timeout or error). *** NICHT rebootet. ***"
        log_critical "DNS kann partiell gedrained sein. Pruefe mit 'dns-maintenance status'"
        log_critical "Wiederherstellung: 'dns-maintenance restore' (legt Records neu an)"
        send_alert "critical" "DRAIN-AND-REBOOT FEHLGESCHLAGEN — Server wurde NICHT rebootet. DNS moeglicherweise partiell gedrained. Manueller Eingriff noetig!"
        exit 1
    fi
}

do_restore() {
    # Pre-drain-Marker entfernen, falls uebrig
    rm -f "$PREDRAIN_MARKER" 2>/dev/null || true

    # Restart-Detection: Service-Restart (system 'running'/'degraded') ohne
    # vorherigen Drain (kein STATE_FILE) bedeutet Restore ist ein No-Op —
    # Slack-Noise unterdruecken. Arbeit trotzdem idempotent als Safety-Net.
    # Fehler-Alerts bleiben UNABHAENGIG vom quiet_mode aktiv.
    local sysstate quiet_mode=0
    sysstate=$(systemctl is-system-running 2>/dev/null || true)
    if [[ "$sysstate" == "running" || "$sysstate" == "degraded" ]] && [[ ! -f "$STATE_FILE" ]]; then
        quiet_mode=1
        log_info "=== Restart detected (state=$sysstate, no drain state-file). Quiet mode: info alerts suppressed. ==="
    fi

    log_info "=== RESTORE started for $SERVER_NAME ==="
    [[ $quiet_mode -eq 0 ]] && send_alert "info" "RESTORE gestartet — DNS-Records werden wiederhergestellt"

    # Apache pruefen
    if ! systemctl is-active --quiet apache2; then
        log_error "Apache2 is not running. Aborting restore."
        send_alert "critical" "RESTORE ABBRUCH: Apache2 laeuft nicht! DNS-Records werden NICHT wiederhergestellt."
        return 1
    fi
    log_info "Apache2 is running"

    local expected_hostnames
    read -ra expected_hostnames <<< "$(get_expected_hostnames "$SERVER_NUM")"
    local total_expected=${#expected_hostnames[@]}
    log_info "Expected hostnames for this server: ${total_expected}"

    local total_created=0
    local total_exists=0
    local total_errors=0

    for domain in "${DOMAINS[@]}"; do
        local token zone_id
        token=$(load_token "$domain")
        zone_id=$(get_zone_id "$domain")

        log_info "Processing zone: bahn.${domain}"

        # Aktuelle Records fuer unsere IPs abrufen
        local existing_a existing_aaaa
        existing_a=$(get_records_for_ip "$zone_id" "$token" "A" "$MY_IPV4")
        existing_aaaa=$(get_records_for_ip "$zone_id" "$token" "AAAA" "$MY_IPV6")

        # Set von existierenden Hostnames bauen
        local existing_a_names existing_aaaa_names
        existing_a_names=$(echo "$existing_a" | jq -r '.[].name' 2>/dev/null | sort -u)
        existing_aaaa_names=$(echo "$existing_aaaa" | jq -r '.[].name' 2>/dev/null | sort -u)

        # Sammle alle noch fehlenden Records fuer diese Zone in einem JSON-Array,
        # um sie mit einem einzigen Batch-POST anzulegen. TTL pro Zone aus ZONE_TTL.
        local zone_ttl="${ZONE_TTL[$domain]:-$DNS_TTL_FALLBACK}"
        local posts_array="[]"
        local comment_str="managed by dns-maintenance on ${SERVER_NAME} ($(date +'%Y-%m-%d %H:%M'))"

        for hostname in "${expected_hostnames[@]}"; do
            # Nur Hostnames dieser Domain verarbeiten
            if [[ "$hostname" != *"bahn.${domain}" ]]; then
                continue
            fi

            if echo "$existing_a_names" | grep -qx "$hostname"; then
                total_exists=$((total_exists + 1))
            else
                posts_array=$(echo "$posts_array" | jq \
                    --arg name "$hostname" \
                    --arg content "$MY_IPV4" \
                    --argjson ttl "$zone_ttl" \
                    --arg comment "$comment_str" \
                    '. + [{type: "A", name: $name, content: $content, ttl: $ttl, proxied: false, comment: $comment}]')
            fi

            if echo "$existing_aaaa_names" | grep -qx "$hostname"; then
                total_exists=$((total_exists + 1))
            else
                posts_array=$(echo "$posts_array" | jq \
                    --arg name "$hostname" \
                    --arg content "$MY_IPV6" \
                    --argjson ttl "$zone_ttl" \
                    --arg comment "$comment_str" \
                    '. + [{type: "AAAA", name: $name, content: $content, ttl: $ttl, proxied: false, comment: $comment}]')
            fi
        done

        local posts_count
        posts_count=$(echo "$posts_array" | jq 'length')

        if [[ "$posts_count" -eq 0 ]]; then
            log_info "All expected records already exist in bahn.${domain}"
            continue
        fi

        local batch_body batch_result batch_success
        batch_body=$(jq -n --argjson p "$posts_array" '{posts: $p}')
        batch_result=$(batch_dns_operations_with_retry "$zone_id" "$token" "$batch_body")
        batch_success=$(echo "$batch_result" | jq -r '.success')

        if [[ "$batch_success" == "true" ]]; then
            log_info "Batch-created ${posts_count} records in bahn.${domain}"
            total_created=$((total_created + posts_count))
        else
            local errors_msg
            errors_msg=$(echo "$batch_result" | jq -c '.errors // []')
            log_error "Batch create failed for bahn.${domain}: ${errors_msg}"
            total_errors=$((total_errors + posts_count))
        fi
    done

    log_info "Restore results: ${total_created} created, ${total_exists} already existed, ${total_errors} errors"

    # Verifikation per dig
    local verify_ok=0
    local verify_fail=0
    local sample_hostnames=("cs.prd.bahn.business" "cs.prd.bahn.services" "cert.prd.bahn.services" "ccc.dev.bahn.business")

    for hostname in "${sample_hostnames[@]}"; do
        local resolved
        resolved=$(dig +short "$hostname" 2>/dev/null)
        if echo "$resolved" | grep -q "$MY_IPV4"; then
            verify_ok=$((verify_ok + 1))
        else
            verify_fail=$((verify_fail + 1))
            log_warning "Verification: ${hostname} does not resolve to ${MY_IPV4} yet"
        fi
    done

    # State-File loeschen wenn alles OK
    if [[ "$total_errors" -eq 0 && "$verify_fail" -eq 0 ]]; then
        rm -f "$STATE_FILE"
        log_info "=== RESTORE complete for ${SERVER_NAME}. ${total_created} created, ${total_exists} existed. ==="
        [[ $quiet_mode -eq 0 ]] && send_alert "info" "RESTORE abgeschlossen: ${total_created} Records erstellt, ${total_exists} existierten bereits. Server ist wieder im DNS."
        return 0
    elif [[ "$total_errors" -gt 0 ]]; then
        log_critical "RESTORE completed with ${total_errors} errors for ${SERVER_NAME}"
        send_alert "critical" "RESTORE mit ${total_errors} FEHLERN abgeschlossen! ${total_created} erstellt, ${total_errors} fehlgeschlagen. Manueller Check noetig!"
        return 1
    else
        log_warning "RESTORE complete but ${verify_fail} dig verifications pending (DNS propagation delay)"
        rm -f "$STATE_FILE"
        send_alert "info" "RESTORE abgeschlossen: ${total_created} Records erstellt. ${verify_fail} DNS-Verifikationen ausstehend (Propagation-Delay)."
        return 0
    fi
}

do_status() {
    echo -e "${BLUE}=== DNS Status for [$SERVER_NUM] $SERVER_NAME ===${NC}"
    echo -e "IPv4: ${MY_IPV4}"
    echo -e "IPv6: ${MY_IPV6}"
    echo ""

    local total_a=0
    local total_aaaa=0

    for domain in "${DOMAINS[@]}"; do
        local token zone_id
        token=$(load_token "$domain")
        zone_id=$(get_zone_id "$domain")

        local a_records aaaa_records a_count aaaa_count
        a_records=$(get_records_for_ip "$zone_id" "$token" "A" "$MY_IPV4")
        aaaa_records=$(get_records_for_ip "$zone_id" "$token" "AAAA" "$MY_IPV6")
        a_count=$(echo "$a_records" | jq 'length')
        aaaa_count=$(echo "$aaaa_records" | jq 'length')

        echo -e "${BLUE}--- bahn.${domain}: ${a_count} A + ${aaaa_count} AAAA ---${NC}"
        echo "$a_records" | jq -r '.[] | "  A    \(.name)"' 2>/dev/null
        echo "$aaaa_records" | jq -r '.[] | "  AAAA \(.name)"' 2>/dev/null

        total_a=$((total_a + a_count))
        total_aaaa=$((total_aaaa + aaaa_count))
    done

    echo ""
    echo -e "${GREEN}Total: ${total_a} A + ${total_aaaa} AAAA = $((total_a + total_aaaa)) records${NC}"

    # State-File Status
    if [[ -f "$STATE_FILE" ]]; then
        echo ""
        echo -e "${YELLOW}WARNING: State file exists (server was drained):${NC}"
        jq -r '"  Drained: \(.drain_started) | Records: \(.total_deleted)"' "$STATE_FILE"
    fi
}

do_verify() {
    echo -e "${BLUE}=== Soll/Ist-Abgleich for [$SERVER_NUM] $SERVER_NAME ===${NC}"
    echo ""

    local expected_hostnames
    read -ra expected_hostnames <<< "$(get_expected_hostnames "$SERVER_NUM")"
    local total_expected=${#expected_hostnames[@]}

    local total_ok=0
    local total_missing=0
    local total_unexpected=0

    for domain in "${DOMAINS[@]}"; do
        local token zone_id
        token=$(load_token "$domain")
        zone_id=$(get_zone_id "$domain")

        local existing_a existing_aaaa
        existing_a=$(get_records_for_ip "$zone_id" "$token" "A" "$MY_IPV4")
        existing_aaaa=$(get_records_for_ip "$zone_id" "$token" "AAAA" "$MY_IPV6")

        local existing_a_names existing_aaaa_names
        existing_a_names=$(echo "$existing_a" | jq -r '.[].name' 2>/dev/null | sort -u)
        existing_aaaa_names=$(echo "$existing_aaaa" | jq -r '.[].name' 2>/dev/null | sort -u)

        echo -e "${BLUE}--- bahn.${domain} ---${NC}"

        for hostname in "${expected_hostnames[@]}"; do
            if [[ "$hostname" != *"bahn.${domain}" ]]; then
                continue
            fi

            local a_ok=false aaaa_ok=false

            if echo "$existing_a_names" | grep -qx "$hostname"; then
                a_ok=true
            fi
            if echo "$existing_aaaa_names" | grep -qx "$hostname"; then
                aaaa_ok=true
            fi

            if [[ "$a_ok" == "true" && "$aaaa_ok" == "true" ]]; then
                total_ok=$((total_ok + 1))
            else
                total_missing=$((total_missing + 1))
                local missing_types=""
                [[ "$a_ok" == "false" ]] && missing_types+="A "
                [[ "$aaaa_ok" == "false" ]] && missing_types+="AAAA "
                echo -e "  ${RED}MISSING${NC} ${hostname} (${missing_types})"
            fi
        done

        # Unerwartete Records pruefen
        while IFS= read -r name; do
            [[ -z "$name" ]] && continue
            local found=false
            for expected in "${expected_hostnames[@]}"; do
                if [[ "$expected" == "$name" ]]; then
                    found=true
                    break
                fi
            done
            if [[ "$found" == "false" ]]; then
                echo -e "  ${YELLOW}UNEXPECTED${NC} A ${name}"
                total_unexpected=$((total_unexpected + 1))
            fi
        done <<< "$existing_a_names"
    done

    echo ""
    echo -e "Expected hostnames: ${total_expected}"
    echo -e "OK (A+AAAA present): ${GREEN}${total_ok}${NC}"
    echo -e "Missing:             ${RED}${total_missing}${NC}"
    echo -e "Unexpected:          ${YELLOW}${total_unexpected}${NC}"

    if [[ "$total_missing" -gt 0 ]]; then
        log_warning "Verify: ${total_missing} expected records missing for ${SERVER_NAME}"
        return 1
    fi
    if [[ "$total_unexpected" -gt 0 ]]; then
        log_warning "Verify: ${total_unexpected} unexpected records found for ${SERVER_NAME}"
    fi
    echo -e "\n${GREEN}All expected records present.${NC}"
    return 0
}

# --- Alerting ---

send_alert() {
    local severity=$1  # critical, warning, info
    local message=$2
    local target_server=${3:-$SERVER_NAME}  # optional: betroffener Server (default: eigener)

    # SSH-URL fuer den betroffenen Server ermitteln
    local ssh_url=""
    for num in 1 2 3 4 5; do
        if [[ "${SERVER_NAMES[$num]:-}" == "$target_server" ]]; then
            ssh_url="${SSH_URLS[$num]:-}"
            break
        fi
    done

    local my_ssh="${SSH_URLS[$SERVER_NUM]:-}"
    local NL=$'\n'
    local detail="Server: *${target_server}*"
    [[ -n "$ssh_url" ]] && detail+=" (<${ssh_url}|SSH>)"
    detail+="${NL}Gemeldet von: *${SERVER_NAME}* (${MY_IPV4:-unknown})"
    [[ "$target_server" != "$SERVER_NAME" && -n "$my_ssh" ]] && detail+=" | <${my_ssh}|SSH>"

    logger -t "$LOG_TAG" -p "${LOG_FACILITY}.${severity}" "ALERT: $message [$target_server]"

    # Slack Webhook
    if [[ -n "$ALERT_WEBHOOK_SLACK" ]]; then
        local emoji="ℹ️"
        [[ "$severity" == "warning" ]] && emoji="⚠️"
        [[ "$severity" == "critical" ]] && emoji="🔴"
        local slack_text="${emoji} *[${severity^^}]* ${message}${NL}${detail}"
        local slack_payload
        slack_payload=$(jq -n --arg text "$slack_text" '{text: $text}')
        local slack_http_code
        slack_http_code=$(curl -s -o /dev/null -w '%{http_code}' -m 5 -X POST "$ALERT_WEBHOOK_SLACK" \
            -H "Content-Type: application/json" \
            -d "$slack_payload" 2>/dev/null) || slack_http_code="000"
        if [[ "$slack_http_code" != "200" ]]; then
            logger -t "$LOG_TAG" -p "${LOG_FACILITY}.err" \
                "WEBHOOK FAILED: Slack returned HTTP ${slack_http_code} for [${severity}] ${message}"
        fi
    else
        logger -t "$LOG_TAG" -p "${LOG_FACILITY}.warning" \
            "WEBHOOK SKIPPED: ALERT_WEBHOOK_SLACK not set — alert not delivered: [${severity}] ${message}"
    fi

    # Canari.me Webhook (optional, nur wenn konfiguriert)
    if [[ -n "$ALERT_WEBHOOK_CANARI" ]]; then
        local canari_payload
        canari_payload=$(jq -n \
            --arg severity "$severity" \
            --arg message "$message" \
            --arg server "$target_server" \
            --arg reporter "${SERVER_NAME:-unknown}" \
            --arg reporter_ip "${MY_IPV4:-unknown}" \
            --arg ssh_url "$ssh_url" \
            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            '{severity: $severity, message: $message, server: $server, reporter: $reporter, reporter_ip: $reporter_ip, ssh_url: $ssh_url, timestamp: $timestamp}')
        local canari_http_code
        canari_http_code=$(curl -s -o /dev/null -w '%{http_code}' -m 5 -X POST "$ALERT_WEBHOOK_CANARI" \
            -H "Content-Type: application/json" \
            -d "$canari_payload" 2>/dev/null) || canari_http_code="000"
        if [[ "$canari_http_code" != "200" ]]; then
            logger -t "$LOG_TAG" -p "${LOG_FACILITY}.err" \
                "WEBHOOK FAILED: Canari returned HTTP ${canari_http_code} for [${severity}] ${message}"
        fi
    fi
}

# --- Webhook Test ---

do_test_webhook() {
    echo -e "${CYAN}=== Webhook Connectivity Test ===${NC}"
    local test_msg="Webhook-Test von ${SERVER_NAME} (${MY_IPV4:-unknown}) um $(date '+%Y-%m-%d %H:%M:%S %Z')"
    local any_fail=false

    # Slack
    echo -n "Slack Webhook: "
    if [[ -z "$ALERT_WEBHOOK_SLACK" ]]; then
        echo -e "${RED}NOT SET${NC} (ALERT_WEBHOOK_SLACK leer)"
        any_fail=true
    else
        local slack_payload
        slack_payload=$(jq -n --arg text "🧪 *[TEST]* ${test_msg}" '{text: $text}')
        local slack_http
        slack_http=$(curl -s -o /dev/null -w '%{http_code}' -m 10 -X POST "$ALERT_WEBHOOK_SLACK" \
            -H "Content-Type: application/json" \
            -d "$slack_payload" 2>/dev/null) || slack_http="000"
        if [[ "$slack_http" == "200" ]]; then
            echo -e "${GREEN}OK${NC} (HTTP ${slack_http}) — Nachricht gesendet"
        else
            echo -e "${RED}FAILED${NC} (HTTP ${slack_http})"
            any_fail=true
        fi
    fi

    # Canari
    echo -n "Canari Webhook: "
    if [[ -z "$ALERT_WEBHOOK_CANARI" ]]; then
        echo -e "${YELLOW}NOT SET${NC} (optional, ALERT_WEBHOOK_CANARI leer)"
    else
        local canari_payload
        canari_payload=$(jq -n \
            --arg severity "info" \
            --arg message "$test_msg" \
            --arg server "$SERVER_NAME" \
            --arg reporter "$SERVER_NAME" \
            --arg reporter_ip "${MY_IPV4:-unknown}" \
            --arg ssh_url "${SSH_URLS[$SERVER_NUM]:-}" \
            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            '{severity: $severity, message: $message, server: $server, reporter: $reporter, reporter_ip: $reporter_ip, ssh_url: $ssh_url, timestamp: $timestamp}')
        local canari_http
        canari_http=$(curl -s -o /dev/null -w '%{http_code}' -m 10 -X POST "$ALERT_WEBHOOK_CANARI" \
            -H "Content-Type: application/json" \
            -d "$canari_payload" 2>/dev/null) || canari_http="000"
        if [[ "$canari_http" == "200" ]]; then
            echo -e "${GREEN}OK${NC} (HTTP ${canari_http}) — Nachricht gesendet"
        else
            echo -e "${RED}FAILED${NC} (HTTP ${canari_http})"
            any_fail=true
        fi
    fi

    echo ""
    if [[ "$any_fail" == "true" ]]; then
        echo -e "${RED}Mindestens ein Webhook fehlgeschlagen — vor drain-and-reboot beheben!${NC}"
        return 1
    else
        echo -e "${GREEN}Alle konfigurierten Webhooks erreichbar.${NC}"
        return 0
    fi
}

# --- Peer Health Check ---

do_probe() {
    local target_ip=$1
    local probe_timeout=${2:-5}

    curl -sk -o /dev/null \
        --connect-timeout 3 \
        --max-time "$probe_timeout" \
        "https://${target_ip}/" \
        -H "Host: cs.prd.bahn.business" \
        >/dev/null 2>&1
    return $?
}

do_peer_drain_remote() {
    local peer_num=$1
    local peer_ipv4="${SERVERS_V4[$peer_num]}"
    local peer_ipv6="${SERVERS_V6[$peer_num]}"
    local peer_name="${SERVER_NAMES[$peer_num]}"

    log_info "=== PEER-DRAIN started for ${peer_name} (${peer_ipv4}) by ${SERVER_NAME} ==="

    local total_deleted=0
    local total_errors=0

    for domain in "${DOMAINS[@]}"; do
        local token zone_id
        token=$(load_token "$domain")
        zone_id=$(get_zone_id "$domain")

        local a_records aaaa_records zone_records zone_count
        a_records=$(get_records_for_ip "$zone_id" "$token" "A" "$peer_ipv4")
        aaaa_records=$(get_records_for_ip "$zone_id" "$token" "AAAA" "$peer_ipv6")
        zone_records=$(echo "$a_records" "$aaaa_records" | jq -s '.[0] + .[1]')
        zone_count=$(echo "$zone_records" | jq 'length')

        if [[ "$zone_count" -eq 0 ]]; then
            log_info "No records for ${peer_name} in bahn.${domain} (already drained?)"
            continue
        fi

        local batch_body batch_result batch_success
        batch_body=$(echo "$zone_records" | jq '{deletes: [.[] | {id: .id}]}')
        batch_result=$(batch_dns_operations_with_retry "$zone_id" "$token" "$batch_body")
        batch_success=$(echo "$batch_result" | jq -r '.success')

        if [[ "$batch_success" == "true" ]]; then
            log_info "Peer-drain: deleted ${zone_count} records for ${peer_name} in bahn.${domain}"
            total_deleted=$((total_deleted + zone_count))
        else
            log_error "Peer-drain: batch delete failed for ${peer_name} in bahn.${domain}"
            total_errors=$((total_errors + zone_count))
        fi
    done

    log_info "=== PEER-DRAIN complete for ${peer_name}: ${total_deleted} deleted, ${total_errors} errors ==="

    if [[ "$total_errors" -gt 0 ]]; then
        return 1
    fi
    return 0
}

do_peer_check() {
    if [[ -f "$PEER_KILL_SWITCH" ]]; then
        log_debug "Peer-check disabled via kill switch ($PEER_KILL_SWITCH)"
        return 0
    fi

    mkdir -p "$PEER_FAILCOUNT_DIR"

    # Selbst-Check: ist mein eigener Apache gesund?
    if ! do_probe "$MY_IPV4" 3; then
        log_warning "SELF-CHECK: own Apache not responding on ${MY_IPV4}:443"
        if systemctl is-active --quiet apache2; then
            log_warning "SELF-CHECK: Apache unit active but not responding — restarting"
            systemctl restart apache2
            send_alert "warning" "Self-check: Apache restarted on ${SERVER_NAME} (unit active but port 443 unresponsive)"
        else
            log_critical "SELF-CHECK: Apache unit is DEAD — restarting"
            systemctl start apache2
            send_alert "critical" "Self-check: Apache was dead on ${SERVER_NAME}, started"
        fi
        sleep 5
        if ! do_probe "$MY_IPV4" 3; then
            send_alert "critical" "Self-check: Apache still not responding on ${SERVER_NAME} after restart"
        fi
    fi

    # Peer-Check: jeden Cluster-Peer pruefen
    for peer_num in 1 2 3 4 5; do
        [[ "$peer_num" -eq "$SERVER_NUM" ]] && continue

        local peer_ext_ip="${SERVERS_V4[$peer_num]}"
        local peer_name="${SERVER_NAMES[$peer_num]}"
        local failcount_file="${PEER_FAILCOUNT_DIR}/${peer_ext_ip}.failcount"
        local cooldown_file="${PEER_FAILCOUNT_DIR}/${peer_ext_ip}.cooldown"
        local drained_file="${PEER_FAILCOUNT_DIR}/${peer_ext_ip}.drained"

        # Cooldown pruefen
        if [[ -f "$cooldown_file" ]]; then
            local cooldown_age=$(( $(date +%s) - $(stat -c %Y "$cooldown_file") ))
            if [[ "$cooldown_age" -lt "$PEER_DRAIN_COOLDOWN" ]]; then
                log_debug "Peer ${peer_name}: in cooldown (${cooldown_age}s/${PEER_DRAIN_COOLDOWN}s)"
                continue
            fi
            rm -f "$cooldown_file"
        fi

        if do_probe "$peer_ext_ip" 5; then
            # Peer ist erreichbar — Failcount reset
            if [[ -f "$failcount_file" ]]; then
                local old_count
                old_count=$(cat "$failcount_file" 2>/dev/null || echo 0)
                if [[ "$old_count" -gt 0 ]]; then
                    log_info "Peer ${peer_name} recovered (was at failcount ${old_count})"
                fi
                echo 0 > "$failcount_file"
            fi
            # War der Peer von uns gedrained? Logge Recovery
            if [[ -f "$drained_file" ]]; then
                log_info "Peer ${peer_name} was emergency-drained, now back up (self-restore expected)"
                send_alert "info" "Peer ${peer_name} ist wieder erreichbar nach Emergency-Drain (Self-Restore erwartet)" "$peer_name"
                rm -f "$drained_file"
            fi
            continue
        fi

        # Peer nicht erreichbar — Failcount erhoehen
        local current_count=0
        [[ -f "$failcount_file" ]] && current_count=$(cat "$failcount_file" 2>/dev/null || echo 0)
        current_count=$((current_count + 1))
        echo "$current_count" > "$failcount_file"

        log_warning "Peer ${peer_name} (${peer_ext_ip}) unreachable — failcount ${current_count}/${PEER_FAIL_THRESHOLD}"

        if [[ "$current_count" -lt "$PEER_FAIL_THRESHOLD" ]]; then
            continue
        fi

        # Schwelle erreicht — SSH-Vote bei allen anderen Peers
        log_warning "Peer ${peer_name} failcount threshold reached — starting quorum vote"

        local votes=1  # eigene Stimme zaehlt
        for voter_num in 1 2 3 4 5; do
            [[ "$voter_num" -eq "$SERVER_NUM" ]] && continue
            [[ "$voter_num" -eq "$peer_num" ]] && continue

            local voter_vlan="${VLAN_IPS[$voter_num]}"
            local voter_name="${SERVER_NAMES[$voter_num]}"

            if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
                "root@${voter_vlan}" "dns-maintenance probe ${peer_ext_ip}" >/dev/null 2>&1; then
                log_info "Vote from ${voter_name}: peer ${peer_name} is UP"
            else
                log_info "Vote from ${voter_name}: peer ${peer_name} is DOWN"
                votes=$((votes + 1))
            fi
        done

        log_info "Quorum vote for ${peer_name}: ${votes}/${PEER_QUORUM} needed"

        if [[ "$votes" -lt "$PEER_QUORUM" ]]; then
            log_warning "Quorum NOT reached for ${peer_name} (${votes}/${PEER_QUORUM}) — no action"
            send_alert "warning" "Peer ${peer_name} nicht erreichbar (${peer_ext_ip}), aber Quorum nicht erreicht (${votes}/${PEER_QUORUM}). Failcount: ${current_count}/${PEER_FAIL_THRESHOLD}" "$peer_name"
            continue
        fi

        # Quorum erreicht — Drain
        send_alert "critical" "QUORUM ERREICHT: ${peer_name} (${peer_ext_ip}) von ${votes} Servern als DOWN bestaetigt — Emergency-Drain wird eingeleitet" "$peer_name"

        if [[ "$PEER_DRY_RUN" == "true" ]]; then
            log_warning "DRY-RUN: would drain ${peer_name} (${votes} votes) — skipping actual drain"
            send_alert "warning" "DRY-RUN: Peer-Drain fuer ${peer_name} uebersprungen (Dry-Run aktiv). Waere sonst gedrained worden!" "$peer_name"
            echo 0 > "$failcount_file"
            touch "$cooldown_file"
            continue
        fi

        # Duplikat-Check: hat jemand schon gedrained?
        local records_exist=false
        local check_token check_zone
        check_token=$(load_token "business")
        check_zone=$(get_zone_id "business")
        local existing
        existing=$(get_records_for_ip "$check_zone" "$check_token" "A" "$peer_ext_ip")
        local existing_count
        existing_count=$(echo "$existing" | jq 'length')
        if [[ "$existing_count" -gt 0 ]]; then
            records_exist=true
        fi

        if [[ "$records_exist" == "false" ]]; then
            log_info "Peer ${peer_name} already drained by another server — skipping"
            echo 0 > "$failcount_file"
            touch "$cooldown_file"
            touch "$drained_file"
            continue
        fi

        if do_peer_drain_remote "$peer_num"; then
            send_alert "critical" "EMERGENCY DRAIN abgeschlossen: ${peer_name} (${peer_ext_ip}) wurde aus DNS entfernt. Server muss sich beim naechsten Boot selbst restoren." "$peer_name"
            touch "$drained_file"
        else
            send_alert "critical" "EMERGENCY DRAIN FEHLGESCHLAGEN fuer ${peer_name} (${peer_ext_ip})! Cloudflare API-Fehler. Manueller Eingriff noetig!" "$peer_name"
        fi

        echo 0 > "$failcount_file"
        touch "$cooldown_file"
    done

    # Fixed-IP-Server Monitoring (Alert only, kein Auto-Drain)
    for label in "${!FIXED_IPS[@]}"; do
        local fixed_ip="${FIXED_IPS[$label]}"
        if ! do_probe "$fixed_ip" 5; then
            local fixed_fail="${PEER_FAILCOUNT_DIR}/fixed-${fixed_ip}.failcount"
            local current=0
            [[ -f "$fixed_fail" ]] && current=$(cat "$fixed_fail" 2>/dev/null || echo 0)
            current=$((current + 1))
            echo "$current" > "$fixed_fail"
            if [[ "$current" -eq "$PEER_FAIL_THRESHOLD" ]]; then
                send_alert "critical" "Fixed-IP Server ${label} (${fixed_ip}) seit 3 Minuten nicht erreichbar! Kein Auto-Drain moeglich (kein Cluster-Server). Manueller Eingriff via KVM/Cloudflare-Tunnel noetig."
            fi
        else
            echo 0 > "${PEER_FAILCOUNT_DIR}/fixed-${fixed_ip}.failcount" 2>/dev/null || true
        fi
    done
}

show_help() {
    echo "DNS Maintenance - Graceful Drain & Restore v1.8"
    echo ""
    echo "Usage: $0 <command>"
    echo ""
    echo "Commands:"
    echo "  drain           Remove this server from DNS (before shutdown)"
    echo "  restore         Add this server back to DNS (after startup)"
    echo "  status          Show current DNS records for this server"
    echo "  verify          Compare expected vs actual records (no changes)"
    echo "  probe <ip>      Health-check a server on port 443 (exit 0=up, 1=down)"
    echo "  peer-check      Check all peers, vote+drain if quorum confirms failure"
    echo ""
    echo "Examples:"
    echo "  $0 drain              # Remove from DNS, wait for drain"
    echo "  $0 drain-and-reboot   # Interactive drain + reboot (SSH-Session bleibt bis zum Reboot-Trigger)"
    echo "  $0 restore            # Restore all DNS records"
    echo "  $0 status             # Show current state"
    echo "  $0 verify             # Check for deviations"
    echo "  $0 test-webhook       # Test Slack/Canari webhook connectivity"
    echo ""
    echo "Environment:"
    echo "  CF_TOKEN_BUSINESS     Cloudflare API token for bahn.business"
    echo "  CF_TOKEN_SERVICES     Cloudflare API token for bahn.services"
    echo ""
    echo "Files:"
    echo "  ${STATE_FILE}  Drain state (created on drain, removed on restore)"
}

# --- Main ---

if [[ $# -lt 1 ]]; then
    show_help
    exit 1
fi

COMMAND="$1"

# Tokens validieren
validate_tokens

# Server erkennen
detect_server

case "$COMMAND" in
    drain)
        do_drain
        ;;
    drain-and-reboot)
        do_drain_and_reboot
        ;;
    restore)
        do_restore
        ;;
    status)
        do_status
        ;;
    verify)
        do_verify
        ;;
    probe)
        if [[ -z "${2:-}" ]]; then
            echo -e "${RED}Usage: $0 probe <ip>${NC}"
            exit 1
        fi
        do_probe "$2"
        exit $?
        ;;
    test-webhook)
        do_test_webhook
        ;;
    peer-check)
        do_peer_check
        ;;
    help|--help|-h)
        show_help
        ;;
    *)
        echo -e "${RED}Unknown command: $COMMAND${NC}"
        show_help
        exit 1
        ;;
esac