diff --git a/scripts/service_watchdog.py b/scripts/service_watchdog.py new file mode 100644 index 0000000..847cc87 --- /dev/null +++ b/scripts/service_watchdog.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +""" +Service Health Watchdog — Monitors systemd services and sends email alerts +when any service goes DOWN or comes back UP. + +Runs via systemd timer every 60 seconds. Only sends email on STATUS CHANGE. +State is tracked in /tmp/service_watchdog_state.json +""" + +import subprocess +import json +import os +import sys +import smtplib +from datetime import datetime +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + +if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') + +# ─── CONFIGURATION ─── +SERVICES = [ + {"name": "springboot", "label": "Spring Boot API", "port": "8085"}, + {"name": "chatbot", "label": "Go AI Chatbot Engine", "port": "8081"}, + {"name": "telegram-bot", "label": "Telegram Bot Service", "port": "8082"}, + {"name": "nginx", "label": "Nginx Web Server", "port": "80/443"}, + {"name": "webhook", "label": "Webhook Receiver", "port": "9000"}, + {"name": "postgresql", "label": "PostgreSQL Database", "port": "5432"}, +] + +STATE_FILE = "/tmp/service_watchdog_state.json" +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +# Email config (reuses same Gmail credentials as deploy emails) +SMTP_SERVER = os.environ.get("SMTP_SERVER", "smtp.gmail.com") +SMTP_PORT = int(os.environ.get("SMTP_PORT", 587)) +SENDER_EMAIL = os.environ.get("SENDER_EMAIL", "krishnaowoxd@gmail.com") +SENDER_PASSWORD = os.environ.get("SENDER_PASSWORD", "qkwylwtcsnlrpree").replace(" ", "") + +# Load recipients from shared recipients.txt +recipients_file = os.path.join(SCRIPT_DIR, "recipients.txt") +recipients = [] +if os.path.exists(recipients_file): + with open(recipients_file, "r", encoding="utf-8") as f: + recipients = [line.strip() for line in f if line.strip() and not line.startswith("#")] + +if not recipients: + recipients = [os.environ.get("RECIPIENT_EMAIL", "dorutoslayer@gmail.com")] + + +def is_service_active(service_name): + """Check if a systemd service is currently active (running).""" + try: + result = subprocess.run( + ["systemctl", "is-active", service_name], + capture_output=True, text=True, timeout=5 + ) + return result.stdout.strip() == "active" + except Exception: + return False + + +def load_previous_state(): + """Load the last known state of all services.""" + if os.path.exists(STATE_FILE): + try: + with open(STATE_FILE, "r") as f: + return json.load(f) + except Exception: + pass + return {} + + +def save_current_state(state): + """Save current state to disk.""" + with open(STATE_FILE, "w") as f: + json.dump(state, f, indent=2) + + +def send_alert_email(changes, current_state): + """Send an HTML email alert for service status changes.""" + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # Determine if any service went DOWN + has_down = any(c["status"] == "DOWN" for c in changes) + has_up = any(c["status"] == "UP" for c in changes) + + if has_down and has_up: + subject = f"[RIT VPS] ⚠️ Service Status Change — {now}" + banner_bg = "#fef3c7" + banner_border = "#f59e0b" + banner_text = "#92400e" + banner_icon = "⚠️" + banner_msg = "Some services changed status" + elif has_down: + subject = f"[RIT VPS] 🔴 SERVICE DOWN ALERT — {now}" + banner_bg = "#fef2f2" + banner_border = "#ef4444" + banner_text = "#b91c1c" + banner_icon = "🚨" + banner_msg = "One or more services went DOWN" + else: + subject = f"[RIT VPS] 🟢 SERVICE RECOVERED — {now}" + banner_bg = "#ecfdf5" + banner_border = "#10b981" + banner_text = "#047857" + banner_icon = "✅" + banner_msg = "Services have recovered and are back online" + + # Build change rows + change_rows = "" + for c in changes: + if c["status"] == "DOWN": + icon = "🔴" + status_label = "WENT DOWN" + row_bg = "#fef2f2" + row_color = "#b91c1c" + else: + icon = "🟢" + status_label = "RECOVERED" + row_bg = "#ecfdf5" + row_color = "#047857" + + change_rows += f""" + + {icon} {c['label']} + {c['name']}.service + :{c['port']} + {status_label} + """ + + # Build full status rows + status_rows = "" + for svc in SERVICES: + is_up = current_state.get(svc["name"], False) + s_icon = "🟢" if is_up else "🔴" + s_label = "Running" if is_up else "Down" + s_color = "#047857" if is_up else "#b91c1c" + status_rows += f""" + + {s_icon} {svc['label']} + :{svc['port']} + {s_label} + """ + + html = f""" + + + + +
+
+ 🛡️ RIT VPS Service Health Monitor +
+ +
+ {banner_icon} + {banner_msg} +
+ +
+ Detected at: {now} +
+ +

Status Changes

+ + + + + + + + + + {change_rows} +
ServiceUnitPortEvent
+ +

Full System Status

+ + + + + + + + + {status_rows} +
ServicePortStatus
+ +
+ 🌐 Check Live Site +
+ +
+ Automated health alert from RIT VPS Watchdog • Checks every 60 seconds +
+
+ + + """ + + msg = MIMEMultipart("alternative") + msg["Subject"] = subject + msg["From"] = SENDER_EMAIL + msg["To"] = ", ".join(recipients) + msg.attach(MIMEText(html, "html")) + + try: + with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as server: + server.starttls() + server.login(SENDER_EMAIL, SENDER_PASSWORD) + server.sendmail(SENDER_EMAIL, recipients, msg.as_string()) + print(f"✅ Alert email sent: {subject}") + except Exception as e: + print(f"❌ Failed to send alert: {e}") + + +def main(): + previous_state = load_previous_state() + current_state = {} + changes = [] + + for svc in SERVICES: + is_up = is_service_active(svc["name"]) + current_state[svc["name"]] = is_up + + # Compare with previous state (skip first run — no alert on initial scan) + if svc["name"] in previous_state: + was_up = previous_state[svc["name"]] + if was_up and not is_up: + changes.append({**svc, "status": "DOWN"}) + print(f"🔴 {svc['label']} ({svc['name']}) went DOWN!") + elif not was_up and is_up: + changes.append({**svc, "status": "UP"}) + print(f"🟢 {svc['label']} ({svc['name']}) recovered!") + + # Save current state for next run + save_current_state(current_state) + + # Send email only if there are status changes + if changes: + send_alert_email(changes, current_state) + else: + print(f"✓ All services stable — no changes detected ({datetime.now().strftime('%H:%M:%S')})") + + +if __name__ == "__main__": + main()