Launch-critical monitoring wiring — alerts go to tiaBaby_Bot via Telegram.
- src/lib/alert.ts: sendAlert(level, title, detail?, {fields, silent}) — HTML
formatted, IST timestamped, best-effort (never throws). Env: TELEGRAM_BOT_TOKEN,
TELEGRAM_CHAT_ID
- GET /api/healthz: public, no-auth liveness probe (200 ok / 503 down) for
Uptime Kuma + Dokploy healthcheck. No sensitive detail
- cron/backup: alert on failure (fatal), warn if dump < 1KB (empty), silent
success confirmation with file + size
- cron/monitor: error-spike rising-edge detection (last 1h > 5 and > 2x prior
hour — stateless, no re-alert on flat rate), DB/migrations/integration checks.
?test=1 sends a Telegram test ping
- cron/visitor-summary: polls Umami REST API (login -> stats/metrics/active),
posts visitor digest to Telegram. ?hours=N window (default 24)
- CLAUDE.md: new env vars + Monitoring & Alerting section
Health up/down flip detection is delegated to Uptime Kuma (pings /api/healthz);
this code covers what Kuma can't see from outside.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
103 lines
3.8 KiB
TypeScript
103 lines
3.8 KiB
TypeScript
import { NextResponse } from "next/server";
|
||
import { sql } from "@/db";
|
||
import { sendAlert } from "@/lib/alert";
|
||
|
||
/**
|
||
* Monitor cron — catches the failures Uptime Kuma can't see from the outside:
|
||
* • Error spikes (rising-edge: last hour vs the hour before)
|
||
* • DB unreachable
|
||
* • Migrations missing / integration env not configured
|
||
*
|
||
* Uptime Kuma handles up/down flip detection by pinging /api/healthz, so this
|
||
* focuses on internal signals. Recommended schedule: hourly.
|
||
*
|
||
* POST/GET /api/cron/monitor (header: x-cron-secret)
|
||
* GET /api/cron/monitor?test=1 — sends a test Telegram ping
|
||
*
|
||
* Stateless by design: error alerts use rising-edge comparison so a sustained
|
||
* (flat) error rate won't re-alert every run — only genuine new spikes do.
|
||
*/
|
||
export const dynamic = "force-dynamic";
|
||
|
||
const SPIKE_MIN = 5; // need at least this many errors in the last hour
|
||
const SPIKE_MULTIPLIER = 2; // …and > 2× the previous hour to count as a spike
|
||
|
||
function authed(request: Request): boolean {
|
||
return request.headers.get("x-cron-secret") === process.env.CRON_SECRET;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
if (!authed(request)) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||
return runMonitor(request);
|
||
}
|
||
|
||
export async function GET(request: Request) {
|
||
if (!authed(request)) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||
return runMonitor(request);
|
||
}
|
||
|
||
async function runMonitor(request: Request) {
|
||
const { searchParams } = new URL(request.url);
|
||
|
||
// Manual test ping — confirms the Telegram wiring end-to-end.
|
||
if (searchParams.get("test")) {
|
||
const ok = await sendAlert("info", "Monitor test ping", "Telegram alerting is wired correctly. 🎉");
|
||
return NextResponse.json({ ok, test: true });
|
||
}
|
||
|
||
const fired: string[] = [];
|
||
|
||
// 1. Database reachable?
|
||
try {
|
||
await sql`SELECT 1`;
|
||
} catch (e) {
|
||
await sendAlert("fatal", "Database unreachable", String(e).slice(0, 300));
|
||
return NextResponse.json({ ok: false, dbOk: false, fired: ["db_down"] });
|
||
}
|
||
|
||
// 2. Error spike — rising edge (last 1h vs the hour before it)
|
||
let recent = 0;
|
||
let prior = 0;
|
||
try {
|
||
const rows = await sql`
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE created_at > NOW() - INTERVAL '1 hour')::int AS recent,
|
||
COUNT(*) FILTER (WHERE created_at <= NOW() - INTERVAL '1 hour'
|
||
AND created_at > NOW() - INTERVAL '2 hours')::int AS prior
|
||
FROM error_events
|
||
`;
|
||
recent = Number(rows[0]?.recent) || 0;
|
||
prior = Number(rows[0]?.prior) || 0;
|
||
if (recent >= SPIKE_MIN && recent > prior * SPIKE_MULTIPLIER) {
|
||
await sendAlert("error", "Error spike detected", undefined, {
|
||
fields: { "Last hour": recent, "Previous hour": prior },
|
||
});
|
||
fired.push("error_spike");
|
||
}
|
||
} catch {
|
||
/* error_events may not exist yet — non-fatal */
|
||
}
|
||
|
||
// 3. Migrations recorded
|
||
try {
|
||
const rows = await sql`SELECT COUNT(*)::int AS count FROM drizzle.__drizzle_migrations`;
|
||
if ((Number(rows[0]?.count) || 0) === 0) {
|
||
await sendAlert("warn", "No migrations recorded", "drizzle.__drizzle_migrations is empty", { silent: true });
|
||
fired.push("no_migrations");
|
||
}
|
||
} catch {
|
||
/* ignore */
|
||
}
|
||
|
||
// 4. Integration env presence
|
||
const missing: string[] = [];
|
||
if (!(process.env.LITELLM_BASE_URL && process.env.LITELLM_API_KEY)) missing.push("AI Gateway");
|
||
if (!(process.env.R2_ACCOUNT_ID && process.env.R2_ACCESS_KEY_ID && process.env.R2_BUCKET_NAME)) missing.push("R2 Storage");
|
||
if (!process.env.RESEND_API_KEY) missing.push("Email");
|
||
if (missing.length) {
|
||
await sendAlert("warn", "Integration config missing", missing.join(", "), { silent: true });
|
||
fired.push("config_missing");
|
||
}
|
||
|
||
return NextResponse.json({ ok: true, dbOk: true, errors: { recent, prior }, fired });
|
||
}
|