diff --git a/messages/en-US.json b/messages/en-US.json index 13b47d135..c566d500e 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1432,6 +1432,7 @@ "alertingTriggerHcToggle": "Health check status changes", "alertingTriggerResourceHealthy": "Resource healthy", "alertingTriggerResourceUnhealthy": "Resource unhealthy", + "alertingTriggerResourceDegraded": "Resource degraded", "alertingSearchHealthChecks": "Search health checks…", "alertingHealthChecksEmpty": "No health checks available.", "alertingTriggerResourceToggle": "Resource status changes", diff --git a/server/db/pg/schema/privateSchema.ts b/server/db/pg/schema/privateSchema.ts index d930b69d0..1aa2a1ef7 100644 --- a/server/db/pg/schema/privateSchema.ts +++ b/server/db/pg/schema/privateSchema.ts @@ -484,6 +484,7 @@ export const alertRules = pgTable("alertRules", { | "health_check_toggle" | "resource_healthy" | "resource_unhealthy" + | "resource_degraded" | "resource_toggle" >() .notNull(), diff --git a/server/db/sqlite/schema/privateSchema.ts b/server/db/sqlite/schema/privateSchema.ts index 9a33e2049..25a7b5bf5 100644 --- a/server/db/sqlite/schema/privateSchema.ts +++ b/server/db/sqlite/schema/privateSchema.ts @@ -425,10 +425,18 @@ export const eventStreamingDestinations = sqliteTable( orgId: text("orgId") .notNull() .references(() => orgs.orgId, { onDelete: "cascade" }), - sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" }).notNull().default(false), - sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" }).notNull().default(false), - sendActionLogs: integer("sendActionLogs", { mode: "boolean" }).notNull().default(false), - sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" }).notNull().default(false), + sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" }) + .notNull() + .default(false), + sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" }) + .notNull() + .default(false), + sendActionLogs: integer("sendActionLogs", { mode: "boolean" }) + .notNull() + .default(false), + sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" }) + .notNull() + .default(false), type: text("type").notNull(), // e.g. "http", "kafka", etc. config: text("config").notNull(), // JSON string with the configuration for the destination enabled: integer("enabled", { mode: "boolean" }) @@ -476,14 +484,19 @@ export const alertRules = sqliteTable("alertRules", { | "health_check_toggle" | "resource_healthy" | "resource_unhealthy" + | "resource_degraded" | "resource_toggle" >() .notNull(), enabled: integer("enabled", { mode: "boolean" }).notNull().default(true), cooldownSeconds: integer("cooldownSeconds").notNull().default(300), allSites: integer("allSites", { mode: "boolean" }).notNull().default(false), - allHealthChecks: integer("allHealthChecks", { mode: "boolean" }).notNull().default(false), - allResources: integer("allResources", { mode: "boolean" }).notNull().default(false), + allHealthChecks: integer("allHealthChecks", { mode: "boolean" }) + .notNull() + .default(false), + allResources: integer("allResources", { mode: "boolean" }) + .notNull() + .default(false), lastTriggeredAt: integer("lastTriggeredAt"), createdAt: integer("createdAt").notNull(), updatedAt: integer("updatedAt").notNull() @@ -531,19 +544,27 @@ export const alertEmailRecipients = sqliteTable("alertEmailRecipients", { recipientId: integer("recipientId").primaryKey({ autoIncrement: true }), emailActionId: integer("emailActionId") .notNull() - .references(() => alertEmailActions.emailActionId, { onDelete: "cascade" }), - userId: text("userId").references(() => users.userId, { onDelete: "cascade" }), - roleId: integer("roleId").references(() => roles.roleId, { onDelete: "cascade" }), + .references(() => alertEmailActions.emailActionId, { + onDelete: "cascade" + }), + userId: text("userId").references(() => users.userId, { + onDelete: "cascade" + }), + roleId: integer("roleId").references(() => roles.roleId, { + onDelete: "cascade" + }), email: text("email") }); export const alertWebhookActions = sqliteTable("alertWebhookActions", { - webhookActionId: integer("webhookActionId").primaryKey({ autoIncrement: true }), + webhookActionId: integer("webhookActionId").primaryKey({ + autoIncrement: true + }), alertRuleId: integer("alertRuleId") .notNull() .references(() => alertRules.alertRuleId, { onDelete: "cascade" }), webhookUrl: text("webhookUrl").notNull(), - config: text("config"), // encrypted JSON with auth config (authType, credentials) + config: text("config"), // encrypted JSON with auth config (authType, credentials) enabled: integer("enabled", { mode: "boolean" }).notNull().default(true), lastSentAt: integer("lastSentAt") }); diff --git a/server/emails/templates/AlertNotification.tsx b/server/emails/templates/AlertNotification.tsx index 5542384a9..899c4a82f 100644 --- a/server/emails/templates/AlertNotification.tsx +++ b/server/emails/templates/AlertNotification.tsx @@ -23,6 +23,7 @@ export type AlertEventType = | "health_check_toggle" | "resource_healthy" | "resource_unhealthy" + | "resource_degraded" | "resource_toggle"; export type AlertNotificationProps = { @@ -114,6 +115,15 @@ function getEventMeta(eventType: AlertEventType): { statusLabel: "Unhealthy", statusColor: "#dc2626" }; + case "resource_degraded": + return { + heading: "Resource Unhealthy", + previewText: "A resource in your organization is not healthy.", + summary: + "A resource in your organization is currently unhealthy.", + statusLabel: "Unhealthy", + statusColor: "#dc2626" + }; case "resource_toggle": return { heading: "Resource Status Changed", @@ -135,7 +145,10 @@ function getEventMeta(eventType: AlertEventType): { } } -function resolveToggleStatus(status: unknown): { label: string; color: string } { +function resolveToggleStatus(status: unknown): { + label: string; + color: string; +} { switch (String(status).toLowerCase()) { case "online": return { label: "Online", color: "#16a34a" }; diff --git a/server/private/lib/alerts/events/healthCheckEvents.ts b/server/private/lib/alerts/events/healthCheckEvents.ts index 9b5c3104b..9f675f4c0 100644 --- a/server/private/lib/alerts/events/healthCheckEvents.ts +++ b/server/private/lib/alerts/events/healthCheckEvents.ts @@ -23,6 +23,7 @@ import { } from "@server/db"; import { eq } from "drizzle-orm"; import { + fireResourceDegradedAlert, fireResourceHealthyAlert, fireResourceUnhealthyAlert } from "./resourceEvents"; @@ -217,6 +218,14 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null undefined, trx ); + } else if (health === "degraded") { + await fireResourceDegradedAlert( + orgId, + resource.resourceId, + resource.name, + undefined, + trx + ); } } } diff --git a/server/private/lib/alerts/events/resourceEvents.ts b/server/private/lib/alerts/events/resourceEvents.ts index 8c20bc5a1..41c77dc11 100644 --- a/server/private/lib/alerts/events/resourceEvents.ts +++ b/server/private/lib/alerts/events/resourceEvents.ts @@ -130,9 +130,9 @@ export async function fireResourceUnhealthyAlert( } /** - * Fire a `resource_toggle` alert for the given resource. + * Fire a `resource_degraded` alert for the given resource. * - * Call this when a resource's enabled/disabled status is toggled so that any + * Call this after a resource has been detected as degraded so that any * matching `alertRules` can dispatch their email and webhook actions. * * @param orgId - Organisation that owns the resource. @@ -140,7 +140,7 @@ export async function fireResourceUnhealthyAlert( * @param resourceName - Human-readable name shown in notifications (optional). * @param extra - Any additional key/value pairs to include in the payload. */ -export async function fireResourceToggleAlert( +export async function fireResourceDegradedAlert( orgId: string, resourceId: number, resourceName?: string | null, @@ -148,8 +148,16 @@ export async function fireResourceToggleAlert( trx: Transaction | typeof db = db ): Promise { try { + await trx.insert(statusHistory).values({ + entityType: "resource", + entityId: resourceId, + orgId: orgId, + status: "degraded", + timestamp: Math.floor(Date.now() / 1000) + }); + await processAlerts({ - eventType: "resource_toggle", + eventType: "resource_degraded", orgId, resourceId, data: { @@ -157,9 +165,20 @@ export async function fireResourceToggleAlert( ...extra } }); + await processAlerts({ + eventType: "resource_toggle", + orgId, + resourceId, + data: { + resourceId, + status: "degraded", + ...(resourceName != null ? { resourceName } : {}), + ...extra + } + }); } catch (err) { logger.error( - `fireResourceToggleAlert: unexpected error for resourceId ${resourceId}`, + `fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`, err ); } diff --git a/server/private/lib/alerts/sendAlertEmail.ts b/server/private/lib/alerts/sendAlertEmail.ts index 598262e38..6f99b102c 100644 --- a/server/private/lib/alerts/sendAlertEmail.ts +++ b/server/private/lib/alerts/sendAlertEmail.ts @@ -88,6 +88,8 @@ function buildSubject(context: AlertContext): string { return "[Alert] Resource Healthy"; case "resource_unhealthy": return "[Alert] Resource Unhealthy"; + case "resource_degraded": + return "[Alert] Resource Degraded"; case "resource_toggle": return "[Alert] Resource Status Changed"; default: { diff --git a/server/private/lib/alerts/sendAlertWebhook.ts b/server/private/lib/alerts/sendAlertWebhook.ts index 2dd0eb600..3975eb09f 100644 --- a/server/private/lib/alerts/sendAlertWebhook.ts +++ b/server/private/lib/alerts/sendAlertWebhook.ts @@ -12,7 +12,10 @@ */ import logger from "@server/logger"; -import { AlertContext, WebhookAlertConfig } from "@server/routers/alertRule/types"; +import { + AlertContext, + WebhookAlertConfig +} from "@server/routers/alertRule/types"; const REQUEST_TIMEOUT_MS = 15_000; const MAX_RETRIES = 3; @@ -56,7 +59,10 @@ export async function sendAlertWebhook( for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { const controller = new AbortController(); - const timeoutHandle = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + const timeoutHandle = setTimeout( + () => controller.abort(), + REQUEST_TIMEOUT_MS + ); let response: Response; try { @@ -75,7 +81,9 @@ export async function sendAlertWebhook( ); } else { const msg = err instanceof Error ? err.message : String(err); - lastError = new Error(`Alert webhook: request to "${url}" failed – ${msg}`); + lastError = new Error( + `Alert webhook: request to "${url}" failed – ${msg}` + ); } if (attempt < MAX_RETRIES) { const delay = RETRY_BASE_DELAY_MS * 2 ** (attempt - 1); @@ -111,11 +119,18 @@ export async function sendAlertWebhook( continue; } - logger.debug(`Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`); + logger.debug( + `Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})` + ); return; } - throw lastError ?? new Error(`Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`); + throw ( + lastError ?? + new Error( + `Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"` + ) + ); } // --------------------------------------------------------------------------- @@ -139,6 +154,8 @@ function deriveStatus( case "health_check_unhealthy": case "resource_unhealthy": return "unhealthy"; + case "resource_degraded": + return "degraded"; case "health_check_toggle": case "resource_toggle": return String(data.status ?? "unknown"); @@ -154,7 +171,9 @@ function deriveStatus( // Header construction (mirrors HttpLogDestination.buildHeaders) // --------------------------------------------------------------------------- -function buildHeaders(webhookConfig: WebhookAlertConfig): Record { +function buildHeaders( + webhookConfig: WebhookAlertConfig +): Record { const headers: Record = { "Content-Type": "application/json" }; diff --git a/server/private/routers/alertEvents/triggerResourceAlert.ts b/server/private/routers/alertEvents/triggerResourceAlert.ts index 42e63b288..a43b8e201 100644 --- a/server/private/routers/alertEvents/triggerResourceAlert.ts +++ b/server/private/routers/alertEvents/triggerResourceAlert.ts @@ -24,7 +24,8 @@ import { eq, and } from "drizzle-orm"; import { fireResourceHealthyAlert, fireResourceUnhealthyAlert, - fireResourceToggleAlert + fireResourceToggleAlert, + fireResourceDegradedAlert } from "#private/lib/alerts/events/resourceEvents"; const paramsSchema = z.strictObject({ @@ -33,7 +34,12 @@ const paramsSchema = z.strictObject({ }); const bodySchema = z.strictObject({ - eventType: z.enum(["resource_healthy", "resource_unhealthy", "resource_toggle"]) + eventType: z.enum([ + "resource_healthy", + "resource_unhealthy", + "resource_degraded", + "resource_toggle" + ]) }); export type TriggerResourceAlertResponse = { @@ -101,8 +107,8 @@ export async function triggerResourceAlert( resourceId, resource.name ?? undefined ); - } else { - await fireResourceToggleAlert( + } else if (eventType === "resource_degraded") { + await fireResourceDegradedAlert( orgId, resourceId, resource.name ?? undefined diff --git a/server/private/routers/alertRule/createAlertRule.ts b/server/private/routers/alertRule/createAlertRule.ts index ceaacf73c..f9b84ebab 100644 --- a/server/private/routers/alertRule/createAlertRule.ts +++ b/server/private/routers/alertRule/createAlertRule.ts @@ -33,7 +33,11 @@ import { encrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; import { CreateAlertRuleResponse } from "@server/routers/alertRule/types"; -export const SITE_EVENT_TYPES = ["site_online", "site_offline", "site_toggle"] as const; +export const SITE_EVENT_TYPES = [ + "site_online", + "site_offline", + "site_toggle" +] as const; export const HC_EVENT_TYPES = [ "health_check_healthy", "health_check_unhealthy", @@ -42,6 +46,7 @@ export const HC_EVENT_TYPES = [ export const RESOURCE_EVENT_TYPES = [ "resource_healthy", "resource_unhealthy", + "resource_degraded", "resource_toggle" ] as const; @@ -92,19 +97,24 @@ const bodySchema = z const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes( val.eventType ); - const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes( - val.eventType - ); + const isResourceEvent = ( + RESOURCE_EVENT_TYPES as readonly string[] + ).includes(val.eventType); if (isSiteEvent && !val.allSites && val.siteIds.length === 0) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "At least one siteId is required for site event types when allSites is false", + message: + "At least one siteId is required for site event types when allSites is false", path: ["siteIds"] }); } - if (isHcEvent && !val.allHealthChecks && val.healthCheckIds.length === 0) { + if ( + isHcEvent && + !val.allHealthChecks && + val.healthCheckIds.length === 0 + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: @@ -129,10 +139,15 @@ const bodySchema = z }); } - if (isResourceEvent && !val.allResources && val.resourceIds.length === 0) { + if ( + isResourceEvent && + !val.allResources && + val.resourceIds.length === 0 + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "At least one resourceId is required for resource event types when allResources is false", + message: + "At least one resourceId is required for resource event types when allResources is false", path: ["resourceIds"] }); } @@ -148,7 +163,8 @@ const bodySchema = z if (isResourceEvent && val.healthCheckIds.length > 0) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "healthCheckIds must not be set for resource event types", + message: + "healthCheckIds must not be set for resource event types", path: ["healthCheckIds"] }); } @@ -164,7 +180,8 @@ const bodySchema = z if (isHcEvent && val.resourceIds.length > 0) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "resourceIds must not be set for health check event types", + message: + "resourceIds must not be set for health check event types", path: ["resourceIds"] }); } @@ -284,9 +301,7 @@ export async function createAlertRule( // Create the email action pivot row and recipients if any recipients // were supplied (userIds, roleIds, or raw emails). const hasRecipients = - userIds.length > 0 || - roleIds.length > 0 || - emails.length > 0; + userIds.length > 0 || roleIds.length > 0 || emails.length > 0; if (hasRecipients) { const [emailActionRow] = await db diff --git a/server/private/routers/alertRule/listAlertRules.ts b/server/private/routers/alertRule/listAlertRules.ts index a31a4d119..9684b88a4 100644 --- a/server/private/routers/alertRule/listAlertRules.ts +++ b/server/private/routers/alertRule/listAlertRules.ts @@ -76,6 +76,7 @@ const SITE_ALERT_EVENT_TYPES = [ const RESOURCE_ALERT_EVENT_TYPES = [ "resource_healthy", "resource_unhealthy", + "resource_degraded", "resource_toggle" ] as const; diff --git a/server/routers/alertRule/types.ts b/server/routers/alertRule/types.ts index a9e66350e..e3f92591d 100644 --- a/server/routers/alertRule/types.ts +++ b/server/routers/alertRule/types.ts @@ -37,6 +37,7 @@ export type GetAlertRuleResponse = { | "health_check_toggle" | "resource_healthy" | "resource_unhealthy" + | "resource_degraded" | "resource_toggle"; enabled: boolean; cooldownSeconds: number; @@ -94,6 +95,7 @@ export type AlertEventType = | "health_check_toggle" | "resource_healthy" | "resource_unhealthy" + | "resource_degraded" | "resource_toggle"; // --------------------------------------------------------------------------- diff --git a/src/components/AlertingRulesTable.tsx b/src/components/AlertingRulesTable.tsx index 52ff3b609..f8fcf468d 100644 --- a/src/components/AlertingRulesTable.tsx +++ b/src/components/AlertingRulesTable.tsx @@ -118,6 +118,8 @@ function triggerLabel(rule: AlertRuleRow, t: (k: string) => string) { return t("alertingTriggerResourceHealthy"); case "resource_unhealthy": return t("alertingTriggerResourceUnhealthy"); + case "resource_degraded": + return t("alertingTriggerResourceDegraded"); case "resource_toggle": return t("alertingTriggerResourceToggle"); default: diff --git a/src/components/ProxyResourcesTable.tsx b/src/components/ProxyResourcesTable.tsx index 324f29552..2b56eb98d 100644 --- a/src/components/ProxyResourcesTable.tsx +++ b/src/components/ProxyResourcesTable.tsx @@ -32,7 +32,6 @@ import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { UpdateResourceResponse } from "@server/routers/resource"; import type { PaginationState } from "@tanstack/react-table"; -import { useQuery } from "@tanstack/react-query"; import { AxiosResponse } from "axios"; import { ArrowDown01Icon, diff --git a/src/components/alert-rule-editor/AlertRuleFields.tsx b/src/components/alert-rule-editor/AlertRuleFields.tsx index 1d420f433..887fbaa5a 100644 --- a/src/components/alert-rule-editor/AlertRuleFields.tsx +++ b/src/components/alert-rule-editor/AlertRuleFields.tsx @@ -1147,6 +1147,7 @@ export function AlertRuleSourceFields({ if ( curTrigger !== "resource_healthy" && curTrigger !== "resource_unhealthy" && + curTrigger !== "resource_degraded" && curTrigger !== "resource_toggle" ) { setValue("trigger", "resource_toggle", { @@ -1367,6 +1368,9 @@ export function AlertRuleTriggerFields({ {t("alertingTriggerResourceUnhealthy")} + + {t("alertingTriggerResourceDegraded")} + ) : ( <> diff --git a/src/lib/alertRuleForm.ts b/src/lib/alertRuleForm.ts index 111487c48..039c367b6 100644 --- a/src/lib/alertRuleForm.ts +++ b/src/lib/alertRuleForm.ts @@ -25,6 +25,7 @@ export type AlertTrigger = | "health_check_toggle" | "resource_healthy" | "resource_unhealthy" + | "resource_degraded" | "resource_toggle"; export type AlertRuleFormAction = @@ -77,6 +78,7 @@ export type AlertRuleApiPayload = { | "health_check_toggle" | "resource_healthy" | "resource_unhealthy" + | "resource_degraded" | "resource_toggle"; enabled: boolean; allSites: boolean; @@ -160,6 +162,7 @@ export function buildFormSchema(t: (k: string) => string) { "health_check_toggle", "resource_healthy", "resource_unhealthy", + "resource_degraded", "resource_toggle" ]), actions: z.array( @@ -243,6 +246,7 @@ export function buildFormSchema(t: (k: string) => string) { const resourceTriggers: AlertTrigger[] = [ "resource_healthy", "resource_unhealthy", + "resource_degraded", "resource_toggle" ]; if ( @@ -344,7 +348,9 @@ export function alertRuleAllResourcesSelected( eventType: string, resourceIds: number[] | undefined ): boolean { - return eventType.startsWith("resource_") && (resourceIds?.length ?? 0) === 0; + return ( + eventType.startsWith("resource_") && (resourceIds?.length ?? 0) === 0 + ); } export function alertRuleAllHealthChecksSelected(