calvin.erfmann/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-06-18 21:31:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support unknown and degraded status

Browse Source
This commit is contained in:
Owen
2026-04-25 15:29:59 -07:00
parent 82212af643
commit 7c7d1f641e
14 changed files with 766 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
server/lib/alerts/events/healthCheckEvents.ts
View File

@@ -20,4 +20,15 @@ export async function fireHealthCheckUnhealthyAlert(
trx?: unknown
): Promise<void> {
return;
}
export async function fireHealthCheckUnknownAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
trx?: unknown
): Promise<void> {
return;
}

27
server/lib/blueprints/proxyResources.ts
View File

@@ -34,6 +34,7 @@ import { hashPassword } from "@server/auth/password";
import { isValidCIDR, isValidIP, isValidUrlGlobPattern } from "../validators";
import { isValidRegionId } from "@server/db/regions";
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
import { fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
import { tierMatrix } from "../billing/tierMatrix";
export type ProxyResourcesResults = {
@@ -169,6 +170,18 @@ export async function updateProxyResources(
.returning();
healthchecksToUpdate.push(newHealthcheck);
// Insert unknown status history when HC is created in disabled state
if (!healthcheckData?.enabled) {
await fireHealthCheckUnknownAlert(
orgId,
newHealthcheck.targetHealthCheckId,
newHealthcheck.name,
newHealthcheck.targetId,
undefined,
trx
);
}
}
// Find existing resource by niceId and orgId
@@ -557,6 +570,20 @@ export async function updateProxyResources(
targetsToUpdate.push(updatedTarget);
}
}
// Insert unknown status history when HC is disabled
const isDisablingHc =
!healthcheckData?.enabled && oldHealthcheck?.hcEnabled;
if (isDisablingHc) {
await fireHealthCheckUnknownAlert(
orgId,
newHealthcheck.targetHealthCheckId,
newHealthcheck.name,
newHealthcheck.targetId,
undefined,
trx
);
}
} else {
await createTarget(existingResource.resourceId, targetData);
}

51
server/lib/statusHistory.ts
View File

@@ -18,7 +18,7 @@ export interface StatusHistoryDayBucket {
uptimePercent: number; // 0-100
totalDowntimeSeconds: number;
downtimeWindows: { start: number; end: number | null; status: string }[];
status: "good" | "degraded" | "bad" | "no_data";
status: "good" | "degraded" | "bad" | "no_data" | "unknown";
}
export interface StatusHistoryResponse {
@@ -54,6 +54,7 @@ export function computeBuckets(
const windows: { start: number; end: number | null; status: string }[] = [];
let dayDowntime = 0;
let dayDegradedTime = 0;
let windowStart = dayStartSec;
let windowStatus = currentStatus;
@@ -63,8 +64,8 @@ export function computeBuckets(
const windowEnd = evt.timestamp;
const isDown =
windowStatus === "offline" ||
windowStatus === "unhealthy" ||
windowStatus === "unknown";
windowStatus === "unhealthy";
const isDegraded = windowStatus === "degraded";
if (isDown) {
dayDowntime += windowEnd - windowStart;
windows.push({
@@ -72,6 +73,13 @@ export function computeBuckets(
end: windowEnd,
status: windowStatus,
});
} else if (isDegraded) {
dayDegradedTime += windowEnd - windowStart;
windows.push({
start: windowStart,
end: windowEnd,
status: windowStatus,
});
}
}
windowStart = evt.timestamp;
@@ -83,8 +91,8 @@ export function computeBuckets(
const finalEnd = Math.min(dayEndSec, nowSec);
const isDown =
windowStatus === "offline" ||
windowStatus === "unhealthy" ||
windowStatus === "unknown";
windowStatus === "unhealthy";
const isDegraded = windowStatus === "degraded";
if (isDown && finalEnd > windowStart) {
dayDowntime += finalEnd - windowStart;
windows.push({
@@ -92,6 +100,13 @@ export function computeBuckets(
end: finalEnd,
status: windowStatus,
});
} else if (isDegraded && finalEnd > windowStart) {
dayDegradedTime += finalEnd - windowStart;
windows.push({
start: windowStart,
end: finalEnd,
status: windowStatus,
});
}
}
@@ -105,7 +120,7 @@ export function computeBuckets(
effectiveDayLength > 0
? Math.max(
0,
((effectiveDayLength - dayDowntime) /
((effectiveDayLength - dayDowntime - dayDegradedTime) /
effectiveDayLength) *
100
)
@@ -113,11 +128,27 @@ export function computeBuckets(
const dateStr = new Date(dayStartSec * 1000).toISOString().slice(0, 10);
const hasAnyData = currentStatus !== null || dayEvents.length > 0;
// The whole observable window is "unknown" if every status we have seen is unknown
const allStatuses = [
...(currentStatus !== null ? [currentStatus] : []),
...dayEvents.map((e) => e.status)
];
const onlyUnknownData =
hasAnyData && allStatuses.every((s) => s === "unknown");
let status: StatusHistoryDayBucket["status"] = "no_data";
if (currentStatus !== null || dayEvents.length > 0) {
if (uptimePct >= 99) status = "good";
else if (uptimePct >= 50) status = "degraded";
else status = "bad";
if (hasAnyData) {
if (onlyUnknownData) {
status = "unknown";
} else if (dayDowntime > 0 && uptimePct < 50) {
status = "bad";
} else if (dayDowntime > 0 || dayDegradedTime > 0) {
status = "degraded";
} else {
status = "good";
}
}
buckets.push({

47
server/private/lib/alerts/events/healthCheckEvents.ts
View File

@@ -25,7 +25,8 @@ import { eq } from "drizzle-orm";
import {
fireResourceDegradedAlert,
fireResourceHealthyAlert,
fireResourceUnhealthyAlert
fireResourceUnhealthyAlert,
fireResourceUnknownAlert
} from "./resourceEvents";
// ---------------------------------------------------------------------------
@@ -148,6 +149,32 @@ export async function fireHealthCheckUnhealthyAlert(
}
}
export async function fireHealthCheckUnknownAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await trx.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await handleResource(orgId, healthCheckTargetId, trx);
} catch (err) {
logger.error(
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
async function handleResource(orgId: string, healthCheckTargetId?: number | null, trx: Transaction | typeof db = db) {
if (!healthCheckTargetId) {
return;
@@ -178,10 +205,16 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
.where(eq(targets.resourceId, resource.resourceId));
let health = "healthy";
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
if (allHealthy) {
if (allUnknown) {
logger.debug(
`Marking resource ${resource.resourceId} as unknown because all health checks are disabled`
);
health = "unknown";
} else if (allHealthy) {
health = "healthy";
} else if (allUnhealthy) {
logger.debug(
@@ -202,7 +235,15 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
.set({ health })
.where(eq(resources.resourceId, resource.resourceId));
if (health === "unhealthy") {
if (health === "unknown") {
await fireResourceUnknownAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
trx
);
} else if (health === "unhealthy") {
await fireResourceUnhealthyAlert(
orgId,
resource.resourceId,

46
server/private/lib/alerts/events/resourceEvents.ts
View File

@@ -183,3 +183,49 @@ export async function fireResourceDegradedAlert(
);
}
}
/**
* Fire a `resource_unknown` alert for the given resource.
*
* Call this when all health checks on a resource are disabled so that the
* resource status transitions to unknown.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceUnknownAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await trx.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "unknown",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceUnknownAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}

63
server/private/lib/alerts/types.ts Normal file
View File

@@ -0,0 +1,63 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
// ---------------------------------------------------------------------------
// Alert event types
// ---------------------------------------------------------------------------
export type AlertEventType =
| "site_online"
| "site_offline"
| "health_check_healthy"
| "health_check_not_healthy";
// ---------------------------------------------------------------------------
// Webhook authentication config (stored as encrypted JSON in the DB)
// ---------------------------------------------------------------------------
export type WebhookAuthType = "none" | "bearer" | "basic" | "custom";
/**
* Stored as an encrypted JSON blob in `alertWebhookActions.config`.
*/
export interface WebhookAlertConfig {
/** Authentication strategy for the webhook endpoint */
authType: WebhookAuthType;
/** Bearer token – used when authType === "bearer" */
bearerToken?: string;
/** Basic credentials – "username:password" – used when authType === "basic" */
basicCredentials?: string;
/** Custom header name – used when authType === "custom" */
customHeaderName?: string;
/** Custom header value – used when authType === "custom" */
customHeaderValue?: string;
/** Extra headers to send with every webhook request */
headers?: Array<{ key: string; value: string }>;
/** HTTP method (default POST) */
method?: string;
}
// ---------------------------------------------------------------------------
// Internal alert event passed through the processing pipeline
// ---------------------------------------------------------------------------
export interface AlertContext {
eventType: AlertEventType;
orgId: string;
/** Set for site_online / site_offline events */
siteId?: number;
/** Set for health_check_* events */
healthCheckId?: number;
/** Human-readable context data included in emails and webhook payloads */
data: Record<string, unknown>;
}

15
server/routers/target/updateTarget.ts
View File

@@ -10,6 +10,7 @@ import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { addPeer } from "../gerbil/peers";
import { addTargets } from "../newt/targets";
import { fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
import { pickPort } from "./helpers";
import { isTargetValid } from "@server/lib/validators";
import { OpenAPITags, registry } from "@server/openApi";
@@ -225,6 +226,11 @@ export async function updateTarget(
hcHealthValue = undefined;
}
const isDisablingHc =
(parsedBody.data.hcEnabled === false ||
parsedBody.data.hcEnabled === null) &&
existingHc.hcEnabled === true;
const [updatedHc] = await db
.update(targetHealthCheck)
.set({
@@ -250,6 +256,15 @@ export async function updateTarget(
.where(eq(targetHealthCheck.targetId, targetId))
.returning();
if (isDisablingHc) {
await fireHealthCheckUnknownAlert(
resource.orgId,
existingHc.targetHealthCheckId,
existingHc.name,
updatedHc.targetId
);
}
if (site.pubKey) {
if (site.type == "wireguard") {
await addPeer(site.exitNodeId!, {