Add logging when manually changing the hc status

This commit is contained in:
Owen
2026-04-26 17:28:57 -07:00
parent 06af53c4d6
commit ca2370e31d
21 changed files with 170 additions and 45 deletions

View File

@@ -3196,5 +3196,6 @@
"alertLabel": "Alert",
"domainPickerWildcardSubdomainNotAllowed": "Wildcard subdomains are not allowed.",
"domainPickerWildcardCertWarning": "Wildcard certificates must be configured separately in Traefik.",
"domainPickerWildcardCertWarningLink": "Learn more"
"domainPickerWildcardCertWarningLink": "Learn more",
"health": "Health"
}

View File

@@ -179,6 +179,7 @@ export async function updateProxyResources(
newHealthcheck.name,
newHealthcheck.targetId,
undefined,
true,
trx
);
}
@@ -581,6 +582,7 @@ export async function updateProxyResources(
newHealthcheck.name,
newHealthcheck.targetId,
undefined,
true,
trx
);
}

View File

@@ -50,7 +50,8 @@ export async function fireHealthCheckHealthyAlert(
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
trx: Transaction | typeof db = db
send: boolean = true,
trx: Transaction | typeof db = db,
): Promise<void> {
try {
await trx.insert(statusHistory).values({
@@ -63,6 +64,10 @@ export async function fireHealthCheckHealthyAlert(
await handleResource(orgId, healthCheckTargetId, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_healthy",
orgId,
@@ -108,6 +113,7 @@ export async function fireHealthCheckUnhealthyAlert(
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
@@ -121,6 +127,10 @@ export async function fireHealthCheckUnhealthyAlert(
await handleResource(orgId, healthCheckTargetId, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_unhealthy",
orgId,
@@ -155,6 +165,7 @@ export async function fireHealthCheckUnknownAlert(
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
@@ -167,6 +178,10 @@ export async function fireHealthCheckUnknownAlert(
});
await handleResource(orgId, healthCheckTargetId, trx);
if (!send) {
return;
}
} catch (err) {
logger.error(
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,

View File

@@ -125,6 +125,7 @@ export async function fireSiteOfflineAlert(
healthCheck.name,
undefined,
undefined,
true,
trx
);
}

View File

@@ -14,7 +14,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { targetHealthCheck, statusHistory } from "@server/db";
import { targetHealthCheck } from "@server/db";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";

View File

@@ -14,7 +14,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { resources, statusHistory } from "@server/db";
import { resources } from "@server/db";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";
@@ -24,7 +24,6 @@ import { eq, and } from "drizzle-orm";
import {
fireResourceHealthyAlert,
fireResourceUnhealthyAlert,
fireResourceToggleAlert,
fireResourceDegradedAlert
} from "#private/lib/alerts/events/resourceEvents";

View File

@@ -14,7 +14,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { sites, statusHistory } from "@server/db";
import { sites } from "@server/db";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";

View File

@@ -22,6 +22,7 @@ import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
import { fireHealthCheckUnhealthyAlert } from "#private/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty()
@@ -146,6 +147,15 @@ export async function createHealthCheck(
})
.returning();
await fireHealthCheckUnhealthyAlert(
record.orgId,
record.targetHealthCheckId,
record.name || "",
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
// Push health check to newt if the site is a newt site
if (siteId) {
const [site] = await db

View File

@@ -22,6 +22,7 @@ import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
import { and, eq, isNull } from "drizzle-orm";
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
import { fireHealthCheckUnhealthyAlert, fireHealthCheckUnknownAlert, fireHealthCheckHealthyAlert } from "#private/lib/alerts";
const paramsSchema = z
.object({
@@ -233,6 +234,37 @@ export async function updateHealthCheck(
)
.returning();
if (updated.hcHealth === "unhealthy" && existingHealthCheck.hcHealth !== "unhealthy") {
await fireHealthCheckUnhealthyAlert(
updated.orgId,
updated.targetHealthCheckId,
updated.name || "",
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updated.hcHealth === "unknown" && existingHealthCheck.hcHealth !== "unknown") {
// if the health is unknown, we want to fire an alert to notify users to enable health checks
await fireHealthCheckUnknownAlert(
updated.orgId,
updated.targetHealthCheckId,
updated.name,
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updated.hcHealth === "healthy" && existingHealthCheck.hcHealth !== "healthy") {
await fireHealthCheckHealthyAlert(
updated.orgId,
updated.targetHealthCheckId,
updated.name,
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
}
// Push updated health check to newt if the site is a newt site
const [newt] = await db
.select()

View File

@@ -6,7 +6,7 @@ import {
} from "@server/db";
import { eq } from "drizzle-orm";
import logger from "@server/logger";
import { fireSiteOfflineAlert } from "@server/lib/alerts";
import { fireSiteOfflineAlert } from "#dynamic/lib/alerts";
/**
* Handles disconnecting messages from sites to show disconnected in the ui

View File

@@ -1,10 +1,7 @@
import {
db,
newts,
sites,
targetHealthCheck,
targets,
statusHistory
sites
} from "@server/db";
import { hasActiveConnections } from "#dynamic/routers/ws";
import { eq, lt, isNull, and, or, ne, not, inArray } from "drizzle-orm";

View File

@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { db, statusHistory } from "@server/db";
import {
siteProvisioningKeys,
siteProvisioningKeyOrg,
@@ -223,6 +223,14 @@ export async function registerNewt(
})
.returning();
await trx.insert(statusHistory).values({
entityType: "site",
entityId: newSite.siteId,
orgId: orgId,
status: "offline",
timestamp: Math.floor(Date.now() / 1000)
});
newSiteId = newSite.siteId;
// Grant admin role access to the new site

View File

@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { clients, db, exitNodes } from "@server/db";
import { clients, db, exitNodes, statusHistory } from "@server/db";
import { roles, userSites, sites, roleSites, Site, orgs } from "@server/db";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
@@ -321,12 +321,7 @@ export async function createSite(
const existingSite = await db
.select()
.from(sites)
.where(
and(
eq(sites.niceId, niceId),
eq(sites.orgId, orgId)
)
)
.where(and(eq(sites.niceId, niceId), eq(sites.orgId, orgId)))
.limit(1);
if (existingSite.length > 0) {
@@ -344,7 +339,8 @@ export async function createSite(
if (type == "newt") {
[newSite] = await trx
.insert(sites)
.values({ // NOTE: NO SUBNET OR EXIT NODE ID PASSED IN HERE BECAUSE ITS NOW CHOSEN ON CONNECT
.values({
// NOTE: NO SUBNET OR EXIT NODE ID PASSED IN HERE BECAUSE ITS NOW CHOSEN ON CONNECT
orgId,
name,
niceId: updatedNiceId!,
@@ -354,6 +350,14 @@ export async function createSite(
status: "approved"
})
.returning();
await trx.insert(statusHistory).values({
entityType: "site",
entityId: newSite.siteId,
orgId: orgId,
status: "offline",
timestamp: Math.floor(Date.now() / 1000)
});
} else if (type == "wireguard") {
// we are creating a site with an exit node (tunneled)
if (!subnet) {

View File

@@ -1,6 +1,11 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db, TargetHealthCheck, targetHealthCheck } from "@server/db";
import {
db,
statusHistory,
TargetHealthCheck,
targetHealthCheck
} from "@server/db";
import { newts, resources, sites, Target, targets } from "@server/db";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
@@ -14,6 +19,7 @@ import { eq } from "drizzle-orm";
import { pickPort } from "./helpers";
import { isTargetValid } from "@server/lib/validators";
import { OpenAPITags, registry } from "@server/openApi";
import { fireHealthCheckHealthyAlert, fireHealthCheckUnhealthyAlert, fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
const createTargetParamsSchema = z.strictObject({
resourceId: z.string().transform(Number).pipe(z.int().positive())
@@ -252,6 +258,36 @@ export async function createTarget(
})
.returning();
if (healthCheck[0].hcHealth === "unhealthy") {
await fireHealthCheckUnhealthyAlert(
healthCheck[0].orgId,
healthCheck[0].targetHealthCheckId,
healthCheck[0].name,
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (healthCheck[0].hcHealth === "unknown") {
// if the health is unknown, we want to fire an alert to notify users to enable health checks
await fireHealthCheckUnknownAlert(
healthCheck[0].orgId,
healthCheck[0].targetHealthCheckId,
healthCheck[0].name,
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (healthCheck[0].hcHealth === "healthy") {
await fireHealthCheckHealthyAlert(
healthCheck[0].orgId,
healthCheck[0].targetHealthCheckId,
healthCheck[0].name,
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
}
if (site.pubKey) {
if (site.type == "wireguard") {
await addPeer(site.exitNodeId!, {

View File

@@ -1,10 +1,6 @@
import {
db,
targets,
resources,
sites,
targetHealthCheck,
statusHistory
targetHealthCheck
} from "@server/db";
import { MessageHandler } from "@server/routers/ws";
import { Newt } from "@server/db";
@@ -142,6 +138,7 @@ export const handleHealthcheckStatusMessage: MessageHandler = async (
targetCheck.name ?? undefined,
targetCheck.targetId,
undefined,
true,
trx
);
} else if (healthStatus.status === "healthy") {
@@ -151,6 +148,7 @@ export const handleHealthcheckStatusMessage: MessageHandler = async (
targetCheck.name ?? undefined,
targetCheck.targetId,
undefined,
true,
trx
);
}

View File

@@ -10,10 +10,11 @@ import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { addPeer } from "../gerbil/peers";
import { addTargets } from "../newt/targets";
import { fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
import { fireHealthCheckHealthyAlert, fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
import { pickPort } from "./helpers";
import { isTargetValid } from "@server/lib/validators";
import { OpenAPITags, registry } from "@server/openApi";
import { fireHealthCheckUnhealthyAlert } from "@server/lib/alerts";
const updateTargetParamsSchema = z.strictObject({
@@ -256,12 +257,33 @@ export async function updateTarget(
.where(eq(targetHealthCheck.targetId, targetId))
.returning();
if (isDisablingHc) {
if (updatedHc.hcHealth === "unhealthy" && existingHc.hcHealth !== "unhealthy") {
await fireHealthCheckUnhealthyAlert(
updatedHc.orgId,
updatedHc.targetHealthCheckId,
updatedHc.name || "",
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updatedHc.hcHealth === "unknown" && existingHc.hcHealth !== "unknown") {
// if the health is unknown, we want to fire an alert to notify users to enable health checks
await fireHealthCheckUnknownAlert(
resource.orgId,
existingHc.targetHealthCheckId,
existingHc.name,
updatedHc.targetId
updatedHc.orgId,
updatedHc.targetHealthCheckId,
updatedHc.name,
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updatedHc.hcHealth === "healthy" && existingHc.hcHealth !== "healthy") {
await fireHealthCheckHealthyAlert(
updatedHc.orgId,
updatedHc.targetHealthCheckId,
updatedHc.name,
undefined,
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
}

View File

@@ -151,6 +151,7 @@ export default async function AlertingHealthChecksPage(
fullDomain: string | null;
niceId: string;
ssl: boolean;
wildcard: boolean;
} | null = null;
if (resourceIdParam) {
try {
@@ -165,7 +166,8 @@ export default async function AlertingHealthChecksPage(
resourceId: r.resourceId,
fullDomain: r.fullDomain,
niceId: r.niceId,
ssl: r.ssl
ssl: r.ssl,
wildcard: r.wildcard
};
}
} catch {

View File

@@ -557,6 +557,7 @@ export default function DomainPicker({
)}
</p>
<PaidFeaturesAlert
showBookADemo={false}
tiers={
tierMatrix[
TierFeature.WildcardSubdomain

View File

@@ -151,7 +151,8 @@ export default function HealthChecksTable({
resourceId: resourceIdNum,
fullDomain: null,
niceId: "",
ssl: false
ssl: false,
wildcard: false
};
}, [initialFilterResource, resourceIdQ, resourceIdNum, t]);

View File

@@ -114,9 +114,10 @@ function getDocsLinkRenderer(href: string) {
type Props = {
tiers: Tier[];
showBookADemo?: boolean;
};
export function PaidFeaturesAlert({ tiers }: Props) {
export function PaidFeaturesAlert({ tiers, showBookADemo = true }: Props) {
const t = useTranslations();
const params = useParams();
const orgId = params?.orgId as string | undefined;
@@ -134,7 +135,9 @@ export function PaidFeaturesAlert({ tiers }: Props) {
const tierLinkRenderer = getTierLinkRenderer(billingHref);
const pangolinCloudLinkRenderer = getPangolinCloudLinkRenderer();
const enterpriseDocsLinkRenderer = getDocsLinkRenderer(ENTERPRISE_DOCS_URL);
const bookADemoLinkRenderer = getBookADemoLinkRenderer();
const bookADemoLinkRenderer = showBookADemo
? getBookADemoLinkRenderer()
: () => null;
if (env.flags.disableEnterpriseFeatures) {
return null;

View File

@@ -63,13 +63,6 @@ import { useDebouncedCallback } from "use-debounce";
import z from "zod";
import { ColumnFilterButton } from "./ColumnFilterButton";
import { ControlledDataTable } from "./ui/controlled-data-table";
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger
} from "@app/components/ui/tooltip";
import type { StatusHistoryResponse } from "@server/lib/statusHistory";
import UptimeMiniBar from "./UptimeMiniBar";
export type TargetHealth = {
@@ -466,7 +459,7 @@ export default function ProxyResourcesTable({
{
id: "status",
accessorKey: "status",
friendlyName: t("status"),
friendlyName: t("health"),
header: () => (
<ColumnFilterButton
options={[
@@ -489,7 +482,7 @@ export default function ProxyResourcesTable({
}
searchPlaceholder={t("searchPlaceholder")}
emptyMessage={t("emptySearchOptions")}
label={t("status")}
label={t("health")}
className="p-3"
/>
),