From 217a59ad103a9e50de8b547fe7e6791be67c3282 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 10 Sep 2026 10:35:42 -0400 Subject: [PATCH] Implement exit node check-in tracking and adjust logging for connection errors --- server/lib/exitNodes/exitNodeCheckIn.ts | 14 ++++++++ server/lib/exitNodes/exitNodeComms.ts | 19 +++++++---- server/lib/exitNodes/index.ts | 1 + server/lib/traefik/TraefikConfigManager.ts | 33 ++++++++++++++----- server/private/lib/exitNodes/exitNodeComms.ts | 19 +++++++---- server/routers/gerbil/getConfig.ts | 3 ++ 6 files changed, 69 insertions(+), 20 deletions(-) create mode 100644 server/lib/exitNodes/exitNodeCheckIn.ts diff --git a/server/lib/exitNodes/exitNodeCheckIn.ts b/server/lib/exitNodes/exitNodeCheckIn.ts new file mode 100644 index 000000000..819ab8749 --- /dev/null +++ b/server/lib/exitNodes/exitNodeCheckIn.ts @@ -0,0 +1,14 @@ +// Tracks, per process lifetime, whether a given exit node has ever checked in +// (called /gerbil/get-config) since this Pangolin instance started. This lets +// callers distinguish "gerbil hasn't come up yet" (expected briefly after a +// restart, since gerbil depends on pangolin's container starting first) from +// "gerbil was reachable and now isn't" (a real problem worth an error log). +const checkedInExitNodeIds = new Set(); + +export function markExitNodeCheckedIn(exitNodeId: number): void { + checkedInExitNodeIds.add(exitNodeId); +} + +export function hasExitNodeCheckedIn(exitNodeId: number): boolean { + return checkedInExitNodeIds.has(exitNodeId); +} diff --git a/server/lib/exitNodes/exitNodeComms.ts b/server/lib/exitNodes/exitNodeComms.ts index bcfbec3ee..24d932161 100644 --- a/server/lib/exitNodes/exitNodeComms.ts +++ b/server/lib/exitNodes/exitNodeComms.ts @@ -1,6 +1,7 @@ import axios from "axios"; import logger from "@server/logger"; import { ExitNode } from "@server/db"; +import { hasExitNodeCheckedIn } from "./exitNodeCheckIn"; interface ExitNodeRequest { remoteType?: string; @@ -72,13 +73,19 @@ export async function sendToExitNode( return response.data; } catch (error) { - if (axios.isAxiosError(error)) { - logger.error( - `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}` - ); + const message = axios.isAxiosError(error) + ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}` + : `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`; + + // The exit node (gerbil) may still be starting up and not yet + // reachable. Until it has checked in at least once, log this at a + // lower level since it's expected; once it has checked in, a + // connection failure is a real problem. + if (hasExitNodeCheckedIn(exitNode.exitNodeId)) { + logger.error(message); } else { - logger.error( - `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}` + logger.warn( + `${message} (exit node has not checked in yet since startup, this is expected briefly)` ); } throw error; diff --git a/server/lib/exitNodes/index.ts b/server/lib/exitNodes/index.ts index 18474e27d..bd7a76166 100644 --- a/server/lib/exitNodes/index.ts +++ b/server/lib/exitNodes/index.ts @@ -1,5 +1,6 @@ export * from "./exitNodes"; export * from "./exitNodeComms"; +export * from "./exitNodeCheckIn"; export * from "./subnet"; export * from "./getCurrentExitNodeId"; export * from "./calculateExitNodeWeight"; diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index c8b48878f..320863f07 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -6,7 +6,10 @@ import * as yaml from "js-yaml"; import axios from "axios"; import { db, exitNodes } from "@server/db"; import { eq } from "drizzle-orm"; -import { getCurrentExitNodeId } from "@server/lib/exitNodes"; +import { + getCurrentExitNodeId, + hasExitNodeCheckedIn +} from "@server/lib/exitNodes"; import { getTraefikConfig } from "#dynamic/lib/traefik"; import { getValidCertificatesForDomains } from "@server/lib/certificates"; import { sendToExitNode } from "#dynamic/lib/exitNodes"; @@ -466,32 +469,46 @@ export class TraefikConfigManager { await this.writeTraefikDynamicConfig(traefikConfig); // Send domains to SNI proxy + let exitNodeForSni: (typeof exitNodes.$inferSelect) | undefined; try { - let exitNode; if (config.getRawConfig().gerbil.exit_node_name) { const exitNodeName = config.getRawConfig().gerbil.exit_node_name!; - [exitNode] = await db + [exitNodeForSni] = await db .select() .from(exitNodes) .where(eq(exitNodes.name, exitNodeName)) .limit(1); } else { - [exitNode] = await db.select().from(exitNodes).limit(1); + [exitNodeForSni] = await db + .select() + .from(exitNodes) + .limit(1); } - if (exitNode) { - await sendToExitNode(exitNode, { + if (exitNodeForSni) { + await sendToExitNode(exitNodeForSni, { localPath: "/update-local-snis", method: "POST", data: { fullDomains: Array.from(domains) } }); } else { - logger.error( + logger.warn( "No exit node found. Has gerbil registered yet?" ); } } catch (err) { - logger.error("Failed to post domains to SNI proxy:", err); + // sendToExitNode already logs the underlying connection + // error at the appropriate level (warn before the exit node + // has checked in since startup, error after), so avoid + // double-logging it as an error here. + if ( + exitNodeForSni && + !hasExitNodeCheckedIn(exitNodeForSni.exitNodeId) + ) { + logger.warn("Failed to post domains to SNI proxy:", err); + } else { + logger.error("Failed to post domains to SNI proxy:", err); + } } // Update active domains tracking diff --git a/server/private/lib/exitNodes/exitNodeComms.ts b/server/private/lib/exitNodes/exitNodeComms.ts index 3adeadd2a..03035bf9f 100644 --- a/server/private/lib/exitNodes/exitNodeComms.ts +++ b/server/private/lib/exitNodes/exitNodeComms.ts @@ -18,6 +18,7 @@ import { eq } from "drizzle-orm"; import { sendToClient } from "#private/routers/ws"; import privateConfig from "#private/lib/config"; import config from "@server/lib/config"; +import { hasExitNodeCheckedIn } from "@server/lib/exitNodes"; interface ExitNodeRequest { remoteType?: string; @@ -138,13 +139,19 @@ export async function sendToExitNode( return response.data; } catch (error) { - if (axios.isAxiosError(error)) { - logger.error( - `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}` - ); + const message = axios.isAxiosError(error) + ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}` + : `Error making ${method} request for exit node at ${hostname}: ${error}`; + + // The exit node (gerbil) may still be starting up and not yet + // reachable. Until it has checked in at least once, log this at a + // lower level since it's expected; once it has checked in, a + // connection failure is a real problem. + if (hasExitNodeCheckedIn(exitNode.exitNodeId)) { + logger.error(message); } else { - logger.error( - `Error making ${method} request for exit node at ${hostname}: ${error}` + logger.warn( + `${message} (exit node has not checked in yet since startup, this is expected briefly)` ); } } diff --git a/server/routers/gerbil/getConfig.ts b/server/routers/gerbil/getConfig.ts index ffc3c6120..6f5eb8785 100644 --- a/server/routers/gerbil/getConfig.ts +++ b/server/routers/gerbil/getConfig.ts @@ -10,6 +10,7 @@ import config from "@server/lib/config"; import { fromError } from "zod-validation-error"; import { getAllowedIps } from "../target/helpers"; import { createExitNode } from "#dynamic/routers/gerbil/createExitNode"; +import { markExitNodeCheckedIn } from "@server/lib/exitNodes"; // Define Zod schema for request validation const getConfigSchema = z.object({ @@ -65,6 +66,8 @@ export async function getConfig( ); } + markExitNodeCheckedIn(exitNode.exitNodeId); + const configResponse = await generateGerbilConfig(exitNode); logger.debug("Sending config: ", configResponse);