Implement exit node check-in tracking and adjust logging for connection errors

server/lib/exitNodes/exitNodeCheckIn.ts

@@ -0,0 +1,14 @@
+// Tracks, per process lifetime, whether a given exit node has ever checked in
+// (called /gerbil/get-config) since this Pangolin instance started. This lets
+// callers distinguish "gerbil hasn't come up yet" (expected briefly after a
+// restart, since gerbil depends on pangolin's container starting first) from
+// "gerbil was reachable and now isn't" (a real problem worth an error log).
+const checkedInExitNodeIds = new Set<number>();
+
+export function markExitNodeCheckedIn(exitNodeId: number): void {
+    checkedInExitNodeIds.add(exitNodeId);
+}
+
+export function hasExitNodeCheckedIn(exitNodeId: number): boolean {
+    return checkedInExitNodeIds.has(exitNodeId);
+}

server/lib/exitNodes/exitNodeComms.ts

@@ -1,6 +1,7 @@
 import axios from "axios";
 import logger from "@server/logger";
 import { ExitNode } from "@server/db";
+import { hasExitNodeCheckedIn } from "./exitNodeCheckIn";
 
 interface ExitNodeRequest {
     remoteType?: string;
@@ -72,13 +73,19 @@ export async function sendToExitNode(
 
         return response.data;
     } catch (error) {
-        if (axios.isAxiosError(error)) {
-            logger.error(
-                `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}`
-            );
+        const message = axios.isAxiosError(error)
+            ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}`
+            : `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`;
+
+        // The exit node (gerbil) may still be starting up and not yet
+        // reachable. Until it has checked in at least once, log this at a
+        // lower level since it's expected; once it has checked in, a
+        // connection failure is a real problem.
+        if (hasExitNodeCheckedIn(exitNode.exitNodeId)) {
+            logger.error(message);
         } else {
-            logger.error(
-                `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`
+            logger.warn(
+                `${message} (exit node has not checked in yet since startup, this is expected briefly)`
             );
         }
         throw error;

server/lib/exitNodes/index.ts

@@ -1,5 +1,6 @@
 export * from "./exitNodes";
 export * from "./exitNodeComms";
+export * from "./exitNodeCheckIn";
 export * from "./subnet";
 export * from "./getCurrentExitNodeId";
 export * from "./calculateExitNodeWeight";

server/lib/traefik/TraefikConfigManager.ts

@@ -6,7 +6,10 @@ import * as yaml from "js-yaml";
 import axios from "axios";
 import { db, exitNodes } from "@server/db";
 import { eq } from "drizzle-orm";
-import { getCurrentExitNodeId } from "@server/lib/exitNodes";
+import {
+    getCurrentExitNodeId,
+    hasExitNodeCheckedIn
+} from "@server/lib/exitNodes";
 import { getTraefikConfig } from "#dynamic/lib/traefik";
 import { getValidCertificatesForDomains } from "@server/lib/certificates";
 import { sendToExitNode } from "#dynamic/lib/exitNodes";
@@ -466,32 +469,46 @@ export class TraefikConfigManager {
             await this.writeTraefikDynamicConfig(traefikConfig);
 
             // Send domains to SNI proxy
+            let exitNodeForSni: (typeof exitNodes.$inferSelect) | undefined;
             try {
-                let exitNode;
                 if (config.getRawConfig().gerbil.exit_node_name) {
                     const exitNodeName =
                         config.getRawConfig().gerbil.exit_node_name!;
-                    [exitNode] = await db
+                    [exitNodeForSni] = await db
                         .select()
                         .from(exitNodes)
                         .where(eq(exitNodes.name, exitNodeName))
                         .limit(1);
                 } else {
-                    [exitNode] = await db.select().from(exitNodes).limit(1);
+                    [exitNodeForSni] = await db
+                        .select()
+                        .from(exitNodes)
+                        .limit(1);
                 }
-                if (exitNode) {
-                    await sendToExitNode(exitNode, {
+                if (exitNodeForSni) {
+                    await sendToExitNode(exitNodeForSni, {
                         localPath: "/update-local-snis",
                         method: "POST",
                         data: { fullDomains: Array.from(domains) }
                     });
                 } else {
-                    logger.error(
+                    logger.warn(
                         "No exit node found. Has gerbil registered yet?"
                     );
                 }
             } catch (err) {
-                logger.error("Failed to post domains to SNI proxy:", err);
+                // sendToExitNode already logs the underlying connection
+                // error at the appropriate level (warn before the exit node
+                // has checked in since startup, error after), so avoid
+                // double-logging it as an error here.
+                if (
+                    exitNodeForSni &&
+                    !hasExitNodeCheckedIn(exitNodeForSni.exitNodeId)
+                ) {
+                    logger.warn("Failed to post domains to SNI proxy:", err);
+                } else {
+                    logger.error("Failed to post domains to SNI proxy:", err);
+                }
             }
 
             // Update active domains tracking

server/private/lib/exitNodes/exitNodeComms.ts

@@ -18,6 +18,7 @@ import { eq } from "drizzle-orm";
 import { sendToClient } from "#private/routers/ws";
 import privateConfig from "#private/lib/config";
 import config from "@server/lib/config";
+import { hasExitNodeCheckedIn } from "@server/lib/exitNodes";
 
 interface ExitNodeRequest {
     remoteType?: string;
@@ -138,13 +139,19 @@ export async function sendToExitNode(
 
             return response.data;
         } catch (error) {
-            if (axios.isAxiosError(error)) {
-                logger.error(
-                    `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
-                );
+            const message = axios.isAxiosError(error)
+                ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
+                : `Error making ${method} request for exit node at ${hostname}: ${error}`;
+
+            // The exit node (gerbil) may still be starting up and not yet
+            // reachable. Until it has checked in at least once, log this at a
+            // lower level since it's expected; once it has checked in, a
+            // connection failure is a real problem.
+            if (hasExitNodeCheckedIn(exitNode.exitNodeId)) {
+                logger.error(message);
             } else {
-                logger.error(
-                    `Error making ${method} request for exit node at ${hostname}: ${error}`
+                logger.warn(
+                    `${message} (exit node has not checked in yet since startup, this is expected briefly)`
                 );
             }
         }

server/routers/gerbil/getConfig.ts

@@ -10,6 +10,7 @@ import config from "@server/lib/config";
 import { fromError } from "zod-validation-error";
 import { getAllowedIps } from "../target/helpers";
 import { createExitNode } from "#dynamic/routers/gerbil/createExitNode";
+import { markExitNodeCheckedIn } from "@server/lib/exitNodes";
 
 // Define Zod schema for request validation
 const getConfigSchema = z.object({
@@ -65,6 +66,8 @@ export async function getConfig(
             );
         }
 
+        markExitNodeCheckedIn(exitNode.exitNodeId);
+
         const configResponse = await generateGerbilConfig(exitNode);
 
         logger.debug("Sending config: ", configResponse);