Improve cert retreival and new newt combined certs

server/routers/newt/buildConfiguration.ts

@@ -19,6 +19,7 @@ import { eq, and, inArray } from "drizzle-orm";
 import config from "@server/lib/config";
 import { decrypt } from "@server/lib/crypto";
 import {
+    batchFetchCertsForSiteResources,
     formatEndpoint,
     generateSubnetProxyTargetV2,
     SubnetProxyTargetV2
@@ -206,11 +207,18 @@ export async function buildClientConfigurationForNewtClient(
         });
     }
 
+    // Batch-fetch certs for every SSL-enabled HTTP resource's domain in one
+    // call rather than letting each resource fetch its own — with thousands
+    // of resources this avoids a concurrent DB/cache stampede for what is
+    // often the very same (e.g. wildcard) certificate.
+    const certByDomain = await batchFetchCertsForSiteResources(allSiteResources);
+
     const resourceTargetsArr = await Promise.all(
         allSiteResources.map((resource) =>
             generateSubnetProxyTargetV2(
                 resource,
-                clientsByResourceId.get(resource.siteResourceId) ?? []
+                clientsByResourceId.get(resource.siteResourceId) ?? [],
+                certByDomain
             )
         )
     );

server/routers/newt/certs.ts

@@ -0,0 +1,75 @@
+import { sendToClient } from "#dynamic/routers/ws";
+import logger from "@server/logger";
+import {
+    canCompress,
+    supportsCertReferences
+} from "@server/lib/clientVersionChecks";
+import { CertRef } from "@server/lib/ip";
+
+/**
+ * Pushes an incremental set of certs to a newt client outside of a full
+ * newt/sync or newt/wg/receive-config, e.g. after a certificate renewal so
+ * that every target referencing it (by tlsCertId) picks up the new material
+ * without waiting for the next full resync.
+ */
+export async function sendCertsAdd(
+    newtId: string,
+    certs: CertRef[],
+    version?: string | null
+) {
+    if (certs.length === 0) {
+        return;
+    }
+
+    if (!supportsCertReferences(version)) {
+        logger.debug(
+            `Newt ${newtId} (version ${version}) does not support cert references, skipping certs/add`
+        );
+        return;
+    }
+
+    await sendToClient(
+        newtId,
+        {
+            type: "newt/certs/add",
+            data: certs
+        },
+        {
+            incrementConfigVersion: true,
+            compress: canCompress(version, "newt")
+        }
+    );
+}
+
+/**
+ * Tells a newt client to drop the given cert IDs, e.g. once the server knows
+ * no target references them anymore.
+ */
+export async function sendCertsRemove(
+    newtId: string,
+    certIds: string[],
+    version?: string | null
+) {
+    if (certIds.length === 0) {
+        return;
+    }
+
+    if (!supportsCertReferences(version)) {
+        logger.debug(
+            `Newt ${newtId} (version ${version}) does not support cert references, skipping certs/remove`
+        );
+        return;
+    }
+
+    await sendToClient(
+        newtId,
+        {
+            type: "newt/certs/remove",
+            data: { ids: certIds }
+        },
+        {
+            incrementConfigVersion: true,
+            compress: canCompress(version, "newt")
+        }
+    );
+}

server/routers/newt/handleNewtGetConfigMessage.ts

@@ -7,7 +7,11 @@ import { eq } from "drizzle-orm";
 import { sendToExitNode } from "#dynamic/lib/exitNodes";
 import { buildClientConfigurationForNewtClient } from "./buildConfiguration";
 import { convertTargetsIfNecessary } from "../client/targets";
-import { canCompress } from "@server/lib/clientVersionChecks";
+import {
+    canCompress,
+    supportsCertReferences
+} from "@server/lib/clientVersionChecks";
+import { dedupeCertsForTargets } from "@server/lib/ip";
 import config from "@server/lib/config";
 import { waitForSiteRebuildIdle } from "@server/lib/rebuildClientAssociations";
 
@@ -119,7 +123,16 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
         exitNode
     );
 
-    const targetsToSend = await convertTargetsIfNecessary(newt.newtId, targets); // for backward compatibility with old newt versions that don't support the new target format
+    // Older newt clients only understand inline tlsCert/tlsKey on each
+    // target, so only switch to certId references once we know the client
+    // can resolve them.
+    let dedupedTargets = targets;
+    let certs: { id: string; cert: string; key: string }[] = [];
+    if (supportsCertReferences(newt.version)) {
+        ({ targets: dedupedTargets, certs } = dedupeCertsForTargets(targets));
+    }
+
+    const targetsToSend = await convertTargetsIfNecessary(newt.newtId, dedupedTargets); // for backward compatibility with old newt versions that don't support the new target format
 
     return {
         message: {
@@ -128,6 +141,7 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
                 ipAddress: site.address,
                 peers,
                 targets: targetsToSend,
+                certs,
                 chainId: chainId
             }
         },

server/routers/newt/sync.ts

@@ -6,7 +6,11 @@ import {
     buildClientConfigurationForNewtClient,
     buildTargetConfigurationForNewtClient
 } from "./buildConfiguration";
-import { canCompress } from "@server/lib/clientVersionChecks";
+import {
+    canCompress,
+    supportsCertReferences
+} from "@server/lib/clientVersionChecks";
+import { dedupeCertsForTargets } from "@server/lib/ip";
 
 export async function sendNewtSyncMessage(newt: Newt, site: Site) {
     const {
@@ -28,6 +32,16 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) {
         site,
         exitNode
     );
+
+    // Older newt clients only understand inline tlsCert/tlsKey on each
+    // target, so only switch to certId references once we know the client
+    // can resolve them.
+    let clientTargets = targets;
+    let certs: { id: string; cert: string; key: string }[] = [];
+    if (supportsCertReferences(newt.version)) {
+        ({ targets: clientTargets, certs } = dedupeCertsForTargets(targets));
+    }
+
     await sendToClient(
         newt.newtId,
         {
@@ -39,7 +53,8 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) {
                 },
                 healthCheckTargets: validHealthCheckTargets,
                 peers: peers,
-                clientTargets: targets,
+                clientTargets: clientTargets,
+                certs: certs,
                 browserGatewayTargets: browserGatewayTargets,
                 remoteExitNodeSubnets: remoteExitNodeSubnets
             }