Improve cert retreival and new newt combined certs

fix labels dropdown flicker if no changes applied

server/lib/certificates.ts

@@ -5,6 +5,7 @@ export async function getValidCertificatesForDomains(
     Array<{
         id: number;
         domain: string;
+        queriedDomain: string;
         wildcard: boolean | null;
         certFile: string | null;
         keyFile: string | null;

server/lib/clientVersionChecks.ts

@@ -18,3 +18,19 @@ export function canCompress(
         return false;
     }
 }
+
+// Whether this newt client understands `tlsCertId` references into the
+// sync message's `certs` array, instead of requiring each target to carry
+// its own inline `tlsCert`/`tlsKey` PEM data. Bump the version floor here to
+// match whatever release first ships the newt-side support.
+export function supportsCertReferences(
+    clientVersion: string | null | undefined
+): boolean {
+    try {
+        if (!clientVersion) return false;
+        if (!semver.valid(clientVersion)) return false;
+        return semver.gte(clientVersion, "1.16.0");
+    } catch {
+        return false;
+    }
+}

server/lib/ip.ts

@@ -5,6 +5,7 @@ import config from "@server/lib/config";
 import z from "zod";
 import logger from "@server/logger";
 import semver from "semver";
+import { createHash } from "crypto";
 import { getValidCertificatesForDomains } from "#dynamic/lib/certificates";
 import { lockManager } from "#dynamic/lib/lock";
 
@@ -648,21 +649,101 @@ export type SubnetProxyTargetV2 = {
     httpTargets?: HTTPTarget[];
     tlsCert?: string;
     tlsKey?: string;
+    tlsCertId?: string; // references an entry in the sync message's top-level `certs` array instead of inlining tlsCert/tlsKey
 };
 
+export type CertRef = { id: string; cert: string; key: string };
+
+/**
+ * Replaces each target's inline tlsCert/tlsKey with a tlsCertId reference
+ * into a deduplicated certs array, so that many targets sharing the same
+ * certificate (e.g. a wildcard cert used by thousands of site resources)
+ * only need that certificate sent once per sync message.
+ */
+export function dedupeCertsForTargets(
+    targetsV2: SubnetProxyTargetV2[]
+): { targets: SubnetProxyTargetV2[]; certs: CertRef[] } {
+    const idByContent = new Map<string, string>();
+    const certs: CertRef[] = [];
+
+    const targets = targetsV2.map((target) => {
+        if (!target.tlsCert || !target.tlsKey) {
+            return target;
+        }
+
+        const contentKey = `${target.tlsCert}|${target.tlsKey}`;
+        let id = idByContent.get(contentKey);
+        if (!id) {
+            id = createHash("sha1").update(contentKey).digest("hex").slice(0, 16);
+            idByContent.set(contentKey, id);
+            certs.push({ id, cert: target.tlsCert, key: target.tlsKey });
+        }
+
+        const { tlsCert, tlsKey, ...rest } = target;
+        return { ...rest, tlsCertId: id };
+    });
+
+    return { targets, certs };
+}
+
 export type HTTPTarget = {
     destAddr: string; // must be an IP or hostname
     destPort: number;
     scheme: "http" | "https";
 };
 
+export type CertByDomain = Map<string, { certFile: string; keyFile: string }>;
+
+/**
+ * Fetches the TLS certificates for every enabled, SSL-enabled HTTP site
+ * resource's fullDomain in a single batched call, instead of one call per
+ * resource. Many resources commonly resolve to the very same certificate
+ * (e.g. a wildcard covering the org's domain), so batching turns what would
+ * be N concurrent DB/cache round-trips into one, and a lookup failure fails
+ * loudly for the whole batch rather than silently dropping the cert on a
+ * random subset of otherwise-identical resources under load.
+ */
+export async function batchFetchCertsForSiteResources(
+    allSiteResources: SiteResource[]
+): Promise<CertByDomain> {
+    const domains = new Set(
+        allSiteResources
+            .filter((r) => r.enabled && r.mode === "http" && r.ssl && r.fullDomain)
+            .map((r) => r.fullDomain as string)
+    );
+
+    const certByDomain: CertByDomain = new Map();
+    if (domains.size === 0) {
+        return certByDomain;
+    }
+
+    try {
+        const certResults = await getValidCertificatesForDomains(domains, true);
+        for (const cert of certResults) {
+            if (cert.certFile && cert.keyFile) {
+                certByDomain.set(cert.queriedDomain, {
+                    certFile: cert.certFile,
+                    keyFile: cert.keyFile
+                });
+            }
+        }
+    } catch (err) {
+        logger.error(
+            `Failed to batch-retrieve certificates for ${domains.size} domain(s): ${err}`
+        );
+    }
+
+    return certByDomain;
+}
+
 export async function generateSubnetProxyTargetV2(
     siteResource: SiteResource,
     clients: {
         clientId: number;
         pubKey: string | null;
         subnet: string | null;
-    }[]
+    }[],
+    certByDomain?: CertByDomain
 ): Promise<SubnetProxyTargetV2[] | undefined> {
     if (!siteResource.enabled) {
         logger.debug(
@@ -750,6 +831,22 @@ export async function generateSubnetProxyTargetV2(
         let tlsKey: string | undefined;
 
         if (siteResource.ssl && siteResource.fullDomain) {
+            if (certByDomain) {
+                // Caller batch-fetched certs for all resources up front (the
+                // common, high-scale path) — just look up this resource's
+                // domain rather than issuing its own DB/cache round-trip.
+                const cert = certByDomain.get(siteResource.fullDomain);
+                if (cert) {
+                    tlsCert = cert.certFile;
+                    tlsKey = cert.keyFile;
+                } else {
+                    logger.warn(
+                        `No valid certificate found for SSL site resource ${siteResource.siteResourceId} with domain ${siteResource.fullDomain}`
+                    );
+                }
+            } else {
+                // No batched map supplied by the caller — fall back to a
+                // single-domain lookup for this resource alone.
                 try {
                     const certs = await getValidCertificatesForDomains(
                         new Set([siteResource.fullDomain]),
@@ -769,6 +866,7 @@ export async function generateSubnetProxyTargetV2(
                     );
                 }
             }
+        }
 
         targets.push({
             sourcePrefixes: [],

server/routers/newt/buildConfiguration.ts

@@ -19,6 +19,7 @@ import { eq, and, inArray } from "drizzle-orm";
 import config from "@server/lib/config";
 import { decrypt } from "@server/lib/crypto";
 import {
+    batchFetchCertsForSiteResources,
     formatEndpoint,
     generateSubnetProxyTargetV2,
     SubnetProxyTargetV2
@@ -206,11 +207,18 @@ export async function buildClientConfigurationForNewtClient(
         });
     }
 
+    // Batch-fetch certs for every SSL-enabled HTTP resource's domain in one
+    // call rather than letting each resource fetch its own — with thousands
+    // of resources this avoids a concurrent DB/cache stampede for what is
+    // often the very same (e.g. wildcard) certificate.
+    const certByDomain = await batchFetchCertsForSiteResources(allSiteResources);
+
     const resourceTargetsArr = await Promise.all(
         allSiteResources.map((resource) =>
             generateSubnetProxyTargetV2(
                 resource,
-                clientsByResourceId.get(resource.siteResourceId) ?? []
+                clientsByResourceId.get(resource.siteResourceId) ?? [],
+                certByDomain
             )
         )
     );

server/routers/newt/certs.ts

@@ -0,0 +1,75 @@
+import { sendToClient } from "#dynamic/routers/ws";
+import logger from "@server/logger";
+import {
+    canCompress,
+    supportsCertReferences
+} from "@server/lib/clientVersionChecks";
+import { CertRef } from "@server/lib/ip";
+
+/**
+ * Pushes an incremental set of certs to a newt client outside of a full
+ * newt/sync or newt/wg/receive-config, e.g. after a certificate renewal so
+ * that every target referencing it (by tlsCertId) picks up the new material
+ * without waiting for the next full resync.
+ */
+export async function sendCertsAdd(
+    newtId: string,
+    certs: CertRef[],
+    version?: string | null
+) {
+    if (certs.length === 0) {
+        return;
+    }
+
+    if (!supportsCertReferences(version)) {
+        logger.debug(
+            `Newt ${newtId} (version ${version}) does not support cert references, skipping certs/add`
+        );
+        return;
+    }
+
+    await sendToClient(
+        newtId,
+        {
+            type: "newt/certs/add",
+            data: certs
+        },
+        {
+            incrementConfigVersion: true,
+            compress: canCompress(version, "newt")
+        }
+    );
+}
+
+/**
+ * Tells a newt client to drop the given cert IDs, e.g. once the server knows
+ * no target references them anymore.
+ */
+export async function sendCertsRemove(
+    newtId: string,
+    certIds: string[],
+    version?: string | null
+) {
+    if (certIds.length === 0) {
+        return;
+    }
+
+    if (!supportsCertReferences(version)) {
+        logger.debug(
+            `Newt ${newtId} (version ${version}) does not support cert references, skipping certs/remove`
+        );
+        return;
+    }
+
+    await sendToClient(
+        newtId,
+        {
+            type: "newt/certs/remove",
+            data: { ids: certIds }
+        },
+        {
+            incrementConfigVersion: true,
+            compress: canCompress(version, "newt")
+        }
+    );
+}

server/routers/newt/handleNewtGetConfigMessage.ts

@@ -7,7 +7,11 @@ import { eq } from "drizzle-orm";
 import { sendToExitNode } from "#dynamic/lib/exitNodes";
 import { buildClientConfigurationForNewtClient } from "./buildConfiguration";
 import { convertTargetsIfNecessary } from "../client/targets";
-import { canCompress } from "@server/lib/clientVersionChecks";
+import {
+    canCompress,
+    supportsCertReferences
+} from "@server/lib/clientVersionChecks";
+import { dedupeCertsForTargets } from "@server/lib/ip";
 import config from "@server/lib/config";
 import { waitForSiteRebuildIdle } from "@server/lib/rebuildClientAssociations";
 
@@ -119,7 +123,16 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
         exitNode
     );
 
-    const targetsToSend = await convertTargetsIfNecessary(newt.newtId, targets); // for backward compatibility with old newt versions that don't support the new target format
+    // Older newt clients only understand inline tlsCert/tlsKey on each
+    // target, so only switch to certId references once we know the client
+    // can resolve them.
+    let dedupedTargets = targets;
+    let certs: { id: string; cert: string; key: string }[] = [];
+    if (supportsCertReferences(newt.version)) {
+        ({ targets: dedupedTargets, certs } = dedupeCertsForTargets(targets));
+    }
+
+    const targetsToSend = await convertTargetsIfNecessary(newt.newtId, dedupedTargets); // for backward compatibility with old newt versions that don't support the new target format
 
     return {
         message: {
@@ -128,6 +141,7 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
                 ipAddress: site.address,
                 peers,
                 targets: targetsToSend,
+                certs,
                 chainId: chainId
             }
         },

server/routers/newt/sync.ts

@@ -6,7 +6,11 @@ import {
     buildClientConfigurationForNewtClient,
     buildTargetConfigurationForNewtClient
 } from "./buildConfiguration";
-import { canCompress } from "@server/lib/clientVersionChecks";
+import {
+    canCompress,
+    supportsCertReferences
+} from "@server/lib/clientVersionChecks";
+import { dedupeCertsForTargets } from "@server/lib/ip";
 
 export async function sendNewtSyncMessage(newt: Newt, site: Site) {
     const {
@@ -28,6 +32,16 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) {
         site,
         exitNode
     );
+
+    // Older newt clients only understand inline tlsCert/tlsKey on each
+    // target, so only switch to certId references once we know the client
+    // can resolve them.
+    let clientTargets = targets;
+    let certs: { id: string; cert: string; key: string }[] = [];
+    if (supportsCertReferences(newt.version)) {
+        ({ targets: clientTargets, certs } = dedupeCertsForTargets(targets));
+    }
+
     await sendToClient(
         newt.newtId,
         {
@@ -39,7 +53,8 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) {
                 },
                 healthCheckTargets: validHealthCheckTargets,
                 peers: peers,
-                clientTargets: targets,
+                clientTargets: clientTargets,
+                certs: certs,
                 browserGatewayTargets: browserGatewayTargets,
                 remoteExitNodeSubnets: remoteExitNodeSubnets
             }

src/hooks/useOptimisticLabels.ts

@@ -95,7 +95,10 @@ export function useOptimisticLabels({
     }
 
     async function refresh() {
+        // Only refresh if the labels have been modified
+        if (pendingActions.length > 0) {
             router.refresh();
+        }
         setPendingActions([]);
     }