From ecf008a8d9baade72af02ea68409285259fd42c3 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 22 Jul 2026 17:22:17 -0400 Subject: [PATCH] Improve cert retreival and new newt combined certs --- server/lib/certificates.ts | 1 + server/lib/clientVersionChecks.ts | 16 +++ server/lib/ip.ts | 124 ++++++++++++++++-- server/routers/newt/buildConfiguration.ts | 10 +- server/routers/newt/certs.ts | 75 +++++++++++ .../newt/handleNewtGetConfigMessage.ts | 18 ++- server/routers/newt/sync.ts | 19 ++- 7 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 server/routers/newt/certs.ts diff --git a/server/lib/certificates.ts b/server/lib/certificates.ts index f03f37f25..6d24d2996 100644 --- a/server/lib/certificates.ts +++ b/server/lib/certificates.ts @@ -5,6 +5,7 @@ export async function getValidCertificatesForDomains( Array<{ id: number; domain: string; + queriedDomain: string; wildcard: boolean | null; certFile: string | null; keyFile: string | null; diff --git a/server/lib/clientVersionChecks.ts b/server/lib/clientVersionChecks.ts index 330959e7c..4c9a11c14 100644 --- a/server/lib/clientVersionChecks.ts +++ b/server/lib/clientVersionChecks.ts @@ -18,3 +18,19 @@ export function canCompress( return false; } } + +// Whether this newt client understands `tlsCertId` references into the +// sync message's `certs` array, instead of requiring each target to carry +// its own inline `tlsCert`/`tlsKey` PEM data. Bump the version floor here to +// match whatever release first ships the newt-side support. +export function supportsCertReferences( + clientVersion: string | null | undefined +): boolean { + try { + if (!clientVersion) return false; + if (!semver.valid(clientVersion)) return false; + return semver.gte(clientVersion, "1.16.0"); + } catch { + return false; + } +} diff --git a/server/lib/ip.ts b/server/lib/ip.ts index fb161df1c..e8f589424 100644 --- a/server/lib/ip.ts +++ b/server/lib/ip.ts @@ -5,6 +5,7 @@ import config from "@server/lib/config"; import z from "zod"; import logger from "@server/logger"; import semver from "semver"; +import { createHash } from "crypto"; import { getValidCertificatesForDomains } from "#dynamic/lib/certificates"; import { lockManager } from "#dynamic/lib/lock"; @@ -648,21 +649,101 @@ export type SubnetProxyTargetV2 = { httpTargets?: HTTPTarget[]; tlsCert?: string; tlsKey?: string; + tlsCertId?: string; // references an entry in the sync message's top-level `certs` array instead of inlining tlsCert/tlsKey }; +export type CertRef = { id: string; cert: string; key: string }; + +/** + * Replaces each target's inline tlsCert/tlsKey with a tlsCertId reference + * into a deduplicated certs array, so that many targets sharing the same + * certificate (e.g. a wildcard cert used by thousands of site resources) + * only need that certificate sent once per sync message. + */ +export function dedupeCertsForTargets( + targetsV2: SubnetProxyTargetV2[] +): { targets: SubnetProxyTargetV2[]; certs: CertRef[] } { + const idByContent = new Map(); + const certs: CertRef[] = []; + + const targets = targetsV2.map((target) => { + if (!target.tlsCert || !target.tlsKey) { + return target; + } + + const contentKey = `${target.tlsCert}|${target.tlsKey}`; + let id = idByContent.get(contentKey); + if (!id) { + id = createHash("sha1").update(contentKey).digest("hex").slice(0, 16); + idByContent.set(contentKey, id); + certs.push({ id, cert: target.tlsCert, key: target.tlsKey }); + } + + const { tlsCert, tlsKey, ...rest } = target; + return { ...rest, tlsCertId: id }; + }); + + return { targets, certs }; +} + export type HTTPTarget = { destAddr: string; // must be an IP or hostname destPort: number; scheme: "http" | "https"; }; +export type CertByDomain = Map; + +/** + * Fetches the TLS certificates for every enabled, SSL-enabled HTTP site + * resource's fullDomain in a single batched call, instead of one call per + * resource. Many resources commonly resolve to the very same certificate + * (e.g. a wildcard covering the org's domain), so batching turns what would + * be N concurrent DB/cache round-trips into one, and a lookup failure fails + * loudly for the whole batch rather than silently dropping the cert on a + * random subset of otherwise-identical resources under load. + */ +export async function batchFetchCertsForSiteResources( + allSiteResources: SiteResource[] +): Promise { + const domains = new Set( + allSiteResources + .filter((r) => r.enabled && r.mode === "http" && r.ssl && r.fullDomain) + .map((r) => r.fullDomain as string) + ); + + const certByDomain: CertByDomain = new Map(); + if (domains.size === 0) { + return certByDomain; + } + + try { + const certResults = await getValidCertificatesForDomains(domains, true); + for (const cert of certResults) { + if (cert.certFile && cert.keyFile) { + certByDomain.set(cert.queriedDomain, { + certFile: cert.certFile, + keyFile: cert.keyFile + }); + } + } + } catch (err) { + logger.error( + `Failed to batch-retrieve certificates for ${domains.size} domain(s): ${err}` + ); + } + + return certByDomain; +} + export async function generateSubnetProxyTargetV2( siteResource: SiteResource, clients: { clientId: number; pubKey: string | null; subnet: string | null; - }[] + }[], + certByDomain?: CertByDomain ): Promise { if (!siteResource.enabled) { logger.debug( @@ -750,23 +831,40 @@ export async function generateSubnetProxyTargetV2( let tlsKey: string | undefined; if (siteResource.ssl && siteResource.fullDomain) { - try { - const certs = await getValidCertificatesForDomains( - new Set([siteResource.fullDomain]), - true - ); - if (certs.length > 0 && certs[0].certFile && certs[0].keyFile) { - tlsCert = certs[0].certFile; - tlsKey = certs[0].keyFile; + if (certByDomain) { + // Caller batch-fetched certs for all resources up front (the + // common, high-scale path) — just look up this resource's + // domain rather than issuing its own DB/cache round-trip. + const cert = certByDomain.get(siteResource.fullDomain); + if (cert) { + tlsCert = cert.certFile; + tlsKey = cert.keyFile; } else { logger.warn( `No valid certificate found for SSL site resource ${siteResource.siteResourceId} with domain ${siteResource.fullDomain}` ); } - } catch (err) { - logger.error( - `Failed to retrieve certificate for site resource ${siteResource.siteResourceId} domain ${siteResource.fullDomain}: ${err}` - ); + } else { + // No batched map supplied by the caller — fall back to a + // single-domain lookup for this resource alone. + try { + const certs = await getValidCertificatesForDomains( + new Set([siteResource.fullDomain]), + true + ); + if (certs.length > 0 && certs[0].certFile && certs[0].keyFile) { + tlsCert = certs[0].certFile; + tlsKey = certs[0].keyFile; + } else { + logger.warn( + `No valid certificate found for SSL site resource ${siteResource.siteResourceId} with domain ${siteResource.fullDomain}` + ); + } + } catch (err) { + logger.error( + `Failed to retrieve certificate for site resource ${siteResource.siteResourceId} domain ${siteResource.fullDomain}: ${err}` + ); + } } } diff --git a/server/routers/newt/buildConfiguration.ts b/server/routers/newt/buildConfiguration.ts index a12d035a5..ee33a2c94 100644 --- a/server/routers/newt/buildConfiguration.ts +++ b/server/routers/newt/buildConfiguration.ts @@ -19,6 +19,7 @@ import { eq, and, inArray } from "drizzle-orm"; import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; import { + batchFetchCertsForSiteResources, formatEndpoint, generateSubnetProxyTargetV2, SubnetProxyTargetV2 @@ -206,11 +207,18 @@ export async function buildClientConfigurationForNewtClient( }); } + // Batch-fetch certs for every SSL-enabled HTTP resource's domain in one + // call rather than letting each resource fetch its own — with thousands + // of resources this avoids a concurrent DB/cache stampede for what is + // often the very same (e.g. wildcard) certificate. + const certByDomain = await batchFetchCertsForSiteResources(allSiteResources); + const resourceTargetsArr = await Promise.all( allSiteResources.map((resource) => generateSubnetProxyTargetV2( resource, - clientsByResourceId.get(resource.siteResourceId) ?? [] + clientsByResourceId.get(resource.siteResourceId) ?? [], + certByDomain ) ) ); diff --git a/server/routers/newt/certs.ts b/server/routers/newt/certs.ts new file mode 100644 index 000000000..8898ee564 --- /dev/null +++ b/server/routers/newt/certs.ts @@ -0,0 +1,75 @@ +import { sendToClient } from "#dynamic/routers/ws"; +import logger from "@server/logger"; +import { + canCompress, + supportsCertReferences +} from "@server/lib/clientVersionChecks"; +import { CertRef } from "@server/lib/ip"; + +/** + * Pushes an incremental set of certs to a newt client outside of a full + * newt/sync or newt/wg/receive-config, e.g. after a certificate renewal so + * that every target referencing it (by tlsCertId) picks up the new material + * without waiting for the next full resync. + */ +export async function sendCertsAdd( + newtId: string, + certs: CertRef[], + version?: string | null +) { + if (certs.length === 0) { + return; + } + + if (!supportsCertReferences(version)) { + logger.debug( + `Newt ${newtId} (version ${version}) does not support cert references, skipping certs/add` + ); + return; + } + + await sendToClient( + newtId, + { + type: "newt/certs/add", + data: certs + }, + { + incrementConfigVersion: true, + compress: canCompress(version, "newt") + } + ); +} + +/** + * Tells a newt client to drop the given cert IDs, e.g. once the server knows + * no target references them anymore. + */ +export async function sendCertsRemove( + newtId: string, + certIds: string[], + version?: string | null +) { + if (certIds.length === 0) { + return; + } + + if (!supportsCertReferences(version)) { + logger.debug( + `Newt ${newtId} (version ${version}) does not support cert references, skipping certs/remove` + ); + return; + } + + await sendToClient( + newtId, + { + type: "newt/certs/remove", + data: { ids: certIds } + }, + { + incrementConfigVersion: true, + compress: canCompress(version, "newt") + } + ); +} diff --git a/server/routers/newt/handleNewtGetConfigMessage.ts b/server/routers/newt/handleNewtGetConfigMessage.ts index d99dd89ef..cb8b8d931 100644 --- a/server/routers/newt/handleNewtGetConfigMessage.ts +++ b/server/routers/newt/handleNewtGetConfigMessage.ts @@ -7,7 +7,11 @@ import { eq } from "drizzle-orm"; import { sendToExitNode } from "#dynamic/lib/exitNodes"; import { buildClientConfigurationForNewtClient } from "./buildConfiguration"; import { convertTargetsIfNecessary } from "../client/targets"; -import { canCompress } from "@server/lib/clientVersionChecks"; +import { + canCompress, + supportsCertReferences +} from "@server/lib/clientVersionChecks"; +import { dedupeCertsForTargets } from "@server/lib/ip"; import config from "@server/lib/config"; import { waitForSiteRebuildIdle } from "@server/lib/rebuildClientAssociations"; @@ -119,7 +123,16 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => { exitNode ); - const targetsToSend = await convertTargetsIfNecessary(newt.newtId, targets); // for backward compatibility with old newt versions that don't support the new target format + // Older newt clients only understand inline tlsCert/tlsKey on each + // target, so only switch to certId references once we know the client + // can resolve them. + let dedupedTargets = targets; + let certs: { id: string; cert: string; key: string }[] = []; + if (supportsCertReferences(newt.version)) { + ({ targets: dedupedTargets, certs } = dedupeCertsForTargets(targets)); + } + + const targetsToSend = await convertTargetsIfNecessary(newt.newtId, dedupedTargets); // for backward compatibility with old newt versions that don't support the new target format return { message: { @@ -128,6 +141,7 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => { ipAddress: site.address, peers, targets: targetsToSend, + certs, chainId: chainId } }, diff --git a/server/routers/newt/sync.ts b/server/routers/newt/sync.ts index 30e2d59f8..e88f71bee 100644 --- a/server/routers/newt/sync.ts +++ b/server/routers/newt/sync.ts @@ -6,7 +6,11 @@ import { buildClientConfigurationForNewtClient, buildTargetConfigurationForNewtClient } from "./buildConfiguration"; -import { canCompress } from "@server/lib/clientVersionChecks"; +import { + canCompress, + supportsCertReferences +} from "@server/lib/clientVersionChecks"; +import { dedupeCertsForTargets } from "@server/lib/ip"; export async function sendNewtSyncMessage(newt: Newt, site: Site) { const { @@ -28,6 +32,16 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) { site, exitNode ); + + // Older newt clients only understand inline tlsCert/tlsKey on each + // target, so only switch to certId references once we know the client + // can resolve them. + let clientTargets = targets; + let certs: { id: string; cert: string; key: string }[] = []; + if (supportsCertReferences(newt.version)) { + ({ targets: clientTargets, certs } = dedupeCertsForTargets(targets)); + } + await sendToClient( newt.newtId, { @@ -39,7 +53,8 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) { }, healthCheckTargets: validHealthCheckTargets, peers: peers, - clientTargets: targets, + clientTargets: clientTargets, + certs: certs, browserGatewayTargets: browserGatewayTargets, remoteExitNodeSubnets: remoteExitNodeSubnets }