mirror of
https://github.com/fosrl/pangolin.git
synced 2026-07-23 14:04:12 +02:00
Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a85e9585e2 | |||
| 71f45b8a80 | |||
| ecf008a8d9 | |||
| 19c1c2042b | |||
| 56fcb80b23 | |||
| 41c68148a9 | |||
| c30fe5b574 | |||
| 113b7a0b84 | |||
| 5d20956a0e | |||
| e5398d441e | |||
| d4138e2141 | |||
| 09d5d9082e | |||
| 5cb316f4e9 | |||
| 6b6c9cf4d8 | |||
| 05617c63c0 | |||
| d4c52bbf2f | |||
| 87f50bf0cc | |||
| 2c66da1b19 | |||
| ab19955502 | |||
| 1db9dcec81 | |||
| 49c2d3163e | |||
| 45b9e13a13 |
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
FROM node:24-alpine
|
FROM node:26-alpine
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ services:
|
|||||||
- 80:80 # Port for traefik because of the network_mode
|
- 80:80 # Port for traefik because of the network_mode
|
||||||
|
|
||||||
traefik:
|
traefik:
|
||||||
image: traefik:v3.6
|
image: traefik:v3.7
|
||||||
container_name: traefik
|
container_name: traefik
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
network_mode: service:gerbil # Ports appear on the gerbil service
|
network_mode: service:gerbil # Ports appear on the gerbil service
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ services:
|
|||||||
- 80:80{{end}}
|
- 80:80{{end}}
|
||||||
|
|
||||||
traefik:
|
traefik:
|
||||||
image: docker.io/traefik:v3.6
|
image: docker.io/traefik:v3.7
|
||||||
container_name: traefik
|
container_name: traefik
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
{{if .InstallGerbil}}network_mode: service:gerbil # Ports appear on the gerbil service{{end}}{{if not .InstallGerbil}}
|
{{if .InstallGerbil}}network_mode: service:gerbil # Ports appear on the gerbil service{{end}}{{if not .InstallGerbil}}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ export async function getValidCertificatesForDomains(
|
|||||||
Array<{
|
Array<{
|
||||||
id: number;
|
id: number;
|
||||||
domain: string;
|
domain: string;
|
||||||
|
queriedDomain: string;
|
||||||
wildcard: boolean | null;
|
wildcard: boolean | null;
|
||||||
certFile: string | null;
|
certFile: string | null;
|
||||||
keyFile: string | null;
|
keyFile: string | null;
|
||||||
|
|||||||
@@ -18,3 +18,19 @@ export function canCompress(
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Whether this newt client understands `tlsCertId` references into the
|
||||||
|
// sync message's `certs` array, instead of requiring each target to carry
|
||||||
|
// its own inline `tlsCert`/`tlsKey` PEM data. Bump the version floor here to
|
||||||
|
// match whatever release first ships the newt-side support.
|
||||||
|
export function supportsCertReferences(
|
||||||
|
clientVersion: string | null | undefined
|
||||||
|
): boolean {
|
||||||
|
try {
|
||||||
|
if (!clientVersion) return false;
|
||||||
|
if (!semver.valid(clientVersion)) return false;
|
||||||
|
return semver.gte(clientVersion, "1.16.0");
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+99
-1
@@ -5,6 +5,7 @@ import config from "@server/lib/config";
|
|||||||
import z from "zod";
|
import z from "zod";
|
||||||
import logger from "@server/logger";
|
import logger from "@server/logger";
|
||||||
import semver from "semver";
|
import semver from "semver";
|
||||||
|
import { createHash } from "crypto";
|
||||||
import { getValidCertificatesForDomains } from "#dynamic/lib/certificates";
|
import { getValidCertificatesForDomains } from "#dynamic/lib/certificates";
|
||||||
import { lockManager } from "#dynamic/lib/lock";
|
import { lockManager } from "#dynamic/lib/lock";
|
||||||
|
|
||||||
@@ -648,21 +649,101 @@ export type SubnetProxyTargetV2 = {
|
|||||||
httpTargets?: HTTPTarget[];
|
httpTargets?: HTTPTarget[];
|
||||||
tlsCert?: string;
|
tlsCert?: string;
|
||||||
tlsKey?: string;
|
tlsKey?: string;
|
||||||
|
tlsCertId?: string; // references an entry in the sync message's top-level `certs` array instead of inlining tlsCert/tlsKey
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type CertRef = { id: string; cert: string; key: string };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replaces each target's inline tlsCert/tlsKey with a tlsCertId reference
|
||||||
|
* into a deduplicated certs array, so that many targets sharing the same
|
||||||
|
* certificate (e.g. a wildcard cert used by thousands of site resources)
|
||||||
|
* only need that certificate sent once per sync message.
|
||||||
|
*/
|
||||||
|
export function dedupeCertsForTargets(
|
||||||
|
targetsV2: SubnetProxyTargetV2[]
|
||||||
|
): { targets: SubnetProxyTargetV2[]; certs: CertRef[] } {
|
||||||
|
const idByContent = new Map<string, string>();
|
||||||
|
const certs: CertRef[] = [];
|
||||||
|
|
||||||
|
const targets = targetsV2.map((target) => {
|
||||||
|
if (!target.tlsCert || !target.tlsKey) {
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentKey = `${target.tlsCert}|${target.tlsKey}`;
|
||||||
|
let id = idByContent.get(contentKey);
|
||||||
|
if (!id) {
|
||||||
|
id = createHash("sha1").update(contentKey).digest("hex").slice(0, 16);
|
||||||
|
idByContent.set(contentKey, id);
|
||||||
|
certs.push({ id, cert: target.tlsCert, key: target.tlsKey });
|
||||||
|
}
|
||||||
|
|
||||||
|
const { tlsCert, tlsKey, ...rest } = target;
|
||||||
|
return { ...rest, tlsCertId: id };
|
||||||
|
});
|
||||||
|
|
||||||
|
return { targets, certs };
|
||||||
|
}
|
||||||
|
|
||||||
export type HTTPTarget = {
|
export type HTTPTarget = {
|
||||||
destAddr: string; // must be an IP or hostname
|
destAddr: string; // must be an IP or hostname
|
||||||
destPort: number;
|
destPort: number;
|
||||||
scheme: "http" | "https";
|
scheme: "http" | "https";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type CertByDomain = Map<string, { certFile: string; keyFile: string }>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetches the TLS certificates for every enabled, SSL-enabled HTTP site
|
||||||
|
* resource's fullDomain in a single batched call, instead of one call per
|
||||||
|
* resource. Many resources commonly resolve to the very same certificate
|
||||||
|
* (e.g. a wildcard covering the org's domain), so batching turns what would
|
||||||
|
* be N concurrent DB/cache round-trips into one, and a lookup failure fails
|
||||||
|
* loudly for the whole batch rather than silently dropping the cert on a
|
||||||
|
* random subset of otherwise-identical resources under load.
|
||||||
|
*/
|
||||||
|
export async function batchFetchCertsForSiteResources(
|
||||||
|
allSiteResources: SiteResource[]
|
||||||
|
): Promise<CertByDomain> {
|
||||||
|
const domains = new Set(
|
||||||
|
allSiteResources
|
||||||
|
.filter((r) => r.enabled && r.mode === "http" && r.ssl && r.fullDomain)
|
||||||
|
.map((r) => r.fullDomain as string)
|
||||||
|
);
|
||||||
|
|
||||||
|
const certByDomain: CertByDomain = new Map();
|
||||||
|
if (domains.size === 0) {
|
||||||
|
return certByDomain;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const certResults = await getValidCertificatesForDomains(domains, true);
|
||||||
|
for (const cert of certResults) {
|
||||||
|
if (cert.certFile && cert.keyFile) {
|
||||||
|
certByDomain.set(cert.queriedDomain, {
|
||||||
|
certFile: cert.certFile,
|
||||||
|
keyFile: cert.keyFile
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`Failed to batch-retrieve certificates for ${domains.size} domain(s): ${err}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return certByDomain;
|
||||||
|
}
|
||||||
|
|
||||||
export async function generateSubnetProxyTargetV2(
|
export async function generateSubnetProxyTargetV2(
|
||||||
siteResource: SiteResource,
|
siteResource: SiteResource,
|
||||||
clients: {
|
clients: {
|
||||||
clientId: number;
|
clientId: number;
|
||||||
pubKey: string | null;
|
pubKey: string | null;
|
||||||
subnet: string | null;
|
subnet: string | null;
|
||||||
}[]
|
}[],
|
||||||
|
certByDomain?: CertByDomain
|
||||||
): Promise<SubnetProxyTargetV2[] | undefined> {
|
): Promise<SubnetProxyTargetV2[] | undefined> {
|
||||||
if (!siteResource.enabled) {
|
if (!siteResource.enabled) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
@@ -750,6 +831,22 @@ export async function generateSubnetProxyTargetV2(
|
|||||||
let tlsKey: string | undefined;
|
let tlsKey: string | undefined;
|
||||||
|
|
||||||
if (siteResource.ssl && siteResource.fullDomain) {
|
if (siteResource.ssl && siteResource.fullDomain) {
|
||||||
|
if (certByDomain) {
|
||||||
|
// Caller batch-fetched certs for all resources up front (the
|
||||||
|
// common, high-scale path) — just look up this resource's
|
||||||
|
// domain rather than issuing its own DB/cache round-trip.
|
||||||
|
const cert = certByDomain.get(siteResource.fullDomain);
|
||||||
|
if (cert) {
|
||||||
|
tlsCert = cert.certFile;
|
||||||
|
tlsKey = cert.keyFile;
|
||||||
|
} else {
|
||||||
|
logger.warn(
|
||||||
|
`No valid certificate found for SSL site resource ${siteResource.siteResourceId} with domain ${siteResource.fullDomain}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No batched map supplied by the caller — fall back to a
|
||||||
|
// single-domain lookup for this resource alone.
|
||||||
try {
|
try {
|
||||||
const certs = await getValidCertificatesForDomains(
|
const certs = await getValidCertificatesForDomains(
|
||||||
new Set([siteResource.fullDomain]),
|
new Set([siteResource.fullDomain]),
|
||||||
@@ -769,6 +866,7 @@ export async function generateSubnetProxyTargetV2(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
targets.push({
|
targets.push({
|
||||||
sourcePrefixes: [],
|
sourcePrefixes: [],
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import { eq, and, inArray } from "drizzle-orm";
|
|||||||
import config from "@server/lib/config";
|
import config from "@server/lib/config";
|
||||||
import { decrypt } from "@server/lib/crypto";
|
import { decrypt } from "@server/lib/crypto";
|
||||||
import {
|
import {
|
||||||
|
batchFetchCertsForSiteResources,
|
||||||
formatEndpoint,
|
formatEndpoint,
|
||||||
generateSubnetProxyTargetV2,
|
generateSubnetProxyTargetV2,
|
||||||
SubnetProxyTargetV2
|
SubnetProxyTargetV2
|
||||||
@@ -206,11 +207,18 @@ export async function buildClientConfigurationForNewtClient(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Batch-fetch certs for every SSL-enabled HTTP resource's domain in one
|
||||||
|
// call rather than letting each resource fetch its own — with thousands
|
||||||
|
// of resources this avoids a concurrent DB/cache stampede for what is
|
||||||
|
// often the very same (e.g. wildcard) certificate.
|
||||||
|
const certByDomain = await batchFetchCertsForSiteResources(allSiteResources);
|
||||||
|
|
||||||
const resourceTargetsArr = await Promise.all(
|
const resourceTargetsArr = await Promise.all(
|
||||||
allSiteResources.map((resource) =>
|
allSiteResources.map((resource) =>
|
||||||
generateSubnetProxyTargetV2(
|
generateSubnetProxyTargetV2(
|
||||||
resource,
|
resource,
|
||||||
clientsByResourceId.get(resource.siteResourceId) ?? []
|
clientsByResourceId.get(resource.siteResourceId) ?? [],
|
||||||
|
certByDomain
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { sendToClient } from "#dynamic/routers/ws";
|
||||||
|
import logger from "@server/logger";
|
||||||
|
import {
|
||||||
|
canCompress,
|
||||||
|
supportsCertReferences
|
||||||
|
} from "@server/lib/clientVersionChecks";
|
||||||
|
import { CertRef } from "@server/lib/ip";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pushes an incremental set of certs to a newt client outside of a full
|
||||||
|
* newt/sync or newt/wg/receive-config, e.g. after a certificate renewal so
|
||||||
|
* that every target referencing it (by tlsCertId) picks up the new material
|
||||||
|
* without waiting for the next full resync.
|
||||||
|
*/
|
||||||
|
export async function sendCertsAdd(
|
||||||
|
newtId: string,
|
||||||
|
certs: CertRef[],
|
||||||
|
version?: string | null
|
||||||
|
) {
|
||||||
|
if (certs.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!supportsCertReferences(version)) {
|
||||||
|
logger.debug(
|
||||||
|
`Newt ${newtId} (version ${version}) does not support cert references, skipping certs/add`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await sendToClient(
|
||||||
|
newtId,
|
||||||
|
{
|
||||||
|
type: "newt/certs/add",
|
||||||
|
data: certs
|
||||||
|
},
|
||||||
|
{
|
||||||
|
incrementConfigVersion: true,
|
||||||
|
compress: canCompress(version, "newt")
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tells a newt client to drop the given cert IDs, e.g. once the server knows
|
||||||
|
* no target references them anymore.
|
||||||
|
*/
|
||||||
|
export async function sendCertsRemove(
|
||||||
|
newtId: string,
|
||||||
|
certIds: string[],
|
||||||
|
version?: string | null
|
||||||
|
) {
|
||||||
|
if (certIds.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!supportsCertReferences(version)) {
|
||||||
|
logger.debug(
|
||||||
|
`Newt ${newtId} (version ${version}) does not support cert references, skipping certs/remove`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await sendToClient(
|
||||||
|
newtId,
|
||||||
|
{
|
||||||
|
type: "newt/certs/remove",
|
||||||
|
data: { ids: certIds }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
incrementConfigVersion: true,
|
||||||
|
compress: canCompress(version, "newt")
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -7,7 +7,11 @@ import { eq } from "drizzle-orm";
|
|||||||
import { sendToExitNode } from "#dynamic/lib/exitNodes";
|
import { sendToExitNode } from "#dynamic/lib/exitNodes";
|
||||||
import { buildClientConfigurationForNewtClient } from "./buildConfiguration";
|
import { buildClientConfigurationForNewtClient } from "./buildConfiguration";
|
||||||
import { convertTargetsIfNecessary } from "../client/targets";
|
import { convertTargetsIfNecessary } from "../client/targets";
|
||||||
import { canCompress } from "@server/lib/clientVersionChecks";
|
import {
|
||||||
|
canCompress,
|
||||||
|
supportsCertReferences
|
||||||
|
} from "@server/lib/clientVersionChecks";
|
||||||
|
import { dedupeCertsForTargets } from "@server/lib/ip";
|
||||||
import config from "@server/lib/config";
|
import config from "@server/lib/config";
|
||||||
import { waitForSiteRebuildIdle } from "@server/lib/rebuildClientAssociations";
|
import { waitForSiteRebuildIdle } from "@server/lib/rebuildClientAssociations";
|
||||||
|
|
||||||
@@ -119,7 +123,16 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
|
|||||||
exitNode
|
exitNode
|
||||||
);
|
);
|
||||||
|
|
||||||
const targetsToSend = await convertTargetsIfNecessary(newt.newtId, targets); // for backward compatibility with old newt versions that don't support the new target format
|
// Older newt clients only understand inline tlsCert/tlsKey on each
|
||||||
|
// target, so only switch to certId references once we know the client
|
||||||
|
// can resolve them.
|
||||||
|
let dedupedTargets = targets;
|
||||||
|
let certs: { id: string; cert: string; key: string }[] = [];
|
||||||
|
if (supportsCertReferences(newt.version)) {
|
||||||
|
({ targets: dedupedTargets, certs } = dedupeCertsForTargets(targets));
|
||||||
|
}
|
||||||
|
|
||||||
|
const targetsToSend = await convertTargetsIfNecessary(newt.newtId, dedupedTargets); // for backward compatibility with old newt versions that don't support the new target format
|
||||||
|
|
||||||
return {
|
return {
|
||||||
message: {
|
message: {
|
||||||
@@ -128,6 +141,7 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
|
|||||||
ipAddress: site.address,
|
ipAddress: site.address,
|
||||||
peers,
|
peers,
|
||||||
targets: targetsToSend,
|
targets: targetsToSend,
|
||||||
|
certs,
|
||||||
chainId: chainId
|
chainId: chainId
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -6,7 +6,11 @@ import {
|
|||||||
buildClientConfigurationForNewtClient,
|
buildClientConfigurationForNewtClient,
|
||||||
buildTargetConfigurationForNewtClient
|
buildTargetConfigurationForNewtClient
|
||||||
} from "./buildConfiguration";
|
} from "./buildConfiguration";
|
||||||
import { canCompress } from "@server/lib/clientVersionChecks";
|
import {
|
||||||
|
canCompress,
|
||||||
|
supportsCertReferences
|
||||||
|
} from "@server/lib/clientVersionChecks";
|
||||||
|
import { dedupeCertsForTargets } from "@server/lib/ip";
|
||||||
|
|
||||||
export async function sendNewtSyncMessage(newt: Newt, site: Site) {
|
export async function sendNewtSyncMessage(newt: Newt, site: Site) {
|
||||||
const {
|
const {
|
||||||
@@ -28,6 +32,16 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) {
|
|||||||
site,
|
site,
|
||||||
exitNode
|
exitNode
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Older newt clients only understand inline tlsCert/tlsKey on each
|
||||||
|
// target, so only switch to certId references once we know the client
|
||||||
|
// can resolve them.
|
||||||
|
let clientTargets = targets;
|
||||||
|
let certs: { id: string; cert: string; key: string }[] = [];
|
||||||
|
if (supportsCertReferences(newt.version)) {
|
||||||
|
({ targets: clientTargets, certs } = dedupeCertsForTargets(targets));
|
||||||
|
}
|
||||||
|
|
||||||
await sendToClient(
|
await sendToClient(
|
||||||
newt.newtId,
|
newt.newtId,
|
||||||
{
|
{
|
||||||
@@ -39,7 +53,8 @@ export async function sendNewtSyncMessage(newt: Newt, site: Site) {
|
|||||||
},
|
},
|
||||||
healthCheckTargets: validHealthCheckTargets,
|
healthCheckTargets: validHealthCheckTargets,
|
||||||
peers: peers,
|
peers: peers,
|
||||||
clientTargets: targets,
|
clientTargets: clientTargets,
|
||||||
|
certs: certs,
|
||||||
browserGatewayTargets: browserGatewayTargets,
|
browserGatewayTargets: browserGatewayTargets,
|
||||||
remoteExitNodeSubnets: remoteExitNodeSubnets
|
remoteExitNodeSubnets: remoteExitNodeSubnets
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -95,7 +95,10 @@ export function useOptimisticLabels({
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function refresh() {
|
async function refresh() {
|
||||||
|
// Only refresh if the labels have been modified
|
||||||
|
if (pendingActions.length > 0) {
|
||||||
router.refresh();
|
router.refresh();
|
||||||
|
}
|
||||||
setPendingActions([]);
|
setPendingActions([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user