Merge pull request #3702 from fosrl/dev

1.22.2

messages/en-US.json

@@ -3477,8 +3477,7 @@
     },
     "priority": "Priority",
     "priorityDescription": "Higher priority routes are evaluated first. Priority = 100 means automatic ordering (system decides). Use another number to enforce manual priority.",
-    "instanceName": "Server ID",
+    "instanceName": "Instance Name",
-    "clearInstanceName": "Reset Server Association",
     "pathMatchModalTitle": "Configure Path Matching",
     "pathMatchModalDescription": "Set up how incoming requests should be matched based on their path.",
     "pathMatchType": "Match Type",

package-lock.json

@@ -50,7 +50,6 @@
                 "@xterm/addon-fit": "^0.11.0",
                 "@xterm/addon-web-links": "^0.12.0",
                 "@xterm/xterm": "^6.0.0",
-                "acme-client": "^5.4.0",
                 "arctic": "3.7.0",
                 "axios": "1.20.0",
                 "better-sqlite3": "11.9.1",
@@ -62,7 +61,6 @@
                 "cors": "2.8.6",
                 "crypto-js": "4.2.0",
                 "d3": "7.9.0",
-                "dns-packet": "^5.6.1",
                 "drizzle-orm": "0.45.2",
                 "express": "5.2.1",
                 "express-rate-limit": "8.7.0",
@@ -126,7 +124,6 @@
                 "@types/cors": "2.8.19",
                 "@types/crypto-js": "4.2.2",
                 "@types/d3": "7.4.3",
-                "@types/dns-packet": "^5.6.5",
                 "@types/express": "5.0.6",
                 "@types/express-session": "1.19.0",
                 "@types/jmespath": "0.15.2",
@@ -2448,12 +2445,6 @@
                 "@jridgewell/sourcemap-codec": "^1.4.14"
             }
         },
-        "node_modules/@leichtgewicht/ip-codec": {
-            "version": "2.0.5",
-            "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz",
-            "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==",
-            "license": "MIT"
-        },
         "node_modules/@levischuck/tiny-cbor": {
             "version": "0.2.11",
             "resolved": "https://registry.npmjs.org/@levischuck/tiny-cbor/-/tiny-cbor-0.2.11.tgz",
@@ -6773,16 +6764,6 @@
                 "@types/d3-selection": "*"
             }
         },
-        "node_modules/@types/dns-packet": {
-            "version": "5.6.5",
-            "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.6.5.tgz",
-            "integrity": "sha512-qXOC7XLOEe43ehtWJCMnQXvgcIpv6rPmQ1jXT98Ad8A3TB1Ue50jsCbSSSyuazScEuZ/Q026vHbrOTVkmwA+7Q==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "@types/node": "*"
-            }
-        },
         "node_modules/@types/esrecurse": {
             "version": "4.3.1",
             "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz",
@@ -7454,22 +7435,6 @@
                 "node": ">= 0.6"
             }
         },
-        "node_modules/acme-client": {
-            "version": "5.4.0",
-            "resolved": "https://registry.npmjs.org/acme-client/-/acme-client-5.4.0.tgz",
-            "integrity": "sha512-mORqg60S8iML6XSmVjqjGHJkINrCGLMj2QvDmFzI9vIlv1RGlyjmw3nrzaINJjkNsYXC41XhhD5pfy7CtuGcbA==",
-            "license": "MIT",
-            "dependencies": {
-                "@peculiar/x509": "^1.11.0",
-                "asn1js": "^3.0.5",
-                "axios": "^1.7.2",
-                "debug": "^4.3.5",
-                "node-forge": "^1.3.1"
-            },
-            "engines": {
-                "node": ">= 16"
-            }
-        },
         "node_modules/acorn": {
             "version": "8.16.0",
             "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz",
@@ -9296,18 +9261,6 @@
                 "node": ">=8"
             }
         },
-        "node_modules/dns-packet": {
-            "version": "5.6.1",
-            "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz",
-            "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==",
-            "license": "MIT",
-            "dependencies": {
-                "@leichtgewicht/ip-codec": "^2.0.1"
-            },
-            "engines": {
-                "node": ">=6"
-            }
-        },
         "node_modules/doctrine": {
             "version": "2.1.0",
             "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
@@ -13187,15 +13140,6 @@
                 "semver": "bin/semver.js"
             }
         },
-        "node_modules/node-forge": {
-            "version": "1.4.0",
-            "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz",
-            "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==",
-            "license": "(BSD-3-Clause OR GPL-2.0)",
-            "engines": {
-                "node": ">= 6.13.0"
-            }
-        },
         "node_modules/node-releases": {
             "version": "2.0.54",
             "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz",

package.json

@@ -73,7 +73,6 @@
         "@xterm/addon-fit": "^0.11.0",
         "@xterm/addon-web-links": "^0.12.0",
         "@xterm/xterm": "^6.0.0",
-        "acme-client": "^5.4.0",
         "arctic": "3.7.0",
         "axios": "1.20.0",
         "better-sqlite3": "11.9.1",
@@ -85,7 +84,6 @@
         "cors": "2.8.6",
         "crypto-js": "4.2.0",
         "d3": "7.9.0",
-        "dns-packet": "^5.6.1",
         "drizzle-orm": "0.45.2",
         "express": "5.2.1",
         "express-rate-limit": "8.7.0",
@@ -149,7 +147,6 @@
         "@types/cors": "2.8.19",
         "@types/crypto-js": "4.2.2",
         "@types/d3": "7.4.3",
-        "@types/dns-packet": "^5.6.5",
         "@types/express": "5.0.6",
         "@types/express-session": "1.19.0",
         "@types/jmespath": "0.15.2",

server/certificates.ts

@@ -1,8 +0,0 @@
-export async function startCertificateManager() {
-    // No-op: ACME certificate generation/management is only available in
-    // builds that include the private/enterprise feature set.
-}
-
-export async function stopCertificateManager() {
-    // No-op counterpart to startCertificateManager.
-}

server/dns.ts

@@ -1,8 +0,0 @@
-export async function startDnsServer() {
-    // No-op: the authoritative DNS server is only available in builds
-    // that include the private/enterprise feature set.
-}
-
-export async function stopDnsServer() {
-    // No-op counterpart to startDnsServer.
-}

server/index.ts

@@ -25,8 +25,6 @@ import { setHostMeta } from "@server/lib/hostMeta";
 import { TraefikConfigManager } from "@server/lib/traefik/TraefikConfigManager";
 import { initCleanup } from "#dynamic/cleanup";
 import { startSchedulers } from "#dynamic/startSchedulers";
-import { startDnsServer } from "#dynamic/dns";
-import { startCertificateManager } from "#dynamic/certificates";
 import license from "#dynamic/license/license";
 import { fetchServerIp } from "@server/lib/serverIpService";
 import { initAiModelCatalog } from "@server/lib/aiModelCatalog";
@@ -47,10 +45,6 @@ async function startServers() {
 
     startSchedulers();
 
-    await startDnsServer();
-
-    await startCertificateManager();
-
     // Start all servers
     const apiServer = createApiServer();
     const internalServer = createInternalServer();

server/lib/dbRetry.ts

@@ -71,7 +71,7 @@ export async function withRetry<T>(
                 const jitter = Math.random() * baseDelay;
                 const delay = baseDelay + jitter;
                 logger.warn(
-                    `Transient DB issue in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
+                    `Transient DB error in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
                     { code: error?.code ?? error?.cause?.code }
                 );
                 await new Promise((resolve) => setTimeout(resolve, delay));

server/lib/readConfigFile.ts

@@ -348,8 +348,8 @@ export const configSchema = z
                     .optional()
                     .pipe(z.string())
                     .transform((url) => url.toLowerCase()),
-                subnet_group: z.string().optional().default("100.89.137.0/18"),
+                subnet_group: z.string().optional().default("100.89.137.0/20"),
-                block_size: z.number().positive().gt(0).optional().default(22),
+                block_size: z.number().positive().gt(0).optional().default(24),
                 site_block_size: z
                     .number()
                     .positive()

server/lib/retry.ts

@@ -1,72 +0,0 @@
-import logger from "@server/logger";
-
-export async function withRetry<T>(
-    fn: () => Promise<T>,
-    options: {
-        retries?: number;
-        baseDelayMs?: number;
-        label?: string;
-        // Called with each caught error to decide whether it's worth
-        // retrying. Defaults to retrying everything (existing behavior) -
-        // pass this to exclude errors that are known to be permanent (e.g.
-        // an upstream rate limit or validation rejection) rather than
-        // transient, so they fail fast instead of wasting retry attempts.
-        shouldRetry?: (error: unknown) => boolean;
-    } = {}
-): Promise<T> {
-    const {
-        retries = 3,
-        baseDelayMs = 250,
-        label = "operation",
-        shouldRetry = () => true
-    } = options;
-
-    let attempt = 0;
-    while (true) {
-        try {
-            return await fn();
-        } catch (error) {
-            attempt++;
-            if (attempt > retries || !shouldRetry(error)) {
-                throw error;
-            }
-
-            // Exponential backoff with jitter so retries don't all land at once.
-            const delay =
-                baseDelayMs * 2 ** (attempt - 1) * (0.5 + Math.random());
-
-            logger.warn(
-                `${label} failed (attempt ${attempt}/${retries + 1}), retrying in ${delay.toFixed(0)}ms`,
-                error
-            );
-
-            await new Promise((resolve) => setTimeout(resolve, delay));
-        }
-    }
-}
-
-// Bounds an operation that has no timeout of its own (e.g. acme-client's
-// axios instance never sets one, so a stalled TCP connection to the ACME
-// server hangs forever instead of erroring). Without this, a single hung
-// call can leave its caller's promise permanently unsettled - fatal for
-// code that gates future work on that promise resolving, like the
-// scheduler's runExclusive() waiting on a batch's Promise.all.
-export async function withTimeout<T>(
-    promise: Promise<T>,
-    ms: number,
-    label = "operation"
-): Promise<T> {
-    let timer: NodeJS.Timeout;
-    const timeout = new Promise<never>((_, reject) => {
-        timer = setTimeout(
-            () => reject(new Error(`${label} timed out after ${ms}ms`)),
-            ms
-        );
-    });
-
-    try {
-        return await Promise.race([promise, timeout]);
-    } finally {
-        clearTimeout(timer!);
-    }
-}

server/lib/traefik/TraefikConfigManager.ts

@@ -358,7 +358,7 @@ export class TraefikConfigManager {
                 this.lastActiveDomains = new Set(domains);
             }
 
-            if (process.env.CERT_MODE === "pangolin" && build != "oss") {
+            if (process.env.USE_PANGOLIN_DNS === "true" && build != "oss") {
                 // Scan current local certificate state
                 this.lastLocalCertificateState =
                     await this.scanLocalCertificateState();

server/lib/traefik/rule.ts

@@ -1,5 +1,7 @@
-// Build the Host()/HostRegexp() Traefik rule for a resource's domain.
+/**
-// Wildcard resources match any single subdomain via HostRegexp.
+ * Build the Host()/HostRegexp() Traefik rule for a resource's domain.
+ * Wildcard resources match any single subdomain via HostRegexp.
+ */
 export function buildHostRule(
     fullDomain: string,
     wildcard?: boolean | null
@@ -12,8 +14,10 @@ export function buildHostRule(
     return `Host(\`${fullDomain}\`)`;
 }
 
-// Append a path-matching clause to a Traefik rule based on the resource's
+/**
-// configured path and pathMatchType.
+ * Append a path-matching clause to a Traefik rule based on the resource's
+ * configured path and pathMatchType.
+ */
 export function appendPathMatch(
     rule: string,
     path: string | null | undefined,
@@ -36,8 +40,10 @@ export function appendPathMatch(
     return rule;
 }
 
-// Compute the router priority for a resource, favoring an explicit override
+/**
-// and otherwise deriving it from the path match specificity.
+ * Compute the router priority for a resource, favoring an explicit override
+ * and otherwise deriving it from the path match specificity.
+ */
 export function computeRoutePriority(
     priority: number | null | undefined,
     path: string | null | undefined,

server/private/certificates.ts

@@ -1,17 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-export {
-    startCertificateManager,
-    stopCertificateManager
-} from "./lib/certificates";

server/private/cleanup.ts

@@ -20,8 +20,6 @@ import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"
 import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
 import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement";
 import { shutdownAiSessionLogger } from "@server/routers/aiGateway/logAiSession";
-import { stopDnsServer } from "./dns";
-import { stopCertificateManager } from "./certificates";
 
 async function cleanup() {
     await stopPingAccumulator();
@@ -33,8 +31,6 @@ async function cleanup() {
     await rateLimitService.cleanup();
     await wsCleanup();
     await logStreamingManager.shutdown();
-    await stopDnsServer();
-    await stopCertificateManager();
 
     process.exit(0);
 }

server/private/dns.ts

@@ -1,44 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import { AuthoritativeDNSServer } from "#private/lib/dns";
-import { privateConfig } from "#private/lib/config";
-
-let dnsServer: AuthoritativeDNSServer | undefined;
-
-export async function startDnsServer() {
-    const dnsConfig = privateConfig.getRawPrivateConfig().dns;
-    if (!dnsConfig || !dnsConfig.enabled) {
-        return;
-    }
-
-    const cacheOptions = {
-        stdTTL: 300, // 5 minutes default TTL
-        checkperiod: 60, // Check for expired keys every 60 seconds
-        useClones: false // Better performance
-    };
-
-    // Create DNS server
-    dnsServer = new AuthoritativeDNSServer(dnsConfig.listen_port, cacheOptions);
-
-    await dnsServer.start();
-}
-
-export async function stopDnsServer() {
-    if (!dnsServer) {
-        return;
-    }
-
-    await dnsServer.stop();
-    dnsServer = undefined;
-}

server/private/lib/cache.ts

@@ -13,7 +13,7 @@
 
 import NodeCache from "node-cache";
 import logger from "@server/logger";
-import { redisManager, regionalRedisManager } from "#private/lib/redis";
+import { redisManager, regionalRedisManager } from "@server/private/lib/redis";
 
 // Create local cache with maxKeys limit to prevent memory leaks
 // With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient
@@ -97,11 +97,11 @@ class AdaptiveCache {
                 const value = await redisManager.get(key);
 
                 if (value !== null) {
-                    // logger.debug(`Cache hit in Redis: ${key}`);
+                    logger.debug(`Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
 
-                // logger.debug(`Cache miss in Redis: ${key}`);
+                logger.debug(`Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(`Redis get error for key ${key}:`, error);
@@ -134,7 +134,7 @@ class AdaptiveCache {
                     const success = await redisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        // logger.debug(`Deleted key from Redis: ${k}`);
+                        logger.debug(`Deleted key from Redis: ${k}`);
                     }
                 }
 
@@ -161,7 +161,7 @@ class AdaptiveCache {
             const success = localCache.del(k);
             if (success > 0) {
                 deletedCount++;
-                // logger.debug(`Deleted key from local cache: ${k}`);
+                logger.debug(`Deleted key from local cache: ${k}`);
             }
         }
 
@@ -229,7 +229,7 @@ class AdaptiveCache {
         }
 
         localCache.flushAll();
-        // logger.debug("Flushed local cache");
+        logger.debug("Flushed local cache");
     }
 
     /**
@@ -332,7 +332,7 @@ class RegionalAdaptiveCache {
                     redisTtl
                 );
                 if (success) {
-                    // logger.debug(`[regional] Set key in Redis: ${key}`);
+                    logger.debug(`[regional] Set key in Redis: ${key}`);
                     return true;
                 }
             } catch (error) {
@@ -353,10 +353,10 @@ class RegionalAdaptiveCache {
             try {
                 const value = await regionalRedisManager.get(key);
                 if (value !== null) {
-                    // logger.debug(`[regional] Cache hit in Redis: ${key}`);
+                    logger.debug(`[regional] Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
-                // logger.debug(`[regional] Cache miss in Redis: ${key}`);
+                logger.debug(`[regional] Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(
@@ -385,7 +385,7 @@ class RegionalAdaptiveCache {
                     const success = await regionalRedisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        // logger.debug(`[regional] Deleted key from Redis: ${k}`);
+                        logger.debug(`[regional] Deleted key from Redis: ${k}`);
                     }
                 }
                 if (deletedCount === keys.length) return deletedCount;
@@ -400,7 +400,7 @@ class RegionalAdaptiveCache {
             const count = regionalLocalCache.del(k);
             if (count > 0) {
                 deletedCount++;
-                // logger.debug(`[regional] Deleted key from local cache: ${k}`);
+                logger.debug(`[regional] Deleted key from local cache: ${k}`);
             }
         }
         return deletedCount;

server/private/lib/certificates/acme-client.ts

@@ -1,298 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import * as acme from "acme-client";
-import * as fs from "fs";
-import { eq } from "drizzle-orm/sql";
-import { privateConfig as config } from "#private/lib/config";
-import { DnsChallenge, db, dnsChallenge } from "@server/db";
-import { withRetry } from "@server/lib/retry";
-import logger from "@server/logger";
-import { acmeRateLimiter } from "./acmeRateLimiter";
-
-// acme-client's own retry/backoff logging (429 retries, 5xx retries, each
-// status-poll tick in waitForValidStatus) is a no-op by default - it only
-// activates via DEBUG=acme-client or this call, neither of which was wired
-// up. Without it, a cert silently retrying a Let's Encrypt rate limit for
-// several minutes is indistinguishable in our logs from one that's actually
-// hung, since our own logging only wraps the call, not what happens inside
-// it. Must run before any AcmeClient method is called.
-acme.setLogger((msg: string) => logger.info(`[acme-client] ${msg}`));
-
-// acme-client's axios retry wrapper treats any response-less request error
-// (timeout, connection reset, DNS blip reaching the ACME server) as
-// retryable, but once its internal retries are exhausted it falls through to
-// `validateStatus(response)` with `response` still undefined, throwing this
-// uninformative TypeError instead of the real network error.
-// https://github.com/publishlab/node-acme-client/blob/master/src/axios.js
-function isUnresponsiveAcmeError(error: unknown): boolean {
-    return (
-        error instanceof TypeError &&
-        error.message ===
-            "Cannot read properties of undefined (reading 'config')"
-    );
-}
-
-function normalizeAcmeError(error: unknown): Error {
-    if (isUnresponsiveAcmeError(error)) {
-        return new Error(
-            "ACME server did not respond after repeated attempts (network error reaching the ACME endpoint)",
-            { cause: error }
-        );
-    }
-    return error instanceof Error ? error : new Error(String(error));
-}
-
-export class AcmeClientManager {
-    private client: acme.Client | null = null;
-    private accountKey: string | null = null;
-
-    async initialize() {
-        try {
-            this.accountKey = await this.loadAccountKey();
-
-            this.client = new acme.Client({
-                directoryUrl: config.getRawConfig().acme!.acme_directory_url,
-                accountKey: this.accountKey
-            });
-
-            // Try to create account or get existing one
-            await this.client.createAccount({
-                termsOfServiceAgreed: true,
-                contact: [`mailto:${config.getRawConfig().acme!.contact_email}`]
-            });
-
-            logger.info("ACME client initialized successfully");
-        } catch (error) {
-            logger.error("Failed to initialize ACME client:", error);
-            throw error;
-        }
-    }
-
-    private async loadAccountKey(): Promise<string> {
-        const keyPath = config.getRawConfig().acme!.acme_account_key_path;
-
-        if (fs.existsSync(keyPath)) {
-            logger.info("Loading existing account key");
-            return fs.readFileSync(keyPath, "utf8");
-        } else {
-            logger.info("Generating new account key");
-            const privateKey = await acme.crypto.createPrivateKey();
-            const privateKeyString = privateKey.toString();
-            fs.writeFileSync(keyPath, privateKeyString);
-            return privateKeyString;
-        }
-    }
-
-    getClient(): acme.Client {
-        if (!this.client) {
-            throw new Error("ACME client not initialized");
-        }
-        return this.client;
-    }
-
-    async createOrder(domain: string, wildcard: boolean = false): Promise<any> {
-        const client = this.getClient();
-
-        const identifiers = wildcard
-            ? [
-                  { type: "dns", value: domain },
-                  { type: "dns", value: `*.${domain}` }
-              ]
-            : [{ type: "dns", value: domain }];
-
-        await acmeRateLimiter.acquire();
-        const order = await client.createOrder({
-            identifiers
-        });
-
-        if (wildcard) {
-            logger.info(`Created wildcard order for domain: ${domain}`);
-        } else {
-            logger.info(`Created order for domain: ${domain}`);
-        }
-        return order;
-    }
-
-    async getAuthorizations(order: any): Promise<any[]> {
-        const client = this.getClient();
-        await acmeRateLimiter.acquire();
-        return client.getAuthorizations(order);
-    }
-
-    async handleDnsChallenge(
-        dnsChallenges: {
-            authz: any;
-            challenge: any;
-        }[]
-    ): Promise<void> {
-        const client = this.getClient();
-
-        let challengeDomains: DnsChallenge[] = [];
-
-        for (const { authz, challenge } of dnsChallenges) {
-            const keyAuthorization =
-                await client.getChallengeKeyAuthorization(challenge);
-
-            // Extract the domain from authorization
-            const domain = authz.identifier.value;
-
-            // Store challenge in database for DNS server to pick up
-            challengeDomains = await withRetry(
-                () =>
-                    db
-                        .insert(dnsChallenge)
-                        .values({
-                            domain: domain,
-                            token: challenge.token,
-                            keyAuthorization,
-                            createdAt: Math.floor(Date.now() / 1000),
-                            expiresAt: Math.floor(
-                                (Date.now() +
-                                    config.getRawConfig().acme!
-                                        .challenge_ttl_ms) /
-                                    1000
-                            )
-                        })
-                        .returning(),
-                { label: `insert dnsChallenge for domain ${domain}` }
-            );
-
-            logger.info(
-                `DNS challenge stored for domain: ${domain} as token ${challenge.token} and keyAuthorization`
-            );
-        }
-
-        await new Promise((resolve) => setTimeout(resolve, 2000));
-
-        const failedDomains: string[] = [];
-
-        for (const { authz, challenge } of dnsChallenges) {
-            const domain = authz.identifier.value;
-            const challengeDomain = `_acme-challenge.${domain}`;
-
-            try {
-                // The ACME server occasionally has a transient network blip
-                // mid-sequence; retry the whole verify/complete/wait sequence
-                // rather than just the DNS challenge propagation wait, since
-                // these calls are safe to repeat against the ACME server.
-                await withRetry(
-                    async () => {
-                        // Verify challenge
-                        await acmeRateLimiter.acquire();
-                        await client.verifyChallenge(authz, challenge);
-
-                        // Complete challenge
-                        logger.info(
-                            `Completing challenge for domain: ${challengeDomain}`
-                        );
-                        await acmeRateLimiter.acquire();
-                        await client.completeChallenge(challenge);
-
-                        // Wait for validation
-                        logger.info(
-                            `Waiting for challenge to be validated for domain: ${challengeDomain}...`
-                        );
-                        await acmeRateLimiter.acquire();
-                        await client.waitForValidStatus(challenge);
-                    },
-                    {
-                        retries: 2,
-                        baseDelayMs: 5000,
-                        label: `ACME challenge completion for domain ${domain}`,
-                        // Only retry the known network-blip crash - a
-                        // genuine validation failure (e.g. challenge marked
-                        // "invalid" because the DNS record wasn't found) is
-                        // permanent and should fail immediately instead of
-                        // burning Let's Encrypt's per-hostname failed-
-                        // validation rate limit on retries that can't help.
-                        shouldRetry: isUnresponsiveAcmeError
-                    }
-                );
-
-                logger.info(`Challenge completed for domain: ${domain}`);
-            } catch (error) {
-                logger.error(
-                    `Failed to complete challenge for domain ${domain}:`,
-                    normalizeAcmeError(error)
-                );
-                failedDomains.push(domain);
-            }
-        }
-
-        for (const challengeDomain of challengeDomains) {
-            await this.removeDnsChallenge(challengeDomain.dnsChallengeId);
-            logger.info(
-                `Removed DNS challenge for domain: ${challengeDomain.domain}`
-            );
-        }
-
-        // A failed dns-01 challenge leaves the order stuck in "pending" -
-        // finalizing it would just fail with a confusing ACME error, so
-        // stop here and let the caller mark the certificate as failed.
-        if (failedDomains.length > 0) {
-            throw new Error(
-                `DNS-01 challenge validation failed for domain(s): ${failedDomains.join(", ")}`
-            );
-        }
-    }
-
-    async removeDnsChallenge(dnsChallengeId: number): Promise<void> {
-        try {
-            await withRetry(
-                () =>
-                    db
-                        .delete(dnsChallenge)
-                        .where(eq(dnsChallenge.dnsChallengeId, dnsChallengeId)),
-                { label: `delete dnsChallenge ${dnsChallengeId}` }
-            );
-        } catch (error) {
-            logger.error(
-                `Failed to clean up DNS challenge for id ${dnsChallengeId}:`,
-                error
-            );
-        }
-    }
-
-    async finalizeCertificate(
-        order: any,
-        domain: string,
-        wildcard: boolean = false
-    ): Promise<{ certificate: string; privateKey: string }> {
-        const client = this.getClient();
-
-        const altNames = wildcard ? [`*.${domain}`, domain] : [domain];
-
-        // Create CSR
-        const [privateKey, csr] = await acme.crypto.createCsr({
-            altNames
-        });
-
-        // Finalize order
-        await acmeRateLimiter.acquire();
-        const finalizedOrder = await client.finalizeOrder(order, csr);
-
-        // Get certificate
-        await acmeRateLimiter.acquire();
-        const certificate = await client.getCertificate(finalizedOrder);
-
-        logger.info(`Certificate obtained for domain: ${domain}`);
-
-        return {
-            certificate: certificate.toString(),
-            privateKey: privateKey.toString()
-        };
-    }
-}
-
-export const acmeClientManager = new AcmeClientManager();

server/private/lib/certificates/acmeRateLimiter.ts

@@ -1,71 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import { privateConfig as config } from "#private/lib/config";
-import logger from "@server/logger";
-import { redis } from "../redis";
-// Caps outgoing ACME API calls to a fixed budget per wall-clock second,
-// shared across all pops workers via Redis (mirrors the lockManager pattern
-// in @lib/lock) - a per-process limiter wouldn't be enough since multiple
-// workers issue certificates against the same Let's Encrypt account.
-const ACQUIRE_SCRIPT = `
-local key = KEYS[1]
-local limit = tonumber(ARGV[1])
-local current = redis.call('INCR', key)
-if current == 1 then
-    redis.call('PEXPIRE', key, 2000)
-end
-if current > limit then
-    return 0
-else
-    return 1
-end
-`;
-
-class AcmeRateLimiter {
-    async acquire(): Promise<void> {
-        const limit =
-            config.getRawConfig().acme?.acme_requests_per_second ?? 15;
-
-        for (;;) {
-            const bucket = Math.floor(Date.now() / 1000);
-            const key = `acme_rate_limit:${bucket}`;
-
-            let allowed: number;
-            try {
-                allowed = (await redis.eval(
-                    ACQUIRE_SCRIPT,
-                    1,
-                    key,
-                    limit.toString()
-                )) as number;
-            } catch (error) {
-                logger.error(
-                    "ACME rate limiter check failed, proceeding without throttling:",
-                    error
-                );
-                return;
-            }
-
-            if (allowed === 1) {
-                return;
-            }
-
-            // Budget for this second is spent - wait for the next window.
-            const waitMs = 1000 - (Date.now() % 1000) + 10;
-            await new Promise((resolve) => setTimeout(resolve, waitMs));
-        }
-    }
-}
-
-export const acmeRateLimiter = new AcmeRateLimiter();

server/private/lib/certificates/certificate-service.ts

@@ -1,511 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import { acmeClientManager } from "./acme-client";
-import { dnsValidator } from "./dns-validator";
-import { getTableColumns } from "drizzle-orm";
-import { eq, and, or, isNull, lt, asc } from "drizzle-orm/sql";
-import { config } from "@server/lib/config";
-import { db, certificates, domains, Certificate } from "@server/db";
-import { encrypt } from "@server/lib/crypto";
-import { withTimeout, withRetry } from "@server/lib/retry";
-import logger from "@server/logger";
-import { lockManager } from "../lock";
-import { pushCertUpdateToAffectedNewts } from "@server/lib/acmeCertSync";
-import crypto from "crypto";
-
-// Number of on-demand DNS validation attempts made right before a
-// certificate is (re)issued, to avoid burning Let's Encrypt rate limits on
-// domains whose DNS has drifted since they were last verified.
-const PRE_CERT_DNS_VALIDATION_ATTEMPTS = 3;
-
-// Hard ceiling on a single certificate's issuance/renewal flow. acme-client's
-// axios instance never sets a request timeout, so a stalled connection to
-// the ACME server hangs forever instead of erroring - and since
-// processPendingCertificates/processRenewalCandidates gate the *next* batch
-// on Promise.all(...) over the current one, one hung certificate would
-// otherwise stall every other domain permanently. Sized generously above the
-// legitimate worst case (acme-client's own bounded backoff is ~3.6min per
-// status-polling loop, and a wildcard cert's two identifiers plus order
-// finalization can chain a few of those) so this only fires on a genuine hang.
-const CERTIFICATE_ISSUANCE_TIMEOUT_MS = 20 * 60 * 1000;
-
-// "requested" is set the instant a cert starts processing and is never
-// queried anywhere else - processPendingCertificates only selects "pending"
-// and processRenewalCandidates only selects "valid". So if the *process*
-// dies mid-flight (OOM, node eviction, a rolling deploy) rather than just
-// hanging, the row is orphaned in "requested" permanently with nothing to
-// ever pick it back up, no matter how good the in-process timeouts are.
-// Threshold is set comfortably above CERTIFICATE_ISSUANCE_TIMEOUT_MS plus the
-// scheduler's own outer backstop so this never reclaims a cert that's still
-// genuinely being worked on.
-const STUCK_CERTIFICATE_THRESHOLD_MS = 40 * 60 * 1000;
-
-export class CertificateService {
-    // Runs at the top of every processPendingCertificates tick so an
-    // interrupted worker's leftovers always get put back in the queue
-    // instead of sitting invisible to every query forever.
-    private async reclaimStuckCertificates(): Promise<void> {
-        const staleBefore =
-            Math.floor(Date.now() / 1000) -
-            Math.floor(STUCK_CERTIFICATE_THRESHOLD_MS / 1000);
-
-        const reclaimed = await db
-            .update(certificates)
-            .set({
-                status: "pending",
-                errorMessage:
-                    'Reclaimed after being stuck in "requested" state - the worker processing it likely restarted or crashed',
-                updatedAt: Math.floor(Date.now() / 1000)
-            })
-            .where(
-                and(
-                    eq(certificates.status, "requested"),
-                    lt(certificates.updatedAt, staleBefore)
-                )
-            )
-            .returning({ domain: certificates.domain });
-
-        if (reclaimed.length > 0) {
-            logger.warn(
-                `Reclaimed ${reclaimed.length} certificate(s) stuck in "requested" state: ${reclaimed
-                    .map((c) => c.domain)
-                    .join(", ")}`
-            );
-        }
-    }
-
-    async processPendingCertificates(): Promise<void> {
-        logger.debug("Checking for pending certificates...");
-
-        await this.reclaimStuckCertificates();
-
-        const pendingCerts = await db
-            .select(getTableColumns(certificates))
-            .from(certificates)
-            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
-            .where(
-                and(
-                    eq(certificates.status, "pending"),
-                    or(
-                        // Certs with no linked domain row (e.g. legacy certs
-                        // imported from acme.json) aren't gated on domain
-                        // verification since there's nothing to check.
-                        isNull(certificates.domainId),
-                        and(
-                            eq(domains.verified, true),
-                            eq(domains.failed, false)
-                        )
-                    )
-                )
-            )
-            .limit(10);
-
-        if (pendingCerts.length === 0) {
-            logger.debug("No pending certificates found");
-            return;
-        }
-
-        logger.info(`Found ${pendingCerts.length} pending certificates`);
-
-        // Process the batch concurrently so one domain stuck retrying a slow
-        // DNS-01 challenge (the ACME client's waitForValidStatus can spend
-        // minutes on a bad domain) doesn't stall the rest of the batch.
-        // processSingleCertificate catches its own errors and each cert uses
-        // an independent per-domain lock, so this is safe to parallelize.
-        await Promise.all(
-            pendingCerts.map((cert) => this.processSingleCertificate(cert))
-        );
-    }
-
-    async processRenewalCandidates(): Promise<void> {
-        logger.debug("Checking for certificates needing renewal...");
-
-        const now = Math.floor(Date.now() / 1000);
-
-        const renewalCandidates = await db
-            .select(getTableColumns(certificates))
-            .from(certificates)
-            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
-            .where(
-                and(
-                    eq(certificates.status, "valid"),
-                    lt(certificates.expiresAt, now + 15 * 24 * 60 * 60), // 15 days from now
-                    or(
-                        // Certs with no linked domain row (e.g. legacy certs
-                        // imported from acme.json) aren't gated on domain
-                        // verification since there's nothing to check.
-                        isNull(certificates.domainId),
-                        and(
-                            eq(domains.verified, true),
-                            eq(domains.failed, false)
-                        )
-                    )
-                )
-            )
-            // Most urgent first, so already-expired certs aren't starved
-            // behind the limit by certs that still have weeks of runway.
-            .orderBy(asc(certificates.expiresAt))
-            .limit(50);
-
-        if (renewalCandidates.length === 0) {
-            logger.debug("No certificates need renewal");
-            return;
-        }
-
-        logger.info(
-            `Found ${renewalCandidates.length} certificates needing renewal`
-        );
-
-        for (const cert of renewalCandidates) {
-            if (cert.expiresAt !== null && cert.expiresAt < now) {
-                logger.warn(
-                    `Certificate for ${cert.domain} is marked "valid" but already expired at ${new Date(cert.expiresAt * 1000).toISOString()} (bad state) - renewing immediately`
-                );
-            }
-        }
-
-        // Process the batch concurrently - see processPendingCertificates for why.
-        await Promise.all(
-            renewalCandidates.map((cert) => this.renewCertificate(cert))
-        );
-    }
-
-    private async processSingleCertificate(cert: Certificate): Promise<void> {
-        const lockKey = `cert:${cert.domain}`;
-
-        const lockToken = await lockManager.acquireLock(lockKey);
-        if (!lockToken) {
-            logger.debug(
-                `Could not acquire lock for certificate: ${cert.domain}`
-            );
-            return;
-        }
-
-        try {
-            logger.info(`Processing certificate for domain: ${cert.domain}`);
-
-            // Update status to processing
-            await db
-                .update(certificates)
-                .set({
-                    status: "requested",
-                    updatedAt: Math.floor(Date.now() / 1000)
-                })
-                .where(eq(certificates.certId, cert.certId));
-            //
-
-            await withTimeout(
-                this.obtainCertificate(cert),
-                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
-                `certificate issuance for ${cert.domain}`
-            );
-        } catch (error) {
-            logger.error(
-                `Failed to process certificate for ${cert.domain}:`,
-                error
-            );
-
-            await db
-                .update(certificates)
-                .set({
-                    status: "failed",
-                    errorMessage:
-                        error instanceof Error
-                            ? error.message
-                            : "Unknown error",
-                    updatedAt: Math.floor(Date.now() / 1000)
-                })
-                .where(eq(certificates.certId, cert.certId));
-        } finally {
-            await lockManager.releaseLock(lockKey, lockToken);
-        }
-    }
-
-    private async renewCertificate(cert: Certificate): Promise<void> {
-        const lockKey = `cert:${cert.domain}`;
-
-        const lockToken = await lockManager.acquireLock(lockKey);
-        if (!lockToken) {
-            logger.debug(
-                `Could not acquire lock for certificate renewal: ${cert.domain}`
-            );
-            return;
-        }
-
-        try {
-            logger.info(`Renewing certificate for domain: ${cert.domain}`);
-
-            // Update last renewal attempt
-            await db
-                .update(certificates)
-                .set({
-                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
-                    updatedAt: Math.floor(Date.now() / 1000)
-                })
-                .where(eq(certificates.certId, cert.certId));
-
-            await withTimeout(
-                this.obtainCertificate(cert),
-                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
-                `certificate renewal for ${cert.domain}`
-            );
-        } catch (error) {
-            logger.error(
-                `Failed to renew certificate for ${cert.domain}:`,
-                error
-            );
-
-            await db
-                .update(certificates)
-                .set({
-                    status: "failed",
-                    errorMessage:
-                        error instanceof Error
-                            ? error.message
-                            : "Unknown error",
-                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
-                    updatedAt: Math.floor(Date.now() / 1000)
-                })
-                .where(eq(certificates.certId, cert.certId));
-        } finally {
-            await lockManager.releaseLock(lockKey, lockToken);
-        }
-    }
-
-    // Re-checks the domain's DNS records right before we spend a Let's
-    // Encrypt order on it, so drift that happened after the domain was
-    // originally verified doesn't burn ACME rate limits. Certs with no
-    // linked domain row (e.g. legacy/manually-managed certs) skip this and
-    // proceed as before, since there are no tracked DNS records to check.
-    private async verifyDomainBeforeIssuance(cert: Certificate): Promise<void> {
-        if (!cert.domainId) {
-            return;
-        }
-
-        const [domain] = await db
-            .select()
-            .from(domains)
-            .where(eq(domains.domainId, cert.domainId))
-            .limit(1);
-
-        if (!domain) {
-            return;
-        }
-
-        for (
-            let attempt = 1;
-            attempt <= PRE_CERT_DNS_VALIDATION_ATTEMPTS;
-            attempt++
-        ) {
-            // Offset `tries` so each attempt round-robins to a different
-            // privateConfigured DNS resolver instead of re-querying the same one.
-            const probe = { ...domain, tries: domain.tries + attempt - 1 };
-            if (
-                await dnsValidator.validateDomain(probe, {
-                    forceRecheck: true
-                })
-            ) {
-                await db
-                    .update(domains)
-                    .set({ verified: true, failed: false, errorMessage: null })
-                    .where(eq(domains.domainId, domain.domainId));
-                return;
-            }
-
-            logger.warn(
-                `Pre-certificate DNS check ${attempt}/${PRE_CERT_DNS_VALIDATION_ATTEMPTS} failed for domain ${domain.baseDomain} (cert: ${cert.domain})`
-            );
-        }
-
-        const errorMessage = `Domain failed DNS validation ${PRE_CERT_DNS_VALIDATION_ATTEMPTS} times before certificate issuance`;
-        await db
-            .update(domains)
-            .set({ verified: false, failed: true, errorMessage })
-            .where(eq(domains.domainId, domain.domainId));
-
-        throw new Error(errorMessage);
-    }
-
-    private async obtainCertificate(cert: Certificate): Promise<void> {
-        await this.verifyDomainBeforeIssuance(cert);
-
-        // Create order
-        const order = await acmeClientManager.createOrder(
-            cert.domain,
-            cert.wildcard || false
-        );
-
-        // Update with order ID
-        await withRetry(
-            () =>
-                db
-                    .update(certificates)
-                    .set({
-                        orderId: order.url,
-                        updatedAt: Math.floor(Date.now() / 1000)
-                    })
-                    .where(eq(certificates.certId, cert.certId)),
-            { label: `update orderId for certificate ${cert.domain}` }
-        );
-
-        // Get authorizations
-        const authorizations = await acmeClientManager.getAuthorizations(order);
-
-        // Aggregate all DNS-01 challenges
-        const dnsChallenges = authorizations.map((authz: any) => {
-            const dnsChallenge = authz.challenges.find(
-                (c: any) => c.type === "dns-01"
-            );
-            if (!dnsChallenge) {
-                throw new Error(
-                    `No DNS-01 challenge found for ${authz.identifier.value}`
-                );
-            }
-            return {
-                authz,
-                challenge: dnsChallenge
-            };
-        });
-
-        // Send all DNS-01 challenges in one request to handleDnsChallenge
-        await acmeClientManager.handleDnsChallenge(dnsChallenges);
-
-        // Finalize certificate
-        const { certificate, privateKey } =
-            await acmeClientManager.finalizeCertificate(
-                order,
-                cert.domain,
-                cert.wildcard || false
-            );
-
-        const encryptionKey = config.getRawConfig().server.secret;
-        if (!encryptionKey) {
-            throw new Error("Encryption key not provided");
-        }
-
-        // Encrypt certificate and private key
-        const encryptedCert = encrypt(certificate, encryptionKey);
-        const encryptedKey = encrypt(privateKey, encryptionKey);
-
-        // Parse certificate to get expiration date
-        const expiresAt = this.extractExpirationDate(certificate);
-
-        // Update database record. This persists the certificate we just
-        // obtained from the ACME server, so it's retried aggressively -
-        // losing this write means re-issuing the cert from scratch.
-        await withRetry(
-            () =>
-                db
-                    .update(certificates)
-                    .set({
-                        status: "valid",
-                        expiresAt: Math.floor(expiresAt.getTime() / 1000),
-                        renewalCount: (cert.renewalCount || 0) + 1,
-                        errorMessage: null,
-                        updatedAt: Math.floor(Date.now() / 1000),
-                        certFile: encryptedCert,
-                        keyFile: encryptedKey
-                    })
-                    .where(eq(certificates.certId, cert.certId)),
-            {
-                retries: 5,
-                label: `persist issued certificate for ${cert.domain}`
-            }
-        );
-
-        logger.info(
-            `Certificate successfully obtained/renewed for domain: ${cert.domain}`
-        );
-
-        await pushCertUpdateToAffectedNewts(
-            cert.domain,
-            cert.domainId ?? null,
-            certificate,
-            privateKey
-        );
-    }
-
-    private extractExpirationDate(certificate: string): Date {
-        try {
-            // Extract the certificate block
-            const pem = certificate
-                .replace(/-----BEGIN CERTIFICATE-----/g, "")
-                .replace(/-----END CERTIFICATE-----/g, "")
-                .replace(/\s+/g, "");
-            const der = Buffer.from(pem, "base64");
-
-            // Use Node.js crypto to parse the certificate
-            const x509 = new crypto.X509Certificate(der);
-            return new Date(x509.validTo);
-        } catch (error) {
-            logger.warn(
-                "Failed to parse certificate expiration date, using default",
-                error
-            );
-            // Default to 90 days from now (Let's Encrypt default)
-            return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
-        }
-    }
-
-    async addCertificateRequest(domain: string): Promise<void> {
-        try {
-            await db.insert(certificates).values({
-                domain,
-                status: "pending",
-                createdAt: Math.floor(Date.now() / 1000),
-                updatedAt: Math.floor(Date.now() / 1000)
-            });
-            logger.info(`Certificate request added for domain: ${domain}`);
-        } catch (error) {
-            if (error instanceof Error && error.message.includes("unique")) {
-                logger.warn(
-                    `Certificate request already exists for domain: ${domain}`
-                );
-            } else {
-                throw error;
-            }
-        }
-    }
-
-    async getCertificateStatus(domain: string) {
-        const cert = await db
-            .select()
-            .from(certificates)
-            .where(eq(certificates.domain, domain))
-            .limit(1);
-
-        return cert[0] || null;
-    }
-
-    async cleanupExpiredChallenges(): Promise<void> {
-        try {
-            const result = await db
-                .delete(certificates)
-                .where(
-                    lt(certificates.expiresAt, Math.floor(Date.now() / 1000))
-                )
-                .returning();
-
-            if (result.length > 0) {
-                logger.info(
-                    `Cleaned up ${result.length} expired DNS challenges`
-                );
-            }
-        } catch (error) {
-            logger.error("Failed to cleanup expired challenges:", error);
-        }
-    }
-}
-
-export const certificateService = new CertificateService();

server/private/lib/certificates/dns-validator.ts

@@ -1,334 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import { eq, and, lt } from "drizzle-orm";
-import * as dns from "dns/promises";
-import { privateConfig as config } from "#private/lib/config";
-import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
-import logger from "@server/logger";
-import { lockManager } from "../lock";
-
-export const DNS_VALIDATOR_MAX_TRIES = 300;
-
-export class DNSValidator {
-    private static readonly MAX_TRIES = DNS_VALIDATOR_MAX_TRIES;
-
-    constructor() {}
-
-    async validateAll(): Promise<void> {
-        // Get all domains that are not yet verified and haven't exceeded max tries
-        const unverifiedDomains: Domain[] = await db
-            .select()
-            .from(domains)
-            .where(
-                and(
-                    eq(domains.verified, false),
-                    lt(domains.tries, DNSValidator.MAX_TRIES)
-                )
-            );
-
-        if (unverifiedDomains.length === 0) {
-            logger.debug("No unverified domains found for DNS validation");
-            return;
-        }
-
-        logger.info(`Validating ${unverifiedDomains.length} DNS records`);
-
-        for (const domain of unverifiedDomains) {
-            const lockKey = `dns:${domain.baseDomain}`;
-            const lockToken = await lockManager.acquireLock(lockKey);
-            if (!lockToken) {
-                logger.debug(
-                    `Could not acquire lock for DNS validation: ${domain.baseDomain}`
-                );
-                continue;
-            }
-            try {
-                const isValid = await this.validateDomain(domain);
-                if (isValid) {
-                    await db
-                        .update(domains)
-                        .set({
-                            verified: true,
-                            failed: false,
-                            tries: 0,
-                            errorMessage: null
-                        })
-                        .where(eq(domains.domainId, domain.domainId));
-                    logger.info(
-                        `Domain ${domain.baseDomain} validated successfully`
-                    );
-                } else {
-                    const newTries = domain.tries + 1;
-                    const shouldMarkAsFailed =
-                        newTries >= DNSValidator.MAX_TRIES;
-
-                    await db
-                        .update(domains)
-                        .set({
-                            tries: newTries,
-                            failed: shouldMarkAsFailed
-                        })
-                        .where(eq(domains.domainId, domain.domainId));
-
-                    if (shouldMarkAsFailed) {
-                        logger.warn(
-                            `Domain ${domain.baseDomain} exceeded maximum tries (${DNSValidator.MAX_TRIES}), marking as failed`
-                        );
-                    } else {
-                        logger.debug(
-                            `Domain ${domain.baseDomain} did not validate (attempt ${newTries}/${DNSValidator.MAX_TRIES})`
-                        );
-                    }
-                }
-            } catch (err) {
-                logger.warn(
-                    `Error validating domain ${domain.baseDomain}:`,
-                    err
-                );
-                // Increment tries even on error
-                const newTries = domain.tries + 1;
-                const shouldMarkAsFailed = newTries >= DNSValidator.MAX_TRIES;
-
-                await db
-                    .update(domains)
-                    .set({
-                        tries: newTries,
-                        failed: shouldMarkAsFailed
-                    })
-                    .where(eq(domains.domainId, domain.domainId));
-            } finally {
-                await lockManager.releaseLock(lockKey, lockToken);
-            }
-        }
-    }
-
-    async validateDomain(
-        domain: Domain,
-        opts: { forceRecheck?: boolean } = {}
-    ): Promise<boolean> {
-        const { forceRecheck = false } = opts;
-        const resolver = new dns.Resolver();
-        const servers = config.getRawConfig().acme?.dns_resolvers;
-        if (!servers || servers.length === 0) {
-            throw new Error("No DNS resolvers configured");
-        }
-        const dnsServer = servers[domain.tries % servers.length]!;
-        resolver.setServers([dnsServer]);
-        logger.debug(
-            `Using DNS server ${dnsServer} for domain ${domain.baseDomain} (try ${domain.tries})`
-        );
-
-        // Get all DNS records for this domain
-        const records: DnsRecord[] = await db
-            .select()
-            .from(dnsRecords)
-            .where(eq(dnsRecords.domainId, domain.domainId));
-
-        if (records.length === 0) {
-            logger.warn(`No DNS records found for domain ${domain.baseDomain}`);
-            return false;
-        }
-
-        if (!forceRecheck && records.every((r) => r.verified)) {
-            logger.info(
-                `All DNS records already verified for domain ${domain.baseDomain}`
-            );
-            return true;
-        }
-
-        logger.info(
-            `Validating ${records.length} DNS records for domain ${domain.baseDomain}`
-        );
-
-        // Collect the full set of expected NS values for this domain so we can
-        // detect extra records that are present in DNS but not in our DB.
-        const expectedNsValues = new Set<string>(
-            records.filter((r) => r.recordType === "NS").map((r) => r.value)
-        );
-
-        // Cache resolved NS records across iterations — there will be 3 NS
-        // records in the DB and we don't need to hit the upstream server 3 times.
-        let previousNs: string[] | null = null;
-
-        for (const record of records) {
-            // Skip already verified records, unless a live recheck was requested
-            if (record.verified && !forceRecheck) {
-                continue;
-            }
-
-            let isValid = false;
-
-            try {
-                if (record.recordType === "NS") {
-                    let nsRecords: string[] | null = previousNs;
-                    if (!nsRecords) {
-                        nsRecords = await resolver.resolveNs(
-                            record.baseDomain || domain.baseDomain
-                        );
-                    }
-                    logger.info(
-                        `NS records for ${
-                            record.baseDomain || domain.baseDomain
-                        }:`,
-                        nsRecords
-                    );
-
-                    // Check if this expected NS value is present in the live records.
-                    // A stale/legacy expected value (e.g. left over from a
-                    // nameserver rebrand) is also accepted as long as the live
-                    // records resolve to some other known-valid nameserver —
-                    // the specific literal hostname stored per-domain isn't
-                    // meaningful once it's a recognized alias.
-                    isValid = nsRecords.some((ns) => ns === record.value);
-
-                    previousNs = nsRecords;
-                } else if (record.recordType === "CNAME") {
-                    const cnameRecords = await resolver.resolveCname(
-                        record.baseDomain || domain.baseDomain
-                    );
-                    logger.info(
-                        `CNAME records for ${
-                            record.baseDomain || domain.baseDomain
-                        }:`,
-                        cnameRecords
-                    );
-
-                    // Check if the CNAME record matches the expected value
-                    isValid =
-                        cnameRecords.length === 1 &&
-                        cnameRecords[0] === record.value;
-                } else if (record.recordType === "TXT") {
-                    const txtRecords = await resolver.resolveTxt(
-                        record.baseDomain || domain.baseDomain
-                    );
-                    logger.info(
-                        `TXT records for ${
-                            record.baseDomain || domain.baseDomain
-                        }:`,
-                        txtRecords
-                    );
-
-                    // TXT records come as an array of arrays, flatten and check
-                    const flatTxtRecords = txtRecords.flat();
-                    isValid = flatTxtRecords.includes(record.value);
-                } else if (record.recordType === "A") {
-                    const aRecords = await resolver.resolve4(
-                        record.baseDomain || domain.baseDomain
-                    );
-                    logger.info(
-                        `A records for ${
-                            record.baseDomain || domain.baseDomain
-                        }:`,
-                        aRecords
-                    );
-
-                    // Check if the A record matches the expected value
-                    isValid = aRecords.includes(record.value);
-                } else {
-                    logger.warn(
-                        `Unsupported record type: ${record.recordType}`
-                    );
-                    continue;
-                }
-            } catch (error) {
-                isValid = false;
-                logger.debug(
-                    `Did not resolve ${record.recordType} record for ${
-                        record.baseDomain || domain.baseDomain
-                    }:`,
-                    error
-                );
-            }
-
-            // Update the individual record verification status. Runs for
-            // both a mismatched value and a failed/thrown DNS lookup, so a
-            // previously-verified record that stops resolving (e.g. NXDOMAIN
-            // after NS delegation is dropped) gets downgraded instead of
-            // leaving stale `verified: true` state behind.
-            if (isValid) {
-                await db
-                    .update(dnsRecords)
-                    .set({ verified: true })
-                    .where(eq(dnsRecords.id, record.id));
-                logger.info(
-                    `DNS record ${record.id} (${record.recordType}) for ${
-                        record.baseDomain || domain.baseDomain
-                    } verified successfully`
-                );
-            } else {
-                if (record.verified) {
-                    await db
-                        .update(dnsRecords)
-                        .set({ verified: false })
-                        .where(eq(dnsRecords.id, record.id));
-                }
-                logger.debug(
-                    `DNS record ${record.id} (${record.recordType}) for ${
-                        record.baseDomain || domain.baseDomain
-                    } does not match expected value: ${record.value}`
-                );
-            }
-        }
-
-        // --- Extra NS record check ---
-        // If we resolved NS records during this pass, verify that the live DNS
-        // has no nameservers beyond the ones we expect.  Individual records may
-        // already be marked verified above, but we must block full domain
-        // verification until the extra records are removed.
-        if (previousNs !== null && expectedNsValues.size > 0) {
-            const extraNsRecords = previousNs.filter(
-                (ns) => !expectedNsValues.has(ns)
-            );
-
-            if (extraNsRecords.length > 0) {
-                const errorMessage = `Extra NS records found that are not expected: ${extraNsRecords.join(", ")}. Remove these nameservers to complete domain verification.`;
-
-                await db
-                    .update(domains)
-                    .set({ errorMessage })
-                    .where(eq(domains.domainId, domain.domainId));
-
-                logger.warn(
-                    `Domain ${domain.baseDomain} has extra NS records that prevent verification: ${extraNsRecords.join(", ")}`
-                );
-
-                return false;
-            }
-
-            // No extras — clear any stale error that was previously written
-            await db
-                .update(domains)
-                .set({ errorMessage: null })
-                .where(eq(domains.domainId, domain.domainId));
-        }
-
-        // Check if all records are now verified
-        const updatedRecords: DnsRecord[] = await db
-            .select()
-            .from(dnsRecords)
-            .where(eq(dnsRecords.domainId, domain.domainId));
-
-        const allRecordsVerified = updatedRecords.every((r) => r.verified);
-
-        logger.info(
-            `Domain ${domain.baseDomain}: ${
-                updatedRecords.filter((r) => r.verified).length
-            }/${updatedRecords.length} records verified`
-        );
-
-        return allRecordsVerified;
-    }
-}
-
-export const dnsValidator = new DNSValidator();

server/private/lib/certificates/domain-reverifier.ts

@@ -1,233 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import { eq, and, or, isNull, lt } from "drizzle-orm";
-import * as dns from "dns/promises";
-import { DNS_VALIDATOR_MAX_TRIES } from "./dns-validator";
-import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
-import logger from "@server/logger";
-import { lockManager } from "../lock";
-import { privateConfig as config } from "#private/lib/config";
-
-// Module-level counter so successive domains in a batch round-robin across servers.
-let serverIndex = 0;
-
-export class DomainReverifier {
-    async reverifyAll(): Promise<void> {
-        const certConfig = config.getRawConfig().acme;
-        if (!certConfig) {
-            logger.debug(
-                "No certificate config — skipping domain reverification"
-            );
-            return;
-        }
-
-        const windowMs = certConfig.domain_reverification_window_ms;
-        const batchSize = certConfig.domain_reverification_batch_size;
-        const windowSecs = Math.floor(windowMs / 1000);
-        const cutoff = Math.floor(Date.now() / 1000) - windowSecs;
-
-        const domainsToCheck: Domain[] = await db
-            .select()
-            .from(domains)
-            .where(
-                and(
-                    eq(domains.verified, true),
-                    or(
-                        isNull(domains.lastCheckedAt),
-                        lt(domains.lastCheckedAt, cutoff)
-                    )
-                )
-            )
-            .limit(batchSize);
-
-        if (domainsToCheck.length === 0) {
-            logger.debug("No verified domains due for reverification");
-            return;
-        }
-
-        logger.info(`Reverifying ${domainsToCheck.length} domains`);
-
-        for (const domain of domainsToCheck) {
-            const lockKey = `dns-reverify:${domain.baseDomain}`;
-            const lockToken = await lockManager.acquireLock(lockKey);
-            if (!lockToken) {
-                logger.debug(
-                    `Could not acquire lock for domain reverification: ${domain.baseDomain}`
-                );
-                continue;
-            }
-
-            try {
-                await this.reverifyDomain(domain, certConfig.dns_resolvers);
-            } catch (err) {
-                logger.warn(
-                    `Unexpected error reverifying domain ${domain.baseDomain}:`,
-                    err
-                );
-                // Still stamp lastCheckedAt so we don't hammer a broken domain every run.
-                await db
-                    .update(domains)
-                    .set({ lastCheckedAt: Math.floor(Date.now() / 1000) })
-                    .where(eq(domains.domainId, domain.domainId));
-            } finally {
-                await lockManager.releaseLock(lockKey, lockToken);
-            }
-        }
-    }
-
-    private async reverifyDomain(
-        domain: Domain,
-        servers: string[]
-    ): Promise<void> {
-        if (!servers || servers.length === 0) {
-            throw new Error("No DNS resolvers configured");
-        }
-
-        // Round-robin across servers; advance the global counter so the next
-        // domain in the same batch gets a different server.
-        const dnsServer = servers[serverIndex % servers.length]!;
-        serverIndex++;
-
-        const resolver = new dns.Resolver();
-        resolver.setServers([dnsServer]);
-
-        logger.debug(
-            `Reverifying domain ${domain.baseDomain} using DNS server ${dnsServer}`
-        );
-
-        const records: DnsRecord[] = await db
-            .select()
-            .from(dnsRecords)
-            .where(eq(dnsRecords.domainId, domain.domainId));
-
-        if (records.length === 0) {
-            logger.warn(
-                `No DNS records found for domain ${domain.baseDomain} during reverification — marking failed`
-            );
-            await this.markFailed(
-                domain.domainId,
-                "No DNS records found during periodic reverification"
-            );
-            return;
-        }
-
-        const expectedNsValues = new Set<string>(
-            records.filter((r) => r.recordType === "NS").map((r) => r.value)
-        );
-
-        let allValid = true;
-        let errorMessage: string | null = null;
-        let resolvedNs: string[] | null = null;
-
-        for (const record of records) {
-            let isValid = false;
-
-            try {
-                if (record.recordType === "NS") {
-                    if (!resolvedNs) {
-                        resolvedNs = await resolver.resolveNs(
-                            record.baseDomain || domain.baseDomain
-                        );
-                    }
-                    isValid = resolvedNs.some((ns) => ns === record.value);
-                } else if (record.recordType === "CNAME") {
-                    const cnameRecords = await resolver.resolveCname(
-                        record.baseDomain || domain.baseDomain
-                    );
-                    isValid =
-                        cnameRecords.length === 1 &&
-                        cnameRecords[0] === record.value;
-                } else if (record.recordType === "TXT") {
-                    const txtRecords = await resolver.resolveTxt(
-                        record.baseDomain || domain.baseDomain
-                    );
-                    isValid = txtRecords.flat().includes(record.value);
-                } else if (record.recordType === "A") {
-                    const aRecords = await resolver.resolve4(
-                        record.baseDomain || domain.baseDomain
-                    );
-                    isValid = aRecords.includes(record.value);
-                } else {
-                    logger.warn(
-                        `Unsupported record type ${record.recordType} during reverification of ${domain.baseDomain}`
-                    );
-                    continue;
-                }
-            } catch (err) {
-                logger.debug(
-                    `DNS lookup failed for ${record.recordType} record on ${record.baseDomain || domain.baseDomain}:`,
-                    err
-                );
-                isValid = false;
-            }
-
-            if (!isValid) {
-                allValid = false;
-                errorMessage = `${record.recordType} record for ${record.baseDomain || domain.baseDomain} no longer resolves to expected value "${record.value}"`;
-                break;
-            }
-        }
-
-        // Check for extra NS records beyond what we expect.
-        if (allValid && resolvedNs !== null && expectedNsValues.size > 0) {
-            const extraNs = resolvedNs.filter(
-                (ns) => !expectedNsValues.has(ns)
-            );
-            if (extraNs.length > 0) {
-                allValid = false;
-                errorMessage = `Extra NS records found: ${extraNs.join(", ")}. Remove these nameservers.`;
-            }
-        }
-
-        const now = Math.floor(Date.now() / 1000);
-
-        if (allValid) {
-            await db
-                .update(domains)
-                .set({ lastCheckedAt: now, errorMessage: null })
-                .where(eq(domains.domainId, domain.domainId));
-            logger.debug(
-                `Domain ${domain.baseDomain} passed periodic reverification`
-            );
-        } else {
-            await this.markFailed(domain.domainId, errorMessage);
-            logger.warn(
-                `Domain ${domain.baseDomain} failed periodic reverification: ${errorMessage}`
-            );
-        }
-    }
-
-    private async markFailed(
-        domainId: string,
-        errorMessage: string | null
-    ): Promise<void> {
-        await db
-            .update(domains)
-            .set({
-                verified: false,
-                failed: true,
-                // Three below MAX_TRIES: keeps the domain out of the DNS
-                // validator's immediate retry loop, while still leaving it
-                // eligible (tries < MAX_TRIES) for a few more validation
-                // passes instead of being excluded forever once tries hits
-                // MAX_TRIES.
-                tries: DNS_VALIDATOR_MAX_TRIES - 3,
-                lastCheckedAt: Math.floor(Date.now() / 1000),
-                errorMessage
-            })
-            .where(eq(domains.domainId, domainId));
-    }
-}
-
-export const domainReverifier = new DomainReverifier();

server/private/lib/certificates/index.ts

@@ -1,45 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import logger from "@server/logger";
-import { privateConfig } from "#private/lib/config";
-import { acmeClientManager } from "./acme-client";
-import { jobScheduler } from "./scheduler";
-
-export async function startCertificateManager() {
-    const acmeConfig = privateConfig.getRawPrivateConfig().acme;
-    if (
-        acmeConfig &&
-        acmeConfig.cert_mode === "pangolin" &&
-        acmeConfig.enable_acme_client
-    ) {
-        logger.info("Starting certificate management server...");
-
-        // Initialize ACME client
-        await acmeClientManager.initialize();
-
-        // Start certificate issuance/renewal jobs
-        await jobScheduler.start();
-    }
-
-    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
-        // DNS record validation/reverification doesn't require certs, so it
-        // runs whenever Pangolin is acting as the authoritative DNS server,
-        // independent of the cert manager above.
-        await jobScheduler.startDnsJobs();
-    }
-}
-
-export async function stopCertificateManager() {
-    await jobScheduler.stop();
-}

server/private/lib/certificates/scheduler.ts

@@ -1,190 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import { withTimeout } from "@server/lib/retry";
-import logger from "@server/logger";
-import { certificateService } from "./certificate-service";
-import { privateConfig as config } from "#private/lib/config";
-import { dnsValidator } from "./dns-validator";
-import { domainReverifier } from "./domain-reverifier";
-
-// Backstop for runExclusive: no single job's own internal timeouts (e.g.
-// certificate-service's per-cert issuance timeout) are relied on here. This
-// is the last line of defense - if *anything* inside a job hangs with no
-// error (a stalled Redis/DB call, a future code path that forgets to bound
-// itself, etc.), state.active must still reset so the next tick can run.
-// Without it, one hung run permanently skips every future tick for that job,
-// since runExclusive only clears state.active after the job promise settles.
-const RUN_EXCLUSIVE_TIMEOUT_MS = 30 * 60 * 1000;
-
-export class JobScheduler {
-    private certIntervals: NodeJS.Timeout[] = [];
-    private dnsIntervals: NodeJS.Timeout[] = [];
-    private certRunning = false;
-    private dnsRunning = false;
-
-    // Guards against a slow batch (e.g. 10 certs whose DNS challenges take a
-    // while) still being processed when the next interval tick fires -
-    // without this, overlapping ticks would each pull their own batch of up
-    // to 10 pending/renewal certs and process them concurrently instead of
-    // waiting for the prior batch to finish.
-    private runExclusive(
-        job: () => Promise<void>,
-        state: { active: boolean },
-        label: string
-    ): () => Promise<void> {
-        return async () => {
-            if (state.active) {
-                logger.debug(
-                    `Skipping ${label} tick - previous run still in progress`
-                );
-                return;
-            }
-            state.active = true;
-            try {
-                await withTimeout(job(), RUN_EXCLUSIVE_TIMEOUT_MS, label);
-            } catch (error) {
-                logger.error(`Error in ${label}:`, error);
-            } finally {
-                state.active = false;
-            }
-        };
-    }
-
-    // Certificate issuance/renewal - requires an ACME client, so this is
-    // only started when Pangolin is actually managing certs.
-    async start(): Promise<void> {
-        if (this.certRunning) {
-            logger.warn("Certificate job scheduler is already running");
-            return;
-        }
-
-        this.certRunning = true;
-        logger.info("Starting certificate job scheduler");
-
-        const newCertState = { active: false };
-        const renewalState = { active: false };
-
-        const runNewCertCheck = this.runExclusive(
-            () => certificateService.processPendingCertificates(),
-            newCertState,
-            "processing pending certificates"
-        );
-        const runRenewalCheck = this.runExclusive(
-            () => certificateService.processRenewalCandidates(),
-            renewalState,
-            "processing renewal candidates"
-        );
-
-        // Schedule new certificate processing
-        const newCertInterval = setInterval(
-            runNewCertCheck,
-            config.getRawConfig().acme!.new_cert_check_interval_ms
-        );
-
-        // Schedule renewal processing (every 24 hours)
-        const renewalInterval = setInterval(
-            runRenewalCheck,
-            config.getRawConfig().acme!.renewal_check_interval_ms
-        );
-
-        this.certIntervals.push(newCertInterval, renewalInterval);
-
-        // Run initial checks
-        setTimeout(async () => {
-            try {
-                await runNewCertCheck();
-                // await runRenewalCheck();
-            } catch (error) {
-                logger.error("Error in initial certificate processing:", error);
-            }
-        }, 1000); // Wait 1 second after startup
-
-        logger.info("Certificate job scheduler started successfully");
-    }
-
-    // DNS record validation/reverification - doesn't touch certs at all, so
-    // this runs independently whenever Pangolin is acting as the
-    // authoritative DNS server, regardless of cert_mode.
-    async startDnsJobs(): Promise<void> {
-        if (this.dnsRunning) {
-            logger.warn("DNS validation job scheduler is already running");
-            return;
-        }
-
-        this.dnsRunning = true;
-        logger.info("Starting DNS validation job scheduler");
-
-        const dnsValidationState = { active: false };
-        const reverifyState = { active: false };
-
-        const runDnsValidation = this.runExclusive(
-            () => dnsValidator.validateAll(),
-            dnsValidationState,
-            "validating DNS records"
-        );
-        const runReverify = this.runExclusive(
-            () => domainReverifier.reverifyAll(),
-            reverifyState,
-            "reverifying domains"
-        );
-
-        // Schedule DNS validation
-        const dnsValidationInterval = setInterval(
-            runDnsValidation,
-            config.getRawConfig().acme?.dns_check_interval_ms ?? 60000
-        );
-
-        // Schedule periodic reverification of already-verified domains
-        const reverifyInterval = setInterval(
-            runReverify,
-            config.getRawConfig().acme?.domain_reverification_interval_ms ??
-                3600000
-        );
-
-        this.dnsIntervals.push(dnsValidationInterval, reverifyInterval);
-
-        // Run an initial validation pass shortly after startup
-        setTimeout(async () => {
-            try {
-                await runDnsValidation();
-            } catch (error) {
-                logger.error("Error in initial DNS validation:", error);
-            }
-        }, 1000);
-
-        logger.info("DNS validation job scheduler started successfully");
-    }
-
-    async stop(): Promise<void> {
-        if (this.certRunning) {
-            logger.info("Stopping certificate job scheduler");
-            this.certRunning = false;
-            this.certIntervals.forEach((interval) => clearInterval(interval));
-            this.certIntervals = [];
-        }
-
-        if (this.dnsRunning) {
-            logger.info("Stopping DNS validation job scheduler");
-            this.dnsRunning = false;
-            this.dnsIntervals.forEach((interval) => clearInterval(interval));
-            this.dnsIntervals = [];
-        }
-    }
-
-    isRunning(): boolean {
-        return this.certRunning || this.dnsRunning;
-    }
-}
-
-export const jobScheduler = new JobScheduler();

server/private/lib/config.ts

@@ -146,20 +146,12 @@ export class PrivateConfig {
             process.env.USE_PANGOLIN_DNS =
                 this.rawPrivateConfig.flags.use_pangolin_dns.toString();
         }
-
-        if (this.rawPrivateConfig.acme?.cert_mode) {
-            process.env.CERT_MODE = this.rawPrivateConfig.acme.cert_mode;
-        }
     }
 
     public getRawPrivateConfig() {
         return this.rawPrivateConfig;
     }
 
-    public getRawConfig() {
-        return this.getRawPrivateConfig();
-    }
-
     // `flags.enable_acme_cert_sync`, `flags.disable_private_http_placeholder`,
     // and `acme` used to live in the private config file. They now live in
     // the public config file. If an operator still has them set in the

server/private/lib/dns/index.ts

@@ -1,14 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-export * from "./server";

server/private/lib/readConfigFile.ts

@@ -95,70 +95,6 @@ export const privateConfigSchema = z
                     .optional()
             })
             .optional(),
-        dns: z
-            .object({
-                enabled: z.boolean().optional().default(false),
-                listen_port: z.number().int().positive().optional().default(53),
-                nameserver_name: z.string(),
-                cname_extension: z.string(),
-                site_extension: z.string().optional(),
-                cname_alternate_extensions: z
-                    .array(z.string())
-                    .optional()
-                    .default([]),
-                alternate_nameservers: z
-                    .array(z.string())
-                    .optional()
-                    .default([]),
-                rate_limit: z
-                    .object({
-                        enabled: z.boolean().optional().default(true),
-                        window_ms: z
-                            .number()
-                            .int()
-                            .min(1000)
-                            .max(600000)
-                            .optional()
-                            .default(60000),
-                        max_requests: z
-                            .number()
-                            .int()
-                            .min(50)
-                            .max(100000)
-                            .optional()
-                            .default(1200),
-                        max_requests_per_query_type: z
-                            .number()
-                            .int()
-                            .min(10)
-                            .max(50000)
-                            .optional()
-                            .default(600)
-                    })
-                    .default({
-                        enabled: true,
-                        window_ms: 60000,
-                        max_requests: 1200,
-                        max_requests_per_query_type: 600
-                    }),
-                static_records: z
-                    .array(
-                        z.object({
-                            domain: z.string(),
-                            type: z.enum(["TXT", "CNAME", "A", "NS"]),
-                            value: z.string(),
-                            ttl: z
-                                .number()
-                                .int()
-                                .positive()
-                                .optional()
-                                .default(300)
-                        })
-                    )
-                    .optional()
-                    .default([])
-            })
-            .optional(),
         gerbil: z
             .object({
                 local_exit_node_reachable_at: z
@@ -189,86 +125,15 @@ export const privateConfigSchema = z
             })
             .optional()
             .prefault({}),
+        // @deprecated Moved to the public config file as `acme`
+        // (server/lib/readConfigFile.ts). Kept here only so existing private
+        // config files keep parsing; any value set here is migrated into the
+        // public config at startup by PrivateConfig (server/private/lib/config.ts).
         acme: z
             .object({
-                cert_mode: z
-                    .enum(["traefik", "pangolin"])
-                    .optional()
-                    .default("traefik"),
-                enable_acme_client: z.boolean().optional().default(false),
-                // @deprecated Moved to the public config file
-                // (server/lib/readConfigFile.ts). Kept here only so existing private
-                // config files keep parsing; any value set here is migrated into the
-                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_json_path: z.string().optional(),
-                // @deprecated Moved to the public config file
-                // (server/lib/readConfigFile.ts). Kept here only so existing private
-                // config files keep parsing; any value set here is migrated into the
-                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_http_endpoint: z.string().optional(),
-                // @deprecated Moved to the public config file
+                sync_interval_ms: z.number().optional()
-                // (server/lib/readConfigFile.ts). Kept here only so existing private
-                // config files keep parsing; any value set here is migrated into the
-                // public config at startup by PrivateConfig (server/private/lib/config.ts).
-                sync_interval_ms: z.number().optional(),
-                acme_directory_url: z
-                    .string()
-                    .url()
-                    .default("https://acme-v02.api.letsencrypt.org/directory"),
-                contact_email: z.string().email(),
-                acme_account_key_path: z
-                    .string()
-                    .default("./config/account.key"),
-                challenge_ttl_ms: z.number().int().positive().default(300000),
-                renewal_check_interval_ms: z
-                    .number()
-                    .int()
-                    .positive()
-                    .default(3600000),
-                new_cert_check_interval_ms: z
-                    .number()
-                    .int()
-                    .positive()
-                    .default(60000),
-                // Kept safely under Let's Encrypt's ~20 req/s limit since this
-                // budget is shared across all pops workers and only covers the
-                // request-issuing calls we make directly (not every request
-                // acme-client makes internally, e.g. while polling for
-                // challenge/order status).
-                acme_requests_per_second: z
-                    .number()
-                    .int()
-                    .positive()
-                    .default(15),
-                dns_check_interval_ms: z
-                    .number()
-                    .int()
-                    .positive()
-                    .default(60000),
-                domain_reverification_interval_ms: z
-                    .number()
-                    .int()
-                    .positive()
-                    .default(3600000), // 1 hour — how often to run the reverification pass
-                domain_reverification_window_ms: z
-                    .number()
-                    .int()
-                    .positive()
-                    .default(259200000), // 72 hours — how old checkedAt must be before rechecking
-                domain_reverification_batch_size: z
-                    .number()
-                    .int()
-                    .positive()
-                    .default(20), // max domains to recheck per pass
-                dns_resolvers: z
-                    .array(z.string())
-                    .optional()
-                    .default([
-                        "8.8.8.8",
-                        "1.1.1.1",
-                        "9.9.9.9",
-                        "208.67.222.222"
-                    ])
             })
             .optional(),
         branding: z

server/private/lib/traefik/getTraefikConfig.ts

@@ -396,7 +396,7 @@ export async function getTraefikConfig(
         );
 
     let validCerts: CertificateResult[] = [];
-    if (privateConfig.getRawPrivateConfig().acme?.cert_mode == "pangolin") {
+    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
         // create a list of all domains to get certs for
         const domains = new Set<string>();
         for (const resource of resourcesMap.values()) {
@@ -522,10 +522,7 @@ export async function getTraefikConfig(
             );
 
             let tls = {};
-            if (
+            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
-                privateConfig.getRawPrivateConfig().acme?.cert_mode !=
-                "pangolin"
-            ) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!resource.subdomain,
@@ -792,8 +789,7 @@ export async function getTraefikConfig(
                 preferWildcardCert
             }) => {
                 if (
-                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                    "pangolin"
                 ) {
                     return buildWildcardTls({
                         fullDomain,
@@ -836,8 +832,7 @@ export async function getTraefikConfig(
             redirectHttpsMiddlewareName,
             resolveTls: (fullDomain) => {
                 if (
-                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                    "pangolin"
                 ) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
@@ -929,10 +924,7 @@ export async function getTraefikConfig(
             const rule = buildHostRule(fullDomain, ir.wildcard);
 
             let tls: any = {};
-            if (
+            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
-                privateConfig.getRawPrivateConfig().acme?.cert_mode !=
-                "pangolin"
-            ) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!ir.subdomain,
@@ -1013,8 +1005,7 @@ export async function getTraefikConfig(
 
                 let tls: any = {};
                 if (
-                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                    "pangolin"
                 ) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
@@ -1089,7 +1080,7 @@ export async function getTraefikConfig(
             .where(eq(exitNodes.exitNodeId, exitNodeId));
 
         let validCertsLoginPages: CertificateResult[] = [];
-        if (privateConfig.getRawPrivateConfig().acme?.cert_mode == "pangolin") {
+        if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
             // create a list of all domains to get certs for
             const domains = new Set<string>();
             for (const lp of exitNodeLoginPages) {
@@ -1135,8 +1126,7 @@ export async function getTraefikConfig(
 
                 const tls = {};
                 if (
-                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                    "pangolin"
                 ) {
                     // TODO: we need to add the wildcard logic here too
                 } else {

server/private/routers/alertRule/testAlertRule.ts

@@ -14,7 +14,7 @@
 import { getRandomItemInArray } from "@app/lib/getRandomItemInArray";
 import response from "@server/lib/response";
 import logger from "@server/logger";
-import { processTestAlerts } from "#private/lib/alerts/processTestAlerts";
+import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts";
 import { type AlertAction } from "@server/routers/alertRule/types";
 import HttpCode from "@server/types/HttpCode";
 import { NextFunction, Request, Response } from "express";

server/private/routers/alertRule/updateAlertRule.ts

@@ -33,11 +33,8 @@ import { OpenAPITags, registry } from "@server/openApi";
 import { and, eq } from "drizzle-orm";
 import { encrypt } from "@server/lib/crypto";
 import config from "@server/lib/config";
-import {
+import { HC_EVENT_TYPES, SITE_EVENT_TYPES, RESOURCE_EVENT_TYPES } from "./createAlertRule";
-    HC_EVENT_TYPES,
+import { invalidateAllRemoteExitNodeSessions } from "@server/private/auth/sessions/remoteExitNode";
-    SITE_EVENT_TYPES,
-    RESOURCE_EVENT_TYPES
-} from "./createAlertRule";
 
 const paramsSchema = z
     .object({
@@ -88,57 +85,35 @@ const bodySchema = z
         const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes(
             val.eventType
         );
-        const isResourceEvent = (
+        const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes(
-            RESOURCE_EVENT_TYPES as readonly string[]
+            val.eventType
-        ).includes(val.eventType);
+        );
 
-        if (
+        if (isSiteEvent && val.siteIds !== undefined && val.siteIds.length === 0 && !val.allSites) {
-            isSiteEvent &&
-            val.siteIds !== undefined &&
-            val.siteIds.length === 0 &&
-            !val.allSites
-        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message:
+                message: "At least one siteId is required for site event types when allSites is false",
-                    "At least one siteId is required for site event types when allSites is false",
                 path: ["siteIds"]
             });
         }
 
-        if (
+        if (isHcEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length === 0 && !val.allHealthChecks) {
-            isHcEvent &&
-            val.healthCheckIds !== undefined &&
-            val.healthCheckIds.length === 0 &&
-            !val.allHealthChecks
-        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message:
+                message: "At least one healthCheckId is required for health check event types when allHealthChecks is false",
-                    "At least one healthCheckId is required for health check event types when allHealthChecks is false",
                 path: ["healthCheckIds"]
             });
         }
 
-        if (
+        if (isResourceEvent && val.resourceIds !== undefined && val.resourceIds.length === 0 && !val.allResources) {
-            isResourceEvent &&
-            val.resourceIds !== undefined &&
-            val.resourceIds.length === 0 &&
-            !val.allResources
-        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message:
+                message: "At least one resourceId is required for resource event types when allResources is false",
-                    "At least one resourceId is required for resource event types when allResources is false",
                 path: ["resourceIds"]
             });
         }
 
-        if (
+        if (isSiteEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
-            isSiteEvent &&
-            val.healthCheckIds !== undefined &&
-            val.healthCheckIds.length > 0
-        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "healthCheckIds must not be set for site event types",
@@ -154,11 +129,7 @@ const bodySchema = z
             });
         }
 
-        if (
+        if (isResourceEvent && val.siteIds !== undefined && val.siteIds.length > 0) {
-            isResourceEvent &&
-            val.siteIds !== undefined &&
-            val.siteIds.length > 0
-        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "siteIds must not be set for resource event types",
@@ -166,15 +137,10 @@ const bodySchema = z
             });
         }
 
-        if (
+        if (isResourceEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
-            isResourceEvent &&
-            val.healthCheckIds !== undefined &&
-            val.healthCheckIds.length > 0
-        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message:
+                message: "healthCheckIds must not be set for resource event types",
-                    "healthCheckIds must not be set for resource event types",
                 path: ["healthCheckIds"]
             });
         }
@@ -187,6 +153,7 @@ const UpdateAlertRuleResponseDataSchema = z.object({
     alertRuleId: z.number()
 });
 
+
 registry.registerPath({
     method: "post",
     path: "/org/{orgId}/alert-rule/{alertRuleId}",
@@ -207,9 +174,7 @@ registry.registerPath({
             description: "Successful response",
             content: {
                 "application/json": {
-                    schema: createApiResponseSchema(
+                    schema: createApiResponseSchema(UpdateAlertRuleResponseDataSchema)
-                        UpdateAlertRuleResponseDataSchema
-                    )
                 }
             }
         }
@@ -285,11 +250,9 @@ export async function updateAlertRule(
         if (name !== undefined) updateData.name = name;
         if (eventType !== undefined) updateData.eventType = eventType;
         if (enabled !== undefined) updateData.enabled = enabled;
-        if (cooldownSeconds !== undefined)
+        if (cooldownSeconds !== undefined) updateData.cooldownSeconds = cooldownSeconds;
-            updateData.cooldownSeconds = cooldownSeconds;
         if (allSites !== undefined) updateData.allSites = allSites;
-        if (allHealthChecks !== undefined)
+        if (allHealthChecks !== undefined) updateData.allHealthChecks = allHealthChecks;
-            updateData.allHealthChecks = allHealthChecks;
         if (allResources !== undefined) updateData.allResources = allResources;
 
         await db
@@ -310,11 +273,7 @@ export async function updateAlertRule(
 
             // Only insert junction rows when allSites is not true
             const effectiveAllSites = allSites ?? false;
-            if (
+            if (!effectiveAllSites && siteIds !== undefined && siteIds.length > 0) {
-                !effectiveAllSites &&
-                siteIds !== undefined &&
-                siteIds.length > 0
-            ) {
                 await db.insert(alertSites).values(
                     siteIds.map((siteId) => ({
                         alertRuleId,
@@ -331,11 +290,7 @@ export async function updateAlertRule(
                 .where(eq(alertHealthChecks.alertRuleId, alertRuleId));
 
             const effectiveAllHealthChecks = allHealthChecks ?? false;
-            if (
+            if (!effectiveAllHealthChecks && healthCheckIds !== undefined && healthCheckIds.length > 0) {
-                !effectiveAllHealthChecks &&
-                healthCheckIds !== undefined &&
-                healthCheckIds.length > 0
-            ) {
                 await db.insert(alertHealthChecks).values(
                     healthCheckIds.map((healthCheckId) => ({
                         alertRuleId,
@@ -352,11 +307,7 @@ export async function updateAlertRule(
                 .where(eq(alertResources.alertRuleId, alertRuleId));
 
             const effectiveAllResources = allResources ?? false;
-            if (
+            if (!effectiveAllResources && resourceIds !== undefined && resourceIds.length > 0) {
-                !effectiveAllResources &&
-                resourceIds !== undefined &&
-                resourceIds.length > 0
-            ) {
                 await db.insert(alertResources).values(
                     resourceIds.map((resourceId) => ({
                         alertRuleId,
@@ -441,10 +392,7 @@ export async function updateAlertRule(
                     webhookActions.map((wa) => ({
                         alertRuleId,
                         webhookUrl: wa.webhookUrl,
-                        config:
+                        config: wa.config != null ? encrypt(wa.config, serverSecret) : null,
-                            wa.config != null
-                                ? encrypt(wa.config, serverSecret)
-                                : null,
                         enabled: wa.enabled
                     }))
                 );

server/private/routers/generatedLicense/clearInstanceName.ts

@@ -31,9 +31,7 @@ export async function clearInstanceName(
     next: NextFunction
 ): Promise<any> {
     try {
-        const parsedParams = clearInstanceNameParamsSchema.safeParse(
+        const parsedParams = clearInstanceNameParamsSchema.safeParse(req.params);
-            req.params
-        );
         if (!parsedParams.success) {
             return next(
                 createHttpError(
@@ -65,8 +63,7 @@ export async function clearInstanceName(
             return next(
                 createHttpError(
                     data.status || HttpCode.BAD_REQUEST,
-                    data.message ||
+                    data.message || "Failed to clear instance name from Fossorial API"
-                        "Failed to clear server ID from Fossorial API"
                 )
             );
         }
@@ -75,7 +72,7 @@ export async function clearInstanceName(
             data: null,
             success: true,
             error: false,
-            message: "Server ID cleared successfully",
+            message: "Instance name cleared successfully",
             status: HttpCode.OK
         });
     } catch (error) {
@@ -83,8 +80,8 @@ export async function clearInstanceName(
         return next(
             createHttpError(
                 HttpCode.INTERNAL_SERVER_ERROR,
-                "An error occurred while clearing the server ID."
+                "An error occurred while clearing the instance name."
             )
         );
     }
-}
+}

server/private/routers/hybrid.ts

@@ -2478,12 +2478,7 @@ hybridRouter.post(
                 destinations: destinations
             });
         } catch (error) {
-            if (!(
+            logger.error(error);
-                error instanceof Error &&
-                error.message === "Exit node not allowed"
-            )) {
-                logger.error(error);
-            }
             return next(
                 createHttpError(
                     HttpCode.INTERNAL_SERVER_ERROR,

server/private/routers/remoteExitNode/exitNodeEvents.ts

@@ -1,23 +0,0 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
-import { EventEmitter } from "events";
-
-export interface ExitNodeOnlineEvent {
-    exitNodeId: number;
-    endpoint: string;
-}
-
-export const EXIT_NODE_ONLINE_EVENT = "exit-node-online";
-
-export const exitNodeEvents = new EventEmitter();

server/private/routers/remoteExitNode/exitNodeReconnectScheduler.ts

@@ -12,27 +12,11 @@
  */
 
 import axios from "axios";
-import { db, newts, sites } from "@server/db";
+import { db, exitNodes, newts, sites } from "@server/db";
 import { eq } from "drizzle-orm";
 import logger from "@server/logger";
 import redisManager from "#private/lib/redis";
-import { sendToClient } from "../ws";
+// import { sendToClient } from "#private/routers/ws";
-import {
-    exitNodeEvents,
-    EXIT_NODE_ONLINE_EVENT,
-    ExitNodeOnlineEvent
-} from "./exitNodeEvents";
-
-exitNodeEvents.on(
-    EXIT_NODE_ONLINE_EVENT,
-    ({ exitNodeId, endpoint }: ExitNodeOnlineEvent) => {
-        scheduleExitNodeReconnect(exitNodeId, endpoint).catch((error) => {
-            logger.error("Failed to schedule exit node reconnect", {
-                error
-            });
-        });
-    }
-);
 
 const INITIAL_DELAY_MS = 15 * 1000; // 15 seconds before first check
 const CHECK_INTERVAL_MS = 10 * 1000; // Check every 10 seconds
@@ -42,7 +26,7 @@ const REDIS_HASH_PREFIX = "exit-node-reconnect:";
 
 interface PendingReconnect {
     startTime: number;
-    endpoint: string;
+    reachableAt: string;
 }
 
 // In-memory tracking for this node
@@ -56,15 +40,15 @@ let schedulerInterval: NodeJS.Timeout | null = null;
  */
 export async function scheduleExitNodeReconnect(
     exitNodeId: number,
-    endpoint: string
+    reachableAt: string
 ): Promise<void> {
     logger.info(
-        `Scheduling newt reconnect for exit node ${exitNodeId} (endpoint: ${endpoint})`
+        `Scheduling newt reconnect for exit node ${exitNodeId} (reachableAt: ${reachableAt})`
     );
 
     const entry: PendingReconnect = {
         startTime: Date.now(),
-        endpoint
+        reachableAt
     };
 
     pendingReconnects.set(exitNodeId, entry);
@@ -79,8 +63,8 @@ export async function scheduleExitNodeReconnect(
         );
         await redisManager.hset(
             `${REDIS_HASH_PREFIX}${exitNodeId}`,
-            "endpoint",
+            "reachableAt",
-            endpoint
+            reachableAt
         );
     }
 }
@@ -117,14 +101,14 @@ async function processPendingReconnects(): Promise<void> {
                     `${REDIS_HASH_PREFIX}${id}`,
                     "startTime"
                 );
-                const endpoint = await redisManager.hget(
+                const reachableAt = await redisManager.hget(
                     `${REDIS_HASH_PREFIX}${id}`,
-                    "endpoint"
+                    "reachableAt"
                 );
-                if (startTimeStr && endpoint) {
+                if (startTimeStr && reachableAt) {
                     toProcess.set(id, {
                         startTime: parseInt(startTimeStr, 10),
-                        endpoint
+                        reachableAt
                     });
                 }
             }
@@ -151,7 +135,7 @@ async function processPendingReconnects(): Promise<void> {
         }
 
         // Check if the exit node HTTP endpoint is reachable
-        const pingUrl = `http://${entry.endpoint}/ping`;
+        const pingUrl = `${entry.reachableAt}/ping`;
         try {
             await axios.get(pingUrl, { timeout: 5000 });
         } catch {
@@ -166,47 +150,47 @@ async function processPendingReconnects(): Promise<void> {
             `Exit node ${exitNodeId} is reachable. Sending newt/wg/reconnect to connected newts.`
         );
 
-        await sendReconnectToNewts(exitNodeId);
+        // await sendReconnectToNewts(exitNodeId);
         await removePending(exitNodeId);
     }
 }
 
-async function sendReconnectToNewts(exitNodeId: number): Promise<void> {
+// async function sendReconnectToNewts(exitNodeId: number): Promise<void> {
-    try {
+// try {
-        const connectedNewts = await db
+//         const connectedNewts = await db
-            .select({ newtId: newts.newtId })
+//             .select({ newtId: newts.newtId })
-            .from(newts)
+//             .from(newts)
-            .innerJoin(sites, eq(newts.siteId, sites.siteId))
+//             .innerJoin(sites, eq(newts.siteId, sites.siteId))
-            .where(eq(sites.exitNodeId, exitNodeId));
+//             .where(eq(sites.exitNodeId, exitNodeId));
 
-        if (connectedNewts.length === 0) {
+//         if (connectedNewts.length === 0) {
-            logger.debug(
+//             logger.debug(
-                `No newts found for exit node ${exitNodeId}, nothing to reconnect`
+//                 `No newts found for exit node ${exitNodeId}, nothing to reconnect`
-            );
+//             );
-            return;
+//             return;
-        }
+//         }
 
-        logger.info(
+//         logger.info(
-            `Sending newt/wg/reconnect to ${connectedNewts.length} newt(s) for exit node ${exitNodeId}`
+//             `Sending newt/wg/reconnect to ${connectedNewts.length} newt(s) for exit node ${exitNodeId}`
-        );
+//         );
 
-        const reconnectMessage = {
+//         const reconnectMessage = {
-            type: "newt/wg/reconnect",
+//             type: "newt/wg/reconnect",
-            data: {}
+//             data: {}
-        };
+//         };
 
-        await Promise.allSettled(
+//         await Promise.allSettled(
-            connectedNewts.map(({ newtId }) =>
+//             connectedNewts.map(({ newtId }) =>
-                sendToClient(newtId, reconnectMessage)
+//                 sendToClient(newtId, reconnectMessage)
-            )
+//             )
-        );
+//         );
-    } catch (error) {
+//     } catch (error) {
-        logger.error(
+//         logger.error(
-            `Failed to send reconnect messages for exit node ${exitNodeId}`,
+//             `Failed to send reconnect messages for exit node ${exitNodeId}`,
-            { error }
+//             { error }
-        );
+//         );
-    }
+//     }
-}
+// }
 
 async function removePending(exitNodeId: number): Promise<void> {
     pendingReconnects.delete(exitNodeId);

server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts

@@ -26,7 +26,7 @@ import {
     validateRemoteExitNodeSessionToken,
     EXPIRES
 } from "#private/auth/sessions/remoteExitNode";
-import { getOrCreateCachedToken } from "#private/lib/tokenCache";
+import { getOrCreateCachedToken } from "@server/private/lib/tokenCache";
 import { verifyPassword } from "@server/auth/password";
 import logger from "@server/logger";
 import config from "@server/lib/config";

server/private/routers/remoteExitNode/handleRemoteExitNodePingMessage.ts

@@ -16,7 +16,7 @@ import { MessageHandler } from "@server/routers/ws";
 import { RemoteExitNode } from "@server/db";
 import { eq } from "drizzle-orm";
 import logger from "@server/logger";
-import { exitNodeEvents, EXIT_NODE_ONLINE_EVENT } from "./exitNodeEvents";
+import { scheduleExitNodeReconnect } from "./exitNodeReconnectScheduler";
 
 /**
  * Handles ping messages from clients and responds with pong
@@ -40,7 +40,7 @@ export const handleRemoteExitNodePingMessage: MessageHandler = async (
     try {
         // Fetch the current state before updating so we can detect the offline→online transition
         const [currentExitNode] = await db
-            .select({ online: exitNodes.online, endpoint: exitNodes.endpoint })
+            .select({ online: exitNodes.online, reachableAt: exitNodes.reachableAt })
             .from(exitNodes)
             .where(eq(exitNodes.exitNodeId, remoteExitNode.exitNodeId))
             .limit(1);
@@ -55,14 +55,12 @@ export const handleRemoteExitNodePingMessage: MessageHandler = async (
             .where(eq(exitNodes.exitNodeId, remoteExitNode.exitNodeId));
 
         // If the exit node was offline and is now coming online, schedule newt reconnects
-        if (
+        if (currentExitNode && !currentExitNode.online && currentExitNode.reachableAt) {
-            currentExitNode &&
+            scheduleExitNodeReconnect(
-            !currentExitNode.online &&
+                remoteExitNode.exitNodeId,
-            currentExitNode.endpoint
+                currentExitNode.reachableAt
-        ) {
+            ).catch((error) => {
-            exitNodeEvents.emit(EXIT_NODE_ONLINE_EVENT, {
+                logger.error("Failed to schedule exit node reconnect", { error });
-                exitNodeId: remoteExitNode.exitNodeId,
-                endpoint: currentExitNode.endpoint
             });
         }
     } catch (error) {

server/routers/domain/createOrgDomain.ts

@@ -300,15 +300,12 @@ export async function createOrgDomain(
                     {
                         value: `${domainId}.${config.getRawConfig().dns.cname_extension}`,
                         baseDomain: baseDomain
-                    }
+                    },
-                ];
+                    {
-
-                if (build == "saas") {
-                    cnameRecords.push({
                         value: `_acme-challenge.${domainId}.${config.getRawConfig().dns.cname_extension}`,
                         baseDomain: `_acme-challenge.${baseDomain}`
-                    });
+                    }
-                }
+                ];
 
                 // Save CNAME records to database
                 for (const cnameRecord of cnameRecords) {

server/routers/newt/handleRequestLogMessage.ts

@@ -1,3 +1,16 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
 import { db } from "@server/db";
 import { MessageHandler } from "@server/routers/ws";
 import { sites, Newt, orgs, clients, clientSitesAssociationsCache, users } from "@server/db";

server/routers/olm/handleOlmServerInitAddPeerHandshake.ts

@@ -85,7 +85,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             );
 
         if (!resources || resources.length === 0) {
-            logger.warn(
+            logger.error(
                 `handleOlmServerInitAddPeerHandshake: Resource not found`
             );
             await sendCancel();
@@ -94,7 +94,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
 
         if (resources.length > 1) {
             // error but this should not happen because the nice id cant contain a dot and the alias has to have a dot and both have to be unique within the org so there should never be multiple matches
-            logger.warn(
+            logger.error(
                 `handleOlmServerInitAddPeerHandshake: Multiple resources found matching the criteria`
             );
             return;
@@ -119,7 +119,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             );
 
         if (currentResourceAssociationCaches.length === 0) {
-            logger.warn(
+            logger.error(
                 `handleOlmServerInitAddPeerHandshake: Client ${client.clientId} does not have access to resource ${resource.siteResourceId}`
             );
             await sendCancel();
@@ -127,7 +127,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
         }
 
         if (!resource.networkId) {
-            logger.warn(
+            logger.error(
                 `handleOlmServerInitAddPeerHandshake: Resource ${resource.siteResourceId} has no network`
             );
             await sendCancel();
@@ -141,7 +141,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             .where(eq(siteNetworks.networkId, resource.networkId));
 
         if (!siteRows || siteRows.length === 0) {
-            logger.warn(
+            logger.error(
                 `handleOlmServerInitAddPeerHandshake: No sites found for resource ${resource.siteResourceId}`
             );
             await sendCancel();
@@ -164,7 +164,9 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
     }
 
     if (sitesToProcess.length === 0) {
-        logger.warn(`handleOlmServerInitAddPeerHandshake: No sites to process`);
+        logger.error(
+            `handleOlmServerInitAddPeerHandshake: No sites to process`
+        );
         await sendCancel();
         return;
     }
@@ -191,7 +193,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
         }
 
         if (!site.exitNodeId) {
-            logger.warn(
+            logger.error(
                 `handleOlmServerInitAddPeerHandshake: Site ${site.siteId} has no exit node, skipping`
             );
             continue;
@@ -203,7 +205,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             .where(eq(exitNodes.exitNodeId, site.exitNodeId));
 
         if (!exitNode) {
-            logger.warn(
+            logger.error(
                 `handleOlmServerInitAddPeerHandshake: Exit node not found for site ${site.siteId}, skipping`
             );
             continue;
@@ -227,7 +229,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
     }
 
     if (!handshakeInitiated) {
-        logger.warn(
+        logger.error(
             `handleOlmServerInitAddPeerHandshake: No accessible sites with valid exit nodes found, cancelling chain`
         );
         await sendCancel();

src/app/[orgId]/layout.tsx

@@ -104,7 +104,7 @@ export default async function OrgLayout(props: {
             subscriptionStatus = subRes.data.data;
         } catch (error) {
             // If subscription fetch fails, keep subscriptionStatus as null
-            // console.error("Failed to fetch subscription status:", error);
+            console.error("Failed to fetch subscription status:", error);
         }
     }
 

src/components/GenerateLicenseKeysTable.tsx

@@ -78,16 +78,13 @@ export default function GenerateLicenseKeysTable({
             );
             toast({
                 title: t("success"),
-                description: "Server ID cleared successfully"
+                description: "Instance name cleared successfully"
             });
             await refreshData();
         } catch (error) {
             toast({
                 title: t("error"),
-                description: formatAxiosError(
+                description: formatAxiosError(error, "Failed to clear instance name"),
-                    error,
-                    "Failed to clear server ID"
-                ),
                 variant: "destructive"
             });
         } finally {
@@ -294,7 +291,7 @@ export default function GenerateLicenseKeysTable({
                                         clearInstanceName(key.licenseKey)
                                     }
                                 >
-                                    {t("clearInstanceName")}
+                                    Clear Instance Name
                                 </DropdownMenuItem>
                             </DropdownMenuContent>
                         </DropdownMenu>

src/lib/pullEnv.ts

@@ -1,4 +1,3 @@
-import { build } from "@server/build";
 import { Env } from "./types/env";
 
 export function pullEnv(): Env {
@@ -35,7 +34,9 @@ export function pullEnv(): Env {
                         : false
             },
             identityProviderMode: process.env.IDENTITY_PROVIDER_MODE as
-                "org" | "global" | undefined
+                | "org"
+                | "global"
+                | undefined
         },
         email: {
             emailEnabled: process.env.EMAIL_ENABLED === "true" ? true : false

tsconfig.enterprise.json

@@ -14,7 +14,7 @@
         "moduleResolution": "bundler",
         "resolveJsonModule": true,
         "isolatedModules": true,
-        "jsx": "react-jsx",
+        "jsx": "preserve",
         "incremental": true,
         "paths": {
             "@server/*": [