diff --git a/messages/en-US.json b/messages/en-US.json index 7ed4906c1..dbafaa9df 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -3477,7 +3477,8 @@ }, "priority": "Priority", "priorityDescription": "Higher priority routes are evaluated first. Priority = 100 means automatic ordering (system decides). Use another number to enforce manual priority.", - "instanceName": "Instance Name", + "instanceName": "Server ID", + "clearInstanceName": "Reset Server Association", "pathMatchModalTitle": "Configure Path Matching", "pathMatchModalDescription": "Set up how incoming requests should be matched based on their path.", "pathMatchType": "Match Type", diff --git a/package-lock.json b/package-lock.json index a66e92461..2826e825f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,7 @@ "@xterm/addon-fit": "^0.11.0", "@xterm/addon-web-links": "^0.12.0", "@xterm/xterm": "^6.0.0", + "acme-client": "^5.4.0", "arctic": "3.7.0", "axios": "1.20.0", "better-sqlite3": "11.9.1", @@ -7453,6 +7454,22 @@ "node": ">= 0.6" } }, + "node_modules/acme-client": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/acme-client/-/acme-client-5.4.0.tgz", + "integrity": "sha512-mORqg60S8iML6XSmVjqjGHJkINrCGLMj2QvDmFzI9vIlv1RGlyjmw3nrzaINJjkNsYXC41XhhD5pfy7CtuGcbA==", + "license": "MIT", + "dependencies": { + "@peculiar/x509": "^1.11.0", + "asn1js": "^3.0.5", + "axios": "^1.7.2", + "debug": "^4.3.5", + "node-forge": "^1.3.1" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", @@ -13170,6 +13187,15 @@ "semver": "bin/semver.js" } }, + "node_modules/node-forge": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, "node_modules/node-releases": { "version": "2.0.54", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", diff --git a/package.json b/package.json index 962773050..daca14203 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "@xterm/addon-fit": "^0.11.0", "@xterm/addon-web-links": "^0.12.0", "@xterm/xterm": "^6.0.0", + "acme-client": "^5.4.0", "arctic": "3.7.0", "axios": "1.20.0", "better-sqlite3": "11.9.1", diff --git a/server/lib/retry.ts b/server/lib/retry.ts new file mode 100644 index 000000000..76d6b8e25 --- /dev/null +++ b/server/lib/retry.ts @@ -0,0 +1,72 @@ +import logger from "@lib/logger"; + +export async function withRetry( + fn: () => Promise, + options: { + retries?: number; + baseDelayMs?: number; + label?: string; + // Called with each caught error to decide whether it's worth + // retrying. Defaults to retrying everything (existing behavior) - + // pass this to exclude errors that are known to be permanent (e.g. + // an upstream rate limit or validation rejection) rather than + // transient, so they fail fast instead of wasting retry attempts. + shouldRetry?: (error: unknown) => boolean; + } = {} +): Promise { + const { + retries = 3, + baseDelayMs = 250, + label = "operation", + shouldRetry = () => true + } = options; + + let attempt = 0; + while (true) { + try { + return await fn(); + } catch (error) { + attempt++; + if (attempt > retries || !shouldRetry(error)) { + throw error; + } + + // Exponential backoff with jitter so retries don't all land at once. + const delay = + baseDelayMs * 2 ** (attempt - 1) * (0.5 + Math.random()); + + logger.warn( + `${label} failed (attempt ${attempt}/${retries + 1}), retrying in ${delay.toFixed(0)}ms`, + error + ); + + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } +} + +// Bounds an operation that has no timeout of its own (e.g. acme-client's +// axios instance never sets one, so a stalled TCP connection to the ACME +// server hangs forever instead of erroring). Without this, a single hung +// call can leave its caller's promise permanently unsettled - fatal for +// code that gates future work on that promise resolving, like the +// scheduler's runExclusive() waiting on a batch's Promise.all. +export async function withTimeout( + promise: Promise, + ms: number, + label = "operation" +): Promise { + let timer: NodeJS.Timeout; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timed out after ${ms}ms`)), + ms + ); + }); + + try { + return await Promise.race([promise, timeout]); + } finally { + clearTimeout(timer!); + } +} diff --git a/server/private/lib/cache.ts b/server/private/lib/cache.ts index 7ddfeb554..1569cc25a 100644 --- a/server/private/lib/cache.ts +++ b/server/private/lib/cache.ts @@ -13,7 +13,7 @@ import NodeCache from "node-cache"; import logger from "@server/logger"; -import { redisManager, regionalRedisManager } from "@server/private/lib/redis"; +import { redisManager, regionalRedisManager } from "#private/lib/redis"; // Create local cache with maxKeys limit to prevent memory leaks // With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient diff --git a/server/private/lib/certificates/acme-client.ts b/server/private/lib/certificates/acme-client.ts new file mode 100644 index 000000000..c1865b30f --- /dev/null +++ b/server/private/lib/certificates/acme-client.ts @@ -0,0 +1,298 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import * as acme from "acme-client"; +import * as fs from "fs"; +import { eq } from "drizzle-orm/sql"; +import { privateConfig as config } from "#private/lib/config"; +import { DnsChallenge, db, dnsChallenge } from "@server/db"; +import { withRetry } from "@server/lib/retry"; +import logger from "@server/logger"; +import { acmeRateLimiter } from "./acmeRateLimiter"; + +// acme-client's own retry/backoff logging (429 retries, 5xx retries, each +// status-poll tick in waitForValidStatus) is a no-op by default - it only +// activates via DEBUG=acme-client or this call, neither of which was wired +// up. Without it, a cert silently retrying a Let's Encrypt rate limit for +// several minutes is indistinguishable in our logs from one that's actually +// hung, since our own logging only wraps the call, not what happens inside +// it. Must run before any AcmeClient method is called. +acme.setLogger((msg: string) => logger.info(`[acme-client] ${msg}`)); + +// acme-client's axios retry wrapper treats any response-less request error +// (timeout, connection reset, DNS blip reaching the ACME server) as +// retryable, but once its internal retries are exhausted it falls through to +// `validateStatus(response)` with `response` still undefined, throwing this +// uninformative TypeError instead of the real network error. +// https://github.com/publishlab/node-acme-client/blob/master/src/axios.js +function isUnresponsiveAcmeError(error: unknown): boolean { + return ( + error instanceof TypeError && + error.message === + "Cannot read properties of undefined (reading 'config')" + ); +} + +function normalizeAcmeError(error: unknown): Error { + if (isUnresponsiveAcmeError(error)) { + return new Error( + "ACME server did not respond after repeated attempts (network error reaching the ACME endpoint)", + { cause: error } + ); + } + return error instanceof Error ? error : new Error(String(error)); +} + +export class AcmeClientManager { + private client: acme.Client | null = null; + private accountKey: string | null = null; + + async initialize() { + try { + this.accountKey = await this.loadAccountKey(); + + this.client = new acme.Client({ + directoryUrl: config.getRawConfig().acme!.acme_directory_url, + accountKey: this.accountKey + }); + + // Try to create account or get existing one + await this.client.createAccount({ + termsOfServiceAgreed: true, + contact: [`mailto:${config.getRawConfig().acme!.contact_email}`] + }); + + logger.info("ACME client initialized successfully"); + } catch (error) { + logger.error("Failed to initialize ACME client:", error); + throw error; + } + } + + private async loadAccountKey(): Promise { + const keyPath = config.getRawConfig().acme!.acme_account_key_path; + + if (fs.existsSync(keyPath)) { + logger.info("Loading existing account key"); + return fs.readFileSync(keyPath, "utf8"); + } else { + logger.info("Generating new account key"); + const privateKey = await acme.crypto.createPrivateKey(); + const privateKeyString = privateKey.toString(); + fs.writeFileSync(keyPath, privateKeyString); + return privateKeyString; + } + } + + getClient(): acme.Client { + if (!this.client) { + throw new Error("ACME client not initialized"); + } + return this.client; + } + + async createOrder(domain: string, wildcard: boolean = false): Promise { + const client = this.getClient(); + + const identifiers = wildcard + ? [ + { type: "dns", value: domain }, + { type: "dns", value: `*.${domain}` } + ] + : [{ type: "dns", value: domain }]; + + await acmeRateLimiter.acquire(); + const order = await client.createOrder({ + identifiers + }); + + if (wildcard) { + logger.info(`Created wildcard order for domain: ${domain}`); + } else { + logger.info(`Created order for domain: ${domain}`); + } + return order; + } + + async getAuthorizations(order: any): Promise { + const client = this.getClient(); + await acmeRateLimiter.acquire(); + return client.getAuthorizations(order); + } + + async handleDnsChallenge( + dnsChallenges: { + authz: any; + challenge: any; + }[] + ): Promise { + const client = this.getClient(); + + let challengeDomains: DnsChallenge[] = []; + + for (const { authz, challenge } of dnsChallenges) { + const keyAuthorization = + await client.getChallengeKeyAuthorization(challenge); + + // Extract the domain from authorization + const domain = authz.identifier.value; + + // Store challenge in database for DNS server to pick up + challengeDomains = await withRetry( + () => + db + .insert(dnsChallenge) + .values({ + domain: domain, + token: challenge.token, + keyAuthorization, + createdAt: Math.floor(Date.now() / 1000), + expiresAt: Math.floor( + (Date.now() + + config.getRawConfig().acme! + .challenge_ttl_ms) / + 1000 + ) + }) + .returning(), + { label: `insert dnsChallenge for domain ${domain}` } + ); + + logger.info( + `DNS challenge stored for domain: ${domain} as token ${challenge.token} and keyAuthorization` + ); + } + + await new Promise((resolve) => setTimeout(resolve, 2000)); + + const failedDomains: string[] = []; + + for (const { authz, challenge } of dnsChallenges) { + const domain = authz.identifier.value; + const challengeDomain = `_acme-challenge.${domain}`; + + try { + // The ACME server occasionally has a transient network blip + // mid-sequence; retry the whole verify/complete/wait sequence + // rather than just the DNS challenge propagation wait, since + // these calls are safe to repeat against the ACME server. + await withRetry( + async () => { + // Verify challenge + await acmeRateLimiter.acquire(); + await client.verifyChallenge(authz, challenge); + + // Complete challenge + logger.info( + `Completing challenge for domain: ${challengeDomain}` + ); + await acmeRateLimiter.acquire(); + await client.completeChallenge(challenge); + + // Wait for validation + logger.info( + `Waiting for challenge to be validated for domain: ${challengeDomain}...` + ); + await acmeRateLimiter.acquire(); + await client.waitForValidStatus(challenge); + }, + { + retries: 2, + baseDelayMs: 5000, + label: `ACME challenge completion for domain ${domain}`, + // Only retry the known network-blip crash - a + // genuine validation failure (e.g. challenge marked + // "invalid" because the DNS record wasn't found) is + // permanent and should fail immediately instead of + // burning Let's Encrypt's per-hostname failed- + // validation rate limit on retries that can't help. + shouldRetry: isUnresponsiveAcmeError + } + ); + + logger.info(`Challenge completed for domain: ${domain}`); + } catch (error) { + logger.error( + `Failed to complete challenge for domain ${domain}:`, + normalizeAcmeError(error) + ); + failedDomains.push(domain); + } + } + + for (const challengeDomain of challengeDomains) { + await this.removeDnsChallenge(challengeDomain.dnsChallengeId); + logger.info( + `Removed DNS challenge for domain: ${challengeDomain.domain}` + ); + } + + // A failed dns-01 challenge leaves the order stuck in "pending" - + // finalizing it would just fail with a confusing ACME error, so + // stop here and let the caller mark the certificate as failed. + if (failedDomains.length > 0) { + throw new Error( + `DNS-01 challenge validation failed for domain(s): ${failedDomains.join(", ")}` + ); + } + } + + async removeDnsChallenge(dnsChallengeId: number): Promise { + try { + await withRetry( + () => + db + .delete(dnsChallenge) + .where(eq(dnsChallenge.dnsChallengeId, dnsChallengeId)), + { label: `delete dnsChallenge ${dnsChallengeId}` } + ); + } catch (error) { + logger.error( + `Failed to clean up DNS challenge for id ${dnsChallengeId}:`, + error + ); + } + } + + async finalizeCertificate( + order: any, + domain: string, + wildcard: boolean = false + ): Promise<{ certificate: string; privateKey: string }> { + const client = this.getClient(); + + const altNames = wildcard ? [`*.${domain}`, domain] : [domain]; + + // Create CSR + const [privateKey, csr] = await acme.crypto.createCsr({ + altNames + }); + + // Finalize order + await acmeRateLimiter.acquire(); + const finalizedOrder = await client.finalizeOrder(order, csr); + + // Get certificate + await acmeRateLimiter.acquire(); + const certificate = await client.getCertificate(finalizedOrder); + + logger.info(`Certificate obtained for domain: ${domain}`); + + return { + certificate: certificate.toString(), + privateKey: privateKey.toString() + }; + } +} + +export const acmeClientManager = new AcmeClientManager(); diff --git a/server/private/lib/certificates/acmeRateLimiter.ts b/server/private/lib/certificates/acmeRateLimiter.ts new file mode 100644 index 000000000..98da76d1f --- /dev/null +++ b/server/private/lib/certificates/acmeRateLimiter.ts @@ -0,0 +1,71 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { privateConfig as config } from "#private/lib/config"; +import logger from "@server/logger"; +import { redis } from "../redis"; +// Caps outgoing ACME API calls to a fixed budget per wall-clock second, +// shared across all pops workers via Redis (mirrors the lockManager pattern +// in @lib/lock) - a per-process limiter wouldn't be enough since multiple +// workers issue certificates against the same Let's Encrypt account. +const ACQUIRE_SCRIPT = ` +local key = KEYS[1] +local limit = tonumber(ARGV[1]) +local current = redis.call('INCR', key) +if current == 1 then + redis.call('PEXPIRE', key, 2000) +end +if current > limit then + return 0 +else + return 1 +end +`; + +class AcmeRateLimiter { + async acquire(): Promise { + const limit = + config.getRawConfig().acme?.acme_requests_per_second ?? 15; + + for (;;) { + const bucket = Math.floor(Date.now() / 1000); + const key = `acme_rate_limit:${bucket}`; + + let allowed: number; + try { + allowed = (await redis.eval( + ACQUIRE_SCRIPT, + 1, + key, + limit.toString() + )) as number; + } catch (error) { + logger.error( + "ACME rate limiter check failed, proceeding without throttling:", + error + ); + return; + } + + if (allowed === 1) { + return; + } + + // Budget for this second is spent - wait for the next window. + const waitMs = 1000 - (Date.now() % 1000) + 10; + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + } +} + +export const acmeRateLimiter = new AcmeRateLimiter(); diff --git a/server/private/lib/certificates/certificate-service.ts b/server/private/lib/certificates/certificate-service.ts new file mode 100644 index 000000000..6b9b838ff --- /dev/null +++ b/server/private/lib/certificates/certificate-service.ts @@ -0,0 +1,511 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { acmeClientManager } from "./acme-client"; +import { dnsValidator } from "./dns-validator"; +import { getTableColumns } from "drizzle-orm"; +import { eq, and, or, isNull, lt, asc } from "drizzle-orm/sql"; +import { config } from "@server/lib/config"; +import { db, certificates, domains, Certificate } from "@server/db"; +import { encrypt } from "@server/lib/crypto"; +import { withTimeout, withRetry } from "@server/lib/retry"; +import logger from "@server/logger"; +import { lockManager } from "../lock"; +import { pushCertUpdateToAffectedNewts } from "@server/lib/acmeCertSync"; +import crypto from "crypto"; + +// Number of on-demand DNS validation attempts made right before a +// certificate is (re)issued, to avoid burning Let's Encrypt rate limits on +// domains whose DNS has drifted since they were last verified. +const PRE_CERT_DNS_VALIDATION_ATTEMPTS = 3; + +// Hard ceiling on a single certificate's issuance/renewal flow. acme-client's +// axios instance never sets a request timeout, so a stalled connection to +// the ACME server hangs forever instead of erroring - and since +// processPendingCertificates/processRenewalCandidates gate the *next* batch +// on Promise.all(...) over the current one, one hung certificate would +// otherwise stall every other domain permanently. Sized generously above the +// legitimate worst case (acme-client's own bounded backoff is ~3.6min per +// status-polling loop, and a wildcard cert's two identifiers plus order +// finalization can chain a few of those) so this only fires on a genuine hang. +const CERTIFICATE_ISSUANCE_TIMEOUT_MS = 20 * 60 * 1000; + +// "requested" is set the instant a cert starts processing and is never +// queried anywhere else - processPendingCertificates only selects "pending" +// and processRenewalCandidates only selects "valid". So if the *process* +// dies mid-flight (OOM, node eviction, a rolling deploy) rather than just +// hanging, the row is orphaned in "requested" permanently with nothing to +// ever pick it back up, no matter how good the in-process timeouts are. +// Threshold is set comfortably above CERTIFICATE_ISSUANCE_TIMEOUT_MS plus the +// scheduler's own outer backstop so this never reclaims a cert that's still +// genuinely being worked on. +const STUCK_CERTIFICATE_THRESHOLD_MS = 40 * 60 * 1000; + +export class CertificateService { + // Runs at the top of every processPendingCertificates tick so an + // interrupted worker's leftovers always get put back in the queue + // instead of sitting invisible to every query forever. + private async reclaimStuckCertificates(): Promise { + const staleBefore = + Math.floor(Date.now() / 1000) - + Math.floor(STUCK_CERTIFICATE_THRESHOLD_MS / 1000); + + const reclaimed = await db + .update(certificates) + .set({ + status: "pending", + errorMessage: + 'Reclaimed after being stuck in "requested" state - the worker processing it likely restarted or crashed', + updatedAt: Math.floor(Date.now() / 1000) + }) + .where( + and( + eq(certificates.status, "requested"), + lt(certificates.updatedAt, staleBefore) + ) + ) + .returning({ domain: certificates.domain }); + + if (reclaimed.length > 0) { + logger.warn( + `Reclaimed ${reclaimed.length} certificate(s) stuck in "requested" state: ${reclaimed + .map((c) => c.domain) + .join(", ")}` + ); + } + } + + async processPendingCertificates(): Promise { + logger.debug("Checking for pending certificates..."); + + await this.reclaimStuckCertificates(); + + const pendingCerts = await db + .select(getTableColumns(certificates)) + .from(certificates) + .leftJoin(domains, eq(certificates.domainId, domains.domainId)) + .where( + and( + eq(certificates.status, "pending"), + or( + // Certs with no linked domain row (e.g. legacy certs + // imported from acme.json) aren't gated on domain + // verification since there's nothing to check. + isNull(certificates.domainId), + and( + eq(domains.verified, true), + eq(domains.failed, false) + ) + ) + ) + ) + .limit(10); + + if (pendingCerts.length === 0) { + logger.debug("No pending certificates found"); + return; + } + + logger.info(`Found ${pendingCerts.length} pending certificates`); + + // Process the batch concurrently so one domain stuck retrying a slow + // DNS-01 challenge (the ACME client's waitForValidStatus can spend + // minutes on a bad domain) doesn't stall the rest of the batch. + // processSingleCertificate catches its own errors and each cert uses + // an independent per-domain lock, so this is safe to parallelize. + await Promise.all( + pendingCerts.map((cert) => this.processSingleCertificate(cert)) + ); + } + + async processRenewalCandidates(): Promise { + logger.debug("Checking for certificates needing renewal..."); + + const now = Math.floor(Date.now() / 1000); + + const renewalCandidates = await db + .select(getTableColumns(certificates)) + .from(certificates) + .leftJoin(domains, eq(certificates.domainId, domains.domainId)) + .where( + and( + eq(certificates.status, "valid"), + lt(certificates.expiresAt, now + 15 * 24 * 60 * 60), // 15 days from now + or( + // Certs with no linked domain row (e.g. legacy certs + // imported from acme.json) aren't gated on domain + // verification since there's nothing to check. + isNull(certificates.domainId), + and( + eq(domains.verified, true), + eq(domains.failed, false) + ) + ) + ) + ) + // Most urgent first, so already-expired certs aren't starved + // behind the limit by certs that still have weeks of runway. + .orderBy(asc(certificates.expiresAt)) + .limit(50); + + if (renewalCandidates.length === 0) { + logger.debug("No certificates need renewal"); + return; + } + + logger.info( + `Found ${renewalCandidates.length} certificates needing renewal` + ); + + for (const cert of renewalCandidates) { + if (cert.expiresAt !== null && cert.expiresAt < now) { + logger.warn( + `Certificate for ${cert.domain} is marked "valid" but already expired at ${new Date(cert.expiresAt * 1000).toISOString()} (bad state) - renewing immediately` + ); + } + } + + // Process the batch concurrently - see processPendingCertificates for why. + await Promise.all( + renewalCandidates.map((cert) => this.renewCertificate(cert)) + ); + } + + private async processSingleCertificate(cert: Certificate): Promise { + const lockKey = `cert:${cert.domain}`; + + const lockToken = await lockManager.acquireLock(lockKey); + if (!lockToken) { + logger.debug( + `Could not acquire lock for certificate: ${cert.domain}` + ); + return; + } + + try { + logger.info(`Processing certificate for domain: ${cert.domain}`); + + // Update status to processing + await db + .update(certificates) + .set({ + status: "requested", + updatedAt: Math.floor(Date.now() / 1000) + }) + .where(eq(certificates.certId, cert.certId)); + // + + await withTimeout( + this.obtainCertificate(cert), + CERTIFICATE_ISSUANCE_TIMEOUT_MS, + `certificate issuance for ${cert.domain}` + ); + } catch (error) { + logger.error( + `Failed to process certificate for ${cert.domain}:`, + error + ); + + await db + .update(certificates) + .set({ + status: "failed", + errorMessage: + error instanceof Error + ? error.message + : "Unknown error", + updatedAt: Math.floor(Date.now() / 1000) + }) + .where(eq(certificates.certId, cert.certId)); + } finally { + await lockManager.releaseLock(lockKey, lockToken); + } + } + + private async renewCertificate(cert: Certificate): Promise { + const lockKey = `cert:${cert.domain}`; + + const lockToken = await lockManager.acquireLock(lockKey); + if (!lockToken) { + logger.debug( + `Could not acquire lock for certificate renewal: ${cert.domain}` + ); + return; + } + + try { + logger.info(`Renewing certificate for domain: ${cert.domain}`); + + // Update last renewal attempt + await db + .update(certificates) + .set({ + lastRenewalAttempt: Math.floor(Date.now() / 1000), + updatedAt: Math.floor(Date.now() / 1000) + }) + .where(eq(certificates.certId, cert.certId)); + + await withTimeout( + this.obtainCertificate(cert), + CERTIFICATE_ISSUANCE_TIMEOUT_MS, + `certificate renewal for ${cert.domain}` + ); + } catch (error) { + logger.error( + `Failed to renew certificate for ${cert.domain}:`, + error + ); + + await db + .update(certificates) + .set({ + status: "failed", + errorMessage: + error instanceof Error + ? error.message + : "Unknown error", + lastRenewalAttempt: Math.floor(Date.now() / 1000), + updatedAt: Math.floor(Date.now() / 1000) + }) + .where(eq(certificates.certId, cert.certId)); + } finally { + await lockManager.releaseLock(lockKey, lockToken); + } + } + + // Re-checks the domain's DNS records right before we spend a Let's + // Encrypt order on it, so drift that happened after the domain was + // originally verified doesn't burn ACME rate limits. Certs with no + // linked domain row (e.g. legacy/manually-managed certs) skip this and + // proceed as before, since there are no tracked DNS records to check. + private async verifyDomainBeforeIssuance(cert: Certificate): Promise { + if (!cert.domainId) { + return; + } + + const [domain] = await db + .select() + .from(domains) + .where(eq(domains.domainId, cert.domainId)) + .limit(1); + + if (!domain) { + return; + } + + for ( + let attempt = 1; + attempt <= PRE_CERT_DNS_VALIDATION_ATTEMPTS; + attempt++ + ) { + // Offset `tries` so each attempt round-robins to a different + // privateConfigured DNS resolver instead of re-querying the same one. + const probe = { ...domain, tries: domain.tries + attempt - 1 }; + if ( + await dnsValidator.validateDomain(probe, { + forceRecheck: true + }) + ) { + await db + .update(domains) + .set({ verified: true, failed: false, errorMessage: null }) + .where(eq(domains.domainId, domain.domainId)); + return; + } + + logger.warn( + `Pre-certificate DNS check ${attempt}/${PRE_CERT_DNS_VALIDATION_ATTEMPTS} failed for domain ${domain.baseDomain} (cert: ${cert.domain})` + ); + } + + const errorMessage = `Domain failed DNS validation ${PRE_CERT_DNS_VALIDATION_ATTEMPTS} times before certificate issuance`; + await db + .update(domains) + .set({ verified: false, failed: true, errorMessage }) + .where(eq(domains.domainId, domain.domainId)); + + throw new Error(errorMessage); + } + + private async obtainCertificate(cert: Certificate): Promise { + await this.verifyDomainBeforeIssuance(cert); + + // Create order + const order = await acmeClientManager.createOrder( + cert.domain, + cert.wildcard || false + ); + + // Update with order ID + await withRetry( + () => + db + .update(certificates) + .set({ + orderId: order.url, + updatedAt: Math.floor(Date.now() / 1000) + }) + .where(eq(certificates.certId, cert.certId)), + { label: `update orderId for certificate ${cert.domain}` } + ); + + // Get authorizations + const authorizations = await acmeClientManager.getAuthorizations(order); + + // Aggregate all DNS-01 challenges + const dnsChallenges = authorizations.map((authz: any) => { + const dnsChallenge = authz.challenges.find( + (c: any) => c.type === "dns-01" + ); + if (!dnsChallenge) { + throw new Error( + `No DNS-01 challenge found for ${authz.identifier.value}` + ); + } + return { + authz, + challenge: dnsChallenge + }; + }); + + // Send all DNS-01 challenges in one request to handleDnsChallenge + await acmeClientManager.handleDnsChallenge(dnsChallenges); + + // Finalize certificate + const { certificate, privateKey } = + await acmeClientManager.finalizeCertificate( + order, + cert.domain, + cert.wildcard || false + ); + + const encryptionKey = config.getRawConfig().server.secret; + if (!encryptionKey) { + throw new Error("Encryption key not provided"); + } + + // Encrypt certificate and private key + const encryptedCert = encrypt(certificate, encryptionKey); + const encryptedKey = encrypt(privateKey, encryptionKey); + + // Parse certificate to get expiration date + const expiresAt = this.extractExpirationDate(certificate); + + // Update database record. This persists the certificate we just + // obtained from the ACME server, so it's retried aggressively - + // losing this write means re-issuing the cert from scratch. + await withRetry( + () => + db + .update(certificates) + .set({ + status: "valid", + expiresAt: Math.floor(expiresAt.getTime() / 1000), + renewalCount: (cert.renewalCount || 0) + 1, + errorMessage: null, + updatedAt: Math.floor(Date.now() / 1000), + certFile: encryptedCert, + keyFile: encryptedKey + }) + .where(eq(certificates.certId, cert.certId)), + { + retries: 5, + label: `persist issued certificate for ${cert.domain}` + } + ); + + logger.info( + `Certificate successfully obtained/renewed for domain: ${cert.domain}` + ); + + await pushCertUpdateToAffectedNewts( + cert.domain, + cert.domainId ?? null, + certificate, + privateKey + ); + } + + private extractExpirationDate(certificate: string): Date { + try { + // Extract the certificate block + const pem = certificate + .replace(/-----BEGIN CERTIFICATE-----/g, "") + .replace(/-----END CERTIFICATE-----/g, "") + .replace(/\s+/g, ""); + const der = Buffer.from(pem, "base64"); + + // Use Node.js crypto to parse the certificate + const x509 = new crypto.X509Certificate(der); + return new Date(x509.validTo); + } catch (error) { + logger.warn( + "Failed to parse certificate expiration date, using default", + error + ); + // Default to 90 days from now (Let's Encrypt default) + return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000); + } + } + + async addCertificateRequest(domain: string): Promise { + try { + await db.insert(certificates).values({ + domain, + status: "pending", + createdAt: Math.floor(Date.now() / 1000), + updatedAt: Math.floor(Date.now() / 1000) + }); + logger.info(`Certificate request added for domain: ${domain}`); + } catch (error) { + if (error instanceof Error && error.message.includes("unique")) { + logger.warn( + `Certificate request already exists for domain: ${domain}` + ); + } else { + throw error; + } + } + } + + async getCertificateStatus(domain: string) { + const cert = await db + .select() + .from(certificates) + .where(eq(certificates.domain, domain)) + .limit(1); + + return cert[0] || null; + } + + async cleanupExpiredChallenges(): Promise { + try { + const result = await db + .delete(certificates) + .where( + lt(certificates.expiresAt, Math.floor(Date.now() / 1000)) + ) + .returning(); + + if (result.length > 0) { + logger.info( + `Cleaned up ${result.length} expired DNS challenges` + ); + } + } catch (error) { + logger.error("Failed to cleanup expired challenges:", error); + } + } +} + +export const certificateService = new CertificateService(); diff --git a/server/private/lib/certificates/dns-validator.ts b/server/private/lib/certificates/dns-validator.ts new file mode 100644 index 000000000..2ddfc0f33 --- /dev/null +++ b/server/private/lib/certificates/dns-validator.ts @@ -0,0 +1,334 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { eq, and, lt } from "drizzle-orm"; +import * as dns from "dns/promises"; +import { privateConfig as config } from "#private/lib/config"; +import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db"; +import logger from "@server/logger"; +import { lockManager } from "../lock"; + +export const DNS_VALIDATOR_MAX_TRIES = 300; + +export class DNSValidator { + private static readonly MAX_TRIES = DNS_VALIDATOR_MAX_TRIES; + + constructor() {} + + async validateAll(): Promise { + // Get all domains that are not yet verified and haven't exceeded max tries + const unverifiedDomains: Domain[] = await db + .select() + .from(domains) + .where( + and( + eq(domains.verified, false), + lt(domains.tries, DNSValidator.MAX_TRIES) + ) + ); + + if (unverifiedDomains.length === 0) { + logger.debug("No unverified domains found for DNS validation"); + return; + } + + logger.info(`Validating ${unverifiedDomains.length} DNS records`); + + for (const domain of unverifiedDomains) { + const lockKey = `dns:${domain.baseDomain}`; + const lockToken = await lockManager.acquireLock(lockKey); + if (!lockToken) { + logger.debug( + `Could not acquire lock for DNS validation: ${domain.baseDomain}` + ); + continue; + } + try { + const isValid = await this.validateDomain(domain); + if (isValid) { + await db + .update(domains) + .set({ + verified: true, + failed: false, + tries: 0, + errorMessage: null + }) + .where(eq(domains.domainId, domain.domainId)); + logger.info( + `Domain ${domain.baseDomain} validated successfully` + ); + } else { + const newTries = domain.tries + 1; + const shouldMarkAsFailed = + newTries >= DNSValidator.MAX_TRIES; + + await db + .update(domains) + .set({ + tries: newTries, + failed: shouldMarkAsFailed + }) + .where(eq(domains.domainId, domain.domainId)); + + if (shouldMarkAsFailed) { + logger.warn( + `Domain ${domain.baseDomain} exceeded maximum tries (${DNSValidator.MAX_TRIES}), marking as failed` + ); + } else { + logger.debug( + `Domain ${domain.baseDomain} did not validate (attempt ${newTries}/${DNSValidator.MAX_TRIES})` + ); + } + } + } catch (err) { + logger.warn( + `Error validating domain ${domain.baseDomain}:`, + err + ); + // Increment tries even on error + const newTries = domain.tries + 1; + const shouldMarkAsFailed = newTries >= DNSValidator.MAX_TRIES; + + await db + .update(domains) + .set({ + tries: newTries, + failed: shouldMarkAsFailed + }) + .where(eq(domains.domainId, domain.domainId)); + } finally { + await lockManager.releaseLock(lockKey, lockToken); + } + } + } + + async validateDomain( + domain: Domain, + opts: { forceRecheck?: boolean } = {} + ): Promise { + const { forceRecheck = false } = opts; + const resolver = new dns.Resolver(); + const servers = config.getRawConfig().acme?.dns_resolvers; + if (!servers || servers.length === 0) { + throw new Error("No DNS resolvers configured"); + } + const dnsServer = servers[domain.tries % servers.length]!; + resolver.setServers([dnsServer]); + logger.debug( + `Using DNS server ${dnsServer} for domain ${domain.baseDomain} (try ${domain.tries})` + ); + + // Get all DNS records for this domain + const records: DnsRecord[] = await db + .select() + .from(dnsRecords) + .where(eq(dnsRecords.domainId, domain.domainId)); + + if (records.length === 0) { + logger.warn(`No DNS records found for domain ${domain.baseDomain}`); + return false; + } + + if (!forceRecheck && records.every((r) => r.verified)) { + logger.info( + `All DNS records already verified for domain ${domain.baseDomain}` + ); + return true; + } + + logger.info( + `Validating ${records.length} DNS records for domain ${domain.baseDomain}` + ); + + // Collect the full set of expected NS values for this domain so we can + // detect extra records that are present in DNS but not in our DB. + const expectedNsValues = new Set( + records.filter((r) => r.recordType === "NS").map((r) => r.value) + ); + + // Cache resolved NS records across iterations — there will be 3 NS + // records in the DB and we don't need to hit the upstream server 3 times. + let previousNs: string[] | null = null; + + for (const record of records) { + // Skip already verified records, unless a live recheck was requested + if (record.verified && !forceRecheck) { + continue; + } + + let isValid = false; + + try { + if (record.recordType === "NS") { + let nsRecords: string[] | null = previousNs; + if (!nsRecords) { + nsRecords = await resolver.resolveNs( + record.baseDomain || domain.baseDomain + ); + } + logger.info( + `NS records for ${ + record.baseDomain || domain.baseDomain + }:`, + nsRecords + ); + + // Check if this expected NS value is present in the live records. + // A stale/legacy expected value (e.g. left over from a + // nameserver rebrand) is also accepted as long as the live + // records resolve to some other known-valid nameserver — + // the specific literal hostname stored per-domain isn't + // meaningful once it's a recognized alias. + isValid = nsRecords.some((ns) => ns === record.value); + + previousNs = nsRecords; + } else if (record.recordType === "CNAME") { + const cnameRecords = await resolver.resolveCname( + record.baseDomain || domain.baseDomain + ); + logger.info( + `CNAME records for ${ + record.baseDomain || domain.baseDomain + }:`, + cnameRecords + ); + + // Check if the CNAME record matches the expected value + isValid = + cnameRecords.length === 1 && + cnameRecords[0] === record.value; + } else if (record.recordType === "TXT") { + const txtRecords = await resolver.resolveTxt( + record.baseDomain || domain.baseDomain + ); + logger.info( + `TXT records for ${ + record.baseDomain || domain.baseDomain + }:`, + txtRecords + ); + + // TXT records come as an array of arrays, flatten and check + const flatTxtRecords = txtRecords.flat(); + isValid = flatTxtRecords.includes(record.value); + } else if (record.recordType === "A") { + const aRecords = await resolver.resolve4( + record.baseDomain || domain.baseDomain + ); + logger.info( + `A records for ${ + record.baseDomain || domain.baseDomain + }:`, + aRecords + ); + + // Check if the A record matches the expected value + isValid = aRecords.includes(record.value); + } else { + logger.warn( + `Unsupported record type: ${record.recordType}` + ); + continue; + } + } catch (error) { + isValid = false; + logger.debug( + `Did not resolve ${record.recordType} record for ${ + record.baseDomain || domain.baseDomain + }:`, + error + ); + } + + // Update the individual record verification status. Runs for + // both a mismatched value and a failed/thrown DNS lookup, so a + // previously-verified record that stops resolving (e.g. NXDOMAIN + // after NS delegation is dropped) gets downgraded instead of + // leaving stale `verified: true` state behind. + if (isValid) { + await db + .update(dnsRecords) + .set({ verified: true }) + .where(eq(dnsRecords.id, record.id)); + logger.info( + `DNS record ${record.id} (${record.recordType}) for ${ + record.baseDomain || domain.baseDomain + } verified successfully` + ); + } else { + if (record.verified) { + await db + .update(dnsRecords) + .set({ verified: false }) + .where(eq(dnsRecords.id, record.id)); + } + logger.debug( + `DNS record ${record.id} (${record.recordType}) for ${ + record.baseDomain || domain.baseDomain + } does not match expected value: ${record.value}` + ); + } + } + + // --- Extra NS record check --- + // If we resolved NS records during this pass, verify that the live DNS + // has no nameservers beyond the ones we expect. Individual records may + // already be marked verified above, but we must block full domain + // verification until the extra records are removed. + if (previousNs !== null && expectedNsValues.size > 0) { + const extraNsRecords = previousNs.filter( + (ns) => !expectedNsValues.has(ns) + ); + + if (extraNsRecords.length > 0) { + const errorMessage = `Extra NS records found that are not expected: ${extraNsRecords.join(", ")}. Remove these nameservers to complete domain verification.`; + + await db + .update(domains) + .set({ errorMessage }) + .where(eq(domains.domainId, domain.domainId)); + + logger.warn( + `Domain ${domain.baseDomain} has extra NS records that prevent verification: ${extraNsRecords.join(", ")}` + ); + + return false; + } + + // No extras — clear any stale error that was previously written + await db + .update(domains) + .set({ errorMessage: null }) + .where(eq(domains.domainId, domain.domainId)); + } + + // Check if all records are now verified + const updatedRecords: DnsRecord[] = await db + .select() + .from(dnsRecords) + .where(eq(dnsRecords.domainId, domain.domainId)); + + const allRecordsVerified = updatedRecords.every((r) => r.verified); + + logger.info( + `Domain ${domain.baseDomain}: ${ + updatedRecords.filter((r) => r.verified).length + }/${updatedRecords.length} records verified` + ); + + return allRecordsVerified; + } +} + +export const dnsValidator = new DNSValidator(); diff --git a/server/private/lib/certificates/domain-reverifier.ts b/server/private/lib/certificates/domain-reverifier.ts new file mode 100644 index 000000000..d18ed4bcb --- /dev/null +++ b/server/private/lib/certificates/domain-reverifier.ts @@ -0,0 +1,233 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { eq, and, or, isNull, lt } from "drizzle-orm"; +import * as dns from "dns/promises"; +import { DNS_VALIDATOR_MAX_TRIES } from "./dns-validator"; +import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db"; +import logger from "@server/logger"; +import { lockManager } from "../lock"; +import { privateConfig as config } from "#private/lib/config"; + +// Module-level counter so successive domains in a batch round-robin across servers. +let serverIndex = 0; + +export class DomainReverifier { + async reverifyAll(): Promise { + const certConfig = config.getRawConfig().acme; + if (!certConfig) { + logger.debug( + "No certificate config — skipping domain reverification" + ); + return; + } + + const windowMs = certConfig.domain_reverification_window_ms; + const batchSize = certConfig.domain_reverification_batch_size; + const windowSecs = Math.floor(windowMs / 1000); + const cutoff = Math.floor(Date.now() / 1000) - windowSecs; + + const domainsToCheck: Domain[] = await db + .select() + .from(domains) + .where( + and( + eq(domains.verified, true), + or( + isNull(domains.lastCheckedAt), + lt(domains.lastCheckedAt, cutoff) + ) + ) + ) + .limit(batchSize); + + if (domainsToCheck.length === 0) { + logger.debug("No verified domains due for reverification"); + return; + } + + logger.info(`Reverifying ${domainsToCheck.length} domains`); + + for (const domain of domainsToCheck) { + const lockKey = `dns-reverify:${domain.baseDomain}`; + const lockToken = await lockManager.acquireLock(lockKey); + if (!lockToken) { + logger.debug( + `Could not acquire lock for domain reverification: ${domain.baseDomain}` + ); + continue; + } + + try { + await this.reverifyDomain(domain, certConfig.dns_resolvers); + } catch (err) { + logger.warn( + `Unexpected error reverifying domain ${domain.baseDomain}:`, + err + ); + // Still stamp lastCheckedAt so we don't hammer a broken domain every run. + await db + .update(domains) + .set({ lastCheckedAt: Math.floor(Date.now() / 1000) }) + .where(eq(domains.domainId, domain.domainId)); + } finally { + await lockManager.releaseLock(lockKey, lockToken); + } + } + } + + private async reverifyDomain( + domain: Domain, + servers: string[] + ): Promise { + if (!servers || servers.length === 0) { + throw new Error("No DNS resolvers configured"); + } + + // Round-robin across servers; advance the global counter so the next + // domain in the same batch gets a different server. + const dnsServer = servers[serverIndex % servers.length]!; + serverIndex++; + + const resolver = new dns.Resolver(); + resolver.setServers([dnsServer]); + + logger.debug( + `Reverifying domain ${domain.baseDomain} using DNS server ${dnsServer}` + ); + + const records: DnsRecord[] = await db + .select() + .from(dnsRecords) + .where(eq(dnsRecords.domainId, domain.domainId)); + + if (records.length === 0) { + logger.warn( + `No DNS records found for domain ${domain.baseDomain} during reverification — marking failed` + ); + await this.markFailed( + domain.domainId, + "No DNS records found during periodic reverification" + ); + return; + } + + const expectedNsValues = new Set( + records.filter((r) => r.recordType === "NS").map((r) => r.value) + ); + + let allValid = true; + let errorMessage: string | null = null; + let resolvedNs: string[] | null = null; + + for (const record of records) { + let isValid = false; + + try { + if (record.recordType === "NS") { + if (!resolvedNs) { + resolvedNs = await resolver.resolveNs( + record.baseDomain || domain.baseDomain + ); + } + isValid = resolvedNs.some((ns) => ns === record.value); + } else if (record.recordType === "CNAME") { + const cnameRecords = await resolver.resolveCname( + record.baseDomain || domain.baseDomain + ); + isValid = + cnameRecords.length === 1 && + cnameRecords[0] === record.value; + } else if (record.recordType === "TXT") { + const txtRecords = await resolver.resolveTxt( + record.baseDomain || domain.baseDomain + ); + isValid = txtRecords.flat().includes(record.value); + } else if (record.recordType === "A") { + const aRecords = await resolver.resolve4( + record.baseDomain || domain.baseDomain + ); + isValid = aRecords.includes(record.value); + } else { + logger.warn( + `Unsupported record type ${record.recordType} during reverification of ${domain.baseDomain}` + ); + continue; + } + } catch (err) { + logger.debug( + `DNS lookup failed for ${record.recordType} record on ${record.baseDomain || domain.baseDomain}:`, + err + ); + isValid = false; + } + + if (!isValid) { + allValid = false; + errorMessage = `${record.recordType} record for ${record.baseDomain || domain.baseDomain} no longer resolves to expected value "${record.value}"`; + break; + } + } + + // Check for extra NS records beyond what we expect. + if (allValid && resolvedNs !== null && expectedNsValues.size > 0) { + const extraNs = resolvedNs.filter( + (ns) => !expectedNsValues.has(ns) + ); + if (extraNs.length > 0) { + allValid = false; + errorMessage = `Extra NS records found: ${extraNs.join(", ")}. Remove these nameservers.`; + } + } + + const now = Math.floor(Date.now() / 1000); + + if (allValid) { + await db + .update(domains) + .set({ lastCheckedAt: now, errorMessage: null }) + .where(eq(domains.domainId, domain.domainId)); + logger.debug( + `Domain ${domain.baseDomain} passed periodic reverification` + ); + } else { + await this.markFailed(domain.domainId, errorMessage); + logger.warn( + `Domain ${domain.baseDomain} failed periodic reverification: ${errorMessage}` + ); + } + } + + private async markFailed( + domainId: string, + errorMessage: string | null + ): Promise { + await db + .update(domains) + .set({ + verified: false, + failed: true, + // Three below MAX_TRIES: keeps the domain out of the DNS + // validator's immediate retry loop, while still leaving it + // eligible (tries < MAX_TRIES) for a few more validation + // passes instead of being excluded forever once tries hits + // MAX_TRIES. + tries: DNS_VALIDATOR_MAX_TRIES - 3, + lastCheckedAt: Math.floor(Date.now() / 1000), + errorMessage + }) + .where(eq(domains.domainId, domainId)); + } +} + +export const domainReverifier = new DomainReverifier(); diff --git a/server/private/lib/certificates/index.ts b/server/private/lib/certificates/index.ts new file mode 100644 index 000000000..f5dfacb90 --- /dev/null +++ b/server/private/lib/certificates/index.ts @@ -0,0 +1,40 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import logger from "@server/logger"; +import { acmeClientManager } from "./acme-client"; +import { jobScheduler } from "./scheduler"; + +// Request a new certificate +// await certificateService.addCertificateRequest(domain); + +logger.info("Starting certificate management server..."); + +// Initialize ACME client +await acmeClientManager.initialize(); + +// Start job scheduler +await jobScheduler.start(); + +// Graceful shutdown +process.on("SIGTERM", async () => { + logger.info("Received SIGTERM, shutting down gracefully"); + await jobScheduler.stop(); + process.exit(0); +}); + +process.on("SIGINT", async () => { + logger.info("Received SIGINT, shutting down gracefully"); + await jobScheduler.stop(); + process.exit(0); +}); diff --git a/server/private/lib/certificates/scheduler.ts b/server/private/lib/certificates/scheduler.ts new file mode 100644 index 000000000..70ec674eb --- /dev/null +++ b/server/private/lib/certificates/scheduler.ts @@ -0,0 +1,163 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { withTimeout } from "@server/lib/retry"; +import logger from "@server/logger"; +import { certificateService } from "./certificate-service"; +import { privateConfig as config } from "#private/lib/config"; +import { dnsValidator } from "./dns-validator"; +import { domainReverifier } from "./domain-reverifier"; + +// Backstop for runExclusive: no single job's own internal timeouts (e.g. +// certificate-service's per-cert issuance timeout) are relied on here. This +// is the last line of defense - if *anything* inside a job hangs with no +// error (a stalled Redis/DB call, a future code path that forgets to bound +// itself, etc.), state.active must still reset so the next tick can run. +// Without it, one hung run permanently skips every future tick for that job, +// since runExclusive only clears state.active after the job promise settles. +const RUN_EXCLUSIVE_TIMEOUT_MS = 30 * 60 * 1000; + +export class JobScheduler { + private intervals: NodeJS.Timeout[] = []; + private running = false; + + // Guards against a slow batch (e.g. 10 certs whose DNS challenges take a + // while) still being processed when the next interval tick fires - + // without this, overlapping ticks would each pull their own batch of up + // to 10 pending/renewal certs and process them concurrently instead of + // waiting for the prior batch to finish. + private runExclusive( + job: () => Promise, + state: { active: boolean }, + label: string + ): () => Promise { + return async () => { + if (state.active) { + logger.debug( + `Skipping ${label} tick - previous run still in progress` + ); + return; + } + state.active = true; + try { + await withTimeout(job(), RUN_EXCLUSIVE_TIMEOUT_MS, label); + } catch (error) { + logger.error(`Error in ${label}:`, error); + } finally { + state.active = false; + } + }; + } + + async start(): Promise { + if (this.running) { + logger.warn("Scheduler is already running"); + return; + } + + this.running = true; + logger.info("Starting job scheduler"); + + const newCertState = { active: false }; + const renewalState = { active: false }; + const dnsValidationState = { active: false }; + const reverifyState = { active: false }; + + const runNewCertCheck = this.runExclusive( + () => certificateService.processPendingCertificates(), + newCertState, + "processing pending certificates" + ); + const runRenewalCheck = this.runExclusive( + () => certificateService.processRenewalCandidates(), + renewalState, + "processing renewal candidates" + ); + const runDnsValidation = this.runExclusive( + () => dnsValidator.validateAll(), + dnsValidationState, + "validating DNS records" + ); + const runReverify = this.runExclusive( + () => domainReverifier.reverifyAll(), + reverifyState, + "reverifying domains" + ); + + // Schedule new certificate processing + const newCertInterval = setInterval( + runNewCertCheck, + config.getRawConfig().acme!.new_cert_check_interval_ms + ); + + // Schedule renewal processing (every 24 hours) + const renewalInterval = setInterval( + runRenewalCheck, + config.getRawConfig().acme!.renewal_check_interval_ms + ); + + // Schedule DNS validation + const dnsValidationInterval = setInterval( + runDnsValidation, + config.getRawConfig().acme?.dns_check_interval_ms + ); + + // Schedule periodic reverification of already-verified domains + const reverifyInterval = setInterval( + runReverify, + config.getRawConfig().acme?.domain_reverification_interval_ms ?? + 3600000 + ); + + this.intervals.push( + newCertInterval, + renewalInterval, + dnsValidationInterval, + reverifyInterval + ); + + // Run initial checks + setTimeout(async () => { + try { + await runNewCertCheck(); + // await runRenewalCheck(); + await runDnsValidation(); + } catch (error) { + logger.error("Error in initial certificate processing:", error); + } + }, 1000); // Wait 5 seconds after startup + + logger.info("Job scheduler started successfully"); + } + + async stop(): Promise { + if (!this.running) { + return; + } + + logger.info("Stopping job scheduler"); + this.running = false; + + // Clear all intervals + this.intervals.forEach((interval) => clearInterval(interval)); + this.intervals = []; + + logger.info("Job scheduler stopped"); + } + + isRunning(): boolean { + return this.running; + } +} + +export const jobScheduler = new JobScheduler(); diff --git a/server/private/lib/readConfigFile.ts b/server/private/lib/readConfigFile.ts index b4606a0b2..18a80af62 100644 --- a/server/private/lib/readConfigFile.ts +++ b/server/private/lib/readConfigFile.ts @@ -209,7 +209,65 @@ export const privateConfigSchema = z // (server/lib/readConfigFile.ts). Kept here only so existing private // config files keep parsing; any value set here is migrated into the // public config at startup by PrivateConfig (server/private/lib/config.ts). - sync_interval_ms: z.number().optional() + sync_interval_ms: z.number().optional(), + acme_directory_url: z + .string() + .url() + .default("https://acme-v02.api.letsencrypt.org/directory"), + contact_email: z.string().email(), + acme_account_key_path: z + .string() + .default("./config/account.key"), + challenge_ttl_ms: z.number().int().positive().default(300000), + renewal_check_interval_ms: z + .number() + .int() + .positive() + .default(3600000), + new_cert_check_interval_ms: z + .number() + .int() + .positive() + .default(60000), + // Kept safely under Let's Encrypt's ~20 req/s limit since this + // budget is shared across all pops workers and only covers the + // request-issuing calls we make directly (not every request + // acme-client makes internally, e.g. while polling for + // challenge/order status). + acme_requests_per_second: z + .number() + .int() + .positive() + .default(15), + dns_check_interval_ms: z + .number() + .int() + .positive() + .default(60000), + domain_reverification_interval_ms: z + .number() + .int() + .positive() + .default(3600000), // 1 hour — how often to run the reverification pass + domain_reverification_window_ms: z + .number() + .int() + .positive() + .default(259200000), // 72 hours — how old checkedAt must be before rechecking + domain_reverification_batch_size: z + .number() + .int() + .positive() + .default(20), // max domains to recheck per pass + dns_resolvers: z + .array(z.string()) + .optional() + .default([ + "8.8.8.8", + "1.1.1.1", + "9.9.9.9", + "208.67.222.222" + ]) }) .optional(), branding: z diff --git a/server/private/routers/alertRule/testAlertRule.ts b/server/private/routers/alertRule/testAlertRule.ts index 5542c60b9..19b50d3d9 100644 --- a/server/private/routers/alertRule/testAlertRule.ts +++ b/server/private/routers/alertRule/testAlertRule.ts @@ -14,7 +14,7 @@ import { getRandomItemInArray } from "@app/lib/getRandomItemInArray"; import response from "@server/lib/response"; import logger from "@server/logger"; -import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts"; +import { processTestAlerts } from "#private/lib/alerts/processTestAlerts"; import { type AlertAction } from "@server/routers/alertRule/types"; import HttpCode from "@server/types/HttpCode"; import { NextFunction, Request, Response } from "express"; diff --git a/server/private/routers/alertRule/updateAlertRule.ts b/server/private/routers/alertRule/updateAlertRule.ts index 87fa4675f..f9296d4d9 100644 --- a/server/private/routers/alertRule/updateAlertRule.ts +++ b/server/private/routers/alertRule/updateAlertRule.ts @@ -33,8 +33,11 @@ import { OpenAPITags, registry } from "@server/openApi"; import { and, eq } from "drizzle-orm"; import { encrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; -import { HC_EVENT_TYPES, SITE_EVENT_TYPES, RESOURCE_EVENT_TYPES } from "./createAlertRule"; -import { invalidateAllRemoteExitNodeSessions } from "@server/private/auth/sessions/remoteExitNode"; +import { + HC_EVENT_TYPES, + SITE_EVENT_TYPES, + RESOURCE_EVENT_TYPES +} from "./createAlertRule"; const paramsSchema = z .object({ @@ -85,35 +88,57 @@ const bodySchema = z const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes( val.eventType ); - const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes( - val.eventType - ); + const isResourceEvent = ( + RESOURCE_EVENT_TYPES as readonly string[] + ).includes(val.eventType); - if (isSiteEvent && val.siteIds !== undefined && val.siteIds.length === 0 && !val.allSites) { + if ( + isSiteEvent && + val.siteIds !== undefined && + val.siteIds.length === 0 && + !val.allSites + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "At least one siteId is required for site event types when allSites is false", + message: + "At least one siteId is required for site event types when allSites is false", path: ["siteIds"] }); } - if (isHcEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length === 0 && !val.allHealthChecks) { + if ( + isHcEvent && + val.healthCheckIds !== undefined && + val.healthCheckIds.length === 0 && + !val.allHealthChecks + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "At least one healthCheckId is required for health check event types when allHealthChecks is false", + message: + "At least one healthCheckId is required for health check event types when allHealthChecks is false", path: ["healthCheckIds"] }); } - if (isResourceEvent && val.resourceIds !== undefined && val.resourceIds.length === 0 && !val.allResources) { + if ( + isResourceEvent && + val.resourceIds !== undefined && + val.resourceIds.length === 0 && + !val.allResources + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "At least one resourceId is required for resource event types when allResources is false", + message: + "At least one resourceId is required for resource event types when allResources is false", path: ["resourceIds"] }); } - if (isSiteEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) { + if ( + isSiteEvent && + val.healthCheckIds !== undefined && + val.healthCheckIds.length > 0 + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: "healthCheckIds must not be set for site event types", @@ -129,7 +154,11 @@ const bodySchema = z }); } - if (isResourceEvent && val.siteIds !== undefined && val.siteIds.length > 0) { + if ( + isResourceEvent && + val.siteIds !== undefined && + val.siteIds.length > 0 + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: "siteIds must not be set for resource event types", @@ -137,10 +166,15 @@ const bodySchema = z }); } - if (isResourceEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) { + if ( + isResourceEvent && + val.healthCheckIds !== undefined && + val.healthCheckIds.length > 0 + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "healthCheckIds must not be set for resource event types", + message: + "healthCheckIds must not be set for resource event types", path: ["healthCheckIds"] }); } @@ -153,7 +187,6 @@ const UpdateAlertRuleResponseDataSchema = z.object({ alertRuleId: z.number() }); - registry.registerPath({ method: "post", path: "/org/{orgId}/alert-rule/{alertRuleId}", @@ -174,7 +207,9 @@ registry.registerPath({ description: "Successful response", content: { "application/json": { - schema: createApiResponseSchema(UpdateAlertRuleResponseDataSchema) + schema: createApiResponseSchema( + UpdateAlertRuleResponseDataSchema + ) } } } @@ -250,9 +285,11 @@ export async function updateAlertRule( if (name !== undefined) updateData.name = name; if (eventType !== undefined) updateData.eventType = eventType; if (enabled !== undefined) updateData.enabled = enabled; - if (cooldownSeconds !== undefined) updateData.cooldownSeconds = cooldownSeconds; + if (cooldownSeconds !== undefined) + updateData.cooldownSeconds = cooldownSeconds; if (allSites !== undefined) updateData.allSites = allSites; - if (allHealthChecks !== undefined) updateData.allHealthChecks = allHealthChecks; + if (allHealthChecks !== undefined) + updateData.allHealthChecks = allHealthChecks; if (allResources !== undefined) updateData.allResources = allResources; await db @@ -273,7 +310,11 @@ export async function updateAlertRule( // Only insert junction rows when allSites is not true const effectiveAllSites = allSites ?? false; - if (!effectiveAllSites && siteIds !== undefined && siteIds.length > 0) { + if ( + !effectiveAllSites && + siteIds !== undefined && + siteIds.length > 0 + ) { await db.insert(alertSites).values( siteIds.map((siteId) => ({ alertRuleId, @@ -290,7 +331,11 @@ export async function updateAlertRule( .where(eq(alertHealthChecks.alertRuleId, alertRuleId)); const effectiveAllHealthChecks = allHealthChecks ?? false; - if (!effectiveAllHealthChecks && healthCheckIds !== undefined && healthCheckIds.length > 0) { + if ( + !effectiveAllHealthChecks && + healthCheckIds !== undefined && + healthCheckIds.length > 0 + ) { await db.insert(alertHealthChecks).values( healthCheckIds.map((healthCheckId) => ({ alertRuleId, @@ -307,7 +352,11 @@ export async function updateAlertRule( .where(eq(alertResources.alertRuleId, alertRuleId)); const effectiveAllResources = allResources ?? false; - if (!effectiveAllResources && resourceIds !== undefined && resourceIds.length > 0) { + if ( + !effectiveAllResources && + resourceIds !== undefined && + resourceIds.length > 0 + ) { await db.insert(alertResources).values( resourceIds.map((resourceId) => ({ alertRuleId, @@ -392,7 +441,10 @@ export async function updateAlertRule( webhookActions.map((wa) => ({ alertRuleId, webhookUrl: wa.webhookUrl, - config: wa.config != null ? encrypt(wa.config, serverSecret) : null, + config: + wa.config != null + ? encrypt(wa.config, serverSecret) + : null, enabled: wa.enabled })) ); diff --git a/server/private/routers/generatedLicense/clearInstanceName.ts b/server/private/routers/generatedLicense/clearInstanceName.ts index ed176a976..21a407078 100644 --- a/server/private/routers/generatedLicense/clearInstanceName.ts +++ b/server/private/routers/generatedLicense/clearInstanceName.ts @@ -31,7 +31,9 @@ export async function clearInstanceName( next: NextFunction ): Promise { try { - const parsedParams = clearInstanceNameParamsSchema.safeParse(req.params); + const parsedParams = clearInstanceNameParamsSchema.safeParse( + req.params + ); if (!parsedParams.success) { return next( createHttpError( @@ -63,7 +65,8 @@ export async function clearInstanceName( return next( createHttpError( data.status || HttpCode.BAD_REQUEST, - data.message || "Failed to clear instance name from Fossorial API" + data.message || + "Failed to clear server ID from Fossorial API" ) ); } @@ -72,7 +75,7 @@ export async function clearInstanceName( data: null, success: true, error: false, - message: "Instance name cleared successfully", + message: "Server ID cleared successfully", status: HttpCode.OK }); } catch (error) { @@ -80,8 +83,8 @@ export async function clearInstanceName( return next( createHttpError( HttpCode.INTERNAL_SERVER_ERROR, - "An error occurred while clearing the instance name." + "An error occurred while clearing the server ID." ) ); } -} \ No newline at end of file +} diff --git a/server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts b/server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts index ec7fa6a26..8a53b08da 100644 --- a/server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts +++ b/server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts @@ -26,7 +26,7 @@ import { validateRemoteExitNodeSessionToken, EXPIRES } from "#private/auth/sessions/remoteExitNode"; -import { getOrCreateCachedToken } from "@server/private/lib/tokenCache"; +import { getOrCreateCachedToken } from "#private/lib/tokenCache"; import { verifyPassword } from "@server/auth/password"; import logger from "@server/logger"; import config from "@server/lib/config"; diff --git a/server/routers/domain/createOrgDomain.ts b/server/routers/domain/createOrgDomain.ts index 8370b5465..4fd4f9957 100644 --- a/server/routers/domain/createOrgDomain.ts +++ b/server/routers/domain/createOrgDomain.ts @@ -300,13 +300,16 @@ export async function createOrgDomain( { value: `${domainId}.${config.getRawConfig().dns.cname_extension}`, baseDomain: baseDomain - }, - { - value: `_acme-challenge.${domainId}.${config.getRawConfig().dns.cname_extension}`, - baseDomain: `_acme-challenge.${baseDomain}` } ]; + if (build == "saas") { + cnameRecords.push({ + value: `_acme-challenge.${domainId}.${config.getRawConfig().dns.cname_extension}`, + baseDomain: `_acme-challenge.${baseDomain}` + }); + } + // Save CNAME records to database for (const cnameRecord of cnameRecords) { recordsToInsert.push({ diff --git a/src/components/GenerateLicenseKeysTable.tsx b/src/components/GenerateLicenseKeysTable.tsx index e6961251b..e4555a4de 100644 --- a/src/components/GenerateLicenseKeysTable.tsx +++ b/src/components/GenerateLicenseKeysTable.tsx @@ -78,13 +78,16 @@ export default function GenerateLicenseKeysTable({ ); toast({ title: t("success"), - description: "Instance name cleared successfully" + description: "Server ID cleared successfully" }); await refreshData(); } catch (error) { toast({ title: t("error"), - description: formatAxiosError(error, "Failed to clear instance name"), + description: formatAxiosError( + error, + "Failed to clear server ID" + ), variant: "destructive" }); } finally { @@ -291,7 +294,7 @@ export default function GenerateLicenseKeysTable({ clearInstanceName(key.licenseKey) } > - Clear Instance Name + {t("clearInstanceName")}