From 81972dbb73d250843a8c5b0cba1889ea36d2c51f Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Apr 2026 10:56:12 -0700 Subject: [PATCH 1/4] Add name to migration Fixes #2943 --- server/setup/scriptsPg/1.18.0.ts | 11 ++++++++++- server/setup/scriptsSqlite/1.18.0.ts | 13 +++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/server/setup/scriptsPg/1.18.0.ts b/server/setup/scriptsPg/1.18.0.ts index df22faa2d..88b2fb5bc 100644 --- a/server/setup/scriptsPg/1.18.0.ts +++ b/server/setup/scriptsPg/1.18.0.ts @@ -16,6 +16,9 @@ export default async function migration() { thc."targetId", t."siteId", s."orgId", + r."name" AS "resourceName", + t."ip", + t."port", thc."hcEnabled", thc."hcPath", thc."hcScheme", @@ -33,13 +36,17 @@ export default async function migration() { thc."hcTlsServerName" FROM "targetHealthCheck" thc JOIN "targets" t ON thc."targetId" = t."targetId" - JOIN "sites" s ON t."siteId" = s."siteId"` + JOIN "sites" s ON t."siteId" = s."siteId" + JOIN "resources" r ON t."resourceId" = r."resourceId"` ); const existingHealthChecks = healthChecksQuery.rows as { targetHealthCheckId: number; targetId: number; siteId: number; orgId: string; + resourceName: string; + ip: string; + port: number; hcEnabled: boolean; hcPath: string | null; hcScheme: string | null; @@ -385,6 +392,7 @@ export default async function migration() { "targetId", "orgId", "siteId", + "name", "hcEnabled", "hcPath", "hcScheme", @@ -405,6 +413,7 @@ export default async function migration() { ${hc.targetId}, ${hc.orgId}, ${hc.siteId}, + ${`Resource ${hc.resourceName} - ${hc.ip}:${hc.port}`}, ${hc.hcEnabled}, ${hc.hcPath}, ${hc.hcScheme}, diff --git a/server/setup/scriptsSqlite/1.18.0.ts b/server/setup/scriptsSqlite/1.18.0.ts index 49ee8c450..a5078e2d3 100644 --- a/server/setup/scriptsSqlite/1.18.0.ts +++ b/server/setup/scriptsSqlite/1.18.0.ts @@ -22,6 +22,9 @@ export default async function migration() { thc."targetId", t."siteId", s."orgId", + r."name" AS "resourceName", + t."ip", + t."port", thc."hcEnabled", thc."hcPath", thc."hcScheme", @@ -39,13 +42,17 @@ export default async function migration() { thc."hcTlsServerName" FROM 'targetHealthCheck' thc JOIN 'targets' t ON thc."targetId" = t."targetId" - JOIN 'sites' s ON t."siteId" = s."siteId"` + JOIN 'sites' s ON t."siteId" = s."siteId" + JOIN 'resources' r ON t."resourceId" = r."resourceId"` ) .all() as { targetHealthCheckId: number; targetId: number; siteId: number; orgId: string; + resourceName: string; + ip: string; + port: number; hcEnabled: number; hcPath: string | null; hcScheme: string | null; @@ -392,6 +399,7 @@ export default async function migration() { "targetId", "orgId", "siteId", + "name", "hcEnabled", "hcPath", "hcScheme", @@ -407,7 +415,7 @@ export default async function migration() { "hcStatus", "hcHealth", "hcTlsServerName" - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ); const insertAll = db.transaction(() => { @@ -417,6 +425,7 @@ export default async function migration() { hc.targetId, hc.orgId, hc.siteId, + `Resource ${hc.resourceName} - ${hc.ip}:${hc.port}`, hc.hcEnabled, hc.hcPath, hc.hcScheme, From d3e4d8cda88e3c90cb1e9c66f743a70b01e2f694 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Apr 2026 11:39:37 -0700 Subject: [PATCH 2/4] Fix pr blueprints not picking up site --- server/lib/blueprints/clientResources.ts | 71 ++++++++++++------------ 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/server/lib/blueprints/clientResources.ts b/server/lib/blueprints/clientResources.ts index 1b2ec2ef7..21476b580 100644 --- a/server/lib/blueprints/clientResources.ts +++ b/server/lib/blueprints/clientResources.ts @@ -131,41 +131,22 @@ export async function updateClientResources( : []; const allSites: { siteId: number }[] = []; + if (resourceData.site) { - let siteSingle; - const resourceSiteId = resourceData.site; - - if (resourceSiteId) { - // Look up site by niceId - [siteSingle] = await trx - .select({ siteId: sites.siteId }) - .from(sites) - .where( - and( - eq(sites.niceId, resourceSiteId), - eq(sites.orgId, orgId) - ) + // Look up site by niceId + const [siteSingle] = await trx + .select({ siteId: sites.siteId }) + .from(sites) + .where( + and( + eq(sites.niceId, resourceData.site), + eq(sites.orgId, orgId) ) - .limit(1); - } else if (siteId) { - // Use the provided siteId directly, but verify it belongs to the org - [siteSingle] = await trx - .select({ siteId: sites.siteId }) - .from(sites) - .where( - and(eq(sites.siteId, siteId), eq(sites.orgId, orgId)) - ) - .limit(1); - } else { - throw new Error(`Target site is required`); + ) + .limit(1); + if (siteSingle) { + allSites.push(siteSingle); } - - if (!siteSingle) { - throw new Error( - `Site not found: ${resourceSiteId} in org ${orgId}` - ); - } - allSites.push(siteSingle); } if (resourceData.sites) { @@ -180,15 +161,31 @@ export async function updateClientResources( ) ) .limit(1); - if (!site) { - throw new Error( - `Site not found: ${siteId} in org ${orgId}` - ); + if (site) { + allSites.push(site); } - allSites.push(site); } } + if (siteId && allSites.length === 0) { + // only add if there are not provided sites + // Use the provided siteId directly, but verify it belongs to the org + const [siteSingle] = await trx + .select({ siteId: sites.siteId }) + .from(sites) + .where(and(eq(sites.siteId, siteId), eq(sites.orgId, orgId))) + .limit(1); + if (siteSingle) { + allSites.push(siteSingle); + } + } + + if (allSites.length === 0) { + throw new Error( + `No valid sites found for private private resource ${resourceNiceId} in org ${orgId}` + ); + } + if (existingResource) { let domainInfo: | { subdomain: string | null; domainId: string } From 416e124c021d52bc8abd7a7263990165e0211167 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Apr 2026 11:53:55 -0700 Subject: [PATCH 3/4] Rotate the secret on the new things using it --- cli/commands/rotateServerSecret.ts | 134 ++++++++++++++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/cli/commands/rotateServerSecret.ts b/cli/commands/rotateServerSecret.ts index d3828f0e5..afac262b2 100644 --- a/cli/commands/rotateServerSecret.ts +++ b/cli/commands/rotateServerSecret.ts @@ -1,5 +1,5 @@ import { CommandModule } from "yargs"; -import { db, idpOidcConfig, licenseKey } from "@server/db"; +import { db, idpOidcConfig, licenseKey, certificates, eventStreamingDestinations, alertWebhookActions } from "@server/db"; import { encrypt, decrypt } from "@server/lib/crypto"; import { configFilePath1, configFilePath2 } from "@server/lib/consts"; import { eq } from "drizzle-orm"; @@ -129,9 +129,15 @@ export const rotateServerSecret: CommandModule< console.log("\nReading encrypted data from database..."); const idpConfigs = await db.select().from(idpOidcConfig); const licenseKeys = await db.select().from(licenseKey); + const certs = await db.select().from(certificates); + const streamingDestinations = await db.select().from(eventStreamingDestinations); + const webhookActions = await db.select().from(alertWebhookActions); console.log(`Found ${idpConfigs.length} OIDC IdP configuration(s)`); console.log(`Found ${licenseKeys.length} license key(s)`); + console.log(`Found ${certs.length} certificate(s)`); + console.log(`Found ${streamingDestinations.length} event streaming destination(s)`); + console.log(`Found ${webhookActions.length} alert webhook action(s)`); // Prepare all decrypted and re-encrypted values console.log("\nDecrypting and re-encrypting values..."); @@ -149,8 +155,27 @@ export const rotateServerSecret: CommandModule< encryptedInstanceId: string; }; + type CertUpdate = { + certId: number; + encryptedCertFile: string | null; + encryptedKeyFile: string | null; + }; + + type StreamingDestinationUpdate = { + destinationId: number; + encryptedConfig: string; + }; + + type WebhookActionUpdate = { + webhookActionId: number; + encryptedConfig: string; + }; + const idpUpdates: IdpUpdate[] = []; const licenseKeyUpdates: LicenseKeyUpdate[] = []; + const certUpdates: CertUpdate[] = []; + const streamingDestinationUpdates: StreamingDestinationUpdate[] = []; + const webhookActionUpdates: WebhookActionUpdate[] = []; // Process idpOidcConfig entries for (const idpConfig of idpConfigs) { @@ -217,6 +242,70 @@ export const rotateServerSecret: CommandModule< } } + // Process certificate entries + for (const cert of certs) { + try { + const encryptedCertFile = cert.certFile + ? encrypt(decrypt(cert.certFile, oldSecret), newSecret) + : null; + const encryptedKeyFile = cert.keyFile + ? encrypt(decrypt(cert.keyFile, oldSecret), newSecret) + : null; + + certUpdates.push({ + certId: cert.certId, + encryptedCertFile, + encryptedKeyFile + }); + } catch (error) { + console.error( + `Error processing certificate ${cert.certId} (${cert.domain}):`, + error + ); + throw error; + } + } + + // Process eventStreamingDestinations entries + for (const dest of streamingDestinations) { + try { + const decryptedConfig = decrypt(dest.config, oldSecret); + const encryptedConfig = encrypt(decryptedConfig, newSecret); + + streamingDestinationUpdates.push({ + destinationId: dest.destinationId, + encryptedConfig + }); + } catch (error) { + console.error( + `Error processing event streaming destination ${dest.destinationId}:`, + error + ); + throw error; + } + } + + // Process alertWebhookActions entries + for (const webhook of webhookActions) { + try { + if (webhook.config == null) continue; + + const decryptedConfig = decrypt(webhook.config, oldSecret); + const encryptedConfig = encrypt(decryptedConfig, newSecret); + + webhookActionUpdates.push({ + webhookActionId: webhook.webhookActionId, + encryptedConfig + }); + } catch (error) { + console.error( + `Error processing alert webhook action ${webhook.webhookActionId}:`, + error + ); + throw error; + } + } + // Perform all database updates in a single transaction console.log("\nUpdating database in transaction..."); await db.transaction(async (trx) => { @@ -250,10 +339,50 @@ export const rotateServerSecret: CommandModule< instanceId: update.encryptedInstanceId }); } + + // Update certificate entries + for (const update of certUpdates) { + await trx + .update(certificates) + .set({ + certFile: update.encryptedCertFile, + keyFile: update.encryptedKeyFile + }) + .where(eq(certificates.certId, update.certId)); + } + + // Update event streaming destination entries + for (const update of streamingDestinationUpdates) { + await trx + .update(eventStreamingDestinations) + .set({ config: update.encryptedConfig }) + .where( + eq( + eventStreamingDestinations.destinationId, + update.destinationId + ) + ); + } + + // Update alert webhook action entries + for (const update of webhookActionUpdates) { + await trx + .update(alertWebhookActions) + .set({ config: update.encryptedConfig }) + .where( + eq( + alertWebhookActions.webhookActionId, + update.webhookActionId + ) + ); + } }); console.log(`Rotated ${idpUpdates.length} OIDC IdP configuration(s)`); console.log(`Rotated ${licenseKeyUpdates.length} license key(s)`); + console.log(`Rotated ${certUpdates.length} certificate(s)`); + console.log(`Rotated ${streamingDestinationUpdates.length} event streaming destination(s)`); + console.log(`Rotated ${webhookActionUpdates.length} alert webhook action(s)`); // Update config file with new secret console.log("\nUpdating config file..."); @@ -270,6 +399,9 @@ export const rotateServerSecret: CommandModule< console.log(`\nSummary:`); console.log(` - OIDC IdP configurations: ${idpUpdates.length}`); console.log(` - License keys: ${licenseKeyUpdates.length}`); + console.log(` - Certificates: ${certUpdates.length}`); + console.log(` - Event streaming destinations: ${streamingDestinationUpdates.length}`); + console.log(` - Alert webhook actions: ${webhookActionUpdates.length}`); console.log( `\n IMPORTANT: Restart the server for the new secret to take effect.` ); From 68f551273204758d9ac3935399180a356b0b616e Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Apr 2026 14:00:32 -0700 Subject: [PATCH 4/4] Handle messaging in the background; dont time out --- .../siteResource/createSiteResource.ts | 21 +++++++++--- .../siteResource/updateSiteResource.ts | 32 +++++++++++++------ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 0da48d160..01f7a0d9c 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -496,11 +496,6 @@ export async function createSiteResource( ); } } - - await rebuildClientAssociationsFromSiteResource( - newSiteResource, - trx - ); // we need to call this because we added to the admin role }); if (!newSiteResource) { @@ -526,6 +521,22 @@ export async function createSiteResource( await createCertificate(domainId, fullDomain, db); } + // Run in the background after the response is sent. Wrapped in its + // own transaction so it always executes on the primary — avoiding any + // replica-lag issues while still allowing the HTTP response to return + // early. + db.transaction(async (trx) => { + await rebuildClientAssociationsFromSiteResource( + newSiteResource!, + trx + ); + }).catch((err) => { + logger.error( + `Error rebuilding client associations for site resource ${newSiteResource!.siteResourceId}:`, + err + ); + }); + return response(res, { data: newSiteResource, success: true, diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index d0efa0cf4..8a3f93326 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -431,9 +431,6 @@ export async function updateSiteResource( }) .returning(); - // wait some time to allow for messages to be handled - await new Promise((resolve) => setTimeout(resolve, 750)); - const sshPamSet = isLicensedSshPam && (authDaemonPort !== undefined || @@ -556,11 +553,6 @@ export async function updateSiteResource( })) ); } - - await rebuildClientAssociationsFromSiteResource( - updatedSiteResource, - trx - ); } else { // Update the site resource const sshPamSet = @@ -690,7 +682,24 @@ export async function updateSiteResource( } logger.info(`Updated site resource ${siteResourceId}`); + } + }); + // Background: wait for removal messages to propagate, then rebuild + // associations for the re-created resource. Own transaction ensures + // execution on the primary against fully committed state. + (async () => { + await db.transaction(async (trx) => { + if (!updatedSiteResource) { + throw new Error("No updated resource found after update"); + } + if (sitesChanged) { + await new Promise((resolve) => setTimeout(resolve, 750)); + await rebuildClientAssociationsFromSiteResource( + updatedSiteResource, + trx + ); + } await handleMessagingForUpdatedSiteResource( existingSiteResource, updatedSiteResource, @@ -700,7 +709,12 @@ export async function updateSiteResource( })), trx ); - } + }); + })().catch((err) => { + logger.error( + `Error rebuilding client associations for site resource ${updatedSiteResource?.siteResourceId}:`, + err + ); }); return response(res, {